[ May 29 21:46:43 Enabled. ] [ May 29 21:46:43 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 29 21:46:43 Method "start" exited with status 0. ] [ May 29 21:46:43 Rereading configuration. ] [ May 29 21:46:43 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.499271859Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.499422501Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.552249998Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.552288758Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.665921091Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.674938473Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:43.683226286Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.285555724Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.427390498Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.427424408Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.427500729Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.4275619Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.42758706Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.42761087Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430279814Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:44.430296234Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430322424Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430329204Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430389735Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430458376Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430557157Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430580707Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.430605098Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.442463675Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.442492526Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"800d63a5-84ab-4e81-94a7-03a851147655 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.541062081Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744597572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744632292Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744637472Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744642452Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744653243Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744657572Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.744675733Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.752862055Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.752882135Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:44.841927382Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043089253Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043137233Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043158863Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043174143Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043180104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043186654Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043195834Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043208954Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043220794Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043227184Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043232034Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043251535Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043287695Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043298795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043303385Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043308915Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043332396Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043338785Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043343486Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043348616Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043365576Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043374846Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043379176Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043384176Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043388986Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043395136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043399506Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043426277Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043432077Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043437287Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043441877Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043447637Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043451867Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043457127Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043464627Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043480327Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043485777Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043490667Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043494967Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043523938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043528658Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043533568Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043541278Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043559048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043563598Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043570719Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043575088Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043613459Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043621769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043625819Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043660149Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04366581Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04367049Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04367561Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04367972Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.0436845Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04368911Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04369396Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043698Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04370241Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04370713Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04371144Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04371757Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04372231Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.04374133Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043746081Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043751041Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043755751Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.043759951Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.053877067Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.053906737Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999941379s","next_cleanup":"Instant { tv_sec: 1862, tv_nsec: 927767799 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.053952648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.053961118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.053987648Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.054001378Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057100247Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057160568Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057395Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1003"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057401931Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057451281Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057487052Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057501312Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057506272Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1063"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057530142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057544242Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057553922Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057573162Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1104"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057577733Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057582173Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057586253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057597553Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.057923397Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058098789Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.05814004Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.05821023Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058220611Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058394673Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058518614Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058568705Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 67239fc6-3779-4bd2-ac1a-314185fe431f, rack_id: 9090f606-344e-4bff-9fcd-4f77886b6762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058678396Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:34385","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 67239fc6-3779-4bd2-ac1a-314185fe431f, rack_id: 9090f606-344e-4bff-9fcd-4f77886b6762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058802828Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058817268Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058841358Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058846518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058850409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058855949Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058861138Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058879009Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058893119Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058899279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.058902929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.061950617Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.071218702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.081688953Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.091593856Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.100422765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.109303746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.118003534Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.126280907Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.134766172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143827495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143865415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143870386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143874986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143893086Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.143900456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.153588736Z","hostname":"sock","pid":1562,"sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.153642687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.153651697Z","hostname":"sock","pid":1562,"sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.153749298Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 79.975379ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.157324163Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66854947-8912-44eb-aaa6-5ae7111672b4\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"66854947-8912-44eb-aaa6-5ae7111672b4\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.165090029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.167903614Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.183188224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.191731281Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_800d63a5-84ab-4e81-94a7-03a851147655/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.191767391Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.19653681Z","hostname":"sock","pid":1562,"sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.196551621Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_800d63a5-84ab-4e81-94a7-03a851147655/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.196557911Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.238792916Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 90.79109ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.239333513Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"460dd27b-a7d8-421a-9200-b82e257ee18f\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"460dd27b-a7d8-421a-9200-b82e257ee18f\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.265556388Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.331712821Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 215.95192ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.332218788Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d2bcbe8-5957-4c24-8944-6a9d61013f98\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d2bcbe8-5957-4c24-8944-6a9d61013f98\" }"} {"msg":"Mounting oxi_800d63a5-84ab-4e81-94a7-03a851147655/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.483152944Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.496711113Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.496794124Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.496812284Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.496841034Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.549168635Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 155.445115ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.549844893Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f1f7eb9a-82b1-4961-9604-414dfb43ea41\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f1f7eb9a-82b1-4961-9604-414dfb43ea41\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.707352522Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 116.626785ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.708154012Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59ec1d8a-4e12-4ecd-b591-70790729a0d9\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59ec1d8a-4e12-4ecd-b591-70790729a0d9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.82627652Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 207.526121ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:45.827136361Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb0df301-97ad-4cf7-b8bb-6bb6a8bbb4de\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb0df301-97ad-4cf7-b8bb-6bb6a8bbb4de\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.035186747Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 260.859483ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.035615282Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"821d58d0-78da-4462-91dc-ca518b7f042d\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:46:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"821d58d0-78da-4462-91dc-ca518b7f042d\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.206912582Z","hostname":"sock","pid":1562,"sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.207471629Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.20748897Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.207566111Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:61502","producer_id":"67239fc6-3779-4bd2-ac1a-314185fe431f"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:46.21392746Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"156.495226ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.255575228Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.255605088Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.255686759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.255696799Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256420288Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256452188Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256478299Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256492789Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256717242Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256889764Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256913664Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256928634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.256945414Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:34385","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.438739245Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.438797846Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.439063899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.43913827Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.43915467Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.4391664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.439236511Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.439275052Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:46.440211243Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"253.465345ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:46.445332607Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.471277259Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52632"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.471426411Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.471516562Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"c8b9714e-32db-40d8-a546-955994585255","remote_addr":"[fd00:1122:3344:101::1]:52632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.500355391Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.500432272Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"2d754a23-2fd7-4b25-ac31-4d73ce929d8e","remote_addr":"[fd00:1122:3344:101::1]:52288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":32,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.501157071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.501183611Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.501486685Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.501691028Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: c7fac74c-ff6b-46fb-86b5-d71ea8414155, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool) }, OmicronPhysicalDiskConfig { id: 2943cfa4-85a5-40f3-8341-640cc8c7a118, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool) }, OmicronPhysicalDiskConfig { id: 0d5f0158-e44e-4ba3-bd4a-567051ccf8d1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool) }, OmicronPhysicalDiskConfig { id: de483768-8ce8-4e96-9af9-1bc8a421d29d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool) }, OmicronPhysicalDiskConfig { id: fd324f08-ddd2-4770-bf19-43e2485deec9, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 020b43a6-e481-4304-8596-25774e86f14b (zpool) }, OmicronPhysicalDiskConfig { id: 79469e5e-cd5b-404a-bedb-fb74819836c6, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool) }, OmicronPhysicalDiskConfig { id: 9a064006-65eb-4c19-8f5a-5c18c229c40c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool) }, OmicronPhysicalDiskConfig { id: b6295793-cf00-4673-ac4a-bd13fcb081e7, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool) }, OmicronPhysicalDiskConfig { id: 21a64282-06ac-4276-8c8b-4c5f2db10bde, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.53004436Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:347"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530127661Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45207"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: c7fac74c-ff6b-46fb-86b5-d71ea8414155, pool_id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 2943cfa4-85a5-40f3-8341-640cc8c7a118, pool_id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 0d5f0158-e44e-4ba3-bd4a-567051ccf8d1, pool_id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: de483768-8ce8-4e96-9af9-1bc8a421d29d, pool_id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: fd324f08-ddd2-4770-bf19-43e2485deec9, pool_id: 020b43a6-e481-4304-8596-25774e86f14b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 79469e5e-cd5b-404a-bedb-fb74819836c6, pool_id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 9a064006-65eb-4c19-8f5a-5c18c229c40c, pool_id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: b6295793-cf00-4673-ac4a-bd13fcb081e7, pool_id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 21a64282-06ac-4276-8c8b-4c5f2db10bde, pool_id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530356724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530372064Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530377914Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530382824Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530387374Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530397554Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530403645Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.530407745Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.650759561Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:46.650790291Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:46.827106034Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"549.653104ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:47.258701359Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:47.445724795Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.435479357s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.636121792Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810118925Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810213067Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810221217Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810227187Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810236917Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810366848Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:47.810373699Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:48.400386544Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:48.632054875Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:48.632260387Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:49.04353245Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Automatically destroying dataset: oxp_aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:49.043889804Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 42.089567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:49.221110628Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 134.829784ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:49.587729466Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:50.025708582Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:50.025824143Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.601439735s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 182.495348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:50.268595371Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 438.164099ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:50.947001167Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:50.949249845Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:51.157485743Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 530.350853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:51.614669958Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.106802766Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:52.548862982Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.548920763Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.548930623Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.548940053Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.548947013Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.548953463Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.145327903s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:52.772840457Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:52.774878312Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.781389533Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:52.781401664Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8f301520-a970-47c3-90c3-bd3006e5615c (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:53.738468323Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:53.738536334Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.581601903s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.251496291Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722149573Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722204874Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722212194Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722239274Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722245304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722339035Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:54.722350145Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:55.718573701Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:55.718621982Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:56.246578396Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:56.478512269Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:56.923893667Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:56.925943783Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_8f301520-a970-47c3-90c3-bd3006e5615c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.042190618Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.050315939Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.050355659Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.050363549Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.050368909Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.050373069Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.059671345Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.059698705Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.155733669Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164189894Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164229165Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164239865Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164287746Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164298126Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164440738Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.164449788Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.199014517Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.206990707Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:57.226138065Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_cc133e1a-85d1-46d3-b1c3-d13166bc9bff","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.328908903Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.336861311Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.336890582Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.336899042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.336904982Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.336914772Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.34642606Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.34644429Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.442742658Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450639216Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450662916Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450669366Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450703347Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450712677Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450860219Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.450872369Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.48636397Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b5814935-58c2-4e3c-9137-4cbaa1ee9fab/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.49522121Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_b5814935-58c2-4e3c-9137-4cbaa1ee9fab","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.618562704Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.626427772Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.626460722Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.626469572Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.626477752Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.626487163Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.63589742Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.63592157Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"020b43a6-e481-4304-8596-25774e86f14b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.7308035Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.738642137Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.738673177Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.738698098Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.738741228Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.738750968Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.73888468Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.73889314Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.77344111Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_020b43a6-e481-4304-8596-25774e86f14b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.781603941Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_020b43a6-e481-4304-8596-25774e86f14b","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.906837968Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.914900398Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.914931879Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.914974179Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.91499821Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.91500394Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.925252777Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:57.925268827Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.017623575Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.025858477Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.025885008Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.025893448Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.025932328Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.025940658Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.02610273Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.02611576Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:58.029910458Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.061679293Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.066872677Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Automatically destroying dataset: oxp_4532d18b-4909-4537-9ce6-98b74fb663f5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.070183648Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4532d18b-4909-4537-9ce6-98b74fb663f5","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.195706199Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.203406645Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.203447065Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.203454695Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.203460065Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.203464745Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.213263407Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.213292748Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.306396435Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314275393Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314325364Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314333954Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314348864Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314359914Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314513786Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.314521106Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.349889116Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8d81843f-0d2d-4c69-8d17-45004d7cf484/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.35821931Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_8d81843f-0d2d-4c69-8d17-45004d7cf484","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.481905948Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.489965478Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.490001208Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.490010088Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.490016619Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.490030419Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.500030453Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.500060963Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"59fcaf06-997c-43d9-9256-26a5eda3000f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.60028483Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.608246998Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.608291779Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.608298449Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.60833619Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.60836026Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.608448851Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.608458401Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.642483484Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_59fcaf06-997c-43d9-9256-26a5eda3000f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.650121169Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_59fcaf06-997c-43d9-9256-26a5eda3000f","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.771344977Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.779160643Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.779216064Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.779224244Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.779231504Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.779238024Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.789065337Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.789096077Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6b416693-d467-424a-86a1-703b05ad3101 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.888201509Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.895971566Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896003376Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896011647Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896034717Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896054357Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896182109Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.896190829Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.932304588Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6b416693-d467-424a-86a1-703b05ad3101/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:58.940190436Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:46:59.032572024Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_6b416693-d467-424a-86a1-703b05ad3101","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.060796345Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068596332Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068622062Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068664143Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068809995Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 800d63a5-84ab-4e81-94a7-03a851147655 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 92dea9e4-ea67-4c7f-80bc-15932c1cd778 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068851095Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068855965Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.068943236Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.06923204Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"6a4c3716-dd5a-48bd-8ebe-bb6cb8d3ca3c","remote_addr":"[fd00:1122:3344:101::1]:45207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":12539057,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.099874711Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.100058493Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48468"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.100362827Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.100377077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.150308078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.150335078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.20031449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.20034134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:46:59.223453527Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.248009543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.248038823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.248049413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.248116964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.290281828Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.290324059Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.290337299Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.335950726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.339569491Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.343090745Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.356885786Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.361130999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:46:59.364837135Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:47:00.03445795Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:00.885508592Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:00.885604183Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:00.885762675Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.367667256Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.847819746Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.847906587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.847941398Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 47.089157ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:01.856258191Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.856299382Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:01.859987818Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 64.780637ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:01.912913606Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 293.85354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:01.986113045Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 59.610186ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:01.991691064Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 25.762739ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:01.996627526Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 65.361389ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.03072774Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 145.093193ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.234366462Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 250.961638ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.269971024Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 412.486754ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.309954232Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 130.48279ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.404805241Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 275.167442ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.531270313Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 520.071929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.544447617Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 818.293165ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.731616084Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 825.82974ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:02.815743081Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 445.627231ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:03.074174333Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 603.041982ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:03.53154202Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 682.067461ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:03.559353726Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 608.04964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:03.651210158Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 937.656224ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:04.143719481Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 1.413476663s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:04.248899288Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 1.223968356s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:04.266854122Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:04.273052419Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:04.273062549Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe0f:a3ef","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:04.818400049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:04.82405756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:04.944355075Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 955.826306ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:05.09116946Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:05.184595232Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:05.370472263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 1.167319048s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:05.500235527Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:05.665931707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"wait for service Some(\"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f\") failed: Property not found. retry in 1.485472686s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:05.671724679Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:05.874101955Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"wait for service Some(\"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3\") failed: Property not found. retry in 1.311145227s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:06.057501455Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.061570045Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.282974808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.465764091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.475741005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.475782226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:06.475793206Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3\") failed: Property not found. retry in 769.733382ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:06.676311399Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.034690354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.041874864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.207083938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.314829017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.336615588Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.436011794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467105381Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467165541Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_a650c55b-039c-4ca8-a067-787cab718ce3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467172621Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_1e385359-7c92-410b-be48-08df0fa9ebc3"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467220442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467618657Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.467983462Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"132cb89d-8ae9-4d55-86b7-0c869870847d","remote_addr":"[fd00:1122:3344:101::1]:48468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":8367853,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.469774844Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.530694671Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.567048103Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.602773378Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.634499402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.634580843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.634930638Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.634946318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709491274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709554345Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709561795Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709586826Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709592756Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709598566Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709606006Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709611546Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709615716Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709621156Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709629136Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709634906Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709642116Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709647947Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709653696Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709662967Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709667287Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709671137Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709675077Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709679117Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709683047Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709687347Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.709691167Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to write DNS configuration (will retry in 69.51613ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:07.721295761Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:568","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.811533043Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.860027736Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:584"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.860038756Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.891924163Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.892607531Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.892612631Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.920766132Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.920884763Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39528"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.921077685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.921082885Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.942121707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.942150897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.959777757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.959799097Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.978300346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.978333376Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.978554519Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.978655291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:07.989691778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.02208888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.022702318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.022714338Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.065836155Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 187, 96]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.065903265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.066616694Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.066631704Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.066942548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.069740483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.093222905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.111717815Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.728457783Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:08.728655276Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 74.483841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:08.865819961Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 143.814583ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:08.950859269Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 162.761024ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:09.107125801Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 416.494176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:09.279689276Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 1.026561997s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:09.705497471Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 532.969443ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:10.742126909Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:10.92316867Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45087"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:10.924155463Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6f159c2f-5465-4cce-b080-f4853deff40a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45087","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":873,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 1.088555209s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:11.284114408Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\") failed: Property not found. retry in 1.485682999s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:12.381829835Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.886946369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.88701395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.887290253Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.887583797Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"19063087-a9c7-4d43-8f03-cbaf56ec0097","remote_addr":"[fd00:1122:3344:101::1]:39528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":5966648,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.889349449Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:625"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.918247988Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.918439681Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38320"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.932219722Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:13.932358444Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"a9edbd9c-14ab-4e25-af9a-b483963a2200","remote_addr":"[fd00:1122:3344:101::1]:38320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":13847,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:13.933765851Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:14.101952502Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:14.102091353Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62549"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.111174468Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.111246029Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"71db486d-b221-4514-96d1-55a7fe7d78a4","remote_addr":"[fd00:1122:3344:101::1]:62549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008999,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:18.114927765Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.28422309Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.284352181Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34547"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.292646584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.292782836Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"f2c74171-29e6-4a47-a896-85cb3711cbcd","remote_addr":"[fd00:1122:3344:101::1]:34547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8304,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:18.294755031Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.431321199Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:18.43142201Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51786"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6538e8d-59bc-4ef4-b2c8-220ca3e6d575': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.439895817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.44009802Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"601d49e5-7d26-49a2-b83a-0bf97750eedb","remote_addr":"[fd00:1122:3344:101::1]:51786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008558,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:22.44252809Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.638087502Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.638219703Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.64680506Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"62191532-fea6-4c1c-980f-618cf4fa55f3","remote_addr":"[fd00:1122:3344:101::1]:52605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":8474,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.649613115Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:22.649622145Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.854246119Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.85434073Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.862534992Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"bc291883-6e9a-47fc-bb8c-378f7f33f9a1","remote_addr":"[fd00:1122:3344:101::1]:58746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":8111,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:22.86555415Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:22.86557224Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.164973952Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.165090773Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.173118533Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"be309704-6729-47c9-a583-f53785cf5763","remote_addr":"[fd00:1122:3344:101::1]:57872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":7884,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.175325551Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:23.175332141Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.393984859Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.394088811Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.401788156Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"939e5c05-1356-4bec-a5bc-1c14fa6f5402","remote_addr":"[fd00:1122:3344:101::1]:54372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":7593,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.403911803Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:23.403918503Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.79616715Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.796268431Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63019"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.803949267Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"bf1b74f6-0c3b-4211-b06a-639929e3c6c0","remote_addr":"[fd00:1122:3344:101::1]:63019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":7585,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:23.806048753Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:23.806055443Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:24.128796465Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:24.128896926Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54709"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:24.136961057Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"f1bdcd06-d2af-4e7a-b6e8-8718b7a369c7","remote_addr":"[fd00:1122:3344:101::1]:54709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":7941,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:24.138769199Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:24.138777439Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:24.362355029Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"304.205075ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.252478569Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.25259031Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.260956084Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"e5dfa1c5-ee8a-4dbd-9e13-615a16ddb68c","remote_addr":"[fd00:1122:3344:101::1]:37293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":8276,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.2630592Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.263550947Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.020551835Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.020660697Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.041123191Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"0f72aecd-27f7-4218-b67a-cee95faf63d9","remote_addr":"[fd00:1122:3344:101::1]:57107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":20304,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3638887493, ip_addr: 216.229.4.69, stratum: 3, ref_time: 1717019244.7225652, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.043318918Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.043327278Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1216"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.072043645Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.072177787Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.072535451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.072542091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.092178256Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.092227226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.110435733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.110479503Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.129410799Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.129451379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.184722146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.184765037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.239819231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.239865652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.294981127Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.295024708Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.346050392Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.346096573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.398078959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.39812263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.398360153Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.398495024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.431548575Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.431580885Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.431587445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.431592555Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.431598815Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.431726077Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e06d5871-2eeb-4bce-8cb7-d8132c14a8fc","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":359384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.4319517Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"113.286797ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e06d5871-2eeb-4bce-8cb7-d8132c14a8fc\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e06d5871-2eeb-4bce-8cb7-d8132c14a8fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.546084579Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.546493784Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.546506794Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.564757311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.564794602Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.582681164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.582720314Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.600296413Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.600343173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.618773783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.618804493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.636889688Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.636928748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.654964623Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.655000383Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.672982847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.673012057Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.690226831Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.690276032Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.690500594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.690702827Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.722821627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.722845557Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.722851817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.722858437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.722863837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:25.722962398Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":176643,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:25.72311516Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"105.080144ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38cc9bb3-622f-4a6f-a8a4-1dcac8918b1c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.004282543Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.004657008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.004666668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.023645024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.023679164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.040963939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.040992229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.058331905Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.058365956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.075928444Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.075957704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.094471674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.094507505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.112114374Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.112139484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.130159528Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.130184509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.147124219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.14716285Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.147364322Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.147496064Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179519592Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179573263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179584733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179610863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179620493Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.179728925Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7b82ee99-e937-4328-a71d-db34ff674a0b","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":175270,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.179956937Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"175.191456ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7b82ee99-e937-4328-a71d-db34ff674a0b\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7b82ee99-e937-4328-a71d-db34ff674a0b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.35632694Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.356703085Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.356715365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.374700099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.374751479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.391865652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.391914263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.409487011Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.409530132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.42710589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.427146861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.445288807Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.445331587Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.462889345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.462934196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.480591135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.480635566Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.497732308Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.497772949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.497954891Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.498066153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.529136189Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.529159879Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.529165379Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.52917073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.529177239Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.529305731Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"65ff1e68-4c32-4515-8969-e38d7e0a2f5d","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":172792,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.529490994Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"212.379973ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65ff1e68-4c32-4515-8969-e38d7e0a2f5d\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"65ff1e68-4c32-4515-8969-e38d7e0a2f5d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.743825988Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.744197142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.744212323Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.761774661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.761818391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.778208675Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.778254876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.795070555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.795103745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.811685562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.811730112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.828717933Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.828744414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.845440621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.845467662Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.863310763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.863338974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.880378516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.880411386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.880605568Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.88074075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.911248649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.91129642Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.91130713Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.91131763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.91132584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:26.911460982Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7cd38bea-b456-46cd-8526-df26939f6af9","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":167463,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:26.911686685Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"197.780628ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7cd38bea-b456-46cd-8526-df26939f6af9\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7cd38bea-b456-46cd-8526-df26939f6af9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.109882969Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.110147742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.110155973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.131239295Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.131279435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.149996118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.150032598Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.168478228Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.168525868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.187530385Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.187576685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.205944853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.205988074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.223930767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.223991428Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.242085763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.242124714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.260423051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.260461531Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.260650764Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.260776565Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.293923647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.293995348Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.294010808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.294023509Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.294035079Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.29411274Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c08ddd2f-852b-4b90-89fb-adebd25e53a9","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":184091,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.294288572Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"141.028109ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c08ddd2f-852b-4b90-89fb-adebd25e53a9\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c08ddd2f-852b-4b90-89fb-adebd25e53a9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.436629462Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.437423452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.437435332Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.456088414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.456121354Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.473975996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.474012207Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.491758047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.491793818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.509361186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.509395607Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.52741039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.527443051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.545716238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.545749159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.56435906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.56438926Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.582814018Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.582849089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.583032091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.583137782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616016151Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616063592Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616072542Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616082682Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616091652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.616176753Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"be89319e-aea9-4c12-a9f0-4ac773d3ded7","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":179395,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:27.616327115Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"374.082058ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be89319e-aea9-4c12-a9f0-4ac773d3ded7\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be89319e-aea9-4c12-a9f0-4ac773d3ded7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.991213716Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.991577841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:27.991592031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.012318229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.012357819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.031794941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.031830132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.051440975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.051478696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.069940865Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.069982286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.087771957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.087814808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.106077795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.106115975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.124775317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.124815758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.142972113Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.143024934Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.143164776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.143275007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175142763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175186694Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175195464Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175204084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175230504Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.175312416Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":183917,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.175458017Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"422.208795ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4877be18-df25-4fbe-9b5e-1b7e7c6b1aa2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.598654668Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.599059824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.599077074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.61812051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.618154321Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.63579065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.635827431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.654446962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.654484993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.67274204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.67277247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.690065665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.690096096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.708273791Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.708307772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.726410387Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.726450968Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.744147537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.744200648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.744428011Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.744576863Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775421756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775458787Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775465787Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775475047Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775484447Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:28.775613719Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e61b2b07-9dcd-424e-8bed-2a07d768e12c","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":176775,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:28.775809541Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"547.965627ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e61b2b07-9dcd-424e-8bed-2a07d768e12c\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e61b2b07-9dcd-424e-8bed-2a07d768e12c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.325256783Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.326483038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.326495018Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.345227901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.345268522Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.363537569Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.363586399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.381863107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.381899217Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.400201865Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.400231625Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.418601654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.418638194Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.436948162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.436978382Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.455154088Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.455184668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.472726956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.472771487Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.47299094Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.473103651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.504859366Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.504892096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.504911047Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.504921237Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.504931607Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.505033328Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5494ba55-d0ed-4f86-a4bd-efd6167c45a8","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":179580,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:29.50517213Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"327.310334ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5494ba55-d0ed-4f86-a4bd-efd6167c45a8\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5494ba55-d0ed-4f86-a4bd-efd6167c45a8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.833586882Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.833933897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.833943027Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.851809109Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.851845069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.8695448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.86958634Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.886874615Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.886910836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.904256591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.904298872Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.92263366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.92266876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.940972278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.941004438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.958092061Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.958120451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.976342157Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.976383558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.976613641Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:29.976768073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008194854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008237504Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008249884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008259754Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008271245Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.008373716Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"aafd9283-6d30-4d06-9806-5abcde7e4942","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":174622,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:30.008538908Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"879.73787ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aafd9283-6d30-4d06-9806-5abcde7e4942\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aafd9283-6d30-4d06-9806-5abcde7e4942\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.889106576Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.889490451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.889514551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.90952348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.90956486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.927720176Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.927755587Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.945583058Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.945612399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.964330271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.964365422Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.981988691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.982019541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.999780972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:30.999823753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.019557498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.019592399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.037035315Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.037075526Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.037264318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.03736812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.068915902Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.068965092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.068977293Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.068989573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.068999553Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.069085694Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"af7ffdf4-b20b-44f5-baa5-0d85b99f709d","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":179794,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.069249046Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"644.483817ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"af7ffdf4-b20b-44f5-baa5-0d85b99f709d\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"af7ffdf4-b20b-44f5-baa5-0d85b99f709d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.714465918Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.714849062Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.714863843Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.733614016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.733666346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.75241699Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.75244816Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.77012673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.77015559Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.787923991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.787953261Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.805861994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.805899395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.823967509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.82399596Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.842476929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.84251086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.860762007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.860807447Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.860995629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.861111901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.893118639Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.89317355Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.89318512Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.89319392Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.89320494Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:31.893279741Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b2251b1c-234e-40c5-8e08-116b3b8244fb","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":178633,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:31.893433363Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.040193964s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2251b1c-234e-40c5-8e08-116b3b8244fb\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2251b1c-234e-40c5-8e08-116b3b8244fb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.93479455Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.935158505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.935180305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.954780169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.954822659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.972822483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.972858313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.990806496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:32.990850027Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.009055023Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.009092584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.027450782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.027481543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.046125784Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.046159665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.064449322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.064478683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.082520867Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.082556627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.08277779Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.082913792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.115983773Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.116021324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.116030433Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.116041874Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.116052674Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.116152405Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ff8f8d98-7316-47ad-836d-0fc81f41207a","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":181223,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:33.116304887Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"723.378896ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff8f8d98-7316-47ad-836d-0fc81f41207a\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ff8f8d98-7316-47ad-836d-0fc81f41207a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.840739014Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.841085388Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.841101118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.861480932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.861515432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.879698688Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.879740209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.897654981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.897693352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.917237135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.917269395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.934816024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.934865814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.952828077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.952863608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.97076382Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.970802691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.988201367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.988246688Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.98841384Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:33.988526141Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020375437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020426078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020433428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020450468Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020459398Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:34.02058647Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e3477172-0956-44c2-99a6-b03e0fad1308","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":179664,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:34.020771332Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.776763671s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3477172-0956-44c2-99a6-b03e0fad1308\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3477172-0956-44c2-99a6-b03e0fad1308\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.798187481Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.798409173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.798415263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.818488353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.818531724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.837411248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.837459779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.855116679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.855153789Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.873492437Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.873546908Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.891865035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.891905556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.911155745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.911188826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.929341931Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.929387682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.947435626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.947483507Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.947693679Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.947823141Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981130855Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981173306Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981181456Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981191816Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981200386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:35.981295797Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d2f6f184-faf7-4bea-8f47-d1219f5f8edb","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":182990,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:35.981454499Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.684032411s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2f6f184-faf7-4bea-8f47-d1219f5f8edb\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2f6f184-faf7-4bea-8f47-d1219f5f8edb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.666121815Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.666472169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.666484679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.687131746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.687174796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.705500134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.705539345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.723174324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.723208034Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.743307954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.743352545Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.764007822Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.764052872Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.782579923Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.782620673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.801061582Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.801102013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.819250008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.819289889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.819476451Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.819605623Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.851245266Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.851274017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.851280647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.851287077Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.851293087Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:37.851407968Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0cee7879-aed7-453e-b7c6-a57b38fae087","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":185117,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:37.85155528Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.039214865s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0cee7879-aed7-453e-b7c6-a57b38fae087\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0cee7879-aed7-453e-b7c6-a57b38fae087\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:39.315968977Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"290.570094ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.892593346Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.89293335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.892946501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.912438003Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.912483553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.92988071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.92991699Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.947480239Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.947513779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.966680338Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.966715258Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.985559972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:39.985594562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.004605469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.004639719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.023380852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.023414473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.041318815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.041361666Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.041529208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.04164407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073205992Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073233912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073240622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073245752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073266073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.073366594Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"8f138a61-20ed-4750-ac47-2d89eb45ec33","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":180624,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:40.073528986Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.278090756s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f138a61-20ed-4750-ac47-2d89eb45ec33\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f138a61-20ed-4750-ac47-2d89eb45ec33\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.573239119Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:40.573339181Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3662d255-3ca2-4b6f-a059-f30fb44dfd96","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41532","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":53,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.352641905Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.353868761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.353882661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.375458479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.37549325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.394465196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.394489626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.412212056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.412236226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.429906916Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.429930717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.44791095Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.44793939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.465714911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.465739212Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.483432992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.483458222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.500677606Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.500719867Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.500935319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.501067211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.53153652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.531592421Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.531600631Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.531609881Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.531618931Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:43.531755273Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e813c706-ec04-4f47-a38e-bf3c9453d6cf","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized)]","latency_us":178954,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:43.531983095Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.514669322s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e813c706-ec04-4f47-a38e-bf3c9453d6cf\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e813c706-ec04-4f47-a38e-bf3c9453d6cf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.047023804Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.04744262Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.0474597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.068242898Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.068291679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.087603379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.087653549Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.10541129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.105454271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.124264205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.124303025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.142827555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.142861846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.160732668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.160761769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.179014165Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.179048766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.198416577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.198456207Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.198623659Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.198743521Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231117093Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231153214Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231158584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231164944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231170294Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:46.231304406Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e1461f15-f166-4a5d-a9a8-d0eb106eb041","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":184102,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:46.231486058Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.327853055s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1461f15-f166-4a5d-a9a8-d0eb106eb041\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1461f15-f166-4a5d-a9a8-d0eb106eb041\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.561641357Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.561980164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.561990852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.581342401Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.58137693Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.600032985Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.600070561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.619258212Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.619283824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.637734233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.637766718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.655392597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.655428578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.673023016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.673051319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.692195284Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.692228509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.710625357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.710668989Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.710926438Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.711053526Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.743736063Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.74376221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.743767714Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.743772514Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.743778364Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:48.74389545Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"500e9aec-a45e-4996-83d8-4f9dfea32942","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":181635,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:48.744116491Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.360879845s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"500e9aec-a45e-4996-83d8-4f9dfea32942\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"500e9aec-a45e-4996-83d8-4f9dfea32942\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.113161872Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.113566311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.113592974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.133859046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.13388735Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.15302439Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.153057428Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.172283859Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.172317347Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.189877518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.189906984Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.208206209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.208232619Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.225775652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.225802886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.243949599Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.243983237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.26267836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.26271527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.26298723Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.263120178Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.29391929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.293952178Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.293958909Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.293965894Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.293974285Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:51.294130296Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"496b2f7a-8e3c-48a0-811a-86a9957c57c4","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":180263,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:51.294328551Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.920021109s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"496b2f7a-8e3c-48a0-811a-86a9957c57c4\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"496b2f7a-8e3c-48a0-811a-86a9957c57c4\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:54.659899138Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"770.826492ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.225673151Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.225960938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.225970922Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.245472964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.245579938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.264872174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.26489954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.282094797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.282125117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.30079717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.300831801Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.318091684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.318119059Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.336156829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.336190944Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.353739674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.353773527Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.371140949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.37118371Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.371361752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.371478416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.403479026Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.40351137Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.40351767Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.403523727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.403529736Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:47:55.403634785Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"bb79fa49-2f43-4356-a15d-17ba4b824af2","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized)]","latency_us":177386,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:47:55.403809227Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"5.472356996s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb79fa49-2f43-4356-a15d-17ba4b824af2\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:47:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb79fa49-2f43-4356-a15d-17ba4b824af2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.89196859Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.893046254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.893051664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.913129077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.913158908Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.930833414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.930920644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.948125519Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.948148517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.963897231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.963918747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.9813364Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.981357906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.997953847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:00.997978991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.01987864Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.019902368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.036312474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.036344255Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.036545839Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.036658565Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.067821103Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.067851899Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.067860112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.067866582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.067874615Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:01.067998873Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4850fc90-cbad-4969-b915-5c7901ce83e4","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized)]","latency_us":175420,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:01.068177488Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"7.448746006s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4850fc90-cbad-4969-b915-5c7901ce83e4\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:48:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4850fc90-cbad-4969-b915-5c7901ce83e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.537285287Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.537689109Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.537697247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.558536425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.558566585Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.578098341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.578127281Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.596720117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.596756651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.616090188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.616113571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.634040796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.634069558Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.652594479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.652617205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.670946583Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.670976256Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.688740681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.688778153Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.688998672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.689099577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.72123774Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.721265238Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.721272804Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.721278916Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.721285076Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:08.721390199Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"11b63411-4ac7-453f-aa03-ab683c6ac31b","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":183420,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:08.721572133Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"10.949542614s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11b63411-4ac7-453f-aa03-ab683c6ac31b\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:48:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11b63411-4ac7-453f-aa03-ab683c6ac31b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:10.510379597Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.415104219s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:10.665522158Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60197"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:10.665578306Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a8265dfd-1b23-46ff-984c-536d45cad767","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60197","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":34,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.702177332Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.702455473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.702465495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.724191315Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.724222421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.742594556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.742622166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.760651195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.760687628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.77930154Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.779326394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.79714077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.797169092Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.814740203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.814764982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.83317202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.833198729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.850835945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.850866573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.851143605Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.851235266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884193271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884233856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884241937Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884249475Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884255437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:19.884345963Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d59587dd-1a8c-4525-bb73-b60fb39c316f","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized)]","latency_us":181517,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:19.884507543Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"9.821905465s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d59587dd-1a8c-4525-bb73-b60fb39c316f\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:48:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d59587dd-1a8c-4525-bb73-b60fb39c316f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:27.005157417Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.265240499s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.734444656Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.73480975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.73483016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.755448894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.75548812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.773960509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.774000662Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.792035547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.79207091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.810451369Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.810489788Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.829169925Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.829215264Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.848061946Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.848101472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.866200634Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.866240666Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.88359074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.883637757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.883884453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.884012199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.914681408Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.914711277Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.914716321Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.914728977Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.914735166Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:29.914839201Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7a42dab2-1296-4d54-91a4-b500899e8448","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized), (\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized)]","latency_us":179763,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:29.915018228Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"16.568318458s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a42dab2-1296-4d54-91a4-b500899e8448\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:48:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a42dab2-1296-4d54-91a4-b500899e8448\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:40.778069443Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56128"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:40.77812559Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"83a145b7-951c-4242-aa3a-46012b79524f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56128","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":31,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.528914054Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.529277322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.529286772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.551560174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.551588833Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.57006073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.570082218Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.588600279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.588619497Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.608750553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.608771798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.626490709Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.626510134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.645050882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.645083499Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.662213075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.662234957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.679880462Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.67991019Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.680161294Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.68028317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.710943409Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.710983844Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.71099171Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.7110001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.711008247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:46.711126065Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"459fdba1-757b-4fe1-a413-4c57313b63e7","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\", TimeNotSynchronized), (\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\", TimeNotSynchronized), (\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\", TimeNotSynchronized), (\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\", TimeNotSynchronized), (\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\", TimeNotSynchronized)]","latency_us":181553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:46.711297882Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"9.638413455s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"459fdba1-757b-4fe1-a413-4c57313b63e7\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:48:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"459fdba1-757b-4fe1-a413-4c57313b63e7\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:47.364360175Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.220970111s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.377028911Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.377357537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.377365777Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.399467568Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.399522591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.418241848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.41827691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.436448154Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.436477685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.454260104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.454294192Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.47343902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.473472115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.491456619Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.491489206Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.509665884Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.50971709Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.527667841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.527717613Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.527942672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.528149396Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.609533801Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.609581491Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.609588176Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.609594466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.609598704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.699777528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.7039384Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.707907397Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.71186183Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.716341352Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.721127195Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.725698379Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.730042841Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.734445112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:56.739823782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.558413245Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.558559993Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.564685116Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.564809881Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.582635924Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.582749578Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.600987189Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.601103768Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.622970897Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:48:59.623073386Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 29.094859ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.714849467Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 33.917784ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.722420833Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 55.620559ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.729802937Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 36.340373ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.736753141Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 60.601726ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.751499639Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 47.763806ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.758050231Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 100.042064ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.765013339Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 125.958493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.78561377Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 133.305945ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:48:59.888933372Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 245.169946ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.074082725Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 172.100539ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.279183571Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 100.259762ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.511097767Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 406.913864ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.61763722Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 363.250998ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.690292204Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 110.865106ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.737398473Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 101.365375ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:00.828320465Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 205.967874ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:01.182377042Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 733.919479ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:01.336499949Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 408.12299ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:01.887062823Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 367.17017ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:02.681974608Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 408.081759ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:02.765535315Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 251.289391ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:02.774639869Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 803.664773ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:02.782951278Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 514.074146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:02.790857125Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.009311829s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.037531241Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 437.199011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.059834518Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.04081541s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.186894588Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 1.409953295s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.318294572Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 856.015783ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.508663588Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 1.140985782s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:03.598168721Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.351961621s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.056480609Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 655.870187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.237214166Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 844.199758ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.374751176Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 1.121547017s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.73862313Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 1.311643632s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.747568222Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.267263076s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:04.90193075Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 1.186346455s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:05.228721453Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 547.065454ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:05.418132459Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 1.249065249s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:05.870392151Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.022376493s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:05.975128623Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 1.217803767s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:06.070912914Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 995.336328ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:06.179810208Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 820.875042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:06.426027709Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.18259569s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.006749879Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 962.820449ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.128507673Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 592.614948ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.183643768Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 742.688196ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.25615083Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 1.100238358s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.297278041Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.663457524Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"9.926976148s"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.170297281s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:07.786048388Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 1.412545362s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.008318971Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 789.986132ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.101277917Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 552.529583ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.198792119Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 813.388612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.406974398Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 848.490632ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.761428345Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54\") failed: Property not found. retry in 1.447961161s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.899999557Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 514.27182ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:08.966786378Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe\") failed: Property not found. retry in 1.441644369s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:09.230283244Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014\") failed: Property not found. retry in 1.440714611s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:09.430358952Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 839.43301ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:09.490834534Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.009835659s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:09.619972129Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.13138345s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:10.341067714Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 903.729518ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:10.63912751Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:10.885597744Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:10.885733236Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"cfe2ff1b-2414-4647-8283-42f6e0f10a68","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60796","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.042496911s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:11.483613919Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 890.539951ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:11.551822407Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.382973959s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:12.453617079Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.215451534s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:12.535753441Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.045106203s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:14.226577732Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 974.693586ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:14.240617817Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.281713596s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:15.227946572Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.173277393s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:15.282912905Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 717.341259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:16.465753392Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.405818359s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:16.519176426Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 1.076140837s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:17.192710589Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 1.129258951s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:17.934308109Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 779.004634ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:18.278934424Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b\") failed: Property not found. retry in 647.220671ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:19.066915691Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a\") failed: Property not found. retry in 930.530209ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:19.07405385Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025121444Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_c89b5943-b498-418a-8384-b86d007c01fe"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025177355Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_786ee988-6273-42be-9cf0-925b90724c54"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025186085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_24aeb804-b94d-490c-a3ab-e1e642240014"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025192835Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025199455Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_f75c1225-4251-4b27-8dea-29b8679e435a"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025236635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.025829373Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.02641024Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a1f5e8b5-49ad-4e2b-b0ae-86e3a0c3e9a8","remote_addr":"[fd00:1122:3344:101::1]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":23631122,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.05779377Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.087072524Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"e5c66b3e-a48c-4039-bc01-daa2646bc97e","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29204,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.087314798Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"67.280661ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e5c66b3e-a48c-4039-bc01-daa2646bc97e\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e5c66b3e-a48c-4039-bc01-daa2646bc97e\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.183647505Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"41a7cad8-0883-418e-8063-53ef085ac5ae","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27075,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.183833458Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"139.876271ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41a7cad8-0883-418e-8063-53ef085ac5ae\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41a7cad8-0883-418e-8063-53ef085ac5ae\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.351146138Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"f92329e5-dd0a-4b99-96cf-ecd2a11f7930","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26044,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.35132883Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"121.005904ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f92329e5-dd0a-4b99-96cf-ecd2a11f7930\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f92329e5-dd0a-4b99-96cf-ecd2a11f7930\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.49928537Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"af9fa4a9-4eea-4033-8b4d-32d2898946c1","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25758,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.499475702Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"216.108364ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"af9fa4a9-4eea-4033-8b4d-32d2898946c1\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"af9fa4a9-4eea-4033-8b4d-32d2898946c1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.745154607Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"f9710943-e826-42ec-920a-1eea347aeeb5","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28300,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.745362949Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"137.328199ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9710943-e826-42ec-920a-1eea347aeeb5\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9710943-e826-42ec-920a-1eea347aeeb5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:20.964763576Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":75404,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:20.964835457Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"344.608997ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8a69e5c-9b4d-486d-b6e4-4d4ad45ce1ff\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:21.340727511Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"2c722084-c176-45c0-b337-acc822bb1e75","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":30456,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:21.340914833Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"212.686903ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c722084-c176-45c0-b337-acc822bb1e75\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c722084-c176-45c0-b337-acc822bb1e75\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:21.584904937Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"fef3a5f3-6f0c-4ed6-b000-06a15d25d282","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29735,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:21.585081509Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"350.448905ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fef3a5f3-6f0c-4ed6-b000-06a15d25d282\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fef3a5f3-6f0c-4ed6-b000-06a15d25d282\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:21.965427257Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"4067392c-12c1-4f72-8cbf-87c5f2452722","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28126,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:21.965586189Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"266.571005ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4067392c-12c1-4f72-8cbf-87c5f2452722\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4067392c-12c1-4f72-8cbf-87c5f2452722\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:22.259753167Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"e71e42fd-a25c-4933-a584-a74dfb6d0868","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26910,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:22.259910309Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"404.480192ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e71e42fd-a25c-4933-a584-a74dfb6d0868\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e71e42fd-a25c-4933-a584-a74dfb6d0868\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:22.69274566Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"b5cb2f89-3b71-4617-8c2b-5df3af8d3c81","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27509,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:22.693457589Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"622.973441ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5cb2f89-3b71-4617-8c2b-5df3af8d3c81\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5cb2f89-3b71-4617-8c2b-5df3af8d3c81\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:23.344179159Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"a66ee7ff-7cd5-4701-b148-bea2144f61e4","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27473,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:23.344356002Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"618.718136ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a66ee7ff-7cd5-4701-b148-bea2144f61e4\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a66ee7ff-7cd5-4701-b148-bea2144f61e4\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:23.992178375Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"389a2393-aad1-4c99-86c9-fbc9d9d724bb","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28451,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:23.992360758Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"734.650751ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"389a2393-aad1-4c99-86c9-fbc9d9d724bb\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"389a2393-aad1-4c99-86c9-fbc9d9d724bb\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:24.75334959Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"6795d49d-4c1b-47fc-828d-d36cb9529ccc","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1f1e65f0-d4fc-47d6-875a-be69c36e708b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25583,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:24.753531512Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958","retry_after":"1.25944816s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6795d49d-4c1b-47fc-828d-d36cb9529ccc\", \"content-length\": \"124\", \"date\": \"Wed, 29 May 2024 21:49:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6795d49d-4c1b-47fc-828d-d36cb9529ccc\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:26.043061974Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:34.275869071Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.776255913Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.776375175Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"1dc7c552-64b4-4f38-b7e8-65700c417178","remote_addr":"[fd00:1122:3344:101::1]:63868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":15762170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.77676816Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.777133354Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"eff9c2ce-24ec-4a71-b5f5-7fa05d13398e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56476","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":197,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:41.781581999Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"26.741615336s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e385359-7c92-410b-be48-08df0fa9ebc3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a650c55b-039c-4ca8-a067-787cab718ce3, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9cfdc7af-ddad-43c0-86bf-8c32b9fc1269, kind: Service { id: b6538e8d-59bc-4ef4-b2c8-220ca3e6d575 }, name: Name(\"ntp-b6538e8d-59bc-4ef4-b2c8-220ca3e6d575\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 96])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 786ee988-6273-42be-9cf0-925b90724c54, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: c89b5943-b498-418a-8384-b86d007c01fe, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: 24aeb804-b94d-490c-a3ab-e1e642240014, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: f75c1225-4251-4b27-8dea-29b8679e435a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }, OmicronZoneConfig { id: ce802882-5f60-4fa0-8546-381dc464eec5, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 73fd9e9c-3929-404c-966e-fc226ac774de, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b0594198-29c3-400f-bbd5-0154b6bd4027, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 01234475-c550-4426-ba63-e228db75b110, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: 03b95682-bf8e-4d86-bca9-1c8acc0d4e8c, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 91d995f5-4121-4031-8f6b-37b0a19cd7a2, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 4287d8cf-0565-4f0e-abd8-12de702d3848, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9982d53a-69ab-47ac-b36a-154a6b0da765, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External } } } }, OmicronZoneConfig { id: d462ebe6-9d90-445d-bf8a-7c34338191d9, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External } } } }, OmicronZoneConfig { id: a9a35d48-4126-4b06-b5c7-ee36d97697af, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External } } } }, OmicronZoneConfig { id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External } } } }, OmicronZoneConfig { id: 5de4c38e-911d-46b7-beee-a39576f2ce66, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External } } } }, OmicronZoneConfig { id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External } } } }, OmicronZoneConfig { id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.811016965Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.811186717Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47072"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.811493591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.811499781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fcefb7d0-f4d6-4f75-8c6c-6ee2d2710c4f, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.836293079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.836339389Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e385359-7c92-410b-be48-08df0fa9ebc3, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.856933225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.856976516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a650c55b-039c-4ca8-a067-787cab718ce3, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.877367119Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.87741033Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 786ee988-6273-42be-9cf0-925b90724c54, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.897155126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.897193906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c89b5943-b498-418a-8384-b86d007c01fe, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.916618628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.916654818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1e65f0-d4fc-47d6-875a-be69c36e708b, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.936533445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.936570946Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24aeb804-b94d-490c-a3ab-e1e642240014, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.956099469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.956143719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f75c1225-4251-4b27-8dea-29b8679e435a, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce802882-5f60-4fa0-8546-381dc464eec5, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.97471756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:41.974754021Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ce802882-5f60-4fa0-8546-381dc464eec5, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.080320183Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.080373494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 01234475-c550-4426-ba63-e228db75b110, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.191508416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.191560096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 01234475-c550-4426-ba63-e228db75b110, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9982d53a-69ab-47ac-b36a-154a6b0da765, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.367396003Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.367452383Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9982d53a-69ab-47ac-b36a-154a6b0da765, dataset_name: DatasetName { pool_name: ZpoolName { id: aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d462ebe6-9d90-445d-bf8a-7c34338191d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.483928612Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.483981422Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d462ebe6-9d90-445d-bf8a-7c34338191d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f301520-a970-47c3-90c3-bd3006e5615c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.724888277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.724957948Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7e9ce69-0abf-4995-ace8-8f9a44bb9c50, dataset_name: DatasetName { pool_name: ZpoolName { id: cc133e1a-85d1-46d3-b1c3-d13166bc9bff (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9a35d48-4126-4b06-b5c7-ee36d97697af, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.822413779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.82245746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9a35d48-4126-4b06-b5c7-ee36d97697af, dataset_name: DatasetName { pool_name: ZpoolName { id: b5814935-58c2-4e3c-9137-4cbaa1ee9fab (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.934113008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:42.934152109Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1651e1ef-46f8-46cf-a194-eed8a22e18b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 020b43a6-e481-4304-8596-25774e86f14b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5de4c38e-911d-46b7-beee-a39576f2ce66, dataset_name: DatasetName { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.029555325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.029612525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5de4c38e-911d-46b7-beee-a39576f2ce66, dataset_name: DatasetName { pool_name: ZpoolName { id: 4532d18b-4909-4537-9ce6-98b74fb663f5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.131633684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.131679135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b73c1d9-c688-4821-bfd4-1080d5b2eea5, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d81843f-0d2d-4c69-8d17-45004d7cf484 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, dataset_name: DatasetName { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.229973847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.230027577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 210fd0f2-c8d5-4a6d-a520-1ba68096679b, dataset_name: DatasetName { pool_name: ZpoolName { id: 59fcaf06-997c-43d9-9256-26a5eda3000f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.322926452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.322971963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d3e79ddb-29ef-48e6-9b00-cd52126fcc80, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b416693-d467-424a-86a1-703b05ad3101 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.403639056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.403697687Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.40392812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.404128412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.523731929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.559061128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.559281431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.559299482Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.560714539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.562148857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.602950594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.603008365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.603014985Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.603018945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.603024825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.788123936Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.788384559Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.78842719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.78845087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.788489011Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.812857074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.812895524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.812900324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.813276819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.818662336Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.824247765Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.829901725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.834681905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.839388393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.843906489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.976318146Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.980008572Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.98468553Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.988950743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.993268847Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:43.997935945Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.002802845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.007942439Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.013784512Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.018042265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.023099177Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.02897049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.034843543Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.039844516Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.04578589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.051956316Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.05707264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.062142963Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.06754342Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.072560963Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.077547725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.134067587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.174931175Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.211841134Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.248223837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.358420557Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.36026681Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.36028634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.444533868Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 201, 64]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.444597158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.448576348Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.448589678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.565264329Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 177, 61]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.565313909Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.56940529Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.569426401Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.668543753Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 190, 130]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.668593723Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.671245987Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.671263177Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.776611877Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 157, 246]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.776678837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.7809062Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.78092905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.87586913Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 234, 2]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.8759249Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.876617079Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.876633229Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.876909573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.876924413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.877163096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.877175886Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.87748247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.877857704Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.878442562Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.881623751Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.884450237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.884469597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.884477407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.889214196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.892620258Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:44.892634378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.058769564Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.058852165Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Configuring new Omicron zone: oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.059414942Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.069124713Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.079081157Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.090020973Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.101472215Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.104133738Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Installing Omicron zone: oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.104250999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.114144693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.122824681Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.129354992Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.214115776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:45.320392837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.083833232Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.083994984Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.94194216Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.942158143Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.950379355Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.950662829Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.985509662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.985690734Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.996327967Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:47.99656949Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.022983938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.023196051Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.094837271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.095047164Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 44.315636ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:48.13253194Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.137976988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.138225971Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.150387092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:48.150607075Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 93.984579ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.115885256Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.177850207Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.183605348Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 26.744865ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.205744113Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.209613112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.209768483Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 282.703936ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.235822778Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 100.08836ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.250077445Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 34.694986ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.265559047Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 57.064798ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.280941159Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 61.682419ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.295832914Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 66.985499ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.316090736Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 60.946399ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.3831707Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 112.440188ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.403552703Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 57.226477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.425549997Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 51.524639ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.444033626Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 49.779427ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.461016487Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.461558634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:49.461912619Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 136.727289ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.476644082Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 68.015523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:49.491650339Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 40.90717ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.545827275Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 147.799915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.622572259Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 202.450431ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.651660711Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 132.750803ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.684775542Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 328.425693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.738108396Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 211.090628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.808385649Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 51.952127ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.843157651Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 81.662528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.891638698Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:50.891713633Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:49:50.904352922Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 332.720604ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:50.942430845Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 53.485232ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.093762337Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 136.221838ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.210986595Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 279.133607ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.350408378Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 28.671829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.366257285Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 548.721152ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.378396796Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 191.321264ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.389744967Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 640.970966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.401863168Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 292.59192ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.414723728Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 102.146905ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.428754732Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 298.410373ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.440457478Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 520.133672ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.45193772Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 185.123474ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.463898199Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 283.582701ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.474088186Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 145.914352ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.484190241Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 59.891428ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.49377612Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 33.982731ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.524870117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 406.82564ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.541776128Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 114.25379ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.561923388Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 105.796042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.569862277Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 598.650504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.592076473Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 304.224174ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.643886737Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 118.72515ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.653042721Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 522.106964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.66179036Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 171.6522ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.686086962Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 267.898091ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.699105104Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 881.152628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.719569448Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 384.79976ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.750114158Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 484.992526ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.768889511Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 588.446499ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.784569086Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 211.427669ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.876140244Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 688.807679ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.942404848Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 855.657097ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.962718261Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 569.074168ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.976291999Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 424.69712ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:51.988557042Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.486254385s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.002934281Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.306974747s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.057404018Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 772.654009ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.101379725Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.021248748s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.151950003Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 862.681335ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.201418329Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 533.013546ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.218584222Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.190909976s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.271334818Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 450.289133ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.388286022Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 449.10904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.435036653Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.203616531s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.561994562Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.207956812s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.616952155Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 568.333483ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.647428214Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.247066435s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.766212221Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.488477281s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.833252304Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 646.61591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.863305707Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 816.447648ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.887691491Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 683.539956ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:52.89892644Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 949.374259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:53.077405709Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 907.76871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:53.188180077Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 877.010332ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:53.234165189Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 553.265064ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:53.383630297Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.495902429s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:53.489986079Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 588.693987ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.715498092Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 955.654393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.741803689Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.136533636s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.762203793Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 849.08357ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.776644413Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.371887023s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.791353475Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.082781557s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.802753237Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.033965031s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.813264517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 861.9371ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.825880484Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 535.92412ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.839467983Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 598.23658ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.850161936Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.212587778s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.862289117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.127237468s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.874552279Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.048557141s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:56.887694863Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.373428231s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.319184048Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.065219848s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.390238361Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.44209924s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.462038534Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.474996452s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.639798854Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.46683993s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.701677034Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 633.839307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.71343932Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 559.766548ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.861363768Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.174727946s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.898390399Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.09041071s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.91296945Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 896.291493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:57.950211763Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 866.124545ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.014678125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.325453892s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.089613536Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.092409648s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.175590255Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.039493336s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.359578123Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.226082375s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.43494974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.434894068s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.46790948Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 540.00334ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.707083934Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 768.930093ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.860923376Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.032492839s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.894329391Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.265144989s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:58.91757935Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 749.204473ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.015068352Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.476618972s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.086368319Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.129648122s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.128941808Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 506.563007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.181369Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 914.734996ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.259270089Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.403361536s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.280803316Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.25692645s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.41302652Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 999.213125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.427985466Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.258892687s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.64384965Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.284909139s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.674089766Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.266663391s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.701065882Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.034317633s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.777869597Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.398716773s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.913848707Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 763.888331ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:49:59.935263693Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 651.066382ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.183697422Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.290952128s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.191757182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.352560756s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.266755655Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 885.091008ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.435597104Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.304879083s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.571429413Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 757.919505ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.679074261Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.190025055s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.692058353Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 676.384004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.707342143Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 552.8158ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.822466974Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 663.111101ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.844950483Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.356237889s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.911060825Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.409873905s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.967488217Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.123018181s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:00.97656491Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.356409181s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.321206605Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 540.685663ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.328177021Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 948.88157ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.38435875Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 698.277858ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.392027965Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 670.124505ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.447607156Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 907.458343ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.491873807Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.182258612s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.517374314Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.258262628s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.628686258Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 874.929043ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.879570017Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.245182768s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.887225032Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.099544417s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:01.894852767Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.240360388s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.100640235Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 615.116847ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.108899978Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 591.344085ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.125681507Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 684.512253ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.275760683Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.23955431s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.341365119Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.096781137s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.386739653Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 963.851214ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.408002207Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 713.871081ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.686342688Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.127388021s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.707781564Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 830.080223ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.726636499Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.177066602s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.734610208Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 890.999176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.763194273Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.034017201s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.896020704Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.059174088s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:02.968995882Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.037935174s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.002513348Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.495611131s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.142351367Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.462170384s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.350847429Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 901.104285ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.380235535Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.480968436s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.409734382Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.119975669s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.491905283Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 705.374551ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.565071283Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 589.879863ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.591037696Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.244357661s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.663225373Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 505.44554ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.844949802Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 558.476384ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.920139117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 526.262877ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:03.939347226Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.190945342s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.037963072Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 646.122136ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.049184101Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 987.862797ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.190703461Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 508.58572ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.278661755Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.076063163s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.289278277Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.045046929s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.359632612Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.319801976s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.473801281Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 992.566439ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.487224478Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 720.042056ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.625461547Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 537.178051ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.651057525Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 990.117366ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.712076344Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.461875219s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.828356129Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.291071581s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.844087024Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 911.447661ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.905112643Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 953.589493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:04.920760598Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.439110496s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.194798605Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.394670765s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.206980376Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 778.269899ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.245252072Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.180345565s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.358538851Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:05.363930798Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:05.364276002Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.30855667s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.381142822Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.195049344s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.416870556Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.460012504s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.493472489Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 64.44205ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.588338678Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 133.876624ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.666115205Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 656.836571ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.726924611Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.260952861s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.877523153Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 299.443671ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.915258552Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.109373874s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.926152318Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 547.503122ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:05.937747382Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 906.694814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.034187431Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 582.452103ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.145503515Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 538.501511ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.224621349Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 916.799921ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.299995976Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.309286706s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.393446398Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:06.410662372Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:06.411032117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:06.411072457Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:06.415975068Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 592.609045ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.495581268Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.499165901s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.549530069Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 33.454079ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.563678524Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 35.042555ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.591568781Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 139.572027ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.608205188Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 649.03896ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.618915721Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.154074509s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.628102425Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 79.137083ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.637256909Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.307152629s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.64777342Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.26892542s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.70650549Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 240.802145ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.740932799Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 810.374323ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.757224481Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 277.397376ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.77084495Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 705.980428ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.78854793Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 537.047761ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.952773842Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 772.618182ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.961545811Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 330.459073ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:06.993730651Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.196058248s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.045031459Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 583.964385ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.060943817Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.287016252s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.0974002Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.07284459s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.147873387Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 961.329322ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.227189684Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.265683392Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.265718993Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.271298262Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.305534172s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.279769638Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.280071661Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.280201613Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:07.284704259Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 610.720354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.334236245Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 56.470383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.422717045Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 44.154415ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.430185578Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 69.496742ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.43676205Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 83.649675ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.48267598Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 146.945619ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.490636999Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 565.272528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.501562395Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 510.026339ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.510572777Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 88.632435ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.517928039Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 196.755421ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.574074137Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.413888487s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.581968315Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 269.55334ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.615887477Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 265.06525ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.648109496Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.145568204s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.656739374Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.294868556s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.712175974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.179369804s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.744689498Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 210.533973ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.782025422Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 680.523387ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:07.790580028Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 224.282084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.377933401Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 778.458508ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.418171411Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.235785759s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.461011144Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 271.080874ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.481943674Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 737.369298ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.495425192Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.298402526s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.509549257Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.328586394s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.528756216Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 535.619297ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.559014632Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.125278897s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.571031302Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 586.677143ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.610517113Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.02351298s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.645798501Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 1.322425217s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.669965572Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 941.611165ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.689216731Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.166913271s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.69872934Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.030661245s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.726281382Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 792.827111ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.737707514Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.000830054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.770238258Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.216848533s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.814039053Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 843.986268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:08.934147646Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 742.931794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.00689033Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 689.12011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.017932398Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.392471332s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.103170248Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 1.276542378s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.20633351Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 741.351769ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.214781245Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 994.569332ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.242703612Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.25804297s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.54009582Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.480523155s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.641372669Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.339986562s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.678855075Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.148781929s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.70499042Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.477330486s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.712516114Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.147499739s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.720032187Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.215217296s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.759710261Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.330418153s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.76845762Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.157768714s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.781160047Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.357486703s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.788846792Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.35396401s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.816536097Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 671.691679ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.867133596Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 739.511368ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.875037254Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.048632429s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:09.965029833Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 879.628902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.001380595Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 953.684963ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.0411977Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 918.393477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.248487177Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 790.362995ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.493138279Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 858.857811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.504978866Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 715.045899ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.550898377Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 602.743877ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.628103027Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 896.672422ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.80931597Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.055999594s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.863610575Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 630.836722ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.876367113Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 926.812195ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.891609033Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.248614724s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.949436762Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 661.652648ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:10.987263182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 744.475393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.006014215Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.420121948s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.022780004Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 754.767894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.031859777Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 784.672705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.10849562Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 977.330799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.13189556Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.389295087s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.154736524Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 673.29273ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.179362191Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.00908859s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.191787165Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.253844894s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.202211175Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 961.336078ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.243524988Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 1.172178632s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.277037085Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 583.514289ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.293623202Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 844.313354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.377006898Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 534.500811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.518052532Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 661.886064ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.663281378Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.2950414s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.718255331Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 664.824786ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.764629878Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.07948791s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.798227465Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 1.458487491s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.831168105Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 507.327237ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.866115869Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 954.706118ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.889028814Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 714.567885ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.90482835Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 626.655679ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:11.934067174Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:11.965848209Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52840"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:11.969018328Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"5f79234c-d84b-4e14-a375-3085bfd768e7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52840","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":2580,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 911.043998ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.074532031Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 515.997326ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.126661129Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.096926479s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.2160442Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 530.915311ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.233895022Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 976.960757ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.251761734Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 615.460829ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.270195133Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 979.27649ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.340757091Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 1.363176362s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.390625271Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 608.813696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.447880322Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.156047494s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.464094324Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 692.973677ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.478861208Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.326143032s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.491634046Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 891.364711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.55701709Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 1.304266277s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.573494855Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 505.17901ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.639521465Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 728.131852ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.673018192Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 807.972704ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.788284184Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 745.321244ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.85874263Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.270230878s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.899449796Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 501.888109ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:12.913518681Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 834.081161ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.003411009Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.37956294s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.028956257Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 658.218015ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.069150596Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.253757612s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.161211931Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 694.16057ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.188585131Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 782.255841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.244334364Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 1.003055512s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.306802481Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.077063587s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.333491113Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.184438993s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.353030126Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.107285086s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.415012667Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.250125572s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.429140512Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 754.563077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.460486042Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.314426907s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.610026142Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 598.698881ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.621456293Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.063423621s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.632597852Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.196460921s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.741774149Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 1.113846907s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.767781903Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.408782399s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.828348916Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 508.801604ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.846880646Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 871.964645ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.888067608Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 871.417408ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:13.899852865Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 971.362669ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.044086478Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.463459115s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.182965845Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 527.499661ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.224861595Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 1.132283159s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.235235055Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 914.852929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.322586111Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 896.898001ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.366821761Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.229863038s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.41982007Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.462325498s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.429966766Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 1.120477536s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.442576982Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.398921447s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.537128138Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.192119938s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.55174801Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 686.833412ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.695234674Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 683.090227ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.709969617Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.049032758s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.763799767Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 537.785943ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.775811996Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 724.952479ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.785478145Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 824.137199ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.892659838Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 739.248451ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.938399766Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.2636599s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:14.95237191Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.254280979s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.029431049Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 1.290272049s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.252640984Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 874.078296ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.264191387Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 893.038127ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.274573997Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 794.426566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.325252656Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 1.120076635s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.383021425Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.298239654s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.395156696Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 571.464715ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.407689001Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 866.035591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.522778312Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 719.339772ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.579289995Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 673.048066ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.662926575Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 1.467173326s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.676086009Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 708.475383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.690328686Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 1.084323777s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.733248689Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 1.24909442s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.759270683Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 942.434329ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.829202742Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 913.732983ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.90785222Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 701.504683ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.948957081Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.346591094s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:15.992912148Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 687.868483ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.135600792Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 662.911901ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.15312722Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 1.475800209s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.183607859Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.016118362s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.228715749Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c\") failed: Property not found. retry in 1.430164507s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.295776373Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 857.114247ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.316156227Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.006489172s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.352983345Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 802.422864ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.404806369Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 740.038956ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.419548572Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 582.889734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.515758709Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80\") failed: Property not found. retry in 971.527612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.553183874Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af\") failed: Property not found. retry in 1.266633042s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.662811557Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 863.453732ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.704614677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b\") failed: Property not found. retry in 1.128789596s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.782581206Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 617.780874ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.827870389Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.068532506s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.839623675Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 660.818465ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.849791831Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 1.071089186s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:16.858899164Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5\") failed: Property not found. retry in 927.426116ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.02023414Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 763.196137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.108566008Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50\") failed: Property not found. retry in 813.959022ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.152711117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027\") failed: Property not found. retry in 1.238207054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.167258608Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 932.207251ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.18186342Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848\") failed: Property not found. retry in 948.984542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.215432267Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 598.526683ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.255231302Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8\") failed: Property not found. retry in 1.487697013s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.35074321Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2\") failed: Property not found. retry in 1.200530782s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.368896165Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\") failed: Property not found. retry in 1.360849256s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.456891279Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66\") failed: Property not found. retry in 931.311983ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.523179454Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765\") failed: Property not found. retry in 1.207682176s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.584564787Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9\") failed: Property not found. retry in 891.631766ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.675280435Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5\") failed: Property not found. retry in 1.212554366s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.869556869Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110\") failed: Property not found. retry in 1.07079292s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.90016583Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\") failed: Property not found. retry in 1.308131069s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.925381784Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c\") failed: Property not found. retry in 1.28553234s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:17.956582211Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de\") failed: Property not found. retry in 647.927514ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:18.129148777Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480858813Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_217665ab-21f9-40d6-8a7e-9bfc61dd5ddd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480924454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_01234475-c550-4426-ba63-e228db75b110"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480934794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5de4c38e-911d-46b7-beee-a39576f2ce66"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480946604Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_ce802882-5f60-4fa0-8546-381dc464eec5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480953595Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_73fd9e9c-3929-404c-966e-fc226ac774de"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480960075Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_2965e03f-f8ef-4bac-89e9-8a71eac94e2c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480984065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_b0594198-29c3-400f-bbd5-0154b6bd4027"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.480995395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_4287d8cf-0565-4f0e-abd8-12de702d3848"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481002525Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481009235Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d7e9ce69-0abf-4995-ace8-8f9a44bb9c50"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481065986Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_9982d53a-69ab-47ac-b36a-154a6b0da765"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481072726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_210fd0f2-c8d5-4a6d-a520-1ba68096679b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481078936Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d3e79ddb-29ef-48e6-9b00-cd52126fcc80"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481087276Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_1651e1ef-46f8-46cf-a194-eed8a22e18b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481093856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_91d995f5-4121-4031-8f6b-37b0a19cd7a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481099726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481109756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_a9a35d48-4126-4b06-b5c7-ee36d97697af"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481948197Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_2b73c1d9-c688-4821-bfd4-1080d5b2eea5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.481967667Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_03b95682-bf8e-4d86-bca9-1c8acc0d4e8c"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.492352326Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.50152826Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.514840406Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"170e80e8-b1ba-467b-9900-88068a554b84","remote_addr":"[fd00:1122:3344:101::1]:47072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":37703488,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.53848249Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1236"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.538864115Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.541495187Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.566491578Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:680"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.570775431Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:693"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:19.648011321Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:817"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:20.795312205Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:20.919347247Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:21.004673348Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:21.211163646Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:21.455588505Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:21.716970615Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:22.0761199Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:22.440523381Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:22.770084988Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:23.167322547Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:23.658747097Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:23.787684009Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"61.080820397s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:24.309127473Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:24.783541782Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:50:25.506031454Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:25.695135516Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:26.057515441Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:50:26.514079787Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:26.721690289Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:26.741981211Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:50:27.5164658Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/9090f606-344e-4bff-9fcd-4f77886b6762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:28.331038157Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:50:28.519186387Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:50:28.72052843Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/67239fc6-3779-4bd2-ac1a-314185fe431f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:50:29.522214667Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"583d01a1-104f-4949-bbb8-a0165d41021a\", \"content-length\": \"173\", \"date\": \"Wed, 29 May 2024 21:50:30 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"9090f606-344e-4bff-9fcd-4f77886b6762\\\"\", request_id: \"583d01a1-104f-4949-bbb8-a0165d41021a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:30.339142294Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.768059053Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.890194671Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37490"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.891956723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.97539092Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"77027e35-e43f-45c4-aa9d-daccbc793088","remote_addr":"[fd00:1122:3344:101::a]:37490","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":84959,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.976411543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.976474224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.979181337Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:32.979455921Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"fa147788-895d-4695-8c48-04fdb2890476","remote_addr":"[fd00:1122:3344:101::a]:37490","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3522,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.988960502Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53256"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.990213827Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.990652523Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991322861Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991332091Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991350022Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991360402Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991389122Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.991419612Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"dbe931f6-7484-4252-bf85-1d8f2aa13ced","remote_addr":"[fd00:1122:3344:101::a]:53256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":2412,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:33.993702371Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:889"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.120067312Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63978"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.12067531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.204278569Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"21592c1e-5aa4-42a3-ab17-7ce5d7f97023","remote_addr":"[fd00:1122:3344:101::a]:63978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":84109,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.204556742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.204610453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.204873346Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.20514084Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"07d91b4d-6853-4545-9546-9ae10e52d50f","remote_addr":"[fd00:1122:3344:101::a]:63978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":639,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334444137Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37270"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334710071Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334770972Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334809542Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334818702Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334834512Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334843142Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334855113Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.334879823Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3af49ac4-046f-4070-93e2-16b77b666f05","remote_addr":"[fd00:1122:3344:101::a]:37270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":344,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.38776636Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40524"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388154015Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388204516Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388242176Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388248076Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388258097Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388270757Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388277787Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.388299307Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8ece6b47-beeb-47df-a81b-9004c77e1814","remote_addr":"[fd00:1122:3344:101::b]:40524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":384,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.413941986Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40189"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.414199619Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.41423506Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.41425195Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.41425908Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.41426768Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.415117531Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.415141791Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:34.415157571Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d6dd36d1-37e8-4c09-b3dc-fc81b03b7195","remote_addr":"[fd00:1122:3344:101::c]:40189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":1057,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:42.01180828Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48490"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:50:42.012316016Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"31933dbd-c532-47a4-9057-534be38bbf30","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48490","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.217326411Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.303800846Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ea2038f9-c5fb-442e-8b97-370b25f0412b","remote_addr":"[fd00:1122:3344:101::c]:58925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":86416,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.303923837Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.357886488Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"37d946f0-9920-4373-9ef5-ea9b5f545f07","remote_addr":"[fd00:1122:3344:101::b]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":53904,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.358729399Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"43107d71-0698-473b-8107-9ac5f1705cfb","remote_addr":"[fd00:1122:3344:101::c]:58925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":731,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.35958737Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1cce8991-505f-4159-9909-76f15c009fa2","remote_addr":"[fd00:1122:3344:101::b]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":776,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.377113478Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.424081131Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4090d39d-835d-4a66-af36-4ab406773bb4","remote_addr":"[fd00:1122:3344:101::a]:34154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46916,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.424635349Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2cf03956-107d-4092-9e29-eb231e12950d","remote_addr":"[fd00:1122:3344:101::a]:34154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":230,"response_code":"200"} {"msg":"Attempting to update network config with generation: 3, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.425631231Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.425793613Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.427232731Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.428424736Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Attempting to update network config with generation: 2, current_generation: 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.428434466Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Attempted network config update with stale generation: attemped_update_generation: 2, current_generation: 3","v":0,"name":"SledAgent","level":50,"time":"2024-05-29T21:51:00.428438366Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.428478786Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"6f574fed-560c-49e9-9fef-ae9536b4d7aa","remote_addr":"[fd00:1122:3344:101::b]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":4234,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.428489806Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"175cafd9-f8b7-44be-9dd3-77317f88ef35","remote_addr":"[fd00:1122:3344:101::c]:58925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"failed to write updated config to boot store: Network config update failed because it is out of date. Attempted update generation: 2, current generation: 3","latency_us":4110,"response_code":"500"} {"msg":"Attempting to update network config with generation: 4, current_generation: 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.434794235Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/800d63a5-84ab-4e81-94a7-03a851147655/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.434811755Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.435069338Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 4","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.43522995Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:00.435267101Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"c60f8768-8909-43ca-b0c2-122ae46b4826","remote_addr":"[fd00:1122:3344:101::a]:34154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":630,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:12.045453261Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:12.045551342Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"20a306f1-41e3-45e3-ba32-15db4938a768","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50294","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.480327293Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.526584978Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"153c99ff-3446-4078-87ee-46ad5de8ab83","remote_addr":"[fd00:1122:3344:101::a]:58888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.52671926Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.533560505Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4facb570-8743-4213-a76f-a9028aa5aa47","remote_addr":"[fd00:1122:3344:101::a]:58888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":6593,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.57975299Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5d71627b-22af-4b15-9c91-650f07a5324d","remote_addr":"[fd00:1122:3344:101::b]:36531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":52940,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.580100914Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"adfbfe18-e315-44c0-8d61-15d175b4be28","remote_addr":"[fd00:1122:3344:101::b]:36531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.688329Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.729166787Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"cce4446e-b321-4605-b402-5f9afecf6585","remote_addr":"[fd00:1122:3344:101::c]:61956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":40780,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:30.729512002Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e8d1cec6-9bd3-4462-8c09-3653624be56a","remote_addr":"[fd00:1122:3344:101::c]:61956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:42.083727009Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:42.083901181Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"af3af33b-eb6a-487a-81fe-8430256ee3ed","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46268","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:54.998994516Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:45465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:55.009644178Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"4baa1524-2580-4a92-ba1b-479d71ee435c","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":10543,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:51:58.888096659Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.233906492Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40833"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.280468181Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a47b86ca-5bfa-4cff-a3a9-3c189c3b61b8","remote_addr":"[fd00:1122:3344:101::b]:40833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46495,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.280646193Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48347"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.280739365Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.284538512Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"dd599def-40b3-4771-b3dc-4f89dce0feec","remote_addr":"[fd00:1122:3344:101::b]:40833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3610,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.323755999Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"09a15d67-50bb-4443-8b29-94d42d3cabef","remote_addr":"[fd00:1122:3344:101::c]:48347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":43015,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.32786318Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5491d6d6-23f1-41c2-9b03-986f6d60e05c","remote_addr":"[fd00:1122:3344:101::c]:48347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":582,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.368280853Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4702a047-9a32-4bae-a8fe-5ef65b1acd5e","remote_addr":"[fd00:1122:3344:101::a]:48089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":87325,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:00.368567517Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"deae6334-ec43-40de-8528-883f1ba95f5d","remote_addr":"[fd00:1122:3344:101::a]:48089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:12.116323627Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:12.116398218Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"950c4367-0db6-4c3d-8b43-87437840916a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55474","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:24.999681108Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"4433369a-4b12-45ec-a1b2-d9c95cfb678b","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":1088,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.423842068Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.470697981Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9d3b842e-17a3-4568-ab7c-ee5f4fcd9b63","remote_addr":"[fd00:1122:3344:101::b]:44946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46800,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.470815142Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.514491295Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5b6bf734-a5d5-45f8-b53a-365b814330a4","remote_addr":"[fd00:1122:3344:101::c]:64010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":43619,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.514682648Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4b2b5451-9313-4016-96a2-ff57f6fbf230","remote_addr":"[fd00:1122:3344:101::b]:44946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.51488773Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0b451e73-28f7-48aa-b68b-f95f4dc3b8f7","remote_addr":"[fd00:1122:3344:101::c]:64010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.795239695Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.83825169Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"29f83a05-6e7e-426c-a08d-3a6d5ac7ce52","remote_addr":"[fd00:1122:3344:101::a]:64316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":42961,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:30.838474353Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"74182a54-07a2-441c-b348-e02761827820","remote_addr":"[fd00:1122:3344:101::a]:64316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:42.147690033Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:42.147761304Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ec096f27-1966-4a18-8ef1-0b4d70d27328","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54545","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:52:54.998720712Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"104ceac0-3923-4668-882d-fa8457446f2a","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":443,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.17424129Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.221831402Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b0c575f1-ecea-4d7c-ba2a-ae896e1998e0","remote_addr":"[fd00:1122:3344:101::b]:40807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":47522,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.222130735Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"684eb870-8db5-4a69-87df-d92823db3baa","remote_addr":"[fd00:1122:3344:101::b]:40807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.367706306Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.414609649Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a12c8620-ff4f-4480-a418-dfb4c8270352","remote_addr":"[fd00:1122:3344:101::a]:38611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46806,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.414794351Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.418020481Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"014d4fe7-6cbf-485d-8a63-20854da810a1","remote_addr":"[fd00:1122:3344:101::a]:38611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3082,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.457165698Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a3267520-8b18-45ce-b43c-207680f4ca88","remote_addr":"[fd00:1122:3344:101::c]:36139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":42291,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:00.457538562Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6d853065-65e5-4f31-b609-f737000a6bea","remote_addr":"[fd00:1122:3344:101::c]:36139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:12.179075849Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:12.17915201Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"04ee2c1e-9553-4a2f-a245-798ce095699b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51467","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:24.997977387Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"a4eab00b-a471-48b9-912d-0f3e3711aaa8","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":732,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.487253957Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:39781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.532487159Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"74ae45cb-d9e2-4881-9464-461dc5f28176","remote_addr":"[fd00:1122:3344:101::b]:39781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":45169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.532674051Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.535898101Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"48c8c2fb-d8cd-4bd9-8918-99534047d04e","remote_addr":"[fd00:1122:3344:101::b]:39781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.575037468Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4fc6703b-4d49-4969-9647-c4afd71fbcb3","remote_addr":"[fd00:1122:3344:101::a]:44935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":42270,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.575423463Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f0541c5a-9bc1-440d-9f34-4660b012c8fa","remote_addr":"[fd00:1122:3344:101::a]:44935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.714796746Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.758975085Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6fd650a6-0fca-4c4b-84c5-6c448d8a2ceb","remote_addr":"[fd00:1122:3344:101::c]:63575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":44115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:30.759765815Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8fe5ad5f-b6ff-4460-a8cd-f35f502a8a70","remote_addr":"[fd00:1122:3344:101::c]:63575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":603,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:42.211576755Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:42.212191743Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"40929abf-dab0-4de1-9a4b-88efc918fb98","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43075","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:53:55.027739219Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"d22e6775-18e1-4407-9db2-941e7846dd7e","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3228,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.197263922Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.243885002Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"edcad443-8ace-4a58-a707-1a5a7200081e","remote_addr":"[fd00:1122:3344:101::c]:39897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46563,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.244176515Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1c2046d7-16bc-4595-be1e-d2add6a6d48d","remote_addr":"[fd00:1122:3344:101::c]:39897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.272895653Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.317626749Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e10de8d5-b479-4c8e-a935-88d92a7ce021","remote_addr":"[fd00:1122:3344:101::b]:62983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":44636,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.317769731Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.320656566Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"729d26e6-e066-4b54-9b16-315fee8749d0","remote_addr":"[fd00:1122:3344:101::b]:62983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":2704,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.360411511Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"588fcd68-0e63-4254-a410-67c2db4895a1","remote_addr":"[fd00:1122:3344:101::a]:56703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":42541,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:00.360770275Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e94fe4ff-c514-49bc-9479-64e9789d8cc6","remote_addr":"[fd00:1122:3344:101::a]:56703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:12.245653271Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:12.245728112Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"17f33c60-4547-4854-957c-7e48c48dc532","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45071","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:25.073976166Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"e58c4f51-d93b-495f-a8cb-79fb43b5f48a","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":601,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.51788438Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.561762176Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2668e390-1911-4e21-a454-98118d2b8763","remote_addr":"[fd00:1122:3344:101::b]:52492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":43819,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.561951838Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64844"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.561966808Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.562367883Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3f85f0a3-9139-4e42-82f9-a1bb0bbdafb4","remote_addr":"[fd00:1122:3344:101::b]:52492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.610433861Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"18f87ec4-48ec-431e-b1b7-c62ba30dc0bd","remote_addr":"[fd00:1122:3344:101::c]:64844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":48249,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.614231208Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b4f3f460-4f0c-49ef-903d-70c4558c562c","remote_addr":"[fd00:1122:3344:101::c]:64844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":690,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.657313304Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"44ddc1a5-a358-49ec-8726-c29ad8b3b49e","remote_addr":"[fd00:1122:3344:101::a]:39351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":90578,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:30.657687998Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"45a4be73-4e96-4a16-a51b-27cbce0a1271","remote_addr":"[fd00:1122:3344:101::a]:39351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:42.278842954Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58866"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:42.278956245Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b07a2080-8771-45ca-b532-4b6014261a40","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58866","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:54:55.073909354Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"8adbaf0f-9c96-4133-87d2-c12063981de4","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":521,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.166758974Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.213757368Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"62ec964e-9e8e-4894-a18f-604f0dc99716","remote_addr":"[fd00:1122:3344:101::c]:47345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46940,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.214044272Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"eb03c831-3831-4515-b4a7-3ad501072c54","remote_addr":"[fd00:1122:3344:101::c]:47345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.273702034Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.321390676Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c9f9d59e-860d-4013-b826-f141a0c062c6","remote_addr":"[fd00:1122:3344:101::b]:50349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":47596,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.321500548Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.325219334Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e062dbda-d97f-4742-af66-0ca672e44d92","remote_addr":"[fd00:1122:3344:101::b]:50349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3539,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.367750273Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c2321414-cb39-4ab0-adc2-7a68669a30cd","remote_addr":"[fd00:1122:3344:101::a]:59194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":46167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:00.368061217Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1233bc39-8e41-4639-b7d1-333af6bb8276","remote_addr":"[fd00:1122:3344:101::a]:59194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:12.31155654Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:12.311658391Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"aee9706d-7d1e-43ec-a92b-25b0eb0f1c24","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41883","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:25.076670558Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"625ef769-0423-4224-94c8-31b0d1bde9a6","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":2767,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430318431Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51417"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430574394Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430624855Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430665995Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430687885Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430693765Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430705425Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430736676Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.430751736Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"05963163-14f1-4b3d-a628-aae5fc8b99c6","remote_addr":"[fd00:1122:3344:101::b]:51417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":393,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.44798682Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58300"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.448127772Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.448170683Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.448206393Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.45202145Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.452199693Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.452265593Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.452333705Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.452397025Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0bfe63dc-290e-4c1e-9eed-af9f0724572c","remote_addr":"[fd00:1122:3344:101::c]:58300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.454812575Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50985"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460536876Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33484"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460769179Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0f9a9a09-8561-4778-9bbb-b421e6605b7d, kind: Service { id: ce802882-5f60-4fa0-8546-381dc464eec5 }, name: Name(\"external-dns-ce802882-5f60-4fa0-8546-381dc464eec5\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 177, 61])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f8b16e9d-5dd1-4ab6-aa5c-5760609d25ce, kind: Service { id: cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7 }, name: Name(\"external-dns-cf9ddcb1-80ea-4c0a-9bfd-9007b54df2a7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 230c8013-992c-4e47-b787-0801f14bba28, kind: Service { id: 2965e03f-f8ef-4bac-89e9-8a71eac94e2c }, name: Name(\"nexus-2965e03f-f8ef-4bac-89e9-8a71eac94e2c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 246])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b5f20d1b-cfd9-4b0c-a72e-35bfd35dbc4d, kind: Service { id: 217665ab-21f9-40d6-8a7e-9bfc61dd5ddd }, name: Name(\"nexus-217665ab-21f9-40d6-8a7e-9bfc61dd5ddd\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 64])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f0ef69c3-bae4-4bd0-a745-1485bbc451f2, kind: Service { id: 73fd9e9c-3929-404c-966e-fc226ac774de }, name: Name(\"nexus-73fd9e9c-3929-404c-966e-fc226ac774de\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.46081938Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.46085922Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.46087298Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460894701Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460917791Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460942252Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.460958522Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e34493db-4a3d-40fd-b626-0e622e401af8","remote_addr":"[fd00:1122:3344:101::a]:33484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":331,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.484869749Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.49938639Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7aa649e4-f090-46c0-b19d-35e1f75033a1","remote_addr":"[fd00:1122:3344:101::b]:50985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":44428,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.50343251Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6697d9cf-0ffa-4107-92aa-2e62aabd5784","remote_addr":"[fd00:1122:3344:101::b]:50985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":1238,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.503527891Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.544059605Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5b6075ab-e1d2-492d-a730-20d4ce62c841","remote_addr":"[fd00:1122:3344:101::c]:38339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":59061,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.548371799Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"cf502cef-9a71-43f3-84eb-a184bb20072b","remote_addr":"[fd00:1122:3344:101::c]:38339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":649,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.588478887Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3cb53acc-1230-488e-88a1-08d4d3536bc0","remote_addr":"[fd00:1122:3344:101::a]:50131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":84816,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:30.588861632Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"884184c4-ce76-496f-9a18-09549c8500b4","remote_addr":"[fd00:1122:3344:101::a]:50131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:42.343670019Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:42.34374918Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6ab940be-6cf2-4f33-ab62-a6b880fd666a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37861","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:55:55.073991105Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"4591703a-c955-4b99-a4b3-8b97eab1426f","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":445,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.183018426Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.230605037Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"445233ce-2fae-43ca-ad08-861d8f092679","remote_addr":"[fd00:1122:3344:101::b]:44174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":47515,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.230714068Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.273703963Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"db74d891-dc50-4988-ba38-9da624cb7642","remote_addr":"[fd00:1122:3344:101::c]:62684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":42924,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.273837275Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.277529981Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"438ed411-6e45-4045-ac6a-0e0c8b46c3fa","remote_addr":"[fd00:1122:3344:101::b]:44174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3495,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.277750103Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9cf1b738-bf60-46c5-9642-6b9b1594f706","remote_addr":"[fd00:1122:3344:101::c]:62684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":3760,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.319713125Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7d46cbee-fcf5-4baf-94f5-b6746eb75e63","remote_addr":"[fd00:1122:3344:101::a]:49520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":45788,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:00.319988178Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a88742e7-f1d3-4e94-93a7-d00354a2cc97","remote_addr":"[fd00:1122:3344:101::a]:49520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.870552096Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47563"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.882062119Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:546","metadata":"InstanceMetadata { silo_id: 7f285fd5-6333-41bb-876e-2e81f40964f1, project_id: 0289c9e5-a51b-4d7a-ad72-2f0ec456b725 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-29T21:56:01.397941Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1417898))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e22f58cde08\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some(\"fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some(\"JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::19]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.882172501Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:580","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.882657227Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1417898))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab35f591-930b-49b8-b3e0-a510c19d1b2c, kind: Instance { id: 98fac4ed-861f-465d-b739-785c65e21b8a }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 74, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(1417898), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e22f58cde08\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some(\"fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some(\"JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::19]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-29T21:56:01.397941Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.883434627Z","hostname":"sock","pid":1562,"uri":"/instances/98fac4ed-861f-465d-b739-785c65e21b8a","method":"PUT","req_id":"8a307cfd-37c4-4fde-94cb-ee8c07c8780d","remote_addr":"[fd00:1122:3344:101::a]:47563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":12835,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.975746194Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:01.976497414Z","hostname":"sock","pid":1562,"uri":"/instances/98fac4ed-861f-465d-b739-785c65e21b8a/state","method":"PUT","req_id":"d676da51-be25-491d-8609-3d866ffad6d6","remote_addr":"[fd00:1122:3344:101::a]:52578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":622,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:02.000792026Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 74, 86]), slot: 0, vni: Vni { inner: 1417898 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:02.000976938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:02.032958016Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:02.05418581Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:03.175006105Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:03.175234408Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 74.159154ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:03.358477306Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 121.151009ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:03.446193387Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 197.661154ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:03.587139049Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 498.341732ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:03.799141574Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 844.935012ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:04.306803577Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 1.239328232s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:05.162294753Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 854.836921ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:06.412478977Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 1.421988085s","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:07.277872386Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:08.727100015Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 52.457579ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:08.735189835Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 75.07528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:08.796255174Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 267.932745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:08.881123089Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\") failed: Property not found. retry in 546.363566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:09.159141835Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.715544274Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.749970912Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC3AK+9WL1YAAAAr71YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAtwCvvVi9WAAAAK+9WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5OGZhYzRlZC04NjFmLTQ2NWQtYjczOS03ODVjNjVlMjFiOGEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2lJeDZ5SksrRmUyUmhDMkZheUptWGRMOFdoby9qb3NLQ2JtMms2Z0JWdCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-e22f58cde08\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, read_only_parent: Some(Volume { block_size: 512, id: f522685a-8c7c-4ea1-9a59-9d48f2b54b12, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7f6e33e9-d32a-4fc1-a9f4-5c6deab98a7c, key: Some(\"fy067CHA0e6MAlslntAnx2FZZdEgCDSQLgfiRtdsXic=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bf2b630c-c4f1-4047-b6c6-bc780c1c4fc3, key: Some(\"JBmRC8Qij6Z/zPLP3kshAVVOwqIOydO5xFm4vevNBeA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::19]:19003\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 98fac4ed-861f-465d-b739-785c65e21b8a, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 0289c9e5-a51b-4d7a-ad72-2f0ec456b725, silo_id: 7f285fd5-6333-41bb-876e-2e81f40964f1 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.750983894Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.859494743Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-29T21:56:09.859908568Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.859933899Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.859944439Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-29T21:56:09.859908568Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.859953669Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-29T21:56:09.859908568Z } }","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-29T21:56:09.917923640Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.91794481Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.91796615Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-29T21:56:09.917923640Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:09.91797532Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-29T21:56:09.917923640Z } }","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:12.377214706Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38695"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:12.377973145Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d3270bba-7662-46b1-a4b1-c40ed186317b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38695","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.74930675Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.749913728Z","hostname":"sock","pid":1562,"uri":"/instances/98fac4ed-861f-465d-b739-785c65e21b8a/state","method":"PUT","req_id":"2c4672fe-1490-4971-b3e0-f05ee7c2acd7","remote_addr":"[fd00:1122:3344:101::a]:46421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":592,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-29T21:56:18.750043330Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.75008056Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.750112831Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-29T21:56:18.750043330Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.750137791Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8ad579b2-dda9-4e63-8490-567c3ccd27af), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-29T21:56:01.449818326Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-29T21:56:18.750043330Z } }","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-29T21:56:18.776758522Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.776779712Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.776795762Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-29T21:56:18.776758522Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }","propolis_id":"8ad579b2-dda9-4e63-8490-567c3ccd27af"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.776814072Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.776819172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.778855608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:18.778901638Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/800d63a5-84ab-4e81-94a7-03a851147655/debug/bundle/zone\", \"/pool/int/92dea9e4-ea67-4c7f-80bc-15932c1cd778/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/aeaf2c17-7b7a-4447-8f2e-63b0c87bf54c/crypt/debug\", \"/pool/ext/8f301520-a970-47c3-90c3-bd3006e5615c/crypt/debug\", \"/pool/ext/cc133e1a-85d1-46d3-b1c3-d13166bc9bff/crypt/debug\", \"/pool/ext/b5814935-58c2-4e3c-9137-4cbaa1ee9fab/crypt/debug\", \"/pool/ext/020b43a6-e481-4304-8596-25774e86f14b/crypt/debug\", \"/pool/ext/4532d18b-4909-4537-9ce6-98b74fb663f5/crypt/debug\", \"/pool/ext/8d81843f-0d2d-4c69-8d17-45004d7cf484/crypt/debug\", \"/pool/ext/59fcaf06-997c-43d9-9256-26a5eda3000f/crypt/debug\", \"/pool/ext/6b416693-d467-424a-86a1-703b05ad3101/crypt/debug\"] }","zone_name":"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:19.50373132Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af\", bundle_id: dbd3bc50-8793-4e45-a865-c157e7a3a9d6 }, time_created: 2024-05-29T21:56:18.779043850Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_8ad579b2-dda9-4e63-8490-567c3ccd27af","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:19.503813511Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.257485814Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.28289608Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-29T21:56:21.282836850Z } }","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.355492573Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62474"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-29T21:56:21.362142586Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.362161876Z","hostname":"sock","pid":1562,"uri":"/v2p/ab35f591-930b-49b8-b3e0-a510c19d1b2c","method":"DELETE","req_id":"ecab2288-24d7-402d-97b0-2d257482c080","remote_addr":"[fd00:1122:3344:101::a]:62474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":6554,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.661095953Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-29T21:56:18.776758522Z }, propolis_id: 8ad579b2-dda9-4e63-8490-567c3ccd27af, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-29T21:56:21.282836850Z } }","instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:21.674904644Z","hostname":"sock","pid":1562,"instance_id":"98fac4ed-861f-465d-b739-785c65e21b8a","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-29T21:56:25.074203447Z","hostname":"sock","pid":1562,"uri":"/67239fc6-3779-4bd2-ac1a-314185fe431f","method":"GET","req_id":"f360186f-f67a-4780-8612-86aaff77f27c","remote_addr":"[fd00:1122:3344:101::d]:45465","local_addr":"[fd00:1122:3344:101::1]:61502","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"67239fc6-3779-4bd2-ac1a-314185fe431f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/2fdf371/dropshot/src/server.rs:878","latency_us":608,"response_code":"200"}