| 1 | 2024.08.27 19:20:10.892185 [ 252 ] {} <Error> ZooKeeperClient: Code: 33. DB::Exception: Cannot read all data. Bytes read: 0. Bytes expected: 4. (CANNOT_READ_ALL_DATA), Stack trace (when copying this message, always include the lines below): | ||
| 2 | |||
| 3 | 0. StackTrace::tryCapture() @ 0x0000000010014f57 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 4 | 1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000ffe797f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 5 | 2. DB::Exception::Exception<unsigned long&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(int, FormatStringHelperImpl<std::type_identity<unsigned long&>::type, std::type_identity<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >::type>, unsigned long&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&) @ 0x000000000ac3631e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 6 | 3. DB::ReadBuffer::readStrict(char*, unsigned long) @ 0x000000000ac360fa in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 7 | 4. Coordination::ZooKeeper::receiveEvent() @ 0x0000000016bafed2 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 8 | 5. Coordination::ZooKeeper::receiveThread() @ 0x0000000016baf7d8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 9 | 6. std::_Function_handler<void (), ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<Coordination::ZooKeeper::ZooKeeper(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node> > const&, zkutil::ZooKeeperArgs const&, std::shared_ptr<DB::ZooKeeperLog>, std::optional<std::function<void (unsigned long, Coordination::ZooKeeper::Node const&)> >&&)::$_1>(Coordination::ZooKeeper::ZooKeeper(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node> > const&, zkutil::ZooKeeperArgs const&, std::shared_ptr<DB::ZooKeeperLog>, std::optional<std::function<void (unsigned long, Coordination::ZooKeeper::Node const&)> >&&)::$_1&&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) @ 0x0000000016bb4b98 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 10 | 7. ThreadPoolImpl<std::thread>::worker(std::_List_iterator<std::thread>) @ 0x0000000019092c5c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 11 | 8. ? @ 0x000000000014d750 in /usr/gcc/13/lib/amd64/libstdc++.so.6.0.32 | ||
| 12 | 9. _thrp_setup @ 0x0000000000114f37 in /lib/amd64/libc.so.1 | ||
| 13 | (version 23.8.7.1) | ||
| 14 | 2024.08.27 19:20:10.892627 [ 173 ] {} <Warning> oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 15 | 2024.08.27 19:20:10.892693 [ 75 ] {} <Warning> oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 16 | 2024.08.27 19:20:10.892730 [ 175 ] {} <Warning> oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 17 | 2024.08.27 19:20:10.892806 [ 179 ] {} <Warning> oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 18 | 2024.08.27 19:20:10.892911 [ 252 ] {} <Error> ZooKeeperClient: Poco::Exception. Code: 1000, e.code() = 134, Net Exception: Socket is not connected, Stack trace (when copying this message, always include the lines below): | ||
| 19 | |||
| 20 | (version 23.8.7.1) | ||
| 21 | 2024.08.27 19:20:10.892649 [ 167 ] {} <Warning> oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 22 | 2024.08.27 19:20:10.920913 [ 51 ] {} <Error> oximeter.measurements_cumulativef64_local (5a85093b-c21a-4b08-bf37-6af0108427b6): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 23 | |||
| 24 | 0. StackTrace::tryCapture() @ 0x0000000010014f57 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 25 | 1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000ffe797f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 26 | 2. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000ffe19c6 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 27 | 3. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000016bb6e38 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 28 | 4. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x0000000016bb14ce in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 29 | 5. Coordination::ZooKeeper::list(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >) @ 0x0000000016bb37d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 30 | 6. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >, Coordination::ListRequestType) @ 0x0000000016b76078 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 31 | 7. zkutil::ZooKeeper::getChildrenImpl(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >, Coordination::ListRequestType) @ 0x0000000016b75c75 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 32 | 8. zkutil::ZooKeeper::tryGetChildrenWatch(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >, Coordination::ListRequestType) @ 0x0000000016b7679a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 33 | 9. zkutil::ZooKeeper::getChildrenWatch(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >, Coordination::ListRequestType) @ 0x0000000016b76660 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 34 | 10. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >) @ 0x00000000159cb716 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 35 | 11. DB::StorageReplicatedMergeTree::mutationsUpdatingTask() @ 0x000000001560a701 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 36 | 12. DB::BackgroundSchedulePoolTaskInfo::execute() @ 0x0000000014a568cf in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 37 | 13. DB::BackgroundSchedulePool::threadFunction() @ 0x0000000014a5967e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 38 | 14. std::_Function_handler<void (), ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_1>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_1&&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) @ 0x0000000014a59dd8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 39 | 15. ThreadPoolImpl<std::thread>::worker(std::_List_iterator<std::thread>) @ 0x0000000019092c5c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 40 | 16. ? @ 0x000000000014d750 in /usr/gcc/13/lib/amd64/libstdc++.so.6.0.32 | ||
| 41 | 17. _thrp_setup @ 0x0000000000114f37 in /lib/amd64/libc.so.1 | ||
| 42 | (version 23.8.7.1) | ||
| 43 | 2024.08.27 19:20:10.930036 [ 107 ] {} <Error> oximeter.measurements_cumulativef64_local (5a85093b-c21a-4b08-bf37-6af0108427b6): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 44 | |||
| 45 | 0. StackTrace::tryCapture() @ 0x0000000010014f57 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 46 | 1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000ffe797f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 47 | 2. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000ffe19c6 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 48 | 3. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000016bb6e38 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 49 | 4. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x0000000016bb14ce in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 50 | 5. Coordination::ZooKeeper::get(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void (Coordination::GetResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)> >) @ 0x0000000016bb2f2d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 51 | 6. zkutil::ZooKeeper::asyncTryGetNoThrow(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void (Coordination::WatchResponse const&)>) @ 0x0000000016b78d10 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 52 | 7. zkutil::ZooKeeper::getImpl(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>) @ 0x0000000016b788e1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 53 | 8. zkutil::ZooKeeper::tryGetWatch(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>, Coordination::Error*) @ 0x0000000016b79266 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 54 | 9. zkutil::ZooKeeper::get(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&) @ 0x0000000016b78ea0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 55 | 10. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr<zkutil::ZooKeeper>, std::function<void (Coordination::WatchResponse const&)>, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x00000000159c640b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 56 | 11. DB::StorageReplicatedMergeTree::queueUpdatingTask() @ 0x000000001560a3a5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 57 | 12. DB::BackgroundSchedulePoolTaskInfo::execute() @ 0x0000000014a568cf in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 58 | 13. DB::BackgroundSchedulePool::threadFunction() @ 0x0000000014a5967e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 59 | 14. std::_Function_handler<void (), ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_1>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_1&&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) @ 0x0000000014a59dd8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 60 | 15. ThreadPoolImpl<std::thread>::worker(std::_List_iterator<std::thread>) @ 0x0000000019092c5c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 61 | 16. ? @ 0x000000000014d750 in /usr/gcc/13/lib/amd64/libstdc++.so.6.0.32 | ||
| 62 | 17. _thrp_setup @ 0x0000000000114f37 in /lib/amd64/libc.so.1 | ||
| 63 | (version 23.8.7.1) | ||
| 64 | 2024.08.27 19:20:12.098307 [ 107 ] {} <Warning> oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 65 | 2024.08.27 19:20:12.098331 [ 41 ] {} <Warning> oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 66 | 2024.08.27 19:20:12.098390 [ 173 ] {} <Warning> oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 67 | 2024.08.27 19:20:12.098492 [ 172 ] {} <Warning> oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 68 | 2024.08.27 19:20:12.098604 [ 47 ] {} <Warning> oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||