2024.09.02 19:49:59.996385 [ 206108 ] {} <Error> virtual bool DB::DDLWorker::initializeMainThread(): Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003, [::1]:19002
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003
Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002
. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<String&>(int, FormatStringHelperImpl<std::type_identity<String&>::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception::Exception<String&>(Coordination::Error, FormatStringHelperImpl<std::type_identity<String&>::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::ZooKeeper(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node>> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr<DB::ZooKeeperLog>, std::optional<std::function<void (unsigned long, Coordination::ZooKeeper::Node const&)>>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::ZooKeeper(Poco::Util::AbstractConfiguration const&, String const&, std::shared_ptr<DB::ZooKeeperLog>) @ 0x00000000137f66f9 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. DB::Context::getZooKeeper() const @ 0x000000001166afca in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. DB::DDLWorker::getAndSetZooKeeper() @ 0x00000000116ec1ed in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. DB::DDLWorker::initializeMainThread() @ 0x00000000116fc81b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::DDLWorker::runMainThread() @ 0x00000000116e7280 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<void (DB::DDLWorker::*)(), DB::DDLWorker*>(void (DB::DDLWorker::*&&)(), DB::DDLWorker*&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000116fd92c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. ? @ 0x00007fdab736fac3 in ?
13. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.006803 [ 205774 ] {} <Warning> oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session.
2024.09.02 19:50:12.006833 [ 205824 ] {} <Warning> oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session.
2024.09.02 19:50:12.006809 [ 205803 ] {} <Warning> oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session.
2024.09.02 19:50:12.006805 [ 205800 ] {} <Warning> oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session.
2024.09.02 19:50:12.006895 [ 205809 ] {} <Warning> oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session.
2024.09.02 19:50:12.009797 [ 205836 ] {} <Error> oximeter.measurements_cumulativef64_local (8387b2c6-e147-451e-8e5f-84f4bbeb2c47): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 242. DB::Exception: Table is in readonly mode (replica path: /clickhouse/tables/1/measurements_cumulativef64_local/replicas/2). (TABLE_IS_READ_ONLY), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x00000000074d2e8d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d6ae in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. ? @ 0x00007fdab736fac3 in ?
7. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.009976 [ 205778 ] {} <Error> oximeter.measurements_cumulativef64_local (8387b2c6-e147-451e-8e5f-84f4bbeb2c47): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x00007fdab736fac3 in ?
15. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.012036 [ 205790 ] {} <Error> oximeter.fields_uuid_local (2a01182b-0909-4138-ac40-9871004f15b4): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::get(String const&, std::function<void (Coordination::GetResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr<zkutil::ZooKeeper>, std::function<void (Coordination::WatchResponse const&)>, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x00007fdab736fac3 in ?
15. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.012105 [ 205807 ] {} <Error> oximeter.fields_uuid_local (2a01182b-0909-4138-ac40-9871004f15b4): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x00007fdab736fac3 in ?
15. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.012143 [ 205773 ] {} <Error> oximeter.version (626c1ca5-d8d8-4409-a032-b2aeb18345c3): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::get(String const&, std::function<void (Coordination::GetResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr<zkutil::ZooKeeper>, std::function<void (Coordination::WatchResponse const&)>, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x00007fdab736fac3 in ?
15. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))
2024.09.02 19:50:12.012176 [ 205767 ] {} <Error> oximeter.version (626c1ca5-d8d8-4409-a032-b2aeb18345c3): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x00007fdab736fac3 in ?
15. ? @ 0x00007fdab7401850 in ?
 (version 23.8.7.24 (official build))