2026.08.17 23:52:06.045862 [ 425085 ] {} virtual bool DB::DDLWorker::initializeMainThread(): Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001, [::1]:19002 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(Poco::Util::AbstractConfiguration const&, String const&, std::shared_ptr) @ 0x00000000137f66f9 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166afca in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::DDLWorker::getAndSetZooKeeper() @ 0x00000000116ec1ed in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::DDLWorker::initializeMainThread() @ 0x00000000116fc81b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::DDLWorker::runMainThread() @ 0x00000000116e7280 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl(void (DB::DDLWorker::*&&)(), DB::DDLWorker*&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000116fd92c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.046564 [ 424320 ] {} oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.046563 [ 424333 ] {} oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.046568 [ 424325 ] {} oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.046579 [ 424318 ] {} oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.046613 [ 424367 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.046968 [ 424398 ] {} oximeter.fields_i64_local (2d79911c-8882-4309-a914-7e64c31972ae): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception Coordination::Exception::fromMessage(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. Coordination::ZooKeeper::get(String const&, std::function, std::shared_ptr>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr, std::function, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 13. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 14. ? @ 0x00007f44aa6b2a83 in ? 15. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.047051 [ 424314 ] {} oximeter.fields_i64_local (2d79911c-8882-4309-a914-7e64c31972ae): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception Coordination::Exception::fromMessage(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function, std::shared_ptr>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector>&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector>&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr, std::shared_ptr>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 13. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 14. ? @ 0x00007f44aa6b2a83 in ? 15. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.922698 [ 430366 ] {} ZooKeeperClient: Code: 33. DB::Exception: Cannot read all data. Bytes read: 0. Bytes expected: 4. (CANNOT_READ_ALL_DATA), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type, std::type_identity::type>, unsigned long&, String&&) @ 0x0000000007354cb4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. DB::ReadBuffer::readStrict(char*, unsigned long) @ 0x0000000007354bb7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&)::$_1>(Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&)::$_1&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000001384c492 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. ? @ 0x00007f44aa6b2a83 in ? 6. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.922808 [ 424420 ] {} oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.922810 [ 424379 ] {} oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.922827 [ 424332 ] {} oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.922842 [ 424329 ] {} oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.922897 [ 430366 ] {} ZooKeeperClient: Poco::Exception. Code: 1000, e.code() = 107, Net Exception: Socket is not connected, Stack trace (when copying this message, always include the lines below): 0. Poco::Net::SocketImpl::error(int, String const&) @ 0x0000000015b5abb2 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. Poco::Net::SocketImpl::shutdown() @ 0x0000000015b5c39c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::ZooKeeper::finalize(bool, bool, String const&) @ 0x000000001383d143 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&)::$_1>(Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&)::$_1&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000001384e087 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. ? @ 0x00007f44aa6b2a83 in ? 6. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.922932 [ 424361 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. 2026.08.17 23:52:16.923046 [ 424391 ] {} oximeter.fields_uuid_local (eaa5beef-895d-41af-915b-bd6fb56613c4): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception Coordination::Exception::fromMessage(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function, std::shared_ptr>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector>&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector>&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr, std::shared_ptr>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 13. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 14. ? @ 0x00007f44aa6b2a83 in ? 15. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:16.923066 [ 424354 ] {} oximeter.fields_uuid_local (eaa5beef-895d-41af-915b-bd6fb56613c4): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception Coordination::Exception::fromMessage(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. Coordination::ZooKeeper::get(String const&, std::function, std::shared_ptr>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr, std::function, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 13. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 14. ? @ 0x00007f44aa6b2a83 in ? 15. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.931620 [ 424420 ] {} oximeter.version (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 209. DB::NetException: Timeout exceeded while reading from socket ([::1]:19003, 10000 ms): while receiving handshake from ZooKeeper. (SOCKET_TIMEOUT) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.931982 [ 424379 ] {} oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.932349 [ 424329 ] {} oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.932661 [ 424332 ] {} oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.933144 [ 424361 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.933173 [ 424361 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:26.933610 [ 425085 ] {} virtual bool DB::DDLWorker::initializeMainThread(): Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::DDLWorker::getAndSetZooKeeper() @ 0x00000000116ec1ed in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::DDLWorker::initializeMainThread() @ 0x00000000116fc81b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::DDLWorker::runMainThread() @ 0x00000000116e7280 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl(void (DB::DDLWorker::*&&)(), DB::DDLWorker*&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000116fd92c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:26.933830 [ 424361 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:27.031717 [ 424411 ] {} oximeter.version (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:27.032071 [ 424326 ] {} oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:27.032163 [ 424411 ] {} oximeter.version (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:27.032430 [ 424409 ] {} oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:27.032491 [ 424326 ] {} oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:27.032761 [ 424341 ] {} oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:27.032853 [ 424409 ] {} oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:27.033157 [ 424341 ] {} oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, Priority, std::optional, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 12. ? @ 0x00007f44aa6b2a83 in ? 13. ? @ 0x00007f44aa744890 in ? (version 23.8.7.24 (official build)) 2026.08.17 23:52:27.233920 [ 424297 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): Table was in readonly mode. Will try to activate it. 2026.08.17 23:52:27.234476 [ 424297 ] {} oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): Failed to establish a new ZK connection. Will try again: Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19003, [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 Code: 999. Coordination::Exception: Keeper server rejected the connection during the handshake. Possibly it's overloaded, doesn't see leader or stale: while receiving handshake from ZooKeeper. (KEEPER_EXCEPTION) (version 23.8.7.24 (official build)), [::1]:19003 Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 1. DB::Exception::Exception(int, FormatStringHelperImpl::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 2. Coordination::Exception::Exception(Coordination::Error, FormatStringHelperImpl::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 3. Coordination::ZooKeeper::ZooKeeper(std::vector> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr, std::optional>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 5. zkutil::ZooKeeper::ZooKeeper(zkutil::ZooKeeperArgs const&, std::shared_ptr) @ 0x00000000137f5ff0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 6. DB::Context::getZooKeeper() const @ 0x000000001166b47c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 7. DB::StorageReplicatedMergeTree::setZooKeeper() @ 0x000000001275b85e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 8. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012f11640 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 9. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 10. void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse 11. void* std::__thread_proxy[abi:v15000]