2024.12.18 23:38:18.604265 [ 252 ] {} <Error> virtual bool DB::DDLWorker::initializeMainThread(): Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:29002, [::1]:29001, [::1]:29003
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29002 (version 23.8.7.1), [::1]:29002
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29001 (version 23.8.7.1), [::1]:29001
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29003 (version 23.8.7.1), [::1]:29003
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29002 (version 23.8.7.1), [::1]:29002
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29001 (version 23.8.7.1), [::1]:29001
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29003 (version 23.8.7.1), [::1]:29003
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29002 (version 23.8.7.1), [::1]:29002
Poco::Exception. Code: 1000, e.code() = 146, Connection refused (version 23.8.7.1), [::1]:29001
Poco::Exception. Code: 1000, e.code() = 146, Connection refused: [::1]:29003 (version 23.8.7.1), [::1]:29003
. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below):

0. StackTrace::tryCapture() @ 0x0000000010014f57 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000ffe797f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. DB::Exception::Exception<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(int, FormatStringHelperImpl<std::type_identity<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>::type>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) @ 0x000000000aded677 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. Coordination::Exception::Exception<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(Coordination::Error, FormatStringHelperImpl<std::type_identity<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>::type>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) @ 0x0000000015562ded in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. Coordination::ZooKeeper::connect(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node> > const&, Poco::Timespan) @ 0x0000000016bad2ca in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. Coordination::ZooKeeper::ZooKeeper(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node> > const&, zkutil::ZooKeeperArgs const&, std::shared_ptr<DB::ZooKeeperLog>, std::optional<std::function<void (unsigned long, Coordination::ZooKeeper::Node const&)> >&&) @ 0x0000000016baaecb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x0000000016b741e9 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. zkutil::ZooKeeper::ZooKeeper(Poco::Util::AbstractConfiguration const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<DB::ZooKeeperLog>) @ 0x0000000016b75a63 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. DB::Context::getZooKeeper() const @ 0x0000000014e80509 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. DB::DDLWorker::getAndSetZooKeeper() @ 0x0000000014ebd50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. DB::DDLWorker::initializeMainThread() @ 0x0000000014ecbd0c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
11. DB::DDLWorker::runMainThread() @ 0x0000000014ebbd51 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
12. ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<void (DB::DDLWorker::*)(), DB::DDLWorker*>(void (DB::DDLWorker::*&&)(), DB::DDLWorker*&&)::{lambda()#1}::operator()() @ 0x0000000014ecd995 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
13. ThreadPoolImpl<std::thread>::worker(std::_List_iterator<std::thread>) @ 0x0000000019092c5c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
14. ? @ 0x000000000014d750 in /usr/gcc/13/lib/amd64/libstdc++.so.6.0.32
15. _thrp_setup @ 0x0000000000114f37 in /lib/amd64/libc.so.1
 (version 23.8.7.1)
2024.12.18 23:38:20.168418 [ 259 ] {} <Error> ServerErrorHandler: Code: 210. DB::NetException: Connection reset by peer, while reading from socket ([::1]:46978). (NETWORK_ERROR), Stack trace (when copying this message, always include the lines below):

0. StackTrace::tryCapture() @ 0x0000000010014f57 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
1. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000ffe797f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
2. DB::NetException::NetException<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(int, FormatStringHelperImpl<std::type_identity<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >::type, std::type_identity<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >::type>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&) @ 0x0000000015b19afe in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
3. DB::ReadBufferFromPocoSocket::nextImpl() @ 0x00000000190bd7af in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
4. DB::TCPHandler::runImpl() @ 0x0000000015b25abc in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
5. DB::TCPHandler::run() @ 0x0000000015b3661c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
6. Poco::Net::TCPServerConnection::start() @ 0x000000001954534c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
7. Poco::Net::TCPServerDispatcher::run() @ 0x0000000019545821 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
8. Poco::PooledThread::run() @ 0x00000000195aa732 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
9. Poco::ThreadImpl::runnableEntry(void*) @ 0x00000000195a81e5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse
10. _thrp_setup @ 0x0000000000114f37 in /lib/amd64/libc.so.1
 (version 23.8.7.1)