| 1 | 2024.09.02 19:49:59.996385 [ 206108 ] {} <Error> virtual bool DB::DDLWorker::initializeMainThread(): Code: 999. Coordination::Exception: All connection tries failed while connecting to ZooKeeper. nodes: [::1]:19001, [::1]:19003, [::1]:19002 | ||
| 2 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 | ||
| 3 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 | ||
| 4 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 | ||
| 5 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 | ||
| 6 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 | ||
| 7 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 | ||
| 8 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19001 | ||
| 9 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19003 | ||
| 10 | Poco::Exception. Code: 1000, e.code() = 111, Connection refused (version 23.8.7.24 (official build)), [::1]:19002 | ||
| 11 | . (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 12 | |||
| 13 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 14 | 1. DB::Exception::Exception<String&>(int, FormatStringHelperImpl<std::type_identity<String&>::type>, String&) @ 0x00000000074d294d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 15 | 2. Coordination::Exception::Exception<String&>(Coordination::Error, FormatStringHelperImpl<std::type_identity<String&>::type>, String&) @ 0x00000000126eb50a in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 16 | 3. Coordination::ZooKeeper::ZooKeeper(std::vector<Coordination::ZooKeeper::Node, std::allocator<Coordination::ZooKeeper::Node>> const&, zkutil::ZooKeeperArgs const&, std::shared_ptr<DB::ZooKeeperLog>, std::optional<std::function<void (unsigned long, Coordination::ZooKeeper::Node const&)>>&&) @ 0x0000000013841f00 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 17 | 4. zkutil::ZooKeeper::init(zkutil::ZooKeeperArgs) @ 0x00000000137f31d3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 18 | 5. zkutil::ZooKeeper::ZooKeeper(Poco::Util::AbstractConfiguration const&, String const&, std::shared_ptr<DB::ZooKeeperLog>) @ 0x00000000137f66f9 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 19 | 6. DB::Context::getZooKeeper() const @ 0x000000001166afca in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 20 | 7. DB::DDLWorker::getAndSetZooKeeper() @ 0x00000000116ec1ed in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 21 | 8. DB::DDLWorker::initializeMainThread() @ 0x00000000116fc81b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 22 | 9. DB::DDLWorker::runMainThread() @ 0x00000000116e7280 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 23 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<void (DB::DDLWorker::*)(), DB::DDLWorker*>(void (DB::DDLWorker::*&&)(), DB::DDLWorker*&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000116fd92c in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 24 | 11. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 25 | 12. ? @ 0x00007fdab736fac3 in ? | ||
| 26 | 13. ? @ 0x00007fdab7401850 in ? | ||
| 27 | (version 23.8.7.24 (official build)) | ||
| 28 | 2024.09.02 19:50:12.006803 [ 205774 ] {} <Warning> oximeter.fields_i64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 29 | 2024.09.02 19:50:12.006833 [ 205824 ] {} <Warning> oximeter.timeseries_schema_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 30 | 2024.09.02 19:50:12.006809 [ 205803 ] {} <Warning> oximeter.version (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 31 | 2024.09.02 19:50:12.006805 [ 205800 ] {} <Warning> oximeter.measurements_cumulativef64_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 32 | 2024.09.02 19:50:12.006895 [ 205809 ] {} <Warning> oximeter.fields_uuid_local (ReplicatedMergeTreeRestartingThread): ZooKeeper session has expired. Switching to a new session. | ||
| 33 | 2024.09.02 19:50:12.009797 [ 205836 ] {} <Error> oximeter.measurements_cumulativef64_local (8387b2c6-e147-451e-8e5f-84f4bbeb2c47): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 242. DB::Exception: Table is in readonly mode (replica path: /clickhouse/tables/1/measurements_cumulativef64_local/replicas/2). (TABLE_IS_READ_ONLY), Stack trace (when copying this message, always include the lines below): | ||
| 34 | |||
| 35 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 36 | 1. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x00000000074d2e8d in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 37 | 2. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d6ae in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 38 | 3. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 39 | 4. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 40 | 5. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 41 | 6. ? @ 0x00007fdab736fac3 in ? | ||
| 42 | 7. ? @ 0x00007fdab7401850 in ? | ||
| 43 | (version 23.8.7.24 (official build)) | ||
| 44 | 2024.09.02 19:50:12.009976 [ 205778 ] {} <Error> oximeter.measurements_cumulativef64_local (8387b2c6-e147-451e-8e5f-84f4bbeb2c47): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 45 | |||
| 46 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 47 | 1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 48 | 2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 49 | 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 50 | 4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 51 | 5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 52 | 6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 53 | 7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 54 | 8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 55 | 9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 56 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 57 | 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 58 | 12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 59 | 13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 60 | 14. ? @ 0x00007fdab736fac3 in ? | ||
| 61 | 15. ? @ 0x00007fdab7401850 in ? | ||
| 62 | (version 23.8.7.24 (official build)) | ||
| 63 | 2024.09.02 19:50:12.012036 [ 205790 ] {} <Error> oximeter.fields_uuid_local (2a01182b-0909-4138-ac40-9871004f15b4): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 64 | |||
| 65 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 66 | 1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 67 | 2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 68 | 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 69 | 4. Coordination::ZooKeeper::get(String const&, std::function<void (Coordination::GetResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 70 | 5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 71 | 6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 72 | 7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 73 | 8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 74 | 9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr<zkutil::ZooKeeper>, std::function<void (Coordination::WatchResponse const&)>, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 75 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 76 | 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 77 | 12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 78 | 13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 79 | 14. ? @ 0x00007fdab736fac3 in ? | ||
| 80 | 15. ? @ 0x00007fdab7401850 in ? | ||
| 81 | (version 23.8.7.24 (official build)) | ||
| 82 | 2024.09.02 19:50:12.012105 [ 205807 ] {} <Error> oximeter.fields_uuid_local (2a01182b-0909-4138-ac40-9871004f15b4): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 83 | |||
| 84 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 85 | 1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 86 | 2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 87 | 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 88 | 4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 89 | 5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 90 | 6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 91 | 7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 92 | 8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 93 | 9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 94 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 95 | 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 96 | 12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 97 | 13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 98 | 14. ? @ 0x00007fdab736fac3 in ? | ||
| 99 | 15. ? @ 0x00007fdab7401850 in ? | ||
| 100 | (version 23.8.7.24 (official build)) | ||
| 101 | 2024.09.02 19:50:12.012143 [ 205773 ] {} <Error> oximeter.version (626c1ca5-d8d8-4409-a032-b2aeb18345c3): void DB::StorageReplicatedMergeTree::queueUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 102 | |||
| 103 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 104 | 1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 105 | 2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 106 | 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 107 | 4. Coordination::ZooKeeper::get(String const&, std::function<void (Coordination::GetResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000013847605 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 108 | 5. zkutil::ZooKeeper::getImpl(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>) @ 0x00000000137fa0bb in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 109 | 6. zkutil::ZooKeeper::tryGetWatch(String const&, String&, Coordination::Stat*, std::function<void (Coordination::WatchResponse const&)>, Coordination::Error*) @ 0x00000000137fa7df in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 110 | 7. zkutil::ZooKeeper::tryGet(String const&, String&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&, Coordination::Error*) @ 0x00000000137fa6c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 111 | 8. zkutil::ZooKeeper::get(String const&, Coordination::Stat*, std::shared_ptr<Poco::Event> const&) @ 0x00000000137fa518 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 112 | 9. DB::ReplicatedMergeTreeQueue::pullLogsToQueue(std::shared_ptr<zkutil::ZooKeeper>, std::function<void (Coordination::WatchResponse const&)>, DB::ReplicatedMergeTreeQueue::PullLogsReason) @ 0x0000000012eddcf5 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 113 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289d59f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 114 | 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 115 | 12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 116 | 13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 117 | 14. ? @ 0x00007fdab736fac3 in ? | ||
| 118 | 15. ? @ 0x00007fdab7401850 in ? | ||
| 119 | (version 23.8.7.24 (official build)) | ||
| 120 | 2024.09.02 19:50:12.012176 [ 205767 ] {} <Error> oximeter.version (626c1ca5-d8d8-4409-a032-b2aeb18345c3): void DB::StorageReplicatedMergeTree::mutationsUpdatingTask(): Code: 999. Coordination::Exception: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): | ||
| 121 | |||
| 122 | 0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c6487f7 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 123 | 1. DB::Exception::Exception<char const (&) [16]>(int, char const (&) [16]) @ 0x000000000c63f980 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 124 | 2. Coordination::Exception Coordination::Exception::fromMessage<char const (&) [16]>(Coordination::Error, char const (&) [16]) @ 0x0000000013817dd3 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 125 | 3. Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x00000000138452ec in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 126 | 4. Coordination::ZooKeeper::list(String const&, Coordination::ListRequestType, std::function<void (Coordination::ListResponse const&)>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x00000000138481b1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 127 | 5. zkutil::ZooKeeper::asyncTryGetChildrenNoThrow(String const&, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f728e in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 128 | 6. zkutil::ZooKeeper::getChildrenImpl(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f6d96 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 129 | 7. zkutil::ZooKeeper::tryGetChildrenWatch(String const&, std::vector<String, std::allocator<String>>&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7a7b in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 130 | 8. zkutil::ZooKeeper::getChildrenWatch(String const&, Coordination::Stat*, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>, Coordination::ListRequestType) @ 0x00000000137f7971 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 131 | 9. DB::ReplicatedMergeTreeQueue::updateMutations(std::shared_ptr<zkutil::ZooKeeper>, std::shared_ptr<std::function<void (Coordination::WatchResponse const&)>>) @ 0x0000000012ee47c8 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 132 | 10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageReplicatedMergeTree::StorageReplicatedMergeTree(String const&, String const&, bool, DB::StorageID const&, String const&, DB::StorageInMemoryMetadata const&, std::shared_ptr<DB::Context>, String const&, DB::MergeTreeData::MergingParams const&, std::unique_ptr<DB::MergeTreeSettings, std::default_delete<DB::MergeTreeSettings>>, bool, DB::RenamingRestrictions, bool)::$_2, void ()>>(std::__function::__policy_storage const*) @ 0x000000001289dec0 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 133 | 11. DB::BackgroundSchedulePool::threadFunction() @ 0x00000000110c859f in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 134 | 12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x00000000110c95d1 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 135 | 13. void* std::__thread_proxy[abi:v15000]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000c7298c4 in /work/oxidecomputer/omicron/out/clickhouse/clickhouse | ||
| 136 | 14. ? @ 0x00007fdab736fac3 in ? | ||
| 137 | 15. ? @ 0x00007fdab7401850 in ? | ||
| 138 | (version 23.8.7.24 (official build)) | ||