From c783468c6a98e6c4440d81de49cd1a3e8939bc42 Mon Sep 17 00:00:00 2001 From: zhannngchen Date: Mon, 18 Aug 2025 13:46:35 +0800 Subject: [PATCH 1/3] [feat](warm up) introduce immediate warmup on read cluster (#54611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem Summary: introduce a config `enable_warmup_immediately_on_new_rowset`, if user set it to `true` warm up will be triggerd automatically and immediately on syncing new rowsets NOTEļ¼š the method `get_rowset_warmup_state()` is used for #53540 --- be/src/cloud/cloud_internal_service.cpp | 13 +- be/src/cloud/cloud_meta_mgr.cpp | 4 +- be/src/cloud/cloud_tablet.cpp | 73 ++++- be/src/cloud/cloud_tablet.h | 11 + be/src/cloud/cloud_warm_up_manager.cpp | 19 +- be/src/cloud/cloud_warm_up_manager.h | 11 +- be/src/cloud/config.cpp | 2 + be/src/cloud/config.h | 2 + be/src/olap/base_tablet.h | 2 +- be/test/cloud/cloud_tablet_test.cpp | 301 ++++++++++++++++++ .../cluster/test_immediate_warmup_basic.out | 16 + .../test_schema_change_add_key_column.csv.gz | Bin 0 -> 72233 bytes .../test_immediate_warmup_basic.groovy | 154 +++++++++ ...est_immediate_warmup_multi_segments.groovy | 232 ++++++++++++++ 14 files changed, 831 insertions(+), 9 deletions(-) create mode 100644 be/test/cloud/cloud_tablet_test.cpp create mode 100644 regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.out create mode 100644 regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_schema_change_add_key_column.csv.gz create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_multi_segments.groovy diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 22fa9e1d4bdae5..f1dc736ef43dc1 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -21,6 +21,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_downloader.h" @@ -223,9 +224,15 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c expiration_time = 0; } + if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpState::TRIGGERED_BY_JOB)) { + LOG(INFO) << "found duplicate warmup task for rowset " << rs_meta.rowset_id() + << ", skip it"; + continue; + } + for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { auto segment_size = rs_meta.segment_file_size(segment_id); - auto download_done = [=](Status st) { + auto download_done = [=, version = rs_meta.version()](Status st) { if (st.ok()) { g_file_cache_event_driven_warm_up_finished_segment_num << 1; g_file_cache_event_driven_warm_up_finished_segment_size << segment_size; @@ -256,6 +263,10 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id.to_string() << ", error: " << st; } + if (tablet->complete_rowset_segment_warmup(rowset_id, st) == WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id + << ") completed"; + } if (wait) { wait->signal(); } diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 5be30dcac792c3..6a332a292faaf2 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -766,7 +766,9 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, // after doing EMPTY_CUMULATIVE compaction, MS cp is 13, get_rowset will return [2-11][12-12]. bool version_overlap = tablet->max_version_unlocked() >= rowsets.front()->start_version(); - tablet->add_rowsets(std::move(rowsets), version_overlap, wlock, warmup_delta_data); + tablet->add_rowsets( + std::move(rowsets), version_overlap, wlock, + warmup_delta_data || config::enable_warmup_immediately_on_new_rowset); RETURN_IF_ERROR(tablet->merge_rowsets_schema()); } diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index a86abb7fe4d16a..4d1884016cbf69 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -89,6 +89,17 @@ bvar::Adder g_file_cache_recycle_cached_data_segment_size( bvar::Adder g_file_cache_recycle_cached_data_index_num( "file_cache_recycle_cached_data_index_num"); +bvar::Adder g_file_cache_warm_up_segment_complete_num( + "file_cache_warm_up_segment_complete_num"); +bvar::Adder g_file_cache_warm_up_segment_failed_num( + "file_cache_warm_up_segment_failed_num"); +bvar::Adder g_file_cache_warm_up_rowset_complete_num( + "file_cache_warm_up_rowset_complete_num"); +bvar::Adder g_file_cache_warm_up_rowset_triggered_by_job_num( + "file_cache_warm_up_rowset_triggered_by_job_num"); +bvar::Adder g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num( + "file_cache_warm_up_rowset_triggered_by_sync_rowset_num"); + CloudTablet::CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta) : BaseTablet(std::move(tablet_meta)), _engine(engine) {} @@ -243,6 +254,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ for (auto& rs : rowsets) { if (version_overlap || warmup_delta_data) { #ifndef BE_TEST + bool warm_up_state_updated = false; // Warmup rowset data in background for (int seg_id = 0; seg_id < rs->num_segments(); ++seg_id) { const auto& rowset_meta = rs->rowset_meta(); @@ -271,6 +283,19 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ g_file_cache_cloud_tablet_submitted_segment_size << rs->rowset_meta()->segment_file_size(seg_id); } + if (!warm_up_state_updated) { + VLOG_DEBUG << "warm up rowset " << rs->version() << "(" << rs->rowset_id() + << ") triggerd by sync rowset"; + if (!add_rowset_warmup_state_unlocked( + *(rs->rowset_meta()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET)) { + LOG(INFO) << "found duplicate warmup task for rowset " + << rs->rowset_id() << ", skip it"; + break; + } + warm_up_state_updated = true; + } + // clang-format off + auto self = std::dynamic_pointer_cast(shared_from_this()); _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { .path = storage_resource.value()->remote_segment_path(*rowset_meta, seg_id), @@ -282,7 +307,8 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ .is_dryrun = config:: enable_reader_dryrun_when_download_file_cache, }, - .download_done {[](Status st) { + .download_done {[=](Status st) { + self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } @@ -437,6 +463,7 @@ void CloudTablet::delete_rowsets(const std::vector& to_delete, _timestamped_version_tracker.add_stale_path_version(rs_metas); for (auto&& rs : to_delete) { _rs_version_map.erase(rs->version()); + _rowset_warm_up_states.erase(rs->rowset_id()); } _tablet_meta->modify_rs_metas({}, rs_metas, false); @@ -1288,5 +1315,49 @@ Status CloudTablet::check_delete_bitmap_cache(int64_t txn_id, return Status::OK(); } +WarmUpState CloudTablet::get_rowset_warmup_state(RowsetId rowset_id) { + std::shared_lock rlock(_meta_lock); + if (_rowset_warm_up_states.find(rowset_id) == _rowset_warm_up_states.end()) { + return WarmUpState::NONE; + } + return _rowset_warm_up_states[rowset_id].first; +} + +bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state) { + std::lock_guard wlock(_meta_lock); + return add_rowset_warmup_state_unlocked(rowset, state); +} + +bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state) { + if (_rowset_warm_up_states.find(rowset.rowset_id()) != _rowset_warm_up_states.end()) { + return false; + } + if (state == WarmUpState::TRIGGERED_BY_JOB) { + g_file_cache_warm_up_rowset_triggered_by_job_num << 1; + } else if (state == WarmUpState::TRIGGERED_BY_SYNC_ROWSET) { + g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num << 1; + } + _rowset_warm_up_states[rowset.rowset_id()] = std::make_pair(state, rowset.num_segments()); + return true; +} + +WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Status status) { + std::lock_guard wlock(_meta_lock); + if (_rowset_warm_up_states.find(rowset_id) == _rowset_warm_up_states.end()) { + return WarmUpState::NONE; + } + VLOG_DEBUG << "complete rowset segment warmup for rowset " << rowset_id << ", " << status; + g_file_cache_warm_up_segment_complete_num << 1; + if (!status.ok()) { + g_file_cache_warm_up_segment_failed_num << 1; + } + _rowset_warm_up_states[rowset_id].second--; + if (_rowset_warm_up_states[rowset_id].second <= 0) { + g_file_cache_warm_up_rowset_complete_num << 1; + _rowset_warm_up_states[rowset_id].first = WarmUpState::DONE; + } + return _rowset_warm_up_states[rowset_id].first; +} + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 38b9cf94e6a042..aa8947475fb257 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -25,6 +25,7 @@ namespace doris { class CloudStorageEngine; +enum class WarmUpState : int; struct SyncRowsetStats { int64_t get_remote_rowsets_num {0}; @@ -281,12 +282,19 @@ class CloudTablet final : public BaseTablet { static std::vector recycle_cached_data( const std::vector& rowsets); + // Add warmup state management + WarmUpState get_rowset_warmup_state(RowsetId rowset_id); + bool add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state); + WarmUpState complete_rowset_segment_warmup(RowsetId rowset_id, Status status); + private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version void update_base_size(const Rowset& rs); Status sync_if_not_running(SyncRowsetStats* stats = nullptr); + bool add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state); + CloudStorageEngine& _engine; // this mutex MUST ONLY be used when sync meta @@ -346,6 +354,9 @@ class CloudTablet final : public BaseTablet { std::mutex _gc_mutex; std::unordered_map _unused_rowsets; std::vector, DeleteBitmapKeyRanges>> _unused_delete_bitmap; + + // for warm up states management + std::unordered_map> _rowset_warm_up_states; }; using CloudTabletSPtr = std::shared_ptr; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 2915b81f2536a3..6d67b6f2193dae 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -110,7 +110,7 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, int64_t expiration_time, std::shared_ptr wait, - bool is_index) { + bool is_index, std::function done_cb) { if (file_size < 0) { auto st = file_system->file_size(path, &file_size); if (!st.ok()) [[unlikely]] { @@ -147,7 +147,8 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, }, .download_done = - [=](Status st) { + [&](Status st) { + if (done_cb) done_cb(st); if (!st) { LOG_WARNING("Warm up error ").error(st); } else if (is_index) { @@ -227,12 +228,24 @@ void CloudWarmUpManager::handle_jobs() { if (expiration_time <= UnixSeconds()) { expiration_time = 0; } + if (!tablet->add_rowset_warmup_state(*rs, WarmUpState::TRIGGERED_BY_JOB)) { + LOG(INFO) << "found duplicate warmup task for rowset " << rs->rowset_id() + << ", skip it"; + continue; + } // 1st. download segment files submit_download_tasks( storage_resource.value()->remote_segment_path(*rs, seg_id), rs->segment_file_size(seg_id), storage_resource.value()->fs, - expiration_time, wait); + expiration_time, wait, false, [tablet, rs, seg_id](Status st) { + VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " + << seg_id << " completed"; + if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), st) == + WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; + } + }); // 2nd. download inverted index files int64_t file_size = -1; diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index c801e77acc787b..73da26b2bfdcb0 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -38,6 +38,13 @@ enum class DownloadType { S3, }; +enum class WarmUpState : int { + NONE, + TRIGGERED_BY_SYNC_ROWSET, + TRIGGERED_BY_JOB, + DONE, +}; + struct JobMeta { JobMeta() = default; JobMeta(const TJobMeta& meta); @@ -95,8 +102,8 @@ class CloudWarmUpManager { void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, int64_t expiration_time, - std::shared_ptr wait, - bool is_index = false); + std::shared_ptr wait, bool is_index = false, + std::function done_cb = nullptr); std::mutex _mtx; std::condition_variable _cond; int64_t _cur_job_id {0}; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 16d6aa7f782b65..5d22c7d8f4cfe2 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -106,5 +106,7 @@ DEFINE_mInt64(warm_up_rowset_sync_wait_min_timeout_ms, "10000"); DEFINE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms, "120000"); +DEFINE_mBool(enable_warmup_immediately_on_new_rowset, "false"); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index a52ac758e671a8..6dd36d0b7b935b 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -148,5 +148,7 @@ DECLARE_mInt64(warm_up_rowset_sync_wait_min_timeout_ms); DECLARE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms); +DECLARE_mBool(enable_warmup_immediately_on_new_rowset); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index cb61252c6fe75a..9c92d48e9a8148 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -56,7 +56,7 @@ struct TabletWithVersion { enum class CompactionStage { NOT_SCHEDULED, PENDING, EXECUTING }; // Base class for all tablet classes -class BaseTablet { +class BaseTablet : public std::enable_shared_from_this { public: explicit BaseTablet(TabletMetaSharedPtr tablet_meta); virtual ~BaseTablet(); diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp new file mode 100644 index 00000000000000..5ec3df0417591c --- /dev/null +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -0,0 +1,301 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cloud/cloud_tablet.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_warm_up_manager.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_factory.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/tablet_meta.h" +#include "util/uid_util.h" + +namespace doris { + +using namespace std::chrono; + +class CloudTabletWarmUpStateTest : public testing::Test { +public: + CloudTabletWarmUpStateTest() : _engine(CloudStorageEngine(EngineOptions {})) {} + + void SetUp() override { + _tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F)); + _tablet = + std::make_shared(_engine, std::make_shared(*_tablet_meta)); + } + void TearDown() override {} + + RowsetSharedPtr create_rowset(Version version, int num_segments = 1) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version(version); + rs_meta->set_rowset_id(_engine.next_rowset_id()); + rs_meta->set_num_segments(num_segments); + RowsetSharedPtr rowset; + Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset); + if (!st.ok()) { + return nullptr; + } + return rowset; + } + +protected: + std::string _json_rowset_meta; + TabletMetaSharedPtr _tablet_meta; + std::shared_ptr _tablet; + CloudStorageEngine _engine; +}; + +// Test get_rowset_warmup_state for non-existent rowset +TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) { + auto rowset = create_rowset(Version(1, 1)); + ASSERT_NE(rowset, nullptr); + + auto non_existent_id = _engine.next_rowset_id(); + + WarmUpState state = _tablet->get_rowset_warmup_state(non_existent_id); + EXPECT_EQ(state, WarmUpState::NONE); +} + +// Test add_rowset_warmup_state with TRIGGERED_BY_JOB state +TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) { + auto rowset = create_rowset(Version(1, 1), 5); + ASSERT_NE(rowset, nullptr); + + bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(result); + + // Verify the state is correctly set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); +} + +// Test add_rowset_warmup_state with TRIGGERED_BY_SYNC_ROWSET state +TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredBySyncRowset) { + auto rowset = create_rowset(Version(2, 2), 3); + ASSERT_NE(rowset, nullptr); + + bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_TRUE(result); + + // Verify the state is correctly set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); +} + +// Test adding duplicate rowset warmup state should fail +TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { + auto rowset = create_rowset(Version(3, 3), 2); + ASSERT_NE(rowset, nullptr); + + // First addition should succeed + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(result1); + + // Second addition should fail + bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_FALSE(result2); + + // State should remain the original one + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); +} + +// Test complete_rowset_segment_warmup for non-existent rowset +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent) { + auto non_existent_id = _engine.next_rowset_id(); + + WarmUpState result = _tablet->complete_rowset_segment_warmup(non_existent_id, Status::OK()); + EXPECT_EQ(result, WarmUpState::NONE); +} + +// Test complete_rowset_segment_warmup with partial completion +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { + auto rowset = create_rowset(Version(4, 4), 3); + ASSERT_NE(rowset, nullptr); + + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + // Complete one segment, should still be in TRIGGERED_BY_JOB state + WarmUpState result1 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + + // Complete second segment, should still be in TRIGGERED_BY_JOB state + WarmUpState result2 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB); + + // Verify current state is still TRIGGERED_BY_JOB + WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB); +} + +// Test complete_rowset_segment_warmup with full completion +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { + auto rowset = create_rowset(Version(5, 5), 2); + ASSERT_NE(rowset, nullptr); + + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_TRUE(add_result); + + // Complete first segment + WarmUpState result1 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + + // Complete second segment, should transition to DONE state + WarmUpState result2 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + EXPECT_EQ(result2, WarmUpState::DONE); + + // Verify final state is DONE + WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(final_state, WarmUpState::DONE); +} + +// Test complete_rowset_segment_warmup with error status +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { + auto rowset = create_rowset(Version(6, 6), 1); + ASSERT_NE(rowset, nullptr); + + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + // Complete with error status, should still transition to DONE when all segments complete + Status error_status = Status::InternalError("Test error"); + WarmUpState result = _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), error_status); + EXPECT_EQ(result, WarmUpState::DONE); + + // Verify final state is DONE even with error + WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(final_state, WarmUpState::DONE); +} + +// Test multiple rowsets warmup state management +TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { + auto rowset1 = create_rowset(Version(7, 7), 2); + auto rowset2 = create_rowset(Version(8, 8), 3); + auto rowset3 = create_rowset(Version(9, 9), 1); + ASSERT_NE(rowset1, nullptr); + ASSERT_NE(rowset2, nullptr); + ASSERT_NE(rowset3, nullptr); + + // Add multiple rowsets + EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB)); + EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); + EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset3->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB)); + + // Verify all states + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), + WarmUpState::TRIGGERED_BY_JOB); + + // Complete rowset1 (2 segments) + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + WarmUpState::DONE); + + // Complete rowset3 (1 segment) + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset3->rowset_id(), Status::OK()), + WarmUpState::DONE); + + // Verify states after completion + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), WarmUpState::DONE); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), WarmUpState::DONE); +} + +// Test warmup state with zero segments (edge case) +TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { + auto rowset = create_rowset(Version(10, 10), 0); + ASSERT_NE(rowset, nullptr); + + // Add rowset with zero segments + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + // State should be immediately ready for completion since there are no segments to warm up + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); + + // Any completion call should handle the edge case gracefully + WarmUpState result = _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + // With 0 segments, the counter should already be 0, so this should transition to DONE + EXPECT_EQ(result, WarmUpState::DONE); +} + +// Test concurrent access to warmup state (basic thread safety verification) +TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { + auto rowset1 = create_rowset(Version(11, 11), 4); + auto rowset2 = create_rowset(Version(12, 12), 3); + ASSERT_NE(rowset1, nullptr); + ASSERT_NE(rowset2, nullptr); + + // Add rowsets from different "threads" (simulated by sequential calls) + EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB)); + EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); + + // Interleaved completion operations + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset2->rowset_id(), Status::OK()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + WarmUpState::TRIGGERED_BY_JOB); + + // Check states are maintained correctly + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), + WarmUpState::TRIGGERED_BY_SYNC_ROWSET); +} +} // namespace doris diff --git a/regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.out b/regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.out new file mode 100644 index 00000000000000..99e1f4ad641944 --- /dev/null +++ b/regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 {"a":1} +2 {"a":111.1111} +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} + +-- !sql -- +1 {"a":1} +2 {"a":111.1111} +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} +6 {"a":1111.11111} + diff --git a/regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_schema_change_add_key_column.csv.gz b/regression-test/data/cloud_p0/cache/multi_cluster/warm_up/cluster/test_schema_change_add_key_column.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..bc9d3dd70ea8a55c752a3592bd509ffa97fd67ef GIT binary patch literal 72233 zcmXVYdq7h6`~UWB|L@L)TElEw$5Lk0&S?a2l zD=%cOSyECfLq%bxY0d=A5EBs<6+;0Lg;gN`~6&B&+B>K$Kfu%|9R>#a{6fEX;Na;v7;vslcJ6tj*C7@I{ed5q}ZdWq$u)?quJWr_Ff zsYb_&+EYJQI-Uo7zjiTeUd4t@L7vfPx^C_3F0e8tt(cYMZFJa0YaHIRCmX3|l2zc( zH>7ql?i30Q-&ySy9IN#yV~y?X{e;r3pxIA2uE`j7dRquxCBNkO??X2ARzB=(=t<)sUFpsWGyyr%GBv12Ipl(f)YC zb2=vF@JTAiwSQ>BO$?(-qw%%b)kRM`WHhU_LY8O=s9_ZItp;PhOry!Eqt3H%$qed2 zBkdf&6mD}K$Ydv_@n7mG9Wpc3t~ohFrKia3bgPZ0Gi6IpqGu?*ic6Tb;5&vfPi~^V zBfO`lNbTxG=3YE6><)j_z$uVf^O8~=-!a4kfgG!Jz=$4rYNzgsZB=y0lC0E5@$=?)SM%MSRsec|G;R)$iuI8F%&AzUA{snG|NUbHJGPNL%l9reXGzexQs2B)cNZv z$vlat%=#eC`CekDwMc8%Q&#itvn}Q#xL}TwLoS&@3hldwbv1I6)%!_@EdEc&kbyF8 zFSNT3C-QZU#KZ#xa^~G>(igKy|eQhIH;e?C7UW!&GC0NxHD(L@A%9 z!pwx16;2jQME)dErN7Gy|RkkKo z_ShT@SJ&cebTI*LaP?A=Aby^VFVJ&jcDvzjpe#+ejUj2q4vYRXGW8s(y*yqK79X<1 zBI#i=WRLeL`v2l*W2qb~g{Zk|UuQW;SJCDTEB@lE3>2B-(g1pCtzAuYe@;tyB}=w) z)zs@!EEqePp?GaAx~h3RLbjSh_!@tT!vCVZQ!MM8Av@eIJ7cw4NBPDRSB)e+XEkpq zhj9EQa|>fsr7^E}U+}*tBnC~OBbmSmL$?nYgQKJ?EeFXA25W++7__di9Q2mkt+Xnt z9RhLEz>(?7KZ@)Fe1$dd{gu zs#K;t$Y0O-idVodSsj^ar$fjQ__|ijsS!<6+Rx1W7wMiB>jVCz+K%p|{y^uG_=V?^ zEmo|GmL+CQnRs7wRx~FM+VdH$UCrD2o(-l&VqW1~8`wde=>HDo(m6<%eMm{+@bTX85QZLeYlKInQ@6syT z2`{R6i?)&;aLG>u_V1m) zt8L~B5zH_~JAu^`FU#?@#IV?E`&X6;v)-LHDUDxIYe}j;#PK{mLn3PP=AtENl-eg1 zV07?(rLc1WcAGD<{-0%`XC#WBj5u~n>P24n2ustuc=!>x>jCd(fp1X@61L13%vk+9 zAAWvrPMeJJhL{22ROPaLT;y5r)8BGQ!%lZt#yKagNSiVW-qt!D(5Ia+TI7DTWb<@q zK+^gt=e1zk2fS+I<@QIqkC9*%EEzrEY-$d#TglQWn{%m@jU<)O+Fn8aK|I3}!zc>v zkw@gBmyY1Y<$IiYs|lL_*o0zjAd)2dU%v4_ z(CrtXD|$>LbbCM@uEbdic5$&GZjwWN%afXu{CG&`r5edT8yAf$A7N$imJ>LP9KsgT zO1)?#L)T*Te-g-iBYWufgI=_U6tB_s+#g}x;Z@7mhv|%9j^`=a&x31kv*%f2q*R(W z&&?83%yvDqjOT?e=ILtI2)!*aD$1g{Nr%v)b9Icue(Ec5ZMCf5>wp^?}} z&RS0SaP73m395mto1JfL~2VPa);wFh$4 zD?!_syCo(`CG-fKE8K`2HeBOD$LU4N?x;ZP6x~O(3d>^-$6`iddBDh>DU^XY-E1}Q z?8$xPil+m%7nl;&L8M~y&n0jm`!Kpr?wcsL5h5j2R6Ee^q@^ugSW(FU|Cesq~;Z1 zi>K^>&5pG1E=-(H`~lXFA@g-7=Q(`f@wH>Gq~>$H14?hfJwYlFIa<{2RrEh?x`xzS zH={-S^X7|E&+`r(9ioRg-DgoI(xF6LZQ{LW1PjEVB~8z14ntALw1f_z{Z)>ylXh<=?VJp6 z>ZPXH|1Q)Ld-k5wFpeqcr&V)HZtFZOF}K;43US4MRd3n|b72$^X@}8J?lMZ z-u1h)6eg;Lg=HoF6rURlqVTG9iDgX-EHP#5a?j&;qns}^O7s4M=76LR4>q~~NSaAo z3lOQy>|D=)pOv^88GqEvDSb@OML+8)5KDi}{zhfJ#a?Eaj{jEV3c@F*E+S$zT<9NK ztbNkxkm19iEDo&Y+rU?1j%Fg)w)uO?TS-OtSIng2?501WmG&%905z}hrX+f_P4FIs5H9S$}YGQm2G zakWllJ%JPyHM*sAHC2-)i8V-`ATkONpGr$%nEPK_o)Jh%- zACXJ^#|`+e^`iUbGVZP@d?=V8yh^CCWH=+_d`acbCmkdeq3g}MfZ}5Cd)Yqq(E_p4 z2YacvfKja#5tIkTvye#iIeA!?CfBK7(u>!HU(1zUNK-DdS?!gJ<(UYaU4ja$qfHrl=n$44Ot&5aLCl7yovvH>O(vy&okHT_jC-|5r)DWhzgyczE z+eW|{@1i=_{Ff<@UH|B#EMy^bv$u9VG1ondZIuD0B5hGGBIKW5JaG@lc(CrUG0P@s(rw? z5q}zBne@nKx#h1CcH+r;0!xq zO==>wc{pREfnYm*3;OlE2+9_z1Uad~o4!=N7k<2{GnZc+GTnw(r#pzgT@t6AD#ahm z=Q;H}Kl+6w*Ln4j@Wdb5Y7zd&3zT4p{cID9y6Ab;FT4dit%F-boz&p-kg^y)c1gPGc8FoIjO7JK$XU~^ z5Cm~)*6jS4tJY^5NJ;wee$Boy-4vi21S`)`)~cWY1Bfr{u9#*ii{)?P>}aIjVpnTb zxcFsKKKB@h-{A4FL;ZK5mHf7$zl(lM5u|*Xh&G1Z{|&3I3WF3C(=+(@4*3DkE9X56 z`^lzr{`#rwI^fr3WU;S(B*hh5YgaZh@9%MQj=atfuKm3fhs*NBuC68s3fXq|o3mKE8#*wo;WjFgrU&|yY z;5#`jo-$&9TTq*r_=7=A`vS&fJpX6eCuobZY?>Vy0{DV-Yi6f(18H&y*RqCrSr zVTy<9iPNA{0~03$OVH!74e5tnz#cigXAGjpnL}ueEZY54-$f&=kY1Ca5sO1sCNxw% z2PT@NpPM{My#!r+ruJXc(n36J#CZ6Soap0*te{k8#^*qFP_nE=EtMTkg#E7rI8ahV1Dzzr~YJa z3H}OMl=UKZ+^px|sn#wD(3V-U^1|OV=a3P%^+@aJA!zXmP1W}^oYq#o@ybgUARBv+ z{VYnUf~!W3EQP-x_ zZ_+<%xGYpSh;Mq2c@_0{yTq&v;dj^v60a+yz$~NE#tI2IMfW$PwQ@=G=7nR3>9i57 zN^|#9;hQJoQgUlp0n|yoM(*1v^J^!}1gFyf2~fO%H}rGqOk!GyrR=0owz+p6v{5l5 ztOKTyF(EAS)VR_1X+3{C!}vm1o5H>^!ur|X_gT?Z+M(O*n_2>eU%f5fNtjHB2|=0m zt!aVwePi!9cQ@rp2JP$H2`5t>wK6Z{@ax9K6FYNMdIBv2_cXC+7j$8yjcy_AZO7M=7K1OX*}MmCOZ0)a*f;GP4VkNGA=p~r6}{mFkk#jS3$jxV3ApfBy)7eF z{7MF!oJ_k{a-H4?_be{$^^X8=Zrbazr58*pxYbCSC1cT|KnpP`S-4Yv3uKwhV{7NT z1R@7Mukmcuh3Ugbhf9BFXG-C@1(qyZ>m656tTS8qnYtaOK{3!o>kcY-K}gKG_Ig;+FUZ8iGMb`g0zq$`7o(hVeUX1K#Sioi)a|kcpHC zP&U7{qOnF@?)Shv;PE8ko3h$CXEcb_wQ{VT&}vF<++7HknMuni3ibHm{ukL_WN>|n z7+U~q?3>ig@sdNyTMJc$_fw<-Z6Ip!`TC(4xt2cHzE_rMSK_`2TVRCf*+~o`T#X#U#Nq%|p@BocXw1prm4#aWO8vtw-9{BA zy{^q&p$}hc&q_P2`gsN|97fgAwkje1NeO$I{Zlm`ZV5X)Xo)`Xl+>$9J3Rt%71i*U z@-Wf}E58q4ynXm6>?@*YIE=TvLABfL zW!Pj|#0V>kH?XMojFASjp_t$K`%vZD&KOx%-ep7=e%b{4B3k&+njYE*o!>0I@^n@> z)e-kuTq>Mol^VLTdix-Cls}L^9Qy=#=vI4Qu#6*vX|O`)5S-&5DrPHRFJjrXaICd} znW*SLT?-1&8!T1(rz!G zi(1%1FUm?g1stLbMzD=DTGL&@)>*tGh5N7wSdOdhHN8s%9F5@*_z#b+)Q4xAk_RrM z(s-L4Gye4_?)q?*W$$_K@v3a6JR|+Pt3p8ql`=apY*6$+ zWz1_@5wjOn8uN|^)gv*&n?mMTr=VrKJxM3NytX5IYiFZ~IV^SdN~3he(S_Zx@Xx%t z=4s&5e^bs(S-BzB?FxEjBbh96d zZ$L2uyR2x|*BH4NUYAYFEN=%&-cS>^>@yB~QuBjIR`TH%BRW4t3_(QWo} zi?YYszFe(?N;0T;=#T(ZY$45W;$qwp)^g_ISQ(s760CEzH|Fq#pc0mz7N!dWOU=_g zXd|jw@Y)l6*zg7C?0R5T%zkG#9M5BPli{HSvRy&{D*S=fF?CfRdBH3cdm`?Sj)$fy zTt!GXEliq$j1!RhQLhXhrwbWh*HWCuyv*#)QGkoe*yr<4s_~F2_C8|hTEPq?Ua<#W zLxG8XHow}wafR^Oj{2>ghYR+0RI;;q%ZJ?}sqIOkw%%iYjCVa>nBw%|{u6$cE|2Jb zma-T81r46z%r(aqJp7~B;>|#UUPO;HRLzw|cL+@TR9X3k&0OPD+cyPPSl_M>U)6h} zgq>yZhF*CJ-1*#|6|_-}_D18_{lQ|i0RI`g*tKVbbsLMh>9-MT6Pz1ps$)H9mFO}f z2OFGqQXjsXq?KOU4v+oH`@FnlzObg9u*Etz{<~|5Kb5fmTTR--dsi4rnya5{j*ndd z<2^XKDtgtrx6Ap0`tID}{;j0Ba(3U^*0+fx03i+D3vwShH;DVPH{GeoFH=gSKZCew z)6^7Y0oVfST*=?of8oVRk?6Si(_L`9JY0H+lu%T{{;rFyuIj6eg6A#MJ!k9QL#H_p z7KDw$>h_JQdT4^f+v%e9VeJl3w~a9P8QoF{CcMjjp(_-4BZZPq2c7}*Ot-`kYYDAu zq$8}=FU9}jR21mL_2FFl)koJMF1p#{!hK^@SVZr(&n5tP*L>y3OI%40V`Q4w4^HJe zZov}DyB_&4IBt@Qcgk+cxl<}#;%sux?dh#@of`!=Bwdkg`WTy@I1?X%5asHr2YS1RAsl7hX?r%UI!jPSB} z%O~bfH-9-Y=nS0Mj4y<6qLT&2Rosno>N@E7&{?)b#$5PFz+gKXyAFKQPAIJD`!%u~ zmek{B$p)ZOp_lHspMC+hHh)Ormp!M!Hm?Z;@y?`Ya8E@TpSdjH6k43n!_9>*KfFxq zKVI<^O_aO`{u}mz6E_6#^|yng!8x?FYySl2wh`{|(y~L)M12Ur(tSh+a!w8f`uV3g zmB1{q510J#B53A!!a_$E7a`a!cK7G9vNyXJ7mbT`3t;$sO}L>TdCY5sHQVMrFf*cb zgtezP_4OnU?AT7<(i@xTSqx^J9?Wo=eCmuHc|TEuf;kH^-2`!!Hu1$1M;99IG)z{k zgO31YI*rgDI1>!MnU*K{u+S=AVOp2m3uR~24l_zje*Jsi=qk#vo3 zVwFk9Zm~n;694G!aEvsx`;;KT7tXMRCU3!Btlfn++02jW${P`HB++$(0k z$UJ~`@s8!?HkezCfCgb!k%&Vq>W9agQl7>7u|nXnsg5j@-&hn}&1Ym3zQJs_*gO^X zqVi+OTk@dWmC$(*jKgmKt@%c=P%p~leePEK@vsh7kYN9rhplb0mAL~sQ7c0-?Kko7 z|R@RQQpmR^(&D<0?H>`AXAFVc&=m}hl@6nQ$&a*1DUEWykmKDjyliuVW%yR<3` z`%yGJ#TgWbt0^^J7Uf>jED|Rl##j;HERV$sM zZ&_%~r1c>yvCnq|I**MT(nSMjK%9yB>Z-45!aUIn)8l>`iagauko^(1zA_p{(}CH3 z^}-i)D%|?oo?7e$9xBZB(#@w(U@8bOkComfxLK4}g|}S4A%F$^;1ile-7OEe=MjeY zl{dr18pcQV(lZlaxIR){*pKKGc*4s&bXJr!?_T0je$jf2{vft;$|8- zaM4D%-?0?;7Tz3pTkY_4KL1Q@yuprdBap`Dn?hjDpzOfEPubNw;nZZBEo%Uv#^7y= zYODuwN_n5>JUYS&Le34>Mjk@vW?L3kCSQQXtIbxU4GnN%o36{}0+46VlQXZ5rEMV5 z8cB6?Ivt=k(_Y8L?J{oyD^@g(9vn`0x>-4%RI30JN2dj9vuiCydQNbWBmkO7QM7Gt z&G+)dt{@i-{QY|TV2QN$Um2$?&J&87w?9H2xY4FMp3e6IL*p!2L8IkT@C+}e<_oj5 zYd*ZmYHYgVqJYyj_%0y#dTNi;tN&#>o+ZKA`)h)+XGP-&PBCH{*|&|(t+w#e(il#8 z?Apg^Fmz#jt;p_X6t zmF=Ym#$R)Z`?t^f9v?oM8dHVfO{%4ny;yyQVaf#Mr0aIiAfX=mcZFG!V z`#3EZ25Ju0l-dl3dsb+Fzi7sRo-+rlR`9@B*^6s2l|a$Q$uHeZZ*2l;q}3mGtvbgb z(7Dy4hKnhau$Mu#De`3##g)!YojEAp0m5@rf6PotvB!-|wil((4cYJko$426DEF=W z34A>gGv?OS0YL&3WLMB0=Fms;eD1SZ2${b)XQ;I0X0kbn7M8SU3mp!oY=pD3wnzPT_v}P zbSlF-E}a4fNglgCGjKV!lg`Cz6-1+d9hp&GUFWf)jtu8&De}R+uoW|+G^4T=9Ijf_ z<@QE#RmR)Hp10lrF74K~9nS~raTV=#MM1~%WcKw1qk@+bAHKBGbL>!uqJ1*)={!-q<)Y?4f}07$yN(-xNTyqDXxy( z6ql+gJsH6MRVGMVwy1CTw_ZpcM)EJSQ5?oUPGA4g@JR^PIqLD3X6^)dd)MhhL)2+} z>H4K|Vk#5D*ErL+B^5l@=zC)%_*c2RR<@J0sys zQ_w&4j|!_N>}X0J{eGM>#1S_6Ge>nsuxjiq#z4SKh?SPlUzs5J{dRqzo>}cA-0eA{I2{66LtT~p@L8HGo!XlCGOjY5Y4TM zRfS}Tt+?xnAJMKsJZ=rN&ekl3)i;g4T`&7pR^GcoJQG096OI1VL z>1sN4zT$@o390bcEwvBeCjm)Wd_u8pT;fo^_}wl1imV)$ZLX&+H>*thpVq`SZl_bJ z3m+yel=X4Aq*;t-G(6;4Hx0#Ze8`{!%+IvBVwJg5`GvZ?ptc`<>&OZl z_9S){8M00~({`aK6oMhgZ#}6e>{qxG(0O^($E50IT;M0*-2isH$M0egFABPTzM@_; z#9^_|?^P915JZwqKL}V*G)O6BTIsJc-b94Q+~Dhw5!!aQ-T-VH2c&qb+0sC`X7~T3`YGnWGYl9qNgzp^JFhEW0DDy9R zDAdDv%HWI=;}x0OerXXm*#w3a`$o8S5+@{Yn2$6`r<2AdocO?oFQt_iK?%H%OKJlI z{3WuV;0MB2=fGzWS&lPQd4>|*OesP6pn)_qS} z$f4vvb8FIaP#qoQQ&63Vw;TO&7Jsu!L;t!z2rvd;E?C05{T`nN-wlQo4pdF}1)8Jj(x-^}Q#=V$1U*4wPe9e7d{&PY|vv@oCyu!;pPA#mVo5 zPr#VEk@6YqcP%xkcIW+`l7OmUbNVvoo%H}HY8tM{&6opDefROAyZk&^T<;f(w+ibE zvLfhLH(Qk(6ksg%BgGnoNB+|qmfnuOB8&6%Q^(T*?;qm{!=F`uZc=r#`-+Z-0(_>b z?=K$P-3+kV9G`@Nyp^83DfF|jeqAGjL(8!lvm_-=3JDP?Xl=Fa6n9A=^~S zu3yV{(XAw6yAZONsQw~#PklT<0d_6D{VhmcN{&1PsL^x)b>&5$Ka z9B-07wiyb~q%0i!8E){te;9>xl-mq9Ec~@ikj=det=NqToCEt%5hQ3uiN8DgJK`*; z0ZFFEHUYRH?Sg`olV>v=>Gd;Ax6ZX0t}0$+p~{yu|E_yqCAe~^GA%v_FmUovueT4>d<#Z-cHg)29z=@z@t?vUp$ZFV zXDb(|wl_m2*Ju3Q8;j!Jb9|S-FL;A7^uP=No!b+AT?d$?Y~B*|Py?VBoO)mW232rN z2EQrzYco=3`|ZqhnGfe(#$9{KASd4iD;`A=iG`k$zY#)7>kZjabO1`x%hVtH4xswV zAD~wAV3WdM-3ot(LaA^3#W5dNC#wCDQcgnP8x*X>yW+h7wwRI^J&Hh35Ir2Gg-Ssw zpYh{-4ixevujbfKLXflVUH7se#~?FHkNa(>2g*I--v|5*JhR#)0n}jMyZGYGJqY44 zf=3|UKGMhW#fnF!>!Sf#Drdgte}UZg_23pLt%cZ)pcw-RfFtdM*CnSR53TY9eZLVz ziq}w>OXA9a5K_l}V_fXrMd#WVk5y;NT=>MCj}&Va#{vHd!^A~s-S(@mr+n^v3m(Q6 zT9+V(rN3s_pg)L&yr>Ntn^H#>M`n61BK`ri@>`AhZZOo(&M(^*4J8h8MoFLaTug;b z$fKw^$MOh{p`hbZB!c$C!6~wLB1%AWymsL2H@QFSy#v{gqW|g$sV~`SlGM+`ie|slmFKT8Q)GEZ7BlZhs2}@Ac zmt0gnB|=_$Ha>dj1D@x56b@H?$*<+#)b7-Js49Y!@5Iy1t(QRKmu)_u=;bQ!`IX*LW zOMrj!hx}Ia>c|9AUHImvWqeFn)DZ&NnmjqJ<{Ly%*}F(YEft!Iyy{-58PUc^E?Bzf z3bSf>_p{awrAzs>gCS4*cFZ%Y5~Nk+e+Z}pxi5<)sn$ZP=D~08o9zY{4zscU8UV** zQuDiqNPuYu1$R-c5lD+^;TTfl>s`Jv@CBgW<;)PN z3~CYWVIQf41Rg8TN?J$b0d|rb`V*mUS>@#)R>q(R7gCA{;@v>kiq;w=n-FakytFxC zc5c^h|1eZu-!AFBKnHIpjP-B5?R8a_gU?7wH@xxT6CM92--DJFE>5B4M2$X$aYo(l z@E)K}No`Gdf(lm-Z*|;E&msti-CY-ZfT6|_pK+n-00P#tBNs*mS4LKl?m&w9OcJz}F;|Cz*o zsNqK5A!z{AuiBxsyUt2#NKXA`L;}@#|8uv|G$^{f&t+dV0vaf1PPnGQ;)#p)`~Hnc zOQYf=vc`XrEmjn!l)?VtDg&aXu76`3CuI7`>!WKXrhW1gqjSJdi(jNZ$=pmqo zemR$IH{njBJFoQ_Js&D8RyKYa%-`KAcjbyDIYueNnTfmqfQ-e*9g?Vg-bDmBueAs)UFv^wkSF5G~TZyv_}7 z0N}I?e*T#fM1j0Aq6)?|ez0=(v-*wh zOCcbh%MP^HLmYEA)p`jEAqhgK?V4rXFd=!s+7v-CYW@Rj@-Ia7L-f=tc$`a$FsN$@ zLpxWgp-IG3bdmF+OFt;Sq`REt378}jB@LPoz$9GlyIBn&X?G{Dd&KmDx)n2@@f>iL zSZ?jJ4FJ(3GHxi4>w6PxmZeP~OIjxaP~Ybl=EBb%U1d@kjJK|$&aTurRr!ZMJ{@S` z4&E9%^olI6d)7`2xS8eP5O)I7n?P#5p#@>{sEzB7aDX`L+~|!nm+?Ivx9A1K&=bVU zK4tW1VcLK#wKuJQ)yHugz4Vb$V1|81E9*e77i0%3(7XklqMk9RDmspe{$rbqdMc0) zILe9cIRrMDQt@I<1u6ks!eYXpzDBrC?77eZ%*8%+!lw|zl`Fqk*o48Xi0v9Bb^!5) z@S!uIxm~oIRBv7g9LrkoWAHN}cKmZ58Y-y?o*|!EH;+q5A%d?%pvWfp*Kj6iC~0oH z7RGO^Ba;&Z_q_Xgo`_^6kw$Jn9D6Rcw>k$7hfK51y8~EjJw~PrhM@4VH}Te;Sqy=U zCt=eV72tf``G6JmPzyb&D@JRp@$d0#{}O0CvH3Q`xZ5-DQ!pO^$~1XO(*tZ&U|HMy z3QfhXS&~yp+kx1&74S!%K*h%gyksza`L(5y2T+kQT-UsjM?k-%T|SjJ9St_L%e(6q zQp~E|#`)@6)So5%{Q84SKwpP`IOIm)fa4wZT7|2S4ShNP7m{QWk_;<;eXS zH6dSt?vMH^2lAlJ(WQREOtV6Bs}hcUe)>G3gvxD|IS}N-*|S|H%uw%!-4b&9P^%?e zth$59l>DMpvR(nQhHccK_Lmqp#r}RLl=uhU(+R4_K*%gvXSDVZld7}$7nf(WAd^b* zoJ!if5||l@_$+CBbSaSdMo3L>JQ%7@pV{Z|p?jzhF1(;zz{@aMJ72hKI z)z#EF0U^i9S+0ARco{W{+e+f7fMKLG>o087Ni)vY1a(5K!}(RJ5Q`jr?*?TXAEpbb zKk*Wcv_Vepn7EVydI$6C|Dx}Ma@Dg$eio6UX#PZGA7UTI#cGPc8-T$@E8L?f2Y+hs z?CR)?L@nkL?SX<=7&|QFSG9RGlbKbT3Kbd<_=e0wgB7ZfcIerZ6_DI=7Z)KS+S)rg zbFEt^BrP4;r|y1-TK?I|@@jCqDH;3nB@7v28Rr%^n4vBhqXk?fv48d0_K{48xJSOH zenZnt&Mu$$U!Fs@_&MIk-c~~dm}iW?sSJgFD{Y%#=V>U)6{Nb$hJx?tR<0XenFTTF zw!>ED9Gn&U8;LL_U}yrU5sP>k(KVkYEMyqmP^itI_*I1hY-lrd zRRwN9rO0`#u^^zB_SK`cuYgkF%caqghY^wOqTVz9a7C8WTN!XFoDYo!F7L|1Uf{~L zit2+39k@trPn%4&rZLQ_wF<>*9z@a7$!*pGGV-FNdR-TW-u{JphzGWTzt#B((LH|k zzYnZMfJXcsySvsvAV>VXm8lW!3zw0eYHu@CB_d{tqZG z1OArzz+~*cx&}!4WJ5IeDH;^wsnTEZ-=ocb52T?fNjLkh-*2>K0Ns)cEjLGc0Vqg^ zh+)7H@Hug#UD?2BC*J<8%|esOhj+tfp@KBPJeUX^n5Rqoa+nGTSGzIoAM;<(1oq~e zhoNG7*6wZ|C7?X#>oz%(poIF?vy~z9A&34M2{ZH$a^^78`9j+a#0T?xo*xEB$SzeuL}WV0XUtS$BcBnJn&&tc(r^b{Ly`vZfTk+CI?|Fv*5;bTGt4NJcX+b1^Y6 z4VahX8R4)D!Xb&-kAFD;h}%N^EOM$Rl)*fq>tYO5JzQvT6!LoWc>P6~;aMMt339FA zgjUTm#=PLGkRM;&X0&BDL&;%dR~10F>;w4KZowuMe5_8Bq)eirVSf~K-JC>6aKZ~z z0Q?mNffA@U%57tF(WD1vvVT4|f)9$8Xs%4=13HjYuh8a2ALYC|y2`8?$_}h)m<=jR z+UF`AXug>{BjhBc9yVkapwTna%e=Z30-O?3;-G=k4wh~x=c4giTGv*37Mwi96@FUj z*8xw6L$=w9VG0sxF)t2j1%@1FyVi3;kb%TpnbZjFPs@!e>z`UBIEUvdI2c5NN2dq6 z1vR6pb7{nH48|jVi~clku7QT50)Lg@;auL=R9_Ae9wpq!h-rcWd&CM&e0d(sJUox` z%)kMGg5gd70G<_mk+Qac0!~PZicW*{eT=^v_QRu0)B(EM9Gt}QahG~&HrzTHw5f3z8je?Q3w@t$7y^E1lOg)e>f z&r&qu&(w9}@8LkAU5tAiuG9)%#K;KgMf4V=eLSOT}^%K-1+rAuBJ=+41YNR*NM^hI-PGxuyPdnlO<4Z5U zUylx~M291;nL%@d_t?tDsZdg=o=IE(Bp@-1*Pyr$G%i+>?n1wqQyaI3b3PS+~Qu-%>;eeIGE{~!8z|B*34`i$d zwqPGiA{OldPAvO8EFVo7T-Vtm$5`m}V5Y6+^$^%8l+^D#I--JDiLSK1TmXsKM4yV= zY~2k0UwoJ`-1`UMLl~#V9r}K1pmuz03bd8wUtmd6a7@6|E>)o$Krjz0ch}{m%R+{B z9};STJHr1}q4MMy)I5c4i=zH5;Es<5 zt-X)X!6H`tQlf?qgQFgl`h|?WgQjQy`dh02Z6W32_a#I@>|;hHLtVrnMg~Xhmw!~i zgn}1g3~##%LXr|@F{dL-O~uCbECMw%9(PQy}3Jj!X|;MWHZj;H8#MbGgnL- zl<&Y`eGT&l6K8bt!USd%Jr=N*a?Wp{2IYT3mG2>F61e6oriykb!fcZA#_lQ^1Z>fV z$sZW6_Hi8YunPclq4>SV)Dk%P*~RdR|5OPx61=0mObvQz?&W30uLUoM^Pg2<;Q;hNkLmc% z5*?7T?$IvU!$hcT)@kOw6EKA zlNaQx@`q%o`L&$C8m1wXS()q5SV85Vf5s;rDn#&P@@@j+PZ=d$xeyNixoUIdr9com ziQTtrA;iIBdb4|xX(<$L-rR`FBT%Y@G8P;#0S`{v`q1GH{FfN#DC8taKqh|v01XOq+D?+F0NOW1dfl*(vvsx`va-T`2QE!G7M`DWCKQstS?oT!^Vo_G@qe`I1&m}(8wWcrq% zUv&HE)-!gsYf~j?WZn_M3Rr?>E2cU>>kc#qsjEFr2R`NaKTDd)+1B<<4$y)i%;XI4d6V zP-m_eIFHggcL$HJO5xcIWx<(gGldyYs_8=yiq|0Oy-zvA1SgKXqFw7kz=@4h=C}`{ zV{z*m?Ps!aV6T3QJ`fy6owy$}C+LhA4t8RTe78h`7TGXc88Ab}I{H4#5nzxzjikex zq#igovrUWtMGfOF=uUe6Lg`iFbx(ZA4s@_xn6QGVffj?|GaTB5VzSkI05UW>B+8}U zz8}VIwS{TFvF^t~jg-%?^dr)K?B}?M3w3da!^+`&N46|X)%ZXJtq^5v@I@6Y?P1&% zapn>55#y1GGIW*{D&TDGAtLNreP+0bvKHvsN0pyngtOwn8U5PU4kG^bf!UPop&o^>DVMH9^Ks4*9b_q zh}jW7=B{PLH*ifUJAKxR-T1^~+)1L|rt39D)w!_$HOb&w_m<##M%G`_lUiK|-& zV~CI_{;b>z)LP9;y(jjf0=aKVt#|$EEexH9x_2tiuYy^)4+rOH?+b(q_@>``VLq6^ zaB^Vt`O&*@a9_TK)|lV27q+`0^b;Pb7ncE6?7Cont4o*VlsAS|7r@w~ooH%`PAr6R zVZ7>73(|gg?lUe6>0!xW_f{^Zqn5_i@GR#5ggkw2{54dVBYVb;bI~X}>3#DNXMf~# zT7v$4Oe%J%9^H*jp0Dscjc7i1^7N*|Fo2cTEU8?D;@ZK=T#F8U#hP4PvI)p%@5NTm z08FW*TF;Sruv^KRR-Lp4a)0R5Gp5D{U4M6Q-*4Y=Yb<##wdeC#phZ||l%pTCCgK;Q z)d~S>Y6weE|)SqdB&5(kscN*)R$El&&XWkP*Ip!3tFw8W- zoFmrd7i(rOIySr3eRv+^56mitt?+8xR}po?z|UE0`UiJ(r^_OBHJ|=7ge-1*V0nYp z)||om#t;6P*-XaY69(GPgTFNwdSbJn$Oy~_@8_G_;S-Mahesy?9pH#tuvr0i#p3zi z@l_D3dUw0zA3m~tTc0Tjdc(Ze%sF@}P>RXv^WBY*T95G_xRxp?fc|T6c9O*dabe}} zv&j(5YSV&T1r!do=}MjweLIr8sA?cBvya1hy{Tf8k4-TY#qgl$wG`5pHzA!6} z?C z`wmw{aoFf1t_Ly~atEB$ChMkabGdNtA|`*kP?}upf@XUB?P&x!`x7X>o_Gx`goK5> zxQn3hWzyGHG`E^ej}lUF02a8?Sw8!B0u%c$po;DQB;+Kj=!1*JfMucI5oarMA*K%u zzd7szfu!)A4D4}1U#bPDbhhd6`9b2-fhKk{*^7O)dWzpx6ho!_8AXrLbhh#6ett7_ z+CHI1x=^k`VHTlo*a`kmlr{O1h@le_7X5mln+N(v#!J$aYzM*ZKA0F){7I``gT9F? zXMSvbCBW2;t);M#LLeFL;?Y#0$8$<{*TUXOR|z zj@4c%Ye+^QcVEUOzl`0CKCK$6jxzqsy9{Ffk)b>G$_K_Ut3=s)ppNwVyQ2*<#JK&` zHYWOF%|Ca#_Eis@VGGRLv6~(x2pl~^r{`6gilCof2jk~sNx$b>!(2QBotTuKa#FIF3ReA*Vl~F1eX+* ztwKekn38l@XEG}?3sN&QfpJO6OjcQjWhO(#4Nyr*StcVxML|VlXF?EEgprg*Sq2zj zkY$EpU>0WK_ni6tm7jo__r3SrbDr~@=bYQ}tqCukomNwVE#+<0WYBBcM?AgC!#SI} zcQ9=Lxx8yeFZW_93Jqpf#uTErYF1P}+jbK5xJ$f$c&|`sJIM&Gt3KUKYIg|4XWL_nrD({x3!Km}%ZCLhbSWGT+{WU6zEYKhLv$l7(d2@TJDbRgF+w z(PPav0Zoe&E{+s<$JJRa1v~7pL@Kx8ck@mlH(*C^7C1n;VQKpt5}U|kL+gkC!23Ee z7qVBC;0$L8pGqT;sbwl(OI{lHe6e12gR^REEl#yjHggs~6n3skdY~O(p}01@jr zA{FF!TXBxX!MDsBT`i zxxcdFSz=}(FcvFSSlpq8bi?7ehow-cS6yEJn)LAmT%yt?&-+9;7G9MWVFLq7n&wc^ zK+Jk=cSfxe=SyOjDTk*FYB#rjO4W!Sy`@_<6zxw<~ME*T(`+X+1UB{I>_h7R}7(@37LaRz1dcIlehl&*Ao5UDi1ssTS zv260V5{SM%rR)3ZVt{^E-spu^GaM$n3_WKY<)ml%3mHAJx!6;g*oL#SaUYylZtUjU zsmNcHY2r<0SC%hv?>OpslLZCB@3&>|ZXm$@|fv=UUH zA!+>y7)a;F#I@EXV{0I1^B=4TmBSz-{2o7Ce>cz}jQwYJF94g)YVW6@*D4P-SC5h~41($$8GmDxA^1Ija& z_hhyT@Lt9@;#bOMB;B6)~W_{`<Wm-(>O)-p8w%iyG9zB*B{G0ZOIha>t_K}L+EKn$(TU$jPQXplU8Aa`c zXq_wC1BD`?Bg%(^4ONSzUjWX1dRl!j`Aej7>oY>`fE#{31jP|=p1mHp7w;(g@xV;=1tLMNr2k=fA09PctXxXM-EE1dC(}t@oGZAQ7KC*XMjzq+jYBBb zZY#6HK9vQ_>SVh>2^#eqd>(??&Eubn9gpO{1|h?zo5xYh@n!INaiURJoqyo)&(CXf~zp z%zlspjNc;opZbLcs*_^7_tCVDb>OrQUl@kGp(Vg8U`fe4$Vv|e55JTE=7p1$b0Q`X zp1uW7pu)1j&S|Oi69~e;$5K@`Aiet~^@e zs!913EM!KiNz@mqywTO~N+^aeaG9aH*;_c(wBkONSjzfBzcCy}@89so#7PpyMzO^5 z)qY;4`dNfUaS9Gv(N@`}fmC?yJCay8*;rexm=}=&VyqUdbIjO2n3#Zn6&UpCO%>9&h7J8`JlK+>3`ZtoBHXs|OKwm7=Bpjz? zws=dm*(6wPqsxX_tgqk?!MZHUK#-~ph#Il@;1hUc$YHOEUx44lz8jkRu`YS>({!s< z5Hd}zevR2=+62@L2S`cvghgb5WI6oNkjx?Kb$86>8NIH>H3be1Zkyrd3|p9LjLT|D=JhCw|PZ$ zC-z}qJIXc;uH_<{#a}Y#)EIyW4U3$ry8?hM8tlgdq7cpKE9(-;YZe)LS(HS$@-)nN z@Q}~)k2);MsvB!d7~+X48L$^au<(LRij9yhiF{iG3Vfg7QwMiE0iSD~?S}v62z#^l zGH5Zx^UYles9-v7)pjAwhkNU=ODl96DD}SnVKf-t@{G+r8~=epJ$@#?&dhlzU;cv}iQTi^niw}#A#$V++9823HMv`%6 z*T-=TVje~A!jlMRrN+a$j4hT3bQK4u3>3G+h9CGu#ZMQ;_o}S}seG?WD|=fCvaavc zzn=dFqtb8Kt>KJ9ZwA%OR>5x(4nohKHU;B7m=-DnJX22%Usn+NHL@uCPs`t>z6ego ztoye~Y03SqecwMwUy}_#J=mr+^cBWVVc-ZwP*>|4|Q71D|ht;f-sJ!<@o z@gz$c0GFztd7^CYus4Ws;NGZRA|FI0M`5|iO*Q^%sK&Ny%yRV~k<_T4@vYvTvRH|G zw|Hd$T!Qdd`(DJUOy!R|MNw=>C?xJTN_%QixL@c0p}WF?>L@l5x0DFv!NPY|4r0Fm z-~4au5NsyZ7RJ}$a|<{d1RLA~0Fc;djHSq~Kb-QMs##KxyFPCcjBnj_MRLWY-duDr z=XfVT4d#Dw{v!!+s&4ex^#}zO;_wTkMQLSr9j|tUuh3t7x_6LR`a-9-BP?u?V_C7+ zd040*y(!Eoy-Ioy>4wzwS|nXM$!ke6{b`u-PX4whP<5zMhvI)GmQo=6d2`PaSaHWX z{Z>;uk>J;kf9ZRDz4k0*gS3~_%XqDR;3CtixFg4>mvRsQYFGEnpdPYVyCqJUinG+D zQ}+93ox5TCeo@hH4xp-GB@)G_YO(&~n=1nnDKn~>R>0ZO0#Kj|>~Xp;M#joyW9&IW zNXhFr+X@5-LP2X2ok`Ll(H;}_p@!riA0b?un;{1a!u5XN5JV#5sG-e}8OHZ%SDH_v z-tatpp}G%VP-Yl-Zjnepr$2rd^Bnm?`>_572VG%B92(T)bBkm1IMYblxUaI>%RcXlF=`Si($ z4Xd?~FaGZ$D*d_qh;MUsKj7i>+Qpl^@+pa574(-5Vd{x^NBj&tTZfJn`L0R}5-X(- zBgP=Nm~h`Igzp*&51!@TyfbUUVf%8A zoCOnU03e(I{wlr)YAEW|9oErQRJkJ9?Ex)E{ZL|lPtD*cpn zL>ToH+JjB$TlmKl7nH7pjBgF46+tmt_xAaxUS0UfP+iNGr^UclQvK4Y7zW-rJ|}sw zjy8Y1psbk^hvLfNU|R8QUS~aeNEaa~&(B+^QBVfaT+s_^$;NY?U&*8c)x93n_6Ar} z3_n}+Oh)30?n5&Njq~)8wNa;XYRJkI|MoKPA#_K@S!Jv<^~Jq94>n$b0?hpbMV^Q} zvFkg$YsP0e{6-c0!KiG?RWv(&oP&`0_(d5SYWzWg8IH#viHCk{&~Z$djMN*zH%{A! zZ!F~GQg<`Ic2b&y2uE1T|e-W$Ww^bu}1Sv*c;4fyZ^A_;e z^CF}p%i+GF;+VSY_>&-(jTSRragrfdb0AelnCk-V?-fEULYt#skvCSOSk^hT7gZPS z;@%zmvD@1MtChbY6Nne;5AW#wk&J6LM@s%v_dnj25u*MoUNW2J6a-WxiM_V33v1!a z4%R%N@HUSRMJl|3>YQZDcsiVc;bhP_usSk!;mI`QPy>#G#RsrUhF;#-A`z-ODo2^{ zIV8x^s;7E4R2?cmPG4ys5%euc~lBdR~psRU;Zp&!$&LN!oqL zUwEt;hnhewFh&7*szKts_irI+tagM`n?NC9WXgSmnk8KHC(f!JwpuSL0 z{fKLWXEny@rWf8I&hb#l8_81C*tZI+AJiJ{K*o4MX@X->@Wl|K9!ClsszQ^ zMEK_5(=wG`I61jNZ#Y)5S~a@@ZZ;|K7x&sSK-;)iT~1L#H0U_F&o7?D5>Ka0O4r@x z18S%QA}VAaFB(89KwRS|PEkrR^tg4iEb;Lv>+0Gc#ic~{tD)Vr-xUJ3l;IUX(G zWqrs2h~txlzn1>kzMi6cPvKPpLFzR<(Gu-gTT5(mMiTYg{OT}eBf$FxoYMb5qIt-K z@TCUs1f0hAB;g; z23J!!ujRHY;sh^Dxb#Rj&K9U;-an|4>%KJHSS^LDJdh>$;yV0* zq0g>G=N0bNj5e}CXt;Qe)?BQ1quJy;cnhT1ORvI zpGUeg(t%ASJ-fr_X?nRX7ne6FF)WTDK#E;XV+)r z1Zodt8k&h$P-WzGK*~7mTfM=`0qD+z`@?(UuGLQ?PeZ;}PH{w*7VFBpKuR!yuuXcj=Q>R!;Js8hg& z<%2ETJ1Ac`?zZzeO6g9bk*xIRfoMaCJF+%8XdU9V-Pbz2_;~N3swV~r$s2t~kM~XB zPjnR%7mJ`dve9i$GbvG(mPO)=_}b(Ox97Fgs+DAU&$iBpKrpm&3;P0`cSqbTiINnD zHNG7Su|xjJPNC1Qo#-u8Kdst#!w1LBxFocI)Ocz9OXWv_j3G!4Sjh*2&~NdbNrT9b zSg4j#og!vh-p`~<M9th`PA#cMGI20bUg*cH{XjS2O=8oB)B1d($Y=db_Eg-V~ZjUw!BQumWo3 zu?Ch)QMI@e{NC7ZlRlqi#v3Nh7$#W8->Nl9ce>xcTm>A&Th8^=T*NmUjftEZoRXAO z{XHW}C?2uijSaBH+BzEvrGxoSI}#H)l;K`JVmL_6?hW?Jr6zx^V{ns`5df3SjwE(L z&syGBlKz~J&n#Qc{fW-?F8(r^V!H*bQ!&z7l{SRr3T`u~w+d9U_N2i}xD8D`g+pEv zKPoLGdGn5w9!2r)tmNCsf(^lj^zVM6wH5&Y6Y_jXkVl*sc;`Iw^F>KauWR4--?9Afj($I$f z#0yamkWFK<)uVhV{9LX=r18oC?a;#hqTE9C1l}nkFu(@5W-3&dBSmE@NqCnrTnv1u zb)6q{k_^n?aV+5$r9>^=iFdQmw0uD*2y4PZcy#~bQ%AZB{yo-rnaC5_#rftE2r-=) zY2mMAlF`QPs2zZxNz32ZWuk{-Z7T~+9Y7O9T&YR5z4$4!Shod&GnMt&^6s5?;IE9| zrv?7!l;2r$NMoPZai=oaBF|l=JnUZrg^9oa=sYbFPu+{&+?Q$Ez*${DRaP^V%Vs)> zNsJo#{?MosC2lGI@}uMoBqFVs970o~g!>U<g4`+0O98CI)8NKf3uGFC4%Q{V_@aJ~D0F7a3DO01+gcFKXXp zP8|2-*~?S6kgo^DFY4#-fHb4--mmPhGUBlJ*0DXa%}H;ODGg4re2QGA!t-X74Hcy2 z_u>j4nub?6s@+eaFC9Od6_pv|gvWk<$HgZTxvy7Z+_2yT&TH(0(4RvJ(O%oP^I~lN z*E%e$)X%m*bd-glH`L(T6AcQgY`e>7A}ItW*LQR-AdBN>_fC*ztfS0%Y}|^t1%AccV4Q3e*3nR3R(jz*1C@2Ymi#tqB1){>=KR}9!R-nPgSbVHadE@=k zs18`fWYFk54<@N=y0hGCwK&>CrSqd1aRg2JzA@{RlD}&2OW2Gbcxu zXJWD90rG9XC~ecz0yKCq^Jh0Wmw*5cd-t-Le>|L*x>v&Z~&I z=N5kqyAk(uy}*dDI!oGUx71IYs=sMMZLs*aQP=n>toGX%d3n@c`6zIhhkj?OP&BgX z)Y_Q8?%_5Vjzb}5d(xrBe1)vpd^p+U&eqn$7pV4}c5Ay2;oYRQso8bNLGn8P$ht#Z zP|fMPCzM1anJ-kecB33GbouC=+cZ{1=G89tM9?-CMP`;ss2<$F8mrraY>V0OxWpL( z2;KeotVgAI({gFtmg<-P!Ko}<()oKl3!@oQ?n~T>7_c()4(R@&6YVQ`O;-xTs0bLv zel8#`f5JJ!y1f<(bz|JfXS<5fXNSQ?6Z0|!mBDsbtgvb1#@;`sJV0U*UB1G*p{=X1 z<*ENP_P4a%@Ap)&OnO%oub4jqGV=RveTmSxxi3B>wH-JpsFT@#l^mqIv{7X{7>h*5 z!)@%)2wMK74*Prk@xa?GUoSmHLkH@072@vbEYM3u=EIlnp>vn%lpB`iHidj}(o@q| z0_sGjoylx9U<({0mi==F@9oca@mnloz@^t6WQF1}4WX%dCrd|&^xPnqnDU94vlu-$ zFp8nWE$DWzi5$3fRqg{Hg!b~#duItrS^uoD8axLQMSVQZt_2P5eaSneiZV1hgc?H< zI#2~_m^_@b#xRLZkPA<^ln}Mhue_T@d`?50?Q>s}Lk*q$3NVGzlFD^i8uH9sw`NNh zkQ`_|*S=PVIHmSjJe)|slF{T{b_mX`b5kL6@C>|@bA_G9A{8YuIrsRp_;92CvF0Ux znphOe1~0Q7Lt9(uG%Ylq=6baA6?M73CQ*|A4w$SgzB}1B1wXJ?H*Nbkjnax8-Tc6I z{6^;~%X(zg$??;sCi(*eu;%RfvYx-F=J3PVvLXhYhhmld5s)bqGPXWCILzyUI*$x0 zF+}`vE$RgTLN@bgxO;2m8F8nW%X|!VblSEp-DkmYq8M8=*==IRD)))@hP;7oZ1SC@ zvsEE6uRB~5a2(odC|6>AUJOI7+>(_Le~ail=Zb3k%588-e7~(L-Ap5uq{E{|@;3&= zYz`8IfDn865ek}IUF#ki&w^3dYX?Oz7^~XEG2C@Fc<=(C=u`>MSBHgv^?rXBB9OK{55YcIy9s9-KFCta-nf7n6e z`^>ddl#)gKx*+r(9qZjY3R8$DxW?(f7}QWkH@-u_{CyGlG2igTVM;W6BQz~(Mf>TOY7U}?_IX29&N`!h0a2c)k zV?diBOD+*~?gM&b9t%A{^Q(rUb_I`<7X02_i@I{Sgn@O%GOE4p!YFI46LGMyoPQbp z_}H`)t4>Y}jKfLeH5~(IiJfS$FUoG?0^`(JvNZ?5`q~)2vROAa8wN5lGrNinmWx^~ zUNud*artknzZcY$bykaH2p(h$3K_}%## z0Y&ZjJo^@#4>u4Qpo$z0d%A5etUDmb=65MPkchR;MgT`4dH2{%Pte#vn;vs@ToB9| zAIyJyANEWVD^7G%c)3`CmifpcwTiN_|KSpWQb_N-u4#eVE zI)B1CDu&Xhd8u`^`!lEftO@sHoQO^&@7~jKe2^aWUHfwyW`TlIT?q-tLS$k`2 z$xPCHi;nuLvCw^wwohFlerH_0%RiVjw=$ht{+%4n8(6)FUmv4C39X$TYgJ811Z*pZ zA|WCV28(#R=$NW^>}7pnQ$h>=#j)&6B<@#+GJI|4HX53k8GYP$3knK|Llhz&d z4>RpdnUeC-i&afG;qCUBO+A4t0?rWo(M*r6!c>oHO(_upy@4C(1ZPg1Focw{0arYt zBpKAX!m+vR!v7NsqGGrxV!`Iw;N z_mQ9Z0a*TdQEkyCDx|xmx5j4rB2~0?bg0CiJZBa$ig<)?+!hlb8_xcO@d7rDfiD@1 zf&bv3@=J$)jai3X2}>w$3HQdE51F&=K3CuzITcQICR~1k1jorYr`rbvn!C!;wTimP zg}S%qpiK$-f_t3D0P-qhMHPX(Ov4OY^SFPoFDl2L-;NHUGR9YY!Bm_?+Sa`L7x#y( z^ChFB{F8mi=-X|yF}0{f+FjYUj4Q)K$yqKP7NBPeLwcAyNFY6Y zv!H;1Y(sIcB1*CvT!UJY;Zw93zxtfHz-UUDw2!nh)6UiLF2}Dx1;p$adxyV+(C|#~ z_A$~k)-2j7l9NjIW^F+E)2;!WkZgxWhu#igQuhjt*@=;{!6fHH!M_+l$<}DA3?W$n z62`evq&n7#qQL2oF6xv;Fjr#urm&CSs;YCX%p!hfruzAweG zt?7Ma#6eX%+O?SM=NtTu?_u3uoUE`@@?xT~9TsJ_Q#v%9+Aj^OAa2Q7;oK2{jHCE) zbyP7@xFElexM}SP@PFD9!uP1X0e5M}t73Sjc0^yzoRid#)n4EpT&UYP+IGyhQPhJY z1zo`WF&kJ&rl|R|olv5{eOv!E0z};@R<~IEPHV4fioi3-;`(_9Z4fX{GA8yj)o3%x z%`eKY+(8h}a98m}4j7>dl>C_k@Drq8g1g$DP*+!J#Cc-J6llacQI2@*S-Wo)Y3vxm zza1Bmfj}v;k)$bJK6j_TBP+ouh`j>8YnDVH-wgiQ&|nImpI@EyKx}8;pLrD1t4bAz zJj6uXs->kCR%YqsDV4LKu|yADuj553EXEyaF|)e`JHYA(%W&j|(sWyVMGo+#f6m(j z@);R)NO@6<1MOSOe&9ZDzP91{Je+j@4a3evizbN<+K*6$#qDZ_kp4&C!f*RcnvMJ* z??#_Fdeu9%a+!3x5Eb$cfj`6K0nJEI@v>B^(AW~y=+LG+QJ zIEaxZrWO=EBvu~X+0HFNI1oc~JC%^N>c}r@p{DA(t#`~B)J57LUK`Kl;~x^igQ4zroMh{{HKz^Wjdlok zmT4NOB=ME4ClU2%I(MJ}Ay#be(e(<#1l@=1{pqlto-DYViH+q)EI)1p4Vmt>LVDaZ z((dpzum1fwnfzsTPeP%IsJIYTKwP^>z}ttN5*G)2PwUV>J=LcY^WAih)Y9rQ6V*EZ zz-y%4GbEKtVQzP%|E{`ciy0QRjY};h_)A8yw|iUzB&rjAlejnrWi$vg;%L08T}$}^ zBG$((g@a>A{tThIXKe%zEyb)|P}wp^ z^qW4w&$_NeZ&G+co09n20qoR}-I4|xAJw5L+2`<71mhhMr5Alof7HCno`(t1IMlN0 z4}T)XhvA<2&mMtPF*)eK#;7+8v+#^8lH-^G8H@K-oX|;wvxMx(PuylvWse{ z$POQ`pFWt$b%J#5p7i6cV)9s)<+|4=e#heXE!dd&tI`v`D|kEm05!g;<0EM9t&F!c zv62*t<$3?M(c#p(v?@rn?v#BP2UEjQ z??0`4Bv1kDiDN}#q~7IgvPwyy-#<}xMQIJVu2Yr^7JK1XNDexdS<)-NAJO#`qis}^ z4Trl)o}|>~S&n7HZTBUw^f@O5597)|_T-2h5iv+VU*SA}Bb1*rbTR=PhAL8{9T>!* zi>KX^YIrQ;u;|9)VC^?1rFfq&`0>^#m> zJzSH3A2MJ@YrTqHNNTrjX|n@%QOoyYxb{VzX4@yPo@#DK+ z(kPlvwWTUD3#ixW9B-qkuav zvS4zm%I5ZCq3U%<3O^v;<|IR5HJS>aFd^Ti-o?QJA(>Ne%>nolRx1kx4j9hjy|3mp zOgjEB`>qn442ADzJSN@(mQS$6?D-K&jc4yGFUTQ?Y zrWdC&<*)?1VRTt?xa20n>QZ6grDWIvdL=h|+ZbjbL;Rs@Q4qbT{^Aq8JMD7+)OSUH z5O!3gj(QaHVH)vGNj}t|d&lPt+mwcc9qJ1H6qvM`(f;bl>*x*RMI2Sv1Yumvj?Dk{ ztfetqP>+YN>_83{RPm}dow71!*2_){sKhnFxPNaq3rNfOlPSBg3jW-h6L;?oaraAP4mM!40HlY(Qhzo18i;#4}dbuxSf6&~Gu^xz)9R22l zQ;$Kwm)q!GSHu3c6`GeO#n{smjAb%)k@PFdZ>XO`ee^@ZY#1$ z{3jE&GfCy{#84(=tCqCNB(>&`Tck>tzc{tDVl3&YVI9C$=; z*%0wP#ydJmlk^o5q`*wp0%1BSu#|PnQ24{6j@O*2dIVHZekpn6a0E2462Y;+wJAUv zZYx`Jw}OWAMdj-EjAz6{14F0Y(;*gIxCe}4&m|pEqWN?S!TlcGHrnb#~^Ly*M5gc z7N$wLQ1%V=VUIgU9ud)Ks-+_JRTdF5GjwI4%iwpJpJcGrfDl&l_!j#NC2&rIeb2PO zGSv1BGwrktimWN^7CDMe6SJZHk^~*T?0IG~4{&KNR=aud%)6X5v;t$wi01J+Bx03i(a9a*1lvs zhh}2+Wxe0?#v(MMOqyk9{RV@4lQI*2cH1>M(p zcw}@D?c@nyvU%klnnq5*@&Q{H+`8!U-wdo0>Yg(*r_95dhR#DI;r#D2&_3+FboFXM->A_csp zX$q~0$?#h|eCa`Oe({t=@lqUvE9$?sXQ^y>)c;Lm02rEe{+``ErflaXBQb165!4;nvnxXWZdAanU0o+j7cjXo< zo$xuVW8#A(8f(qJoV{O9J-3_De#83nGp7wQS*A*??m!ycaO4NQAlCc7nZ1}yILX0cP86a zo0OzE!QVAp0c0=v*WZXW7gP*>2z=8J_g#(_2NM@lY2TvWUj~22LIG5bYIK)Spl??{ z5O_8qB8Pb?p6tyBI8*2!IZDyN5|t&HZA;8cv0>0jg>>7!V$xF!eyc@j!Hl3uh~t() zM^vYPxb0%pOXXhhqWszBZ_r?+*s0ydBHO;gC{SOWG*&N=RhPXSd!lRLZtRVL1JbM; zyLQu=&c+h`0cyv5!Wp*B5<?6yo*Jnp9gx9LpSf|_6@1OFtZM_q^v?kJaq*djm zBdYoySS`8(>QHP~UG+f#^(5qq&O4A0J*mWlda4m~Ftr6J4ZV7-XkuT6Z??|6YP$L$ z_x|1=Sg!!Ga*BGWk8oSylj!Iy<6opVD?7%HJBu9t^=Te3@T^f5e!cgz}kt=|XL&9uS?hH_BV zh7NVE!U2oSo6G+Z;E*4&C`Rgz71XS2o6|=@N?}LbmZBN$EkIU-R7y~Y+WD@QU}gjd<9-;2dR`E} z>%nJeGLxL}R<3}Kh5KoM)eyk>*yQag8YAm2n6UrAuD6iEJ*kjFWrD#F5D-Gq~$W?Hp)$__hMY*dYJy2bM{Z<}Tt4 zc8$xzdz+B<2vZD5;%laDV$_)C3rer4Nvo=?J6dXs43Np*o$X9Bsjo|#6;hh{)Sp&4 z{gWvIM2p(@${GO*r?z#k?#l5-)m*XoLmxnY-<&^cXe?bXZXUS;q;9)%s18kOs=(lU z`z(85d_{PrJ`=t$$ATr)1;BYst1_(Rqu8j~(tL#wRgKba>{rvk0*+15=5b%Nj%A&S z_SR?7ZvbdXn8^L69E4v7#L_^bO-%m~uYbP&$ zwMu2;EF6XkiMo+yaEBmPOvcq%KWWdw)^HI15dnp@upY#iR937HxLatv7%&!Zr5aCL zC;ZcCEeb@n6>H-9!_WhP{LJND4NXNFt9xj=h9csRuDbasvxVE*sC>w^_~eyd^@my{ zrH%85fwQW;@szuWOhYk2D(cPNHMJlF8d z<8W!p1maNe9-GZT-1`KFW`;`F`9S{nN%YRB_zBUZOLP6(oySW4wc*8vbht9ywBH~H zZQa|*4tW# zc5=Nf!(4Ow14$5iJ^v~vfvWJ5H?CN6f2YbtWLD_U2s9HbRx3tLQ-ddb z4xadrsLI*KaGJIo)b8$gq8{B16}_$V3SEDP)F4MVTv%<7MRcy1oc^i=>nrKJnLdkh zsp80Vm;LyM>U4?CJMkkp3*INnljAO@6#y+hrbV}GJdLXdgxjo)aAg>#j0kMmI(4m=v9K#3u(>UH66 z3h{w<_7yuR;00N)h`Baq15UW*dGCu-u*P|U-50l!5}+?+u^CCnvr8|_PkLSim)hj( zzlubUpE$+rpoC|0Fh(NKNihFouLk|SnUx<{a8 zrBv5c%l!$#()EGNnQ)W(HKR<54ic3wyKwbX5Fm2SHQxp_DH~l%hFbCHQ=ti_OH_&j ztL;}2WX2%;%{R=jg|SXm`7#<9TF%_z8gqZFm>~HiafI15!9S!+LqGDZMCj}W}To) zp#H52>#Kt_A|oBYke3kuNQ;o-ohkVMXG1e5BezAB46f!ev8~qGn0cvBR>_ z&v&9&ZIk(6a(T|GTk+K6g>{)MSS#jq zuCvL7F$_)5p4NzJtXEX!GzYsIkZD1fe99^aZ-%PL^x&&tJdvr)r`uM7ItLDA%Mg<> zjE?oDdNPZaHtpyD2S@+<+BKQW8lK@i*M4Gfq5eZj;6dL<<=0H-$vBhJnlP(#?Y=nb zv9@zkO2~6JS|w<@X@(i+3g2zQGZw&cb(b`gguElSdwoe59h7Cx3+zYjL*Y2fHG&2U zZnK*qs~!i+zFC#Z>}DaGD9)bV@n5t6*|o!+tYorm*H6Oy&~AW?w?CXfHq9*SAfbK)-Pg5bi3tEy@jFp=_*{Y!X zk@%4*b4Wo_;#!$w>h}?jW(sKd7Sm!e>(xmly)_5x%1w3Fref9Xv9qwFt-|5dddjof zUmbfvcM8>-Z?~}~F8pM^^+bvg8ZK$z+k50F4z}bU>S&8tOe@hEL8XEg@4B)G9#}oM zH&YGu&;~VV<`#-OAS_C@`8fGcVh?y`D&<3rID{r?a5+tXqVX52rKlp<1s+M?U=lxk z7;G7Fm`s|FA|#HKu0Z*7Pb!vz)A?+<4Ao9G?^DBJgll3nW3~{?iJj_;J3#z}^|YQ+ znkHYb`#dL}99l9%dmkbuNZKxQ7SPa+Vc!hlO_Jiei{9>}{=rP$X{|gRgOQ;DE>hFn zPPX^Gcg+97LYDE0DoJmNBt>@v^ntOq>6xkiWSko;_N>LojKYiEEv5zsnzCh@P-UK^ z%crPBWg%*+RD0}XRa27QFq?mhWR7CP(UZ@~mw9UEh(x6L<%+BCeVql9lFhUd+PWcm zMyq7$H5q&@Q*KrICyi=}k_p~FfNXEVan{&f8l2i`)-4LALTR~tli(=bM`ie{ciV@U zF6LDGlF&i4jy|`={BxVzFeZZ$Gw1*RUWxv_8-;(v_4T^0ys;Ke$ExMS`@1gSaH{o7 z^W1?4(|s=sn*jCDv#fnenM3oc?9TdrxK><3R8bNl`Vubdy>_($H6RB1#v;hGmh1{l zc+y0Dm1Viicg6uNS}c+)JP8Hk&z(M3gGH;CxQxC`^4UP)tpF>kDqpXPo~XG2zl-_2 z;`x~Z93fus99p$N{@m-Ep&0gT~f1f*>Q;dLX3vGyJXPR~aI@dB$I; zRtJ(c?iz{;@6eRYXM%k1)(?RdI>!vhp3*q3`t@gq_TjuqWJM{~0{eVpUBrh+h%wQB z`N(l>BW$ZFb)c2x1kC(XE^Ddnx?b{@_y%cUCo3QD1I1{3IFW8sk0Ln)Sw-_&X}GcA zH;vu!T;yU|;g3bctA$yuR-_K&*|3gs8WiGJ(9!^eJ;=9=s>xc$)hlF@c)vH-_cU=J14MQuAN{D)I;NBiB z(3lwHQ#yH9JHU3GZrRRJ!nO^kJbsiR6)Ti&K0NawM3^AQ^+zIGNYmC`7?KAvSTWM8 zHceo)o8=!L4P1sXJ7Suwj?2@kUw@C!$PZ|^I`IuKWl+Fe)uTRYrM5`6d&M6ll+5dq zNJMyS#HzVz3<~a)_S)_K&9F~p!~IQ?3eYYiR=p>RgArb0%`^UVVNm|s{hyBr%urLw z7a^AZ40tu2@@E#BymF&__5QiG>POF%Wij+!d7*v@?^rnQJV&5yln1d&&L*WoHVr+X? z+DVif@FedaIDm_z?98lYZbPJM;q1vAS_7+cRaKu4r*d@0BK>O_ImG`2kKH(WVhEl# z`M2nX7v!qax+j^(Vv&7-o{e`Lr@DGd*NS)&bLzXcvF1{xu-3Bl9h$}xZW}-BK&nH2 z#^rwE1YJUUW*-ry`7+XWjTyFDK59Tm3$T6F*&PD8vQmSpCG8L zxs-zCu~O%Xom9BWH-EaY0U8Kr!F540k22yJqdIbnXjv0%D&0!5mucz$QMumhXsn}J z2FH|hDE<-6NFO>rH}Oao`N_irUyaPC{LMW2O`^cU_H?mL_wI$hH~RNu5mILRLk5RQ zC@y<-ElHiwUQ8B0bfreZOx=cq78v}6&2sLK`oM>rS-*!HwLeZ+oH4tIbV0|}wN1Q5 zD};MQ-mP8--}r>1tfeHR?%sdAI+DXG&zF4g+#gZP(~N(~G_>h6v3f5J0tM}#BXxx- z7thk{NEDs+m2-c{m=DbE9(u%UY6ab6qASl)Ay%a)eim0m0k!6JiJ-?CH668K9qR-g z>tWx4jD8OU-;6~)-z(BE0j%lt!QmLnQ$&k;7b&vf_R_12A7midGM!>Vgw%h9@?qLV z^co>r>5B>PwRD<$nY-^N=+J8&DxQg0K7V+GYEo8~NqoKrW)s$NOhA%Df7!$GoNo$C z@UOhm_PI2YGOVHT(trR4+wz8RHFY)a6S)3d1qWu_e8=_i!FYAywyrEB z0)=s+TO@#j1BrD~KSWtFe_@pL7#!Hx=jm=}oF8;-sO*O;Kw&0+Wdn$_K}VPEbg7wC z?^%89Vp}RMFcF4IbRD?p!|1|09K*zU4x9N*d=Ae0x^tLZmr66jlNlKS!zO-pQ|;^| zT9+Ytf9x$}`?wEh%1FajTvKl3d@VbR%%kFE!9((Z_G;yyR7~|$xB2T4v(l>88ULW9 zI>W11`~nkgpjR3Q8Hd~eb7d0;FgCt4^hoOt$U{LCd8z*tAffGdcJ;c0YDmB9Yh0Qd zfHQDo=1)F=CWV_0k~mE5QNpX~MlWXh-`={XutZ-UOcD=3rTDtg%BBtOupzEOoiB2M zI3wK7x_W<)8h`zz&9-rOI=t=Oub9&jGX@!0w@v4LY^NTM%OT)i%xoIUAyjmX{(xhh z#7x-rg0it|_6atzwCiqHLu{TgAxz?b(t+nrkCHwyoyLnrz3nVJrDML2)?C>sH? zSRT%I%|obh7Ib@5-9+Mx>-SuI4(uV>#N8?DY7tm_-JFO!_p^y&;ny$<{Gy1@O`ZtY zpXMvl*bZH#u;X0O>X0+^5y}nWN2?ByO1$LHkW)odU^_hA#|);)sfs-ro7$sH#Z-^A zltV4z=yj$+ziBxIWhhtT8bJ&1RbmhalRs@w>*t}OqY50i@SxnH-#K(y?^#rX>b|~< zE4R@X7Bh|I!A*umNiyr&h%?P9*%zUP9yr!G*K&lk+ZS{JPqN?>eFF=cN1$W2a`?)f zK&el|)FibNU?zvH>B_y7Ctn0Tr(TNywyuft_d7m8!!@*o=S6nBr2;td7KXZyZlx(V zW^VfbUj6N?YeKpridB4$DkII;6F7tT)=0_b`}WCz$w4-oiQfbE1$+EL5u$nXn6@D7;-%V^ z6QX697Q<59?mdVP&lYsMzdsHSUM}Co_>5}%kcPOKCZ3gTU#TN*UgFUGuCIxixMZB@ z&6vfD?VW#jQy8kt2Fw@XU@WjsnYh3(2gfx3l;+<-IA4L!)Ejg;=L?4JKN?>Ni7xj9 zQF?_>Ev5(kfItKp+&{%sj?*y|sng1BkT{@->D_{KrZ42rd>SJgJ9XHZL;RP_YxI?K zIxOIXJT<3FkDwK?u)CFjgqKg`&<_f<&eYw>Z8gyU!O#w2n-bRrV$!TgCK_u?SH}-J z_kbeB^(tn%i|{i}%-_pw2^HCSUFM4MEJJB-*XyYrJmQB|W*mTK>|h)B##&g5aU(mF zG$W{&^nMmoftCHsT#yv~oW@K?9^BUBfsq^!!uL1RcxET&i`f=(;IH)j#+w4lx~5nY z-;vgnXNCQaI4vHi_o z=gw{yo`F{^fBN{dN-zkH=O!Uis_}6;kWZ>xb>&jQU@}4!rz6Q#YXl-mDDQS=0{Iv4 z(nGW8cp5ol?l&AC2Fo!ZHr(Y}8XVQVogH7J0Oap<+xj!wQ^O*|4Oc!mY{NiS`lpX* z&|9^x#DWkprv0x4B)RIZ+Gx298Wm{y`Ag_> zM7zX-3yDZ;oAXso(Iod0?;qsj;U|w99utXKTshiYWC7{Ql)N{d26{7%SO2x#9fzzu z>bHm%(t`|L*dgz+g>))#NB$}sa*q1iN?CUtP@$T`olZ#| zQ@qG_g%km?KBrXqO)Lrv^^AlQLo5V^@$ft5<^=JYuUFYgjUbCKr={nl&uhF`4Ev9; zra?St;VPeQK=L+p{{uGNZ`%@(D0q%3+NFw{n!_}aE2wOSF`fqB$FlFr265(eZTTgk z$4H%OB`8_09T8ZjP|kv1=4DP$FgukGbSRhbs?iO(WjmdH(&vPok(q^p}pc z6x9ca54n+47LgiHaMIo4;p9hrQbOq-43*oUqv@pS=4h8CXJgG}dG<1>`_sZ%qoPSr z%29(e-qV67)y2eg=}4!<>BuOdD;`jTjh(4}4C8*k+Nl@yJ9k}Epz(humBP|vlJ_ay zWSxr%zd*c#DFmI))?&T_e zJ%AgWlkq|t>}u%v%Bh=j_-71W%O_0P{H&Vw9gQ3rUlQ8bF9lKojp=B*-!Sdgg<5FH zgqlbZ>Sj~-oIes4;1l|zEk?*KsHYDZ3AO2h+lx`b^N)Q`XQe|geUk6nPEi^ZT`|<= zIi=to+LM_X#QXKj!!7P|5PBr#-7`ZmCe3~Fe9WS!pW)y!jWhU_wZ*U{d4fkRH5WHC zjB}I_C*xYR$*f1f8qNY{l+Ul7Gc$u2Cvg`f$f5p>R8tb&af6#v(oTZy!m!$CAK7*MNzi+WDjNT+W0Wyy ziWs<-sNWgjCRBmLbL~`!YlXGGFZ%1T6Phi9%QpjXKq^#v*;Kg84N2afxDI;zZP!!w z09Xa`^O}fRpKvzr6QyAd>?Th$YGC4-I@ZL*1(VdO?xa~i#0o97d;R(&U@?X-`Ao7U z_lf`96v&<1&APQ|(R?axUP+LEdR7Uw@7V!q^*`B=_Od@fk}6O(-Bg|G%Q{^<=t`{t zpDR0iQP8xKUysi+4O?CNz>Xjmh!v0AByU7s+!YuE&(uGs!nKn!m9CY+e7qx}W%x;jP3B7}g~WRnKk5AoA6xQoZ<{TRWHe4Ujrlg* z^GPN&)}d6-R^P>-3HO*?r-h~o!fLVmR8kv^QsLU0;4Z?!@R(b+W;g>Lnm1kWuq>T) z2vxR!QVV@dmD>c$FuJ+=p}ed>T0cqZ6KEkA**O~TJfy7Gd- zb*jM-*>X?A-V6-IDYn}nXY^yPTzyVL%R!qJn4|mRfrw&Sr&&DRavV81Q&eydJ72&F zP5C=-4ZwB8Br-#dE@a=g0mQsv|5i^%>{F3AFE=!mu<uSf&Sw-F1{T4feD?33c4UNxnIBAQ( zqR$=jy+HliEq*y)m+r>f)%_xF=Mp@7i~AEa519IJpi~!$3Qy{K{aHEaWI{TmQ`xIcdZCCWq7r3%aPEhcGU#@fXXc9 zL2wn&e%+)a3 zgc9{FQ9LN|46l1`#dMiXm0X%Gh;C|&luBE1ZG?ED#{mIho6doiA4>tz*f3%pc1L@1y7i9o< zP8x}H9-g+j-c`f7fv#&n=g|#mkWVzk?de#=fz}y!LE7CfS#Pp>b@?#fXxP`Q!HUM` zJqnafd>DZJjbOf;F$Eu>-1KVMN+Z&8jS5|U8D zDELXq0MkAiYYLej2j7JRI>@ zxnDcCa|>+U^wcjXlcsBoe{WS(Q%%J@V=!h+@+|Q3c19<%Lp)~jWSVRr`m?&BoQz~7 zHmedSrBRNH2b@w%#fbG))0aXYQ+ZUzCh~l%jX@L^??f}Lc4ONTfKq+ zFqAVQ-Dx|y#$UUI@af|3BF)-t{7BxZ_~WDv8tqVFzu1iDdVxV}2uNc- z_g%SiFCebTk*6wz5%+%D#@_=!h(+3(#D6iHV3&B%$8a2ZjOWOfRs~I)gJtsYD_(<| zjZK1u6q#EStNjYiVWNY~)`t^@$oOpTl8Q-RA-ss{xwFD`cx_luTBhWHCo}bYdS=%G z;ilrXd7hlkgGFuQsn5fiy{_TV*M5|K#@$FoQ{z^1Z_hhDImcQH_a0q)NYob$Mf5UbuNj8lrIG%j_~Or3l8>X|4MzJ`jMBOYOw8) zPvG|3%}sUt?^jNzXzQMOjHt#1;E74SKyXM5hkj}5YukIgg9Wwqa|$+3qq zNd5hnkry2`KJ@)Q8yXEQ_oAuVpw!g6ypQ@D5X{N8v6nX86m+sXYA9#Hq|` zyQ}mSZi{;_j{=AX@}x)XqToRGPK(^@L_LbJJ9`EGz<~myv!p9?jC&|!yA}LN$rY1ZuU1&CuJAA zne&!nJDx)KRR|{rL?PBoTq|}W0=nnt9_P)JX}(S$onP-b2Q&Yyl-l2EKv+^zw#K^{ zr>$--jMLM>rH{{&Qz=U_e(xOr7dRc>V!3N$I=bNl)5p1t;y>tTtE)KB6i}b)Xa}Wg z>|9@$LHdeMqXX)dZKDV=oGlNc^(cK;o!i$;y9y*;nw6~;xD&=RtTq*>wmpI8O}Hb) z&1Jt?wxR6eDpfjKCQ(i2nULc#wszpDYFElJ7+zKIxjbywtnF-UwxIqO&*A_5Qch<) zdOR*GoOW&`XPl8&AhFcTH`TvF_eU}NQhlKf9c^gC(tkZsyfC4>6oYY@reS^g`-7fv zK20~S2DEvgi|<(I&YK}rH@g}7QUPr@js4>SzeWaNEO)Ug)G9|-rQyTR&xZ;UFQKh-LDx+P$I2{Y5N@a@TE#l% zR)fFP|DV?3xdsF_R_TmWB;bV`=kjNnUeg~9I0XG!973hmdqw`V$|kvYPGl>3jA=Z0 zQ68Pz-Y+{s{b`c0dz$*Vn-DD&vgvhLL`$8pe{?*7DN9BZw^UN5)BkV%!>c1GKC!>; z7)RbGpEXToN8@d&`@PQ+7LSr#9lwEEkdl{IFe-fzPt<)%cO?&(L*B{}WE94dq1?^W zPML>ixLF?RdWgRQbkLC3Vrq%NVc&d|OY$ZcU2i=38@Xgki+sA1dJt@^GU2bh>3bo4 zm%@k{w*}3W3j38Mf?gyTl|Qto!)MbFS?4=t+kdBjZj&BUHKA?<7cAD^&~lFeB>Rwd z=g)OAlXhPv3Q{Xp+5zylLDs zHYWRBM31JcFS~6`KT6hB_)i?OkXwygXX<%-Q~L(owaz)+siZ86NordO z1TSBHa;Ig8CL{|iv!ygsg@rRcPGe#+t&Ogh#K=o61C|X6(JS;Itgd(0#Vc?>-U{F6 z)SVovSn{mlA`5e{5m|ZEJhn?SN9#+8%J98aY3?y}ANpNWvrQ8Ls$#MfV;uu_Hypyn zFl4O7&?i$?!27tJ8=M1dgPoIpr1^YZ<`+^(ShdHwOEK0~>}Bb`-p4P*TV8x4qY(T) ztf20}Af7u>QS{e_e*8k0oZq~r43pL*<3!b6$UA}T*dMr`KEL@)vDI`dccAkI%Tf`j zuaWfhJyS-;2og^D69{e!L-JCZwf;C=P1sQJx2b>=O;)MBWaADw?6M{B6n;z_LHJG-S11oQ>w%@JXu~ZoC^o!^_lA@bQ%t>@f?hH1Q&m-@X??%9u?^u|Imo!A0Cn- zCK8wq%@S>Z-$0XSWEy9fh=NeT=c;wDLuj<2HQ$F&`bh31Q3A|9%k)jJX~2XGEaENM zQ}io&YA)_bbfJ0NSB$~$Y`oK8u}htJb6bG`qYfP2^#eww^dG5?mz<$ivhWZIdjic) z$JO~Jyc2`Xq3pnC=jc+#OnMwzhkmcxr@AfH*NTZJaksurOBtK_k5_dpNyU#hzhA+i z1%<8rb4)CP&Q16IEc-K>z_{5EoRwc%h3wMH(oj|lN|4W6{0jr0Z|AK$Kc;VR-7#8~ zXvalCElb+f`=Jj{4-HmDQ+XfYFAqvI9*vl z&Q;WD(4ID;xKETW+pco!5D*p`1>UgOiSJ2#dQL!Oo*!#=p^SJXwrBS=YJ=->HtbhA z;r&>w%UY%a@zub)=Y5Z;A;>N$TFmGSF?Q!&ylFrkR{<@U8-=K|86C`TP(wm7vsXlD zHGTiO?0F-YX2e=kj_O|+IPV@}GmAu8E1FKS5qhilrA^g00~YHc*Dem_uBL%4!fN>;|>Fx~d53EE32E=sZzG~?~v{fu; z-y5XHpqv149xPelkYHUkCVh34MLT^)(i;X|AqS;C?@-zom>&9R=?5f?*!fv4MBOpP z%MSc`F5O#)z}q{}G6l^MUTz%^DPvW-+W>XRzRr)0ry7(ZgIU4E8Kruq*RZK)6wQ8B zcj~~h^}Zb$;zxXQGH1xE0$XN`ZfI)ML7AI~$5z*fJNbQn*Dc>o|b zWEEX`F&@BsjT<-0+c}U1ml)(y`KuzCq5UDQ}u!sI5qXld(zPe3;n*lT(NzN}OYHRM?AdcJN_^c$-e| z^j=3xT=H2hY{sZJ3i}ehPY!yoAjvtM^F8(1NbAQ7VhaJZ+q|^ih#mgTb-y+pAza<8 z5hYBd9^XvA`o#xoy>V7KWif5w3Tp@D;+Ggxzy{NWE~g>eFj94SnV|PQD%)}RM1@xB zHkxNhc2mgM;ZKr!Pgn)rFWS(08f7vX1TdY6X~TL-uh=s2XLeO zGb=d3PIOZ|m;tip;6T!!x4H(2kXfYdU0aMvD4g@N*px&QLR z6kIafDvv?|u1Ud~zJD8cgESdE<9jR{U&z^_vCS0F7^Z8D?g5f`SJVB3Eh9~!DKKYC ze~?m|<+0I;dc--W2Oi#_xlkxaEg53zYb3NAaa zdIA}RgWpf_Appec!-k7=swvs^0sAOIOChdCO{NWk>_tp?7~`eZ+!rzc(Tz@TYngb< zqp3G`>3}+2T(_h+(8ZE(eY%_=KFfCJIjYX5Z`@~I$YX%_zUz}uZFLRyQxxNj2nU7J zY7182$b(&;8BP16QQeG6>kgz&%j;=V8s6~4DyxTUN2}ArAY|gw5PoMj$;umbz8Q2$ zUgn89|<%rUBEiTjRUs( z@pI^YS43>Fs*YJIDJ?CUs-1_FVFZ&q{v=>9%((OAOMA?|W5v=ifu_!P*}U^muJwsx z?JkeTlxc+I=SqmfWAe(Br~ecOZTNmSY0Pl#*^I65iKUry`!*45e`-^)B8JYFe@^og z?Q#s3R5-m1T!YLiP6Y8`gn-@~j?aOAnf~=Al42W7i!xKAxsCa_3;5Q*I*9Vbn-IE zgW5l={)jo?Jx24-=JnAxH7_y@T|;z>YPlM*c`KF4nys%?iD>!RV`=1!TOiO|yw?S|7YGsU3eVFLSJgFgH>L%EoljGvew`35f3L^=bk8YcvP2 z&W9akdlLM(%pB{Q@2&KIT}YA1^i_$48rN+!wOZoKqT+1Ok^D|N^r{8A*APLm(;qVI z-h-Asl3m!sz-*|y5{m`T1V&`NVQ6VYfz&1LGFw2pDT7P3(};%&?3upwGEu?Lt~)X& zC4ydi{q!TBG*o<{&e-01^CI;Vlx=pdFjK0sXlTinIAmsq&X7-of1iQBym?aIMR)&>Vvq*MErQ>oXjvti0Nw*qu!^kJG9i5-gyeN942XR&N1CX*c&`hKh6)e zMG%c))AUd|?woT%n9QS|iGLh;-XqvTUd(hIIz3ixT(0@N=L9eUltKcN4H`IAP1x(( zBn<2+)r@YBLyn-EQnB~~AJ9u;WBj!gjAJG1BtMxoHgsLzc@DTD-EoiRTndf9rE)(> z)_j2hHudE_N=##^S}o2c(N3IFt9e#0VY4m`DXKmn?Dae|8p$!?+N%R4Gl}!z#7+~B zqwdD2J1_Tz3Q#KrJ+p3M?0}sYH)uetJ z@g|(sJ8bKGdO^EVe-N;#y{v>cG;9l~_=IJ+S59I?BhFgx6WS-%7|K-FkMWZ}?}+~; zZgUs~=ua)c+$x{14piF#g=kMjTyAlrMRM&~_ahT0NV2qF_E?MJiN_0UY@!iYg%|P) z?mDWx++=O;BK>pcMJ{jX0|@ZN+`JWvF`Z7sxvp>Fr#mMEGyn(+O2hAl&A}@zQvKZA zcL{@G#!g#yDXKf{mK+|sFNO6@_oy@=Z?Xv*8!pwaZC$7ayscF1aTDNO`LSZE5&=6g z?bi_M%7e>?E@91M2WxsR?X+Y$mOv;NZ0p&QEojh7Z_y9Mwd)Xcq;1@&Z2Um}++j;v zF&}JxoMcjfu@YInt$s&>l_kUJA4U}VVP?(zhS&Dj8%Q>&^QjhEj!}j@JGmR68Q96R zqNDgqVcxe-v+{0wqLg%@YAiku0kak5ekK29 z2~5X$G~TP{0emQ2v0W>vd3tKnl!;KbsLcI3(3N0KPfopK(K3g6<~tG!EU zxS=xQYr#qVSoeAH<~p2>`RVpefpEtjj%m6FNQv~hzUyBH!V_2LIGx{~=wg}M;Zaf@+8z>UxvqNqiO$}TTZWaOoF4AMd1Gq&4;BVh`M-kSNn>`?1bQ)3Xu>csjk$AO>{+4aAESf9%DI z2j^jf2f3ZFp3*m%paz5a@3D^`4DK45PgL_02b~t`SIR^u zTHNrE^~ixuo2-W!AEcOQfaZsvDkcr!pj|XmE&nFgC+bdP`d<|D5(T4|2*+gW#hKG6 z5LBYv)rY&WHRq1oTp)Vm;N-JBB-|NN|Km!s%|8q>2f zq}j28g?zC#Wez z+n<`nKK4DdLZo+$Z`)QpgfyNX5GtuI9V5PCv5y5&iN;xRGwopg|Di$TTTz6e{!FY?m$;M>9N9D6nd#}8?ea9Rk} zx%~IIJ51o8sOvO9uf-arM7 z!W)XOEq0^Y(Cgo#Yv)l#|I?Gg^~f)gnec{*6XVB3FCB$He4vmPt?~l5+RV4{ilkA> z&d=mdJ6P)o*hH4NGvG}V|Ksk%_mnasT_OYwp-GtXw;a!``=a9UG!neryFs~Fjqi2V zBuNcuB|}mhYz7D$ju^g2iT`|^3Z^Ven>;aqt4kn0Sey87AnxnE+)eU+sP|-7SS~Z;H#}Q^ zaF!d@hq1I~(HUyFdZzP;{2$ufdWLhh-}W7ll_8|3_>D)T`5142J+ESos@I!B2S_FS|5G8Ydz&o-29rYVS#zeURp}R!VZ8 z3$KiSBOMDQu?z;QZO|gzS>%{b(wk_TG&4vi{9%q`U8Wk!2oZc`C0g%;^@n1&b%9wC z7$wcfghWJ;UqXkwO$3pT@GA$xDgRU!)gys z<^l=mzKJ|fJroMlR{U`Y98v35<*YFkBYCsB+VB@KplK1akqLxsF#p?Lh&CZ@Y~ky% z`RA}Z{>nm3>-W2BKcqxD%=k}8$8Q9Qty}I9G#GNaKI>!XHDTw=(910>xB>pA!&TG> zZwjxR;u#6k;FTZyQ3i6VA=uQv5*_h&#s`Mi321Sz(|z)A3estHPJIn23cD+^)F-h( z!7g!C@OLx|1!mIzGNOP>>V@S3nr`0J6A=D0+@9y&@}kCau+!!tTj9`e$P9UxOCLky z!K9RIx-Z5p8;;W(VY*n|TC%AQEs5Axwy}c9y51}ERG5Mg^#-s*HsdNmTD)YH0qq?k z%Ly^wh+Ba|UZs7tR{(7JLj5=)k+y^GN*h{vw$-U2egQ3C8_xN2{LKj1Cib_a855BZ zdky^kkX$PyoNurX;nV8gGJHxV!{oMnV1nyg%Rg$EHUB0LQq1JTR%{rN@u}{Q(F_C`!*X^8^)y4+%WIe7I{xnd#;P?` zFt<+qeBynoqnLj#yGz1ekJIvY5zinfjM$`JNY_4VZx-pq=L!#}u~9I92-3i=Cnyxw z9{kac?L*l;uP+8aA~!6-hUeS&1!5_`tJR4CJFKia710UyENk!{GcO#k4y$+Mm+_(I zw@kB5n2FNVJ4Jtmh+f+16X{z#Hqtu8?dL&2B41QH2}DK0`kwh4t$(D@{naVheYi=1cV z_9o}qk^I(+(8mIr*Ocg0M=9&7WNo*IwnJhTKXJ&F#x>BAIhd#Q1RlIr{zdDoy`CiI zQxVDlM`FA;zTuD^u=CKQ8s|8^CCFgMp^ub$DF9LG#;Y=#q3@mP;x;u4kB}{?OtQvx zwNX8c6Zh_=w-wP2RUIa?SAIq4p)qpv)@l3i#z0y$s2f&4+&6FVEQycAdiA_DmF8pE zk1LX7Dla6QrN4XZJxFa!iuy&Szo{W@G%=31A!jE(HF0WoG)>ZmsGEOjD5nabjnnb0 z=t1;w*6#jQHQAhiw5m?m!NW!bQMCWf)Rrwu^1lUfhIKSOF7Zm)8+V-k3&%1rUyX*+ z+~vv6#Gj)D0O67f(a%)7B9Br(b)nO#hEzK=ZYDDGmRNnO!HOM}nXuFkvQw>EBMk&f#YhPSm2RGDh9D@UyqM*_>lnL znT>bRP>>gV%xhT#e9v=H=Zf6|P_Uz4*YAvRI3cqynqt3c?3eA`Vk5ahec`b?4H@Nl zTYXJ0_&&r#r!eI4w2+CM!p? zG9x7T36-9rE?kkzec-v}7mftn04hu7CKcCtC*tjeGlwm#z!iFWk%{=;9 zwlzg{h3bGsJN12}pYmdlAy$irx^8Ogj!pClKV}aQrf0_07GOp7gp$oBYvurxyrT5rK7P@6|g(yfEk(_`gT$RL@Q>$UYkb zexlI1$GH-CDI#IY;A|LARG>#QlNyyq?Fy{(!Eq&)mRx#PgkAj*`4ca zD~C)I#;;mexM8{|uVsU!7bWUZHZMC0FcRCk(Hu}PwvKmaVcf^?i*}y>8+1Pt4Y{kX z6$Msmm~L|ILV+gQ9)YmKV7>tU0lOgT5mHEnCWMWSaOQOd31{)u(7hzVHd8OYGOIf(3qmWPYdI`@3 z?y2~lGJe@jY8AM(HoEQt$`_{B(Ya%bTX}LmgW7$vqyvlH0E}AIhQAxFU@e6XC!KB? zEz#lV^M#?1pn&{9O>R36xsUNs{q<0%toBcb#fgMt%esIog$c=XQJs{^kZuzV8)7^HD(< zZsJI~VULmx&?RXOaIYfs3V(Z;11I~*maxB{(5Js)@*KM*%YS3eD@LITK4GL^jWW5p zbi}tOn{G*9ndudRqP})Y$cwFbU6mrn>(zLD4Q-0+l-T@zRmib)b1|G?q?B$6mkE z?_sx@^hMz0tck(Kwvl7-k{EHRF3LX0!&zMVfrEuYzvzFbm~hlk`(A7!FA$6N7~ z3wQ4S>drJd_!tzW^Fj+!U~X;^j?b{J3AgzZD{>b3oy3}!+JlFxlw-5Pr*9}|B@o0k_;)2X=p%gHA1MYMJ$TT(krd*CjbCLc zG3J?@u77B(qaWFKv*MT~t%t%Iv0%e1T!PBdjjj*T@-cGGBQl0)6p5GRM_WPXLA|PC zzq^vQdIxO#l77{3RWRf8cd1isan%nqtf4*G`r7%fj-WOaG3HofxI|QYCWcR?^v8kE z_@ASzJh3ZozAp)hAWzU+Y3>5q8-orvW!B(hd#ekt;0d~OkLKNmcVg&TeER=ISCz`= zi>W#~Xl>c@BRV`a*!g4uMZsr__wZkWn8x_!i_Rh{sa>!AG`oTJH<;e;IZ(D9pX*w; zGNcGdHat6P(2LR#_L^rKw}s#jgpnL8YL~@oNFjzO@iF0kt@FkzvV~i#6t~A#ZJ@dC z5+15+eQUIz8LXm!HRs1N$`Gm|X1X>}=X~p2(?R?%_FjA1sK@#i*T0P4M9W_|vVl(v zK$D|CV_@k;=ADzR)pQPfYZ)ABWUzU3vhg^^~+V!lC{uuFkZSx>=ln+H-nC=^5M?{1=`Rs zf~=cSX8}&Yx&6CzlGz*eJ+iz?3KGb8z9KD-(0VGwjo_QpkFE0BPkjT9Skh((;!3;m zT0`I11nTCO4x}!NKH1_tCf!YkxF4>B{U-ClqIBbdk_PTx-Zb8#*83pl+n@gC>+ zp)ob=2VuPnXXBXAPq+Vnk-*L7&gZXa(wOsQlxV;ajL`70tQO(Q z@w7r4ZuTTm-3PCwA=p#ikTou3Y%?2QsqY zy5ky0+(AUi;jmmJ8r+)b?GJwfhzoq?B_=SM=JEZ{VPFh`?0W9_lLa>~ZDBX7i7KV*fn^M5a7Wi_Ci` zHkFPlC2=;FN66@I^4tp0?k#$u?r%zX*7o$Q6H9?LMx51`Xz6ese`T-&ep?k}J*+o? z*K}Xr5m981#8I7abv8{@JQpMle?-Yxj%2{&uU_2Y;|$K-Y;YtErfaNh3LU5R{&F26 zX;1T+ah%8S^;JG!^1QtaxZN7(J=gSGun>hgD|H$Jp%MbMdmL$p$MJ~8t=ZH`f|7Fs zG5^IQ)08R)!sqP|UpxK@{g*o41m>Dzj1fEyta1w@6kaHsRd*iOrBG#E=x`iI4Bdj% z!7Hd2qrN_qf)-86Wa%-9?asey^-*RLqNCum>?o=NB#+Ngjs6@omhVJOCK)_@CbsG-AWYy(nBNV+~HXhsJH1^MktKb*phnq zD{2HvR#%DVQTD8#9rN4awFm=)KPEV2lXcZq?+=x%0afLpOg$F;3C=k8YjcVbnMJN2Qq5z)T$r79e*rNN8gW5kRlfAB!9wiOU%_k3QToR@`R*VNqut%xph zc~zg;J;jYM`gQLf!K4t@xcStOIFiPB==gvW0HRSh`r*yD2xyF2w0*sTID?XxYwX`q zg9dz{_{^soHH#ie&cz_KvVu@moeYF2g=YuvtGY)5HiNsnqe=D=sT=18>h*TqV*POf zbXYwfd(R&pvtQc1Rf*0`jhHnkplqZybs%>K=Ivn(J=!Z~d|Jq<;aOw*wDzyI7zNKz zH`+4r4o#pnh}p9Ii|HVpO~2`0(SE7g1y>uLIB?aF7Mt}H2qC;Bw^wtuyC3PV=rhx4 z_1E;dJPD1k7&}bgrqZZCidc_aP{`?8Zdfhf2FPtKd_w_l2&y~s?`frY6>D{=oRkZ~ zjNh5>>EMI=r42rhvhZJWBwaO8DR0tM(8|%7qcwVH-s54_ zCbDr@>%&lO()DNPr)98jp_{;*_^;xI3#9{@DNjtDq6qWL!ENRxbkRB9ObV7di{wze zCH30u8BzXJ=ZRuToZ){@9?oyKTZT8#BGta9$H^pEvh9>j@Q4}t&pO#WSb{7ilX)_* zHp0Bu>MOT%5g2*l@0(hseqWNMx-32u^_{N2Nf5=%7J@X6n`Q1ReH*{P+WWZVeTki} z=i_Vv`w(c|%0C2BjW<=a?#Eoi9L*Dt!V2f2ls0Tr+p2gyVmFn#(KQnFMHl~bUiB*%S&sHY=ubfD^zr?R`Ijf@s zu3xl}X~*43{VOjIw~kN?f?)LX$i@)z@SSJQS#9BYs%AuaHs25#V%=ufvksXD@PWMv z!Ig46T#K!{75piGG-djdQlxja?#ybx->LgP@rdyE>Y7fGBa=V$#n{u1zKrOAOwX`G zmiDS0bW8i(txCe{@YN8Gd=kb#i}BM5=BsQuDk~Prre4DVS)FF4j2DGJ5z9{=V&A4+ zBfodl&JTpbll)|54bNGUsJCFqHDwWksHHcUf)une*ajP{X=`Dao*7`kKj~%7&wi9> z!z-3?XE^-=2V8itah6+7r$}wTzAE;D^eiS#-)J9wl!Jn5Q{=V)d2t!(pN#4j1RkKu zDYJf`7<1yXDXJ_(FU~-|`I*m$l4}TaZ)^-m*8h!ww5a<1l6+Jgu&U^L!$v&5!R33&FF=9%dA|Ciw9}NYA%GNd%C7e4f&wnw=#>aKEW!-aIgQ$J!=g~)Tr#Sta=q* z_vHhw?QsWXrcHVA#~tulDK6|?WD}il-A&VETj4ehRB=8xF9j*TrN^Z(1-@ExN~rXn z%hNaL4IAnpL!dfSyDwn_5QgrIl~h{}eTBg-euoB=Wd7p(*k4pS07a?8uyv3OPuRZM z|MkV8CQfRjsM0-M#@obml*<#!BLc-`R{A1kn~1M(;TT)ddm%fx))OinVo79qrUG`o z>yT}`=S&D|#0~6In9l-1ZbR>Sd|8BKM1NU~do$Qt?Npr$VmkG6yI2`p*MC3o**$fO z6`1~~>%1n(=ms4#tspllJ`;Op1#goNi|zP z7cQe6#l{cJ_=1~$s%ZPN2tVW*~_2A(NWI@lz z@%!G#U8^azP=%s_M_RAi$Uv*Vx$HlQrQvs6&j3GzR}?BWQFU#L=9^E;c*CuSGD~2+ zg5AvFT>uw8b{fUMe9o|^S#!$iYo0!k)AfN+Vi+luzDkCRLhIBOCne6V%BQgPmZn_! z9r)UY0yo9`_#@v=AqmW9^!l$>{ESx=o6YZ9iSOvs<%Dm%;Jz76F$P+6EZi) zny25L_1m}xs8pk@{DyAE6NH&w)$Q;J>gn+1?+ru`=AD6K+kEBM+f)!TPnF&`l~YMD z;T^*Y@*>~qf4-0k->0+Dbn>Q1S~T4Y{0|MEfwx$3O_qgE!?1d|Itcod+>=dbD3-Qn z?B!QK#r;{gkfEaaGDTe0E~6fyQrUgJ6yB_(#v)vI12!)9>(*yqgyF7_WVj24(8Kjp z$+RdOOGY{8cc;w&U4O+1`cw4bKodc{`%c5 zhAVJIPYSOre~W2&AFrCBs7`zj&^otX5}H^ZVO|=@av1$$EFgEzJPmVs+jC++ezVjn zHH%vSP*%I3l7a9dt~{lAFbh5_ah&JVfAN^OvfQa=U`ns|Fh_`@u1dFx^=TdcM(bN% zbNeokEvAi^yWtmxGe0s&Mo;IVncVjJmJu;0!{>@w_|Q@NRak4w*E% z7SyHRT)-)dIB%|L-s=K57*<1X$&IkH zU}+8-wq^gK>(5sVo{8ST)8A%F$8!+%>x9FXN`!z27mGUrv;#1ZOjf~{m21KCYFr9C z74QdH!;2Z^ZX&#=;+5GAJBv$3S(<*MHsk<0y-#}u!Ap(=SM_fSKoQyO!wwQ}Y@^F^ z%Qj5gA0SH?m~dB<{l43Kd58l8X*NmWFI=S4_`OnC_(k(j-MqMY1l))AgeaK!C~NH| zOMFz~O8Y>LrG`yYT^ z+<2V~_zjH(YW*BOu$Ilhm0J!7^X7Ezn;l2-gcsM?3ls|}aC54>8`?z90pdp2-{EA0 znbGY5{$!fN#t(ziflNBzslJd4k22`qU0975fKRW>(c?#{eV-N+yzpWsjBh{`>6lhf zETTV0EEIrAzCdSP3qqc%Rnv8Gf49xpSHZZoOHm72e7P zPKlV6xEp*AoDO@xQiKvM@A;$fAfD&YMEyva1ConDLq)k`66H>wywxhWso|VC4fj2e zv1!_?TzMZEMrsX-hF3y-TYA+OB}dipt3!(huTUUQRwP;V6;8oYnF;Ebqlc)SC&5Do z+!$x5T;Ph=*XLgE{h<{CRPP@r{p3Z6n0D`J&s8V4i3FZ3`F{+s+Jn-b$0N9T2IH3o z3EZXFcz#NK!3%_X)(L%^Cmce;rBq{O$wAOGhq#UrHw+`WYX0xiEj;~MfkR;!PTuLK z(zyDA{PGCqHOKJ|L;0|d22HzrFkVx#u$<>+Z~^(hf0zB02L-3l*%QsWo)~kNGlvv> zD_aZd9wVd`P2D9}wsr_eC&k!PrE$@Bya!6TAxRPgenKvMcKgrZ5Mzpk&xvvi!$@Sy z5sUJ;LduiQ?c-~;a8}71e_QUQP1N>_YwH)Yz2Ko;@@0?Yt3g@ct6)lwXZJ! zng{74Wd--z@5#~dy&=p&bE+A)nRCZSfYzQWmvy#NOoQtI#O6uNNB4MO^e}&@^2-Ez z#)4}W#uo@IebI)M^~dG{0fxox(1Ze9ht%O`Y51KYZeD|j zi&XE9+^Wf+`*qwld~4=RQ(Obm*`awa)w3K{U`=ac!?w-W@zLVif$_ASSXr7lE4>bH zG(6~?=HqFq$&$pTTQ2cy;K|*KDr(|W%8(hhDg7mZJpGB#fP}hQ-DHVfp@&!O$d{0k z4(qFNUf9XvYWtg%?7S_;r?`MF&nCsCk7ZBg3;iOH_ltNgQPp>mhVX`s-)&ol)68)7 zxWUYKX%j`KZkLSeK(3YF+k1kEtkaVd#x%&CdT|?ge8c`$@CYim?K_qbG?Xu`u^Ro* zgp2EZ*Ejy#D!g>Zd)a-s!MvF4h<0bZgQ$5d<<>4H?xVey6|NdbPLk23=tPc)dWhSV z@J{N4_5H8;O&pIhmAbW!Q|@jpT>K()&CSiR9;b z+^i{gaA}mh)#6b@X~|x(CQVdHDA;z1!BrZH5=Ki)buPK5zA)`m=fApb@%{x2Jy~}& z?_sg-rmPHOmZLQQ>Y&f|r6u(i1DnQ{OsShKN8VNVNBq48BPxC`hJR4jA}F`G4OpjS z%3;|_6$NjYT;X5xnmnps!z$UH`?ay7lO)D~ylkZ!13!{Vs{lNur~aHFs67)`DWmv} zT|02}4OTu!EPib55IM)auUD@OdIB6D{$Fy?_drjbi_)ArHL&I2y`_vC%@+KGxn=v} zWQV{OSR|KoS0H9PGe)-1S2r{m%f;}x#UHa@IB9;yL_%F#e2Wt*^fogiz7nh7L#sji zE&C+g@WgLSB?Ar!C{PVav@ ziv^@Ge${fq5za@$3)$QI&m4g-Sai*PLzSAE7%?-}LY~Q0^Y*I3gmF~ym!Txt#X{Yq}5w=dIzQ z`r{kqytmLQkH0_?=J9MG=gG z&3K&G2UCkq2#gTuF$OCZ(%reJnVDS-Pmvhin%$(fC%!6X`{5=SOf++=Un2*ZlGmDe zJ>@|W{`Jv&tq@HzKN#d?$LabtBWbNzaE?>^Eu>O~F*?_jw^P!{RN1P?ey>y{b*}pP zRPeDY?mT8+=%cPaRpYD@ zr*fnZ|Ah1)w{&I=nta5nHc^DJ` zJNlq%J7S@EuOQQTzZTb~c4uav1^K_xvj1i=+C3-`NRH$x2tBG+Mf?K*Gdv|N^+?M# zKnhihu@EF%G;6(;CYE(j#(Qm7a`mD&0)#TFXy^Duz>eMNyyaGudqw%RNRiuwt;}8- zj+92i9NDdaNf^ssGq)i8lnRFPKggzzyo>hlje?*lZhdby%oX>hujAE+V2FgAyJ;mm zfp=Sm4?2!Mmrzu-(w|G5NzIR6dizKbIlH!Jb?@&3Vk;FE2!ipk56c%ue2k;)jNIW` zo(<&0F01eza)CE7eq}o&#vP{{b#sIHG{+Lu_2j&=&w=Ohe_HGICm>rMn#h^lxBwVV zJu(n7P}(M9o(VYXOBKML{q^i;2y)hZ z+aS!ug&fKswKO6N@;hu;(ous4+H@i(6CYHZ_g+UX8L<8?w&8(U&+z^1U^@Ooi<`yJ z{!Bp2!lR)Cm&D!QfQIc-!;MS_LK^%1Lq+_DQ^C(6 z5+3W}Fo{;w;Ws-o(?s3Sc1UarRlO><<>`an+y3*%>5Ob~yQ8Drsehn0G-(;8!RB%q zK}KZUK^hBFs=@>1;Fay;DC?BoAD=!#jQg$=DkHTKa{_u3{Q zLVU~WO7l7iggwfL@gP+Og}A8mGd`ARHt-KPEI=4A4JQQmJSGaeP4t^|Iee((f11?z zOMrix<@#^|2)3zR?K?OF1!W+2~rlh>SOvw8J8L9qLmmz)OoUofWwbw#H+yFHw}JopvZJ zz7b@8N3uY5ngAg8^Q?KVPeUxjM^YYl19*|=okTy}i_*rQTu1l3^YGCqS2RxDd zPD5!`$u!3d$?Bba#+!z`6Rh%`uZdgg`xe&obMM{aB z+dCsiDBFp;P;HT5EG-#*KOj4hfTZ6f*H>%YrO~p0Wt;-t4`U}2&7d)H0 zl2r-HP5)Hq@ElZ7ZD(ZZ7Gz}C<*RwQ8UT>Oi;*837b35TpWUZxl1wLoJ z?$djiBGk5LF9Y4OVj!8~)JSHa*hM{GVtf*kWPXf9$e1d3bx#Gq5qN}eL119`;y&?L z_<_Av2d(${Y=AQhZHtQLA+)tl<<#gVA-P;S;ML-U3XW{P|F7J?@XEe^=~`uvGREm9 zVd)DH-lCMhV?r85VEX(fa+eHkf#Lgg?HUMyE&0Sxm+6V{*;uGtq>4B#Q;z;4^M@Hw zvGY~+PReIzsP_6G8fIy#wHH8C?N#?(bFSKZ5F?&Hw0f)YRz_!x*uHM3_Lr#~1|_5F zQ`h1uf$*RvJo2VASlfR;tvZ>~s$SNwX@{~fx-xt)-R&zevEaEwr+zZxfHKPf$ zEJDRwr_==MjORRpN_No$v~;C?gbb`Rr9H5nWKg_3`{JuZQ8f%W709I#a}~q3j$lrT zIP&g-L6CN+t}ZZ5E?xuUwQ*CGbKW9!*RyLX5N}J-O=}Z_xPA5aM;KA?HQ4r|Z=>?d zu*!z<&@>4&nIPVY3IvdyOxLU?DyP?^SUdLG{U+K)->S{^W6hU*Wz2&h86Pzi*#-7mU#FOYsea0p2|6z>b_Bc)%Ls6wO-Be8b4Mg+@*m zyueAPx#l(_5Z9nh#ZJ8#h}Uc8pg;^}N>%iQD7K#R8}}kH!jjtL-?NY~8I*k!x^-v; zOqihPVGEK#$}V@u_gC;uY)`^OCs`;UCZsuC^OX^94>re#TENY@SPYI-uZ1C7 zePxgkKi3yHkj^Xdqy)Xet0aKpNELfPNw_}Q`e1Dj?wwuYbluU>WsbO-llcQ@Z}y_5 z-QPc}t_?m)-}76}QNaZu*_gGKANwOPc1&ykFuY>~Za%xt_-8tZSh0D5q5v5`mR?BW zMY-oyj&LZXU!=kNzv zL5)9+VfV*NgwTA=Rd63V#*%qTvr0bGa0s*Rv==;Uq324$$lFbJn^+JEZ zz}OT}5D-Fzp}@w=5M;Q)#&|ZivHSe^uPDZz-^=&=d3%1pc+DLMzFGalYe2NiW0V9|Q!N%7K^@T!dT#_mrGFvcPswEZxFodeB+DeP`MR^*x|{ zLA@j%LPn;1Ufz>QZxG7rW8K1u5m@40?nC(&u;H?ZAJRHejG{%>t43isYA1y*={$Y zZtdp`QUiMc3One#E?w%6@H$cF{eAuYXHb$kd)SkoLDR+g;FEh=uMy&vG0W5-`PbJz zvG5|yfZeJu#$UCCqLle3a|7|nFt_bKs!3qYfaf3X!dCF%c}}c5%WvkXr$?e{;)t7J z*eV}L#!U#4%YuWwVTb@qTT8@tU|i0y^J_DNLoF^f_&!5pzO`O|ALax^39%#IhSCCb z;+O-)HnR>yOc3f-JwfWd>=U_h2ca?hb|-I5N3cyHPcNmvL2F?4S2mlW#5ynZeq|cc zyMN`8*<_rWt)5xBVummu&6x<*3qvT}j_kyF5cI6m8O!bxe>7I=lR1^dA`yupE(miDcb4o0+M73XsohA93;g`*LHkNupf($I^N+ik1$Z%RU7sPxQ9uzvvTWNx7Lml}VsOgshP+i6cUj zqb|aGgBA*j<>iO;n?z`$M}G@lg}h52ofm&z4^r>{^!DyLl%C2qx8CQ~_%3Hgl$l(> z)=KubqYB9Wj(#6$C$YaHw|J)mkb^d*eYL6*I8EDCWOxpcE^RVap#t=Qr|y|egUkoZ zGnaph{K$UJ_>~!ib)wV~&4R0lQt?D>!a)J{q&aCY!el&$09^l3bu+=SD)ln<4nYzyfm?e6C6xz*KfI9Wy9)*WFg-BWC8t?LE3?+Ecq-gp`ng!C|PHZ<`% zMZ!>%fxRA5HLjMlW)+`?(lDNrzzKwtVf3yLd(&{T9`hHQSGR%*35Ey4g+Q1?J$EDX zi9pgewq3S{C^NHhI_}7R%K95F~TH}SfhL~L)}sU-*2_O>56*&2*~Tpm1W z9KDS(bDR$8N?Sv;3*pMG-55=I;9Y$YiG_%l2)|GM9`H-~3c~=dfhxGip zWHDDz!@y5+c1HIT15fT|@4GRILcyD$R)?eUQ^%sC*?9WLhc35&*M~)gSjy?;8^AMM zXIr(KV1MD()lz&R|Hj$;(T)Lk))mg7G{3o z-~z_-sp0X}QW)rxTdd$7Lh_GYqm6iGtvaJ+AOx7ynx8^HEOH;HWb|;7@@aG*K+uS$ zis8zQyqkJ6(MtZ;=hFL|^^a()cD+R>s28#n!)*A)V-*N{6E+o_Ii!I+E8D>9xqV zi>%xHaAP<2%N47Qbv6$h+V-;8fb@HG@r%7}Upfq9*Li2v-}XQlQ>mS5yGcXd?Q=jd z%_Wk>WYl;ihIVvs`*Hqz2yl$}{wRj2(MY{j=;5iHBhv{~5!Aim?i}`ypvvX!he1rT zPlZmkA1m<93MBlMl~A%Q0kJG*LKJ53)Eegm^x(`|uaFLN1@7YVPZwW`0l`#X@hWft zND7^5=AIw6*Yvwtpkd{q?|5Y>(F7m+GE^LxOVf(v6?UjO`qb$mqSGqb@5tBc@!`P= z>vwsf5u_nujDB7vuIfyRfY!YUxQBWjK+!a<($T63$ z&xVM|cBP=i3%_Ack6=<$ARM^Zkvi9cq^%EUS8{(FTHUCk2?uNvPU&T31n(rG;KZ+I zc|<~5>*+DoYv_?u$_wlY^!}o|)_2eXI>fGl6{&fsR?9xh@B@qRW1J8;_(XGY+1ba{ z=S%>;bhn>y*{rZjb^jrxy-NB&x7Cn`s3`B6)*NIBllPcA8>&cHs^{{XB)G<>DqLF; z0q)cX#l4rz!FCX^?z0TQ$)S)pL)cU}ohkK|1d^^aJf9c=Lh|I=K0&vA2_blsJx8|VCo^uB9*ouC%9%eH z&1s{Ex%x?#P)Z^#XmMDB}OGBCPa;48}-Kcdvl1N|7j)kHT%xOf5M3grdK3Jp60_=*VUx% z<$R7E#Qt_kY2z%+FXKw4=_?3CS+Lmz@$i8V?-YaMSst1puIjrB3`Y{|;doqSe+And# z55_lDC=aT<)MX@MhI5(=da+#f<`He!ZmcKKrug|J&6m(uVH7cl@Y-hK7L!npZG>Ey zrD)!>BN!s$<3mDmZDbv6(n052IxIZHiiGSDIJtxOF8MIUSK-U~jGy->oqX75GIR!a zuu<=~$ry&ez*-eucGiGXrzwwC^JstozB1e0@HVg$MNPVXUWlDP^gOMiH(-r!z1bNa ziZq`nGf_jpj={Z)`k4qqB2G3>hYu;zhiHy$Wkc{%zPS4`y%+yWd6{zqI7ExvZH1B0 zH*xN^5hj)z&}aPC7QYZujJ{8(o6`lY#-py=T6n(VqXByj?&u0~1=BD8Qpcs;$M8f-iQk9J}b+3MQk|_P7yo0AIm; z#q`zEd#H7l8M3Rq>wxVw?8qtDc)|Lsg_p8GvvhPqq&$@eTm9JtwF~xCP0E_-u>u%+ zMQQSJR?|xIy-bHIB5DwX@Tja$qq?!phLSERStNe%+2;}RFAJcy$*ZBX%oqSiCzcz% zfT(5K`4Wxx5o9XU`e)t*CRQaiY@rAURIE<8NH}%Xt;ZI^as!oPdWIH<2Und+iE&GG z#Q@%Ut03!b%twLXZIC?tLo9UaUdZqMaUbXw+3~%pi&P;>Krkhq%FbPI6+39Ta?)M= z8Ja=vjvpLVKSQrN(_?AkeTpU&u1jgE$B_c$#S8ZG)ix;x`C0OjdEUg2OY&C52RsAQ z9scS@vksj?qVrE>kmoMs4;^#|*3ogNb7`!bajwnK@)Fst9?rX|!CSz~;kxs2eY*L)-RTRhwuN(YEGQ-6YSiJ!Q+sBCe6Q3ahbA)xwR+AySzerqzVGUaog>OMwGbDEv3un}pSqSSw}t`qoZ??M+27P6zPM=X{PWIH*$E zBpNT^iG)6(e}_8}NxJfq!wfet8r{NQ)h8j}1t)DDTfWB^ALu7IZu>5eND+3{uTE_N z7@^3F)T1qUhC2&Bjx2vod$F*`L5V7);8qMb~$p92CsTB@8P#_pqbnU8ULPS%+^FnE{jp1@bxnFIm`Z5{2v z!Zv+6nB?~aF)T$MD?n3?(s<>(q-ky~!~}J^5C%KYxj~F~Wo)fhtrgyxVO!GAa}WAT z89(ipEU<5`X?^BTKpZvkxJg~+Pn_sXyFSzvXOLvL)pirxlsZ=+6UlM)uVF>fimlpq zlo^9&V^$H#O31=AF**lYr6~1|gAlERcc%7{Bw*$ePbbY8Fq}h2Y4a8o+C$1ZUC<43 zhwS2eDV~s!D~pRv0ptvl#@WMtk1~k3++3*%0b-M_4Tx%rLP=GwacCP^$JDb)g}oJI z>K|BS6)q@z9j7kmm?H3Zhs|G(9EO_I;+4ydotyX6^4v#wxu`R?^tRs?#2w31hhEL-jo|c_v49f_141x;vTfsGBr(y|eq1o~Dz-vP^nUuEGYJ@j^a|%${Au~{ z8E3X0KCfwAF@rq>6F;v0#|M;Zgk*SU=1N&K$1EC!(C(3UJnidL5_VACI1y`#hP!TW YqWw@MI>e5rr4FaE1y2sno98s||DR3q#{d8T literal 0 HcmV?d00001 diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.groovy new file mode 100644 index 00000000000000..7bbe3f01895424 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_basic.groovy @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_immediate_warmup_basic', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'block_file_cache_monitor_interval_sec=1', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def updateBeConf = {cluster, key, value -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + def (code, out, err) = update_be_config(ip, port, key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + updateBeConf(clusterName2, "enable_warmup_immediately_on_new_rowset", "true") + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + sql """ + create table test ( + col0 int not null, + col1 variant NOT NULL + ) DUPLICATE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true"); + """ + + clearFileCacheOnAllBackends() + sleep(15000) + + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test values (2, '{"a" : 111.1111}')""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + qt_sql """select * from test""" + assertEquals(5, getBrpcMetricsByCluster(clusterName2, "file_cache_download_submitted_num")) + assertEquals(5, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_triggered_by_sync_rowset_num")) + assertEquals(0, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_triggered_by_job_num")) + + // switch to source cluster and trigger compaction + sql """use @${clusterName1}""" + trigger_and_wait_compaction("test", "cumulative") + sql """insert into test values (6, '{"a" : 1111.11111}')""" + sleep(2000) + + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + qt_sql """select * from test""" + // wait until the injection complete + sleep(1000) + + assertEquals(7, getBrpcMetricsByCluster(clusterName2, "file_cache_download_submitted_num")) + assertEquals(7, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_triggered_by_sync_rowset_num")) + assertEquals(0, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_triggered_by_job_num")) + sleep(5000) + assertEquals(7, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_complete_num")) + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_multi_segments.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_multi_segments.groovy new file mode 100644 index 00000000000000..fc1416984d2e87 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_immediate_warmup_multi_segments.groovy @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_immediate_warmup_multi_segments', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'block_file_cache_monitor_interval_sec=1', + 'tablet_rowset_stale_sweep_time_sec=0', + 'vacuum_stale_rowsets_interval_s=10', + 'doris_scanner_row_bytes=1', + ] + options.enableDebugPoints() + options.cloudMode = true + + def testTable = "test" + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def updateBeConf = {cluster, key, value -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + def (code, out, err) = update_be_config(ip, port, key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + def injectS3FileReadSlow = {cluster, sleep_s -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + def injectName = 'S3FileReader::read_at_impl.io_slow' + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + GetDebugPoint().enableDebugPoint(ip, port as int, NodeType.BE, injectName, [sleep:sleep_s, execute:1]) + } + } + + def getTabletStatus = { cluster, tablet_id, rowsetIndex, lastRowsetSegmentNum, enableAssert = false -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[4] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + assertTrue(tabletJson.rowsets.size() >= rowsetIndex) + def rowset = tabletJson.rowsets.get(rowsetIndex - 1) + logger.info("rowset: ${rowset}") + + int start_index = rowset.indexOf("]") + int end_index = rowset.indexOf("DATA") + def segmentNumStr = rowset.substring(start_index + 1, end_index).trim() + logger.info("segmentNumStr: ${segmentNumStr}") + if (enableAssert) { + assertEquals(lastRowsetSegmentNum, Integer.parseInt(segmentNumStr)) + } else { + return lastRowsetSegmentNum == Integer.parseInt(segmentNumStr); + } + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + updateBeConf(clusterName2, "enable_warmup_immediately_on_new_rowset", "true") + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + sql """ DROP TABLE IF EXISTS ${testTable} """ + sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( + `k1` int(11) NULL, + `k2` int(11) NULL, + `v3` int(11) NULL, + `v4` int(11) NULL + ) unique KEY(`k1`, `k2`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + clearFileCacheOnAllBackends() + sleep(15000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + try { + // load 1 + streamLoad { + table "${testTable}" + set 'column_separator', ',' + set 'compress_type', 'GZ' + file 'test_schema_change_add_key_column.csv.gz' + time 10000 // limit inflight 10s + + check { res, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(res) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(8192, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + def rowCount1 = sql """ select count() from ${testTable}; """ + logger.info("rowCount1: ${rowCount1}") + // check generate 3 segments + getTabletStatus(clusterName1, tablet_id, 2, 3, true) + + // switch to read cluster, trigger a sync rowset + injectS3FileReadSlow(clusterName2, 10) + // the query will be blocked by the injection, we call it async + def future = thread { + sql """use @${clusterName2}""" + sql """select * from test""" + } + sleep(1000) + assertEquals(1, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_triggered_by_sync_rowset_num")) + assertEquals(2, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_segment_complete_num")) + assertEquals(0, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_complete_num")) + + future.get() + assertEquals(3, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_segment_complete_num")) + assertEquals(1, getBrpcMetricsByCluster(clusterName2, "file_cache_warm_up_rowset_complete_num")) + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } +} From 62c564911dd99f51f0e31322d1d98bfd8f1f558b Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 18 Sep 2025 17:19:56 +0800 Subject: [PATCH 2/3] [feature](cloud) Support query with freshness tolerance (#53540) --- be/src/cloud/cloud_internal_service.cpp | 64 +- be/src/cloud/cloud_schema_change_job.cpp | 2 +- be/src/cloud/cloud_storage_engine.cpp | 1 + be/src/cloud/cloud_storage_engine.h | 13 + be/src/cloud/cloud_tablet.cpp | 283 ++++- be/src/cloud/cloud_tablet.h | 91 +- be/src/cloud/cloud_warm_up_manager.cpp | 44 +- be/src/cloud/pb_convert.cpp | 12 + be/src/common/config.cpp | 4 + be/src/common/config.h | 4 + be/src/io/cache/block_file_cache.cpp | 47 + be/src/io/cache/block_file_cache.h | 11 + be/src/io/cache/cached_remote_file_reader.cpp | 14 + be/src/io/cache/file_cache_common.h | 2 + be/src/io/io_common.h | 2 + be/src/olap/base_tablet.h | 28 +- be/src/olap/rowset/rowset.cpp | 3 + be/src/olap/rowset/rowset.h | 2 + be/src/olap/rowset/rowset_meta.h | 17 + be/src/olap/tablet.cpp | 4 +- be/src/olap/tablet.h | 2 +- be/src/olap/tablet_meta.h | 6 +- be/src/olap/version_graph.cpp | 207 ++++ be/src/olap/version_graph.h | 63 + be/src/pipeline/exec/olap_scan_operator.cpp | 18 +- be/src/runtime/runtime_state.h | 14 + be/src/vec/exec/scan/new_olap_scanner.cpp | 15 +- .../cloud_tablet_query_prefer_cache_test.cpp | 804 ++++++++++++ ...cloud_tablet_query_with_tolerance_test.cpp | 1074 +++++++++++++++++ be/test/cloud/cloud_tablet_test.cpp | 84 +- be/test/olap/tablet_test.cpp | 4 +- cloud/src/meta-service/meta_service_job.cpp | 40 +- cloud/src/meta-service/meta_service_txn.cpp | 9 + cloud/src/meta-service/txn_lazy_committer.cpp | 4 + cloud/test/meta_service_job_test.cpp | 14 + cloud/test/meta_service_test.cpp | 63 + .../apache/doris/mysql/privilege/Auth.java | 18 + .../mysql/privilege/CommonUserProperties.java | 22 + .../doris/mysql/privilege/UserProperty.java | 37 + .../mysql/privilege/UserPropertyMgr.java | 18 + .../org/apache/doris/qe/SessionVariable.java | 39 + .../doris/catalog/UserPropertyTest.java | 4 + .../doris/planner/ResourceTagQueryTest.java | 2 +- gensrc/proto/olap_file.proto | 4 + gensrc/thrift/PaloInternalService.thrift | 3 + .../test_enable_prefer_cached_rowset.out | 32 + .../test_query_freshness_tolerance.out | 24 + ...armup_delay_compaction_query_tolerance.out | 9 + .../test_warmup_delay_idx_query_tolerance.out | 23 + .../test_warmup_delay_sc_query_tolerance.out | 11 + ...lay_timeout_compaction_query_tolerance.out | 23 + .../warmup/test_warmup_download_fail.out | 10 + .../test_enable_prefer_cached_rowset.groovy | 178 +++ .../test_query_freshness_tolerance.groovy | 183 +++ ...up_delay_compaction_query_tolerance.groovy | 316 +++++ ...st_warmup_delay_idx_query_tolerance.groovy | 332 +++++ ...est_warmup_delay_sc_query_tolerance.groovy | 308 +++++ ..._timeout_compaction_query_tolerance.groovy | 335 +++++ .../warmup/test_warmup_download_fail.groovy | 254 ++++ .../test_read_cluster_var_property.groovy | 214 ++++ 60 files changed, 5367 insertions(+), 101 deletions(-) create mode 100644 be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp create mode 100644 be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.out create mode 100644 regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.out create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.groovy create mode 100644 regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.groovy create mode 100644 regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index f1dc736ef43dc1..2e058571ba0d87 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -26,6 +26,7 @@ #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" +#include "util/debug_points.h" namespace doris { @@ -225,7 +226,7 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c } if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpState::TRIGGERED_BY_JOB)) { - LOG(INFO) << "found duplicate warmup task for rowset " << rs_meta.rowset_id() + LOG(INFO) << "found duplicate warmup task for rowset " << rowset_id.to_string() << ", skip it"; continue; } @@ -233,6 +234,18 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { auto segment_size = rs_meta.segment_file_size(segment_id); auto download_done = [=, version = rs_meta.version()](Status st) { + DBUG_EXECUTE_IF("CloudInternalServiceImpl::warm_up_rowset.download_segment", { + auto sleep_time = dp->param("sleep", 3); + LOG_INFO("[verbose] block download for rowset={}, version={}, sleep={}", + rowset_id.to_string(), version.to_string(), sleep_time); + std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); + }); + DBUG_EXECUTE_IF( + "CloudInternalServiceImpl::warm_up_rowset.download_segment.inject_error", { + st = Status::InternalError("injected error"); + LOG_INFO("[verbose] inject error, tablet={}, rowset={}, st={}", + tablet_id, rowset_id.to_string(), st.to_string()); + }); if (st.ok()) { g_file_cache_event_driven_warm_up_finished_segment_num << 1; g_file_cache_event_driven_warm_up_finished_segment_size << segment_size; @@ -263,9 +276,10 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id.to_string() << ", error: " << st; } - if (tablet->complete_rowset_segment_warmup(rowset_id, st) == WarmUpState::DONE) { - VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id - << ") completed"; + if (tablet->complete_rowset_segment_warmup(rowset_id, st, 1, 0) == + WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" + << rowset_id.to_string() << ") completed"; } if (wait) { wait->signal(); @@ -278,13 +292,10 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c .offset = 0, .download_size = segment_size, .file_system = storage_resource.value()->fs, - .ctx = - { - .is_index_data = false, - .expiration_time = expiration_time, - .is_dryrun = - config::enable_reader_dryrun_when_download_file_cache, - }, + .ctx = {.is_index_data = false, + .expiration_time = expiration_time, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_warmup = true}, .download_done = std::move(download_done), }; g_file_cache_event_driven_warm_up_submitted_segment_num << 1; @@ -294,9 +305,18 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c } _engine.file_cache_block_downloader().submit_download_task(download_meta); - auto download_inverted_index = [&](std::string index_path, uint64_t idx_size) { + auto download_inverted_index = [&, tablet](std::string index_path, uint64_t idx_size) { auto storage_resource = rs_meta.remote_storage_resource(); - auto download_done = [=](Status st) { + auto download_done = [=, version = rs_meta.version()](Status st) { + DBUG_EXECUTE_IF( + "CloudInternalServiceImpl::warm_up_rowset.download_inverted_idx", { + auto sleep_time = dp->param("sleep", 3); + LOG_INFO( + "[verbose] block download for rowset={}, inverted index " + "file={}, sleep={}", + rowset_id.to_string(), index_path, sleep_time); + std::this_thread::sleep_for(std::chrono::seconds(sleep_time)); + }); if (st.ok()) { g_file_cache_event_driven_warm_up_finished_index_num << 1; g_file_cache_event_driven_warm_up_finished_index_size << idx_size; @@ -330,6 +350,11 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c << "download inverted index failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id.to_string() << ", error: " << st; } + if (tablet->complete_rowset_segment_warmup(rowset_id, st, 0, 1) == + WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" + << rowset_id.to_string() << ") completed"; + } if (wait) { wait->signal(); } @@ -338,18 +363,15 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c .path = io::Path(index_path), .file_size = static_cast(idx_size), .file_system = storage_resource.value()->fs, - .ctx = - { - .is_index_data = false, // DORIS-20877 - .expiration_time = expiration_time, - .is_dryrun = config:: - enable_reader_dryrun_when_download_file_cache, - }, + .ctx = {.is_index_data = false, // DORIS-20877 + .expiration_time = expiration_time, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_warmup = true}, .download_done = std::move(download_done), }; g_file_cache_event_driven_warm_up_submitted_index_num << 1; g_file_cache_event_driven_warm_up_submitted_index_size << idx_size; - + tablet->update_rowset_warmup_state_inverted_idx_num(rowset_id, 1); if (wait) { wait->add_count(); } diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index 9b629c0b038a35..583cc9a905b614 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -141,7 +141,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque if (request.alter_version > 1) { // [0-1] is a placeholder rowset, no need to convert RETURN_IF_ERROR(_base_tablet->capture_rs_readers({2, start_resp.alter_version()}, - &rs_splits, false)); + &rs_splits, CaptureRowsetOps {})); } Defer defer2 {[&]() { _new_tablet->set_alter_version(-1); diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index ed11a5c52aee0f..fe4c1f5f09994d 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -99,6 +99,7 @@ CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) std::make_shared(); _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = std::make_shared(); + _startup_timepoint = std::chrono::system_clock::now(); } CloudStorageEngine::~CloudStorageEngine() { diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index 2b97c0b34b90b8..cfa13cf89ea609 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -156,6 +157,16 @@ class CloudStorageEngine final : public BaseStorageEngine { Status unregister_compaction_stop_token(CloudTabletSPtr tablet, bool clear_ms); + std::chrono::time_point startup_timepoint() const { + return _startup_timepoint; + } + +#ifdef BE_TEST + void set_startup_timepoint(const std::chrono::time_point& tp) { + _startup_timepoint = tp; + } +#endif + private: void _refresh_storage_vault_info_thread_callback(); void _vacuum_stale_rowsets_thread_callback(); @@ -227,6 +238,8 @@ class CloudStorageEngine final : public BaseStorageEngine { EngineOptions _options; std::mutex _store_lock; + + std::chrono::time_point _startup_timepoint; }; } // namespace doris diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 4d1884016cbf69..4913f657f31603 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -18,6 +18,7 @@ #include "cloud/cloud_tablet.h" #include +#include #include #include #include @@ -27,8 +28,11 @@ #include #include +#include #include #include +#include +#include #include #include #include @@ -39,6 +43,7 @@ #include "cloud/cloud_warm_up_manager.h" #include "common/config.h" #include "common/logging.h" +#include "common/status.h" #include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" #include "olap/base_tablet.h" @@ -63,6 +68,20 @@ using namespace ErrorCode; bvar::Adder g_unused_rowsets_count("unused_rowsets_count"); +bvar::Adder g_capture_prefer_cache_count("capture_prefer_cache_count"); +bvar::Adder g_capture_with_freshness_tolerance_count( + "capture_with_freshness_tolerance_count"); +bvar::Adder g_capture_with_freshness_tolerance_fallback_count( + "capture_with_freshness_tolerance_fallback_count"); +bvar::Window> g_capture_prefer_cache_count_window( + "capture_prefer_cache_count_window", &g_capture_prefer_cache_count, 30); +bvar::Window> g_capture_with_freshness_tolerance_count_window( + "capture_with_freshness_tolerance_count_window", &g_capture_with_freshness_tolerance_count, + 30); +bvar::Window> g_capture_with_freshness_tolerance_fallback_count_window( + "capture_with_freshness_tolerance_fallback_count_window", + &g_capture_with_freshness_tolerance_fallback_count, 30); + static constexpr int LOAD_INITIATOR_ID = -1; bvar::Adder g_file_cache_cloud_tablet_submitted_segment_size( @@ -93,12 +112,18 @@ bvar::Adder g_file_cache_warm_up_segment_complete_num( "file_cache_warm_up_segment_complete_num"); bvar::Adder g_file_cache_warm_up_segment_failed_num( "file_cache_warm_up_segment_failed_num"); +bvar::Adder g_file_cache_warm_up_inverted_idx_complete_num( + "file_cache_warm_up_inverted_idx_complete_num"); +bvar::Adder g_file_cache_warm_up_inverted_idx_failed_num( + "file_cache_warm_up_inverted_idx_failed_num"); bvar::Adder g_file_cache_warm_up_rowset_complete_num( "file_cache_warm_up_rowset_complete_num"); bvar::Adder g_file_cache_warm_up_rowset_triggered_by_job_num( "file_cache_warm_up_rowset_triggered_by_job_num"); bvar::Adder g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num( "file_cache_warm_up_rowset_triggered_by_sync_rowset_num"); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_all_segments_latency( + "file_cache_warm_up_rowset_all_segments_latency"); CloudTablet::CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta) : BaseTablet(std::move(tablet_meta)), _engine(engine) {} @@ -115,17 +140,27 @@ std::string CloudTablet::tablet_path() const { Status CloudTablet::capture_rs_readers(const Version& spec_version, std::vector* rs_splits, - bool skip_missing_version) { + const CaptureRowsetOps& opts) { DBUG_EXECUTE_IF("CloudTablet.capture_rs_readers.return.e-230", { LOG_WARNING("CloudTablet.capture_rs_readers.return e-230").tag("tablet_id", tablet_id()); return Status::Error(-230, "injected error"); }); std::shared_lock rlock(_meta_lock); *rs_splits = DORIS_TRY(capture_rs_readers_unlocked( - spec_version, CaptureRowsetOps {.skip_missing_versions = skip_missing_version})); + spec_version, CaptureRowsetOps {.skip_missing_versions = opts.skip_missing_versions})); return Status::OK(); } +[[nodiscard]] Result> CloudTablet::capture_consistent_versions_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const { + if (options.query_freshness_tolerance_ms > 0) { + return capture_versions_with_freshness_tolerance(version_range, options); + } else if (options.enable_prefer_cached_rowset && !enable_unique_key_merge_on_write()) { + return capture_versions_prefer_cache(version_range); + } + return BaseTablet::capture_consistent_versions_unlocked(version_range, options); +} + Status CloudTablet::merge_rowsets_schema() { // Find the rowset with the max version auto max_version_rowset = @@ -156,6 +191,130 @@ Status CloudTablet::merge_rowsets_schema() { return Status::OK(); } +Result> CloudTablet::capture_versions_prefer_cache( + const Version& spec_version) const { + g_capture_prefer_cache_count << 1; + Versions version_path; + std::shared_lock rlock(_meta_lock); + auto st = _timestamped_version_tracker.capture_consistent_versions_prefer_cache( + spec_version, version_path, + [&](int64_t start, int64_t end) { return rowset_is_warmed_up_unlocked(start, end); }); + if (!st.ok()) { + return ResultError(st); + } + int64_t path_max_version = version_path.back().second; + VLOG_DEBUG << fmt::format( + "[verbose] CloudTablet::capture_versions_prefer_cache, capture path: {}, " + "tablet_id={}, spec_version={}, path_max_version={}", + fmt::join(version_path | std::views::transform([](const auto& version) { + return fmt::format("{}", version.to_string()); + }), + ", "), + tablet_id(), spec_version.to_string(), path_max_version); + return version_path; +} + +bool CloudTablet::rowset_is_warmed_up_unlocked(int64_t start_version, int64_t end_version) const { + if (start_version > end_version) { + return false; + } + Version version {start_version, end_version}; + auto it = _rs_version_map.find(version); + if (it == _rs_version_map.end()) { + it = _stale_rs_version_map.find(version); + if (it == _stale_rs_version_map.end()) { + LOG_WARNING( + "fail to find Rowset in rs_version or stale_rs_version for version. " + "tablet={}, version={}", + tablet_id(), version.to_string()); + return false; + } + } + const auto& rs = it->second; + if (rs->visible_timestamp() < _engine.startup_timepoint()) { + // We only care about rowsets that are created after startup time point. For other rowsets, + // we assume they are warmed up. + return true; + } + return is_rowset_warmed_up(rs->rowset_id()); +}; + +Result> CloudTablet::capture_versions_with_freshness_tolerance( + const Version& spec_version, const CaptureRowsetOps& options) const { + g_capture_with_freshness_tolerance_count << 1; + using namespace std::chrono; + auto query_freshness_tolerance_ms = options.query_freshness_tolerance_ms; + auto freshness_limit_tp = system_clock::now() - milliseconds(query_freshness_tolerance_ms); + // find a version path where every edge(rowset) has been warmuped + Versions version_path; + std::shared_lock rlock(_meta_lock); + if (enable_unique_key_merge_on_write()) { + // For merge-on-write table, newly generated delete bitmap marks will be on the rowsets which are in newest layout. + // So we can ony capture rowsets which are in newest data layout. Otherwise there may be data correctness issue. + RETURN_IF_ERROR_RESULT( + _timestamped_version_tracker.capture_consistent_versions_with_validator_mow( + spec_version, version_path, [&](int64_t start, int64_t end) { + return rowset_is_warmed_up_unlocked(start, end); + })); + } else { + RETURN_IF_ERROR_RESULT( + _timestamped_version_tracker.capture_consistent_versions_with_validator( + spec_version, version_path, [&](int64_t start, int64_t end) { + return rowset_is_warmed_up_unlocked(start, end); + })); + } + int64_t path_max_version = version_path.back().second; + auto should_be_visible_but_not_warmed_up = [&](const auto& rs_meta) -> bool { + if (rs_meta->version() == Version {0, 1}) { + // skip rowset[0-1] + return false; + } + bool ret = rs_meta->start_version() > path_max_version && + rs_meta->visible_timestamp() < freshness_limit_tp; + if (ret && config::read_cluster_cache_opt_verbose_log) { + std::time_t t1 = system_clock::to_time_t(rs_meta->visible_timestamp()); + std::tm tm1 = *std::localtime(&t1); + std::ostringstream oss1; + oss1 << std::put_time(&tm1, "%Y-%m-%d %H:%M:%S"); + + std::time_t t2 = system_clock::to_time_t(freshness_limit_tp); + std::tm tm2 = *std::localtime(&t2); + std::ostringstream oss2; + oss2 << std::put_time(&tm2, "%Y-%m-%d %H:%M:%S"); + LOG_INFO( + "[verbose] CloudTablet::capture_rs_readers_with_freshness_tolerance, " + "find a rowset which should be visible but not warmed up, tablet_id={}, " + "path_max_version={}, rowset_id={}, version={}, visible_time={}, " + "freshness_limit={}, version_graph={}, rowset_warmup_digest={}", + tablet_id(), path_max_version, rs_meta->rowset_id().to_string(), + rs_meta->version().to_string(), oss1.str(), oss2.str(), + _timestamped_version_tracker.debug_string(), rowset_warmup_digest()); + } + return ret; + }; + // use std::views::concat after C++26 + bool should_fallback = std::ranges::any_of(_tablet_meta->all_rs_metas(), + should_be_visible_but_not_warmed_up) || + std::ranges::any_of(_tablet_meta->all_stale_rs_metas(), + should_be_visible_but_not_warmed_up); + if (should_fallback) { + rlock.unlock(); + g_capture_with_freshness_tolerance_fallback_count << 1; + // if there exists a rowset which satisfies freshness tolerance and its start version is larger than the path max version + // but has not been warmuped up yet, fallback to capture rowsets as usual + return BaseTablet::capture_consistent_versions_unlocked(spec_version, options); + } + VLOG_DEBUG << fmt::format( + "[verbose] CloudTablet::capture_versions_with_freshness_tolerance, capture path: {}, " + "tablet_id={}, spec_version={}, path_max_version={}", + fmt::join(version_path | std::views::transform([](const auto& version) { + return fmt::format("{}", version.to_string()); + }), + ", "), + tablet_id(), spec_version.to_string(), path_max_version); + return version_path; +} + // There are only two tablet_states RUNNING and NOT_READY in cloud mode // This function will erase the tablet from `CloudTabletMgr` when it can't find this tablet in MS. Status CloudTablet::sync_rowsets(int64_t query_version, bool warmup_delta_data, @@ -304,18 +463,30 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ .ctx = { .expiration_time = expiration_time, - .is_dryrun = config:: - enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_warmup = true }, .download_done {[=](Status st) { - self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st); + DBUG_EXECUTE_IF("CloudTablet::add_rowsets.download_data.callback.block_compaction_rowset", { + if (rs->version().second > rs->version().first) { + auto sleep_time = dp->param("sleep", 3); + LOG_INFO( + "[verbose] block download for rowset={}, " + "version={}, sleep={}", + rs->rowset_id().to_string(), + rs->version().to_string(), sleep_time); + std::this_thread::sleep_for( + std::chrono::seconds(sleep_time)); + } + }); + self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 1, 0); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } }}, }); - auto download_idx_file = [&](const io::Path& idx_path, int64_t idx_size) { + auto download_idx_file = [&, self](const io::Path& idx_path, int64_t idx_size) { io::DownloadFileMeta meta { .path = idx_path, .file_size = idx_size, @@ -323,15 +494,30 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ .ctx = { .expiration_time = expiration_time, - .is_dryrun = config:: - enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_warmup = true }, - .download_done {[](Status st) { + .download_done {[=](Status st) { + DBUG_EXECUTE_IF("CloudTablet::add_rowsets.download_idx.callback.block", { + // clang-format on + auto sleep_time = dp->param("sleep", 3); + LOG_INFO( + "[verbose] block download for " + "rowset={}, inverted_idx_file={}, " + "sleep={}", + rs->rowset_id().to_string(), + idx_path.string(), sleep_time); + std::this_thread::sleep_for( + std::chrono::seconds(sleep_time)); + // clang-format off + }); + self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 0, 1); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } }}, }; + self->update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_meta->rowset_id(), 1); _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); g_file_cache_cloud_tablet_submitted_index_num << 1; g_file_cache_cloud_tablet_submitted_index_size << idx_size; @@ -463,7 +649,6 @@ void CloudTablet::delete_rowsets(const std::vector& to_delete, _timestamped_version_tracker.add_stale_path_version(rs_metas); for (auto&& rs : to_delete) { _rs_version_map.erase(rs->version()); - _rowset_warm_up_states.erase(rs->rowset_id()); } _tablet_meta->modify_rs_metas({}, rs_metas, false); @@ -590,6 +775,7 @@ void CloudTablet::remove_unused_rowsets() { continue; } tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); + _rowset_warm_up_states.erase(rs->rowset_id()); rs->clear_cache(); removed_rowsets.push_back(std::move(rs)); g_unused_rowsets_count << -1; @@ -1317,19 +1503,34 @@ Status CloudTablet::check_delete_bitmap_cache(int64_t txn_id, WarmUpState CloudTablet::get_rowset_warmup_state(RowsetId rowset_id) { std::shared_lock rlock(_meta_lock); - if (_rowset_warm_up_states.find(rowset_id) == _rowset_warm_up_states.end()) { + if (!_rowset_warm_up_states.contains(rowset_id)) { return WarmUpState::NONE; } - return _rowset_warm_up_states[rowset_id].first; + return _rowset_warm_up_states[rowset_id].state; +} + +bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state, + std::chrono::steady_clock::time_point start_tp) { + std::lock_guard wlock(_meta_lock); + return add_rowset_warmup_state_unlocked(rowset, state, start_tp); } -bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state) { +void CloudTablet::update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id, int64_t delta) { std::lock_guard wlock(_meta_lock); - return add_rowset_warmup_state_unlocked(rowset, state); + update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_id, delta); +} + +void CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId rowset_id, + int64_t delta) { + if (!_rowset_warm_up_states.contains(rowset_id)) { + return; + } + _rowset_warm_up_states[rowset_id].num_inverted_idx += delta; } -bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state) { - if (_rowset_warm_up_states.find(rowset.rowset_id()) != _rowset_warm_up_states.end()) { +bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state, + std::chrono::steady_clock::time_point start_tp) { + if (_rowset_warm_up_states.contains(rowset.rowset_id())) { return false; } if (state == WarmUpState::TRIGGERED_BY_JOB) { @@ -1337,26 +1538,56 @@ bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, War } else if (state == WarmUpState::TRIGGERED_BY_SYNC_ROWSET) { g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num << 1; } - _rowset_warm_up_states[rowset.rowset_id()] = std::make_pair(state, rowset.num_segments()); + _rowset_warm_up_states[rowset.rowset_id()] = { + .state = state, .num_segments = rowset.num_segments(), .start_tp = start_tp}; return true; } -WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Status status) { +WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Status status, + int64_t segment_num, + int64_t inverted_idx_num) { std::lock_guard wlock(_meta_lock); - if (_rowset_warm_up_states.find(rowset_id) == _rowset_warm_up_states.end()) { + if (!_rowset_warm_up_states.contains(rowset_id)) { return WarmUpState::NONE; } VLOG_DEBUG << "complete rowset segment warmup for rowset " << rowset_id << ", " << status; - g_file_cache_warm_up_segment_complete_num << 1; - if (!status.ok()) { - g_file_cache_warm_up_segment_failed_num << 1; + if (segment_num > 0) { + g_file_cache_warm_up_segment_complete_num << segment_num; + if (!status.ok()) { + g_file_cache_warm_up_segment_failed_num << segment_num; + } + } + if (inverted_idx_num > 0) { + g_file_cache_warm_up_inverted_idx_complete_num << inverted_idx_num; + if (!status.ok()) { + g_file_cache_warm_up_inverted_idx_failed_num << inverted_idx_num; + } } - _rowset_warm_up_states[rowset_id].second--; - if (_rowset_warm_up_states[rowset_id].second <= 0) { + _rowset_warm_up_states[rowset_id].done(segment_num, inverted_idx_num); + if (_rowset_warm_up_states[rowset_id].has_finished()) { g_file_cache_warm_up_rowset_complete_num << 1; - _rowset_warm_up_states[rowset_id].first = WarmUpState::DONE; + auto cost = std::chrono::duration_cast( + std::chrono::steady_clock::now() - + _rowset_warm_up_states[rowset_id].start_tp) + .count(); + g_file_cache_warm_up_rowset_all_segments_latency << cost; + _rowset_warm_up_states[rowset_id].state = WarmUpState::DONE; + } + return _rowset_warm_up_states[rowset_id].state; +} + +bool CloudTablet::is_rowset_warmed_up(const RowsetId& rowset_id) const { + auto it = _rowset_warm_up_states.find(rowset_id); + if (it == _rowset_warm_up_states.end()) { + return false; } - return _rowset_warm_up_states[rowset_id].first; + return it->second.state == WarmUpState::DONE; +} + +void CloudTablet::add_warmed_up_rowset(const RowsetId& rowset_id) { + _rowset_warm_up_states[rowset_id] = {.state = WarmUpState::DONE, + .num_segments = 1, + .start_tp = std::chrono::steady_clock::now()}; } #include "common/compile_check_end.h" diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index aa8947475fb257..109408bf4bb441 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -21,6 +21,7 @@ #include "olap/base_tablet.h" #include "olap/partial_update_info.h" +#include "olap/rowset/rowset.h" namespace doris { @@ -60,7 +61,33 @@ class CloudTablet final : public BaseTablet { bool vertical) override; Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, - bool skip_missing_version) override; + const CaptureRowsetOps& opts) override; + + [[nodiscard]] Result> capture_consistent_versions_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const override; + + // Capture versions with cache preference optimization. + // This method prioritizes using cached/warmed-up rowsets when building version paths, + // avoiding cold data reads when possible. It uses capture_consistent_versions_prefer_cache + // to find a consistent version path that prefers already warmed-up rowsets. + Result> capture_versions_prefer_cache(const Version& spec_version) const; + + // Capture versions with query freshness tolerance. + // This method finds a consistent version path where all rowsets are warmed up, + // but allows fallback to normal capture if there are newer rowsets that should be + // visible (based on freshness tolerance) but haven't been warmed up yet. + // For merge-on-write tables, uses special validation to ensure data correctness. + // + // IMPORTANT: The returned version may be smaller than the requested version if newer + // data hasn't been warmed up yet. This can cause different tablets in the same query + // to read from different versions, potentially leading to inconsistent query results. + // + // @param options.query_freshness_tolerance_ms: Time tolerance in milliseconds. Rowsets that + // became visible within this time range (after current_time - query_freshness_tolerance_ms) + // can be skipped if not warmed up. However, if older rowsets (before this time point) + // are not warmed up, the method will fallback to normal capture. + Result> capture_versions_with_freshness_tolerance( + const Version& spec_version, const CaptureRowsetOps& options) const; size_t tablet_footprint() override { return _approximate_data_size.load(std::memory_order_relaxed); @@ -284,8 +311,36 @@ class CloudTablet final : public BaseTablet { // Add warmup state management WarmUpState get_rowset_warmup_state(RowsetId rowset_id); - bool add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state); - WarmUpState complete_rowset_segment_warmup(RowsetId rowset_id, Status status); + bool add_rowset_warmup_state( + const RowsetMeta& rowset, WarmUpState state, + std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); + void update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id, int64_t delta); + void update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId rowset_id, int64_t delta); + WarmUpState complete_rowset_segment_warmup(RowsetId rowset_id, Status status, + int64_t segment_num, int64_t inverted_idx_num); + + bool is_rowset_warmed_up(const RowsetId& rowset_id) const; + + void add_warmed_up_rowset(const RowsetId& rowset_id); + + std::string rowset_warmup_digest() const { + std::string res; + auto add_log = [&](const RowsetSharedPtr& rs) { + auto tmp = fmt::format("{}{}", rs->rowset_id().to_string(), rs->version().to_string()); + if (_rowset_warm_up_states.contains(rs->rowset_id())) { + tmp += fmt::format( + ", state={}, segments_warmed_up={}/{}, inverted_idx_warmed_up={}/{}", + _rowset_warm_up_states.at(rs->rowset_id()).state, + _rowset_warm_up_states.at(rs->rowset_id()).num_segments_warmed_up, + _rowset_warm_up_states.at(rs->rowset_id()).num_segments, + _rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx_warmed_up, + _rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx); + } + res += fmt::format("[{}],", tmp); + }; + traverse_rowsets_unlocked(add_log, true); + return res; + } private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version @@ -293,7 +348,12 @@ class CloudTablet final : public BaseTablet { Status sync_if_not_running(SyncRowsetStats* stats = nullptr); - bool add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state); + bool add_rowset_warmup_state_unlocked( + const RowsetMeta& rowset, WarmUpState state, + std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); + + // used by capture_rs_reader_xxx functions + bool rowset_is_warmed_up_unlocked(int64_t start_version, int64_t end_version) const; CloudStorageEngine& _engine; @@ -356,7 +416,28 @@ class CloudTablet final : public BaseTablet { std::vector, DeleteBitmapKeyRanges>> _unused_delete_bitmap; // for warm up states management - std::unordered_map> _rowset_warm_up_states; + struct RowsetWarmUpInfo { + WarmUpState state; + int64_t num_segments = 0; + int64_t num_inverted_idx = 0; + int64_t num_segments_warmed_up = 0; + int64_t num_inverted_idx_warmed_up = 0; + std::chrono::steady_clock::time_point start_tp; + + void done(int64_t num_segments, int64_t num_inverted_idx) { + num_segments_warmed_up += num_segments; + num_inverted_idx_warmed_up += num_inverted_idx; + } + + bool has_finished() const { + return (num_segments_warmed_up >= num_segments) && + (num_inverted_idx_warmed_up >= num_inverted_idx); + } + }; + std::unordered_map _rowset_warm_up_states; + + mutable std::shared_mutex _warmed_up_rowsets_mutex; + std::unordered_set _warmed_up_rowsets; }; using CloudTabletSPtr = std::shared_ptr; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 6d67b6f2193dae..3a1a05749861f4 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -141,11 +141,9 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, .offset = offset, .download_size = current_chunk_size, .file_system = file_system, - .ctx = - { - .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, - }, + .ctx = {.expiration_time = expiration_time, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_warmup = true}, .download_done = [&](Status st) { if (done_cb) done_cb(st); @@ -241,8 +239,8 @@ void CloudWarmUpManager::handle_jobs() { expiration_time, wait, false, [tablet, rs, seg_id](Status st) { VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " << seg_id << " completed"; - if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), st) == - WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup( + rs->rowset_id(), st, 1, 0) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } }); @@ -267,8 +265,20 @@ void CloudWarmUpManager::handle_jobs() { } } } - submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, - expiration_time, wait, true); + tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1); + submit_download_tasks( + idx_path, file_size, storage_resource.value()->fs, + expiration_time, wait, true, [=](Status st) { + VLOG_DEBUG << "warmup rowset " << rs->version() + << " segment " << seg_id + << "inverted idx:" << idx_path << " completed"; + if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), + st, 0, 1) == + WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << rs->version() + << " completed"; + } + }); } } else { if (schema_ptr->has_inverted_index()) { @@ -276,8 +286,20 @@ void CloudWarmUpManager::handle_jobs() { storage_resource.value()->remote_idx_v2_path(*rs, seg_id); file_size = idx_file_info.has_index_size() ? idx_file_info.index_size() : -1; - submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, - expiration_time, wait, true); + tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1); + submit_download_tasks( + idx_path, file_size, storage_resource.value()->fs, + expiration_time, wait, true, [=](Status st) { + VLOG_DEBUG << "warmup rowset " << rs->version() + << " segment " << seg_id + << "inverted idx:" << idx_path << " completed"; + if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), + st, 0, 1) == + WarmUpState::DONE) { + VLOG_DEBUG << "warmup rowset " << rs->version() + << " completed"; + } + }); } } } diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 352b9ae935654b..0da239557e2f85 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -88,6 +88,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in) out->mutable_inverted_index_file_info()->CopyFrom(in.inverted_index_file_info()); out->set_source_rowset_id(in.source_rowset_id()); out->set_source_tablet_id(in.source_tablet_id()); + if (in.has_visible_ts_ms()) { + out->set_visible_ts_ms(in.visible_ts_ms()); + } } void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) { @@ -176,6 +179,9 @@ static void fill_schema_with_dict(const RowsetMetaCloudPB& in, RowsetMetaPB* out *unique_id_map.at(dict_val.parent_unique_id())->add_sparse_columns() = dict_val; VLOG_DEBUG << "fill dict sparse column" << dict_val.ShortDebugString(); } + if (in.has_visible_ts_ms()) { + out->set_visible_ts_ms(in.visible_ts_ms()); + } } RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in, @@ -246,6 +252,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in, out->mutable_inverted_index_file_info()->CopyFrom(in.inverted_index_file_info()); out->set_source_rowset_id(in.source_rowset_id()); out->set_source_tablet_id(in.source_tablet_id()); + if (in.has_visible_ts_ms()) { + out->set_visible_ts_ms(in.visible_ts_ms()); + } } void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in, @@ -304,6 +313,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in, out->mutable_inverted_index_file_info()->Swap(in.mutable_inverted_index_file_info()); out->set_source_rowset_id(in.source_rowset_id()); out->set_source_tablet_id(in.source_tablet_id()); + if (in.has_visible_ts_ms()) { + out->set_visible_ts_ms(in.visible_ts_ms()); + } } TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) { diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 18116506856b7f..cee699e8a9e43b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1573,6 +1573,10 @@ DEFINE_mBool(enable_wal_tde, "false"); DEFINE_mBool(enable_prefill_output_dbm_agg_cache_after_compaction, "true"); DEFINE_mBool(enable_prefill_all_dbm_agg_cache_after_compaction, "true"); +DEFINE_mBool(print_stack_when_cache_miss, "false"); + +DEFINE_mBool(read_cluster_cache_opt_verbose_log, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 7b76d436694d5a..6ff07645336595 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1639,6 +1639,10 @@ DECLARE_mBool(enable_wal_tde); DECLARE_mBool(enable_prefill_output_dbm_agg_cache_after_compaction); DECLARE_mBool(enable_prefill_all_dbm_agg_cache_after_compaction); +DECLARE_mBool(print_stack_when_cache_miss); + +DECLARE_mBool(read_cluster_cache_opt_verbose_log); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index e7b050d5edea7f..6f3b3061990a3d 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -203,12 +203,38 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path, _cache_base_path.c_str(), "file_cache_num_read_blocks_1h", _num_read_blocks.get(), 3600); + _no_warmup_num_read_blocks = std::make_shared>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_read_blocks"); + _no_warmup_num_hit_blocks = std::make_shared>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_hit_blocks"); + + _no_warmup_num_hit_blocks_5m = std::make_shared>>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_hit_blocks_5m", + _no_warmup_num_hit_blocks.get(), 300); + _no_warmup_num_read_blocks_5m = std::make_shared>>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_read_blocks_5m", + _no_warmup_num_read_blocks.get(), 300); + _no_warmup_num_hit_blocks_1h = std::make_shared>>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_hit_blocks_1h", + _no_warmup_num_hit_blocks.get(), 3600); + _no_warmup_num_read_blocks_1h = std::make_shared>>( + _cache_base_path.c_str(), "file_cache_no_warmup_num_read_blocks_1h", + _no_warmup_num_read_blocks.get(), 3600); + _hit_ratio = std::make_shared>(_cache_base_path.c_str(), "file_cache_hit_ratio", 0.0); _hit_ratio_5m = std::make_shared>(_cache_base_path.c_str(), "file_cache_hit_ratio_5m", 0.0); _hit_ratio_1h = std::make_shared>(_cache_base_path.c_str(), "file_cache_hit_ratio_1h", 0.0); + + _no_warmup_hit_ratio = std::make_shared>( + _cache_base_path.c_str(), "file_cache_no_warmup_hit_ratio", 0.0); + _no_warmup_hit_ratio_5m = std::make_shared>( + _cache_base_path.c_str(), "file_cache_no_warmup_hit_ratio_5m", 0.0); + _no_warmup_hit_ratio_1h = std::make_shared>( + _cache_base_path.c_str(), "file_cache_no_warmup_hit_ratio_1h", 0.0); + _disk_limit_mode_metrics = std::make_shared>( _cache_base_path.c_str(), "file_cache_disk_limit_mode", 0); _need_evict_cache_in_advance_metrics = std::make_shared>( @@ -794,9 +820,15 @@ FileBlocksHolder BlockFileCache::get_or_set(const UInt128Wrapper& hash, size_t o } DCHECK(!file_blocks.empty()); *_num_read_blocks << file_blocks.size(); + if (!context.is_warmup) { + *_no_warmup_num_read_blocks << file_blocks.size(); + } for (auto& block : file_blocks) { if (block->state_unsafe() == FileBlock::State::DOWNLOADED) { *_num_hit_blocks << 1; + if (!context.is_warmup) { + *_no_warmup_num_hit_blocks << 1; + } } } } @@ -1940,6 +1972,21 @@ void BlockFileCache::run_background_monitor() { _hit_ratio_1h->set_value((double)_num_hit_blocks_1h->get_value() / _num_read_blocks_1h->get_value()); } + + if (_no_warmup_num_hit_blocks->get_value() > 0) { + _no_warmup_hit_ratio->set_value((double)_no_warmup_num_hit_blocks->get_value() / + (double)_no_warmup_num_read_blocks->get_value()); + } + if (_no_warmup_num_hit_blocks_5m->get_value() > 0) { + _no_warmup_hit_ratio_5m->set_value( + (double)_no_warmup_num_hit_blocks_5m->get_value() / + (double)_no_warmup_num_read_blocks_5m->get_value()); + } + if (_no_warmup_num_hit_blocks_1h->get_value() > 0) { + _no_warmup_hit_ratio_1h->set_value( + (double)_no_warmup_num_hit_blocks_1h->get_value() / + (double)_no_warmup_num_read_blocks_1h->get_value()); + } } } } diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index d152e7403c0310..e8e768f7ce2325 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -532,9 +532,20 @@ class BlockFileCache { std::shared_ptr> _num_hit_blocks; std::shared_ptr> _num_removed_blocks; + std::shared_ptr> _no_warmup_num_read_blocks; + std::shared_ptr> _no_warmup_num_hit_blocks; + + std::shared_ptr>> _no_warmup_num_hit_blocks_5m; + std::shared_ptr>> _no_warmup_num_read_blocks_5m; + std::shared_ptr>> _no_warmup_num_hit_blocks_1h; + std::shared_ptr>> _no_warmup_num_read_blocks_1h; + std::shared_ptr> _hit_ratio; std::shared_ptr> _hit_ratio_5m; std::shared_ptr> _hit_ratio_1h; + std::shared_ptr> _no_warmup_hit_ratio; + std::shared_ptr> _no_warmup_hit_ratio_5m; + std::shared_ptr> _no_warmup_hit_ratio_1h; std::shared_ptr> _disk_limit_mode_metrics; std::shared_ptr> _need_evict_cache_in_advance_metrics; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index a9b87734222249..aaa1e5f4feb920 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -149,7 +149,21 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* return Status::OK(); } ReadStatistics stats; + MonotonicStopWatch read_at_sw; + read_at_sw.start(); auto defer_func = [&](int*) { + if (config::print_stack_when_cache_miss) { + if (io_ctx->file_cache_stats == nullptr && !stats.hit_cache && !io_ctx->is_warmup) { + LOG_INFO("[verbose] {}", Status::InternalError("not hit cache")); + } + } + if (!stats.hit_cache && config::read_cluster_cache_opt_verbose_log) { + LOG_INFO( + "[verbose] not hit cache, path: {}, offset: {}, size: {}, cost: {} ms, warmup: " + "{}", + path().native(), offset, bytes_req, read_at_sw.elapsed_time_milliseconds(), + io_ctx->is_warmup); + } if (io_ctx->file_cache_stats && !is_dryrun) { // update stats in io_ctx, for query profile _update_stats(stats, io_ctx->file_cache_stats, io_ctx->is_inverted_index); diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index f9ac525d0bef86..abbc4ff12fb735 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -148,6 +148,7 @@ struct CacheContext { cache_type = FileCacheType::NORMAL; } query_id = io_context->query_id ? *io_context->query_id : TUniqueId(); + is_warmup = io_context->is_warmup; } CacheContext() = default; bool operator==(const CacheContext& rhs) const { @@ -159,6 +160,7 @@ struct CacheContext { int64_t expiration_time {0}; bool is_cold_data {false}; ReadStatistics* stats; + bool is_warmup {false}; }; template diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index 6934aa6a75a519..82e9ae30ecada2 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -85,6 +85,8 @@ struct IOContext { // if is_dryrun, read IO will download data to cache but return no data to reader // useful to skip cache data read from local disk to accelarate warm up bool is_dryrun = false; + // if `is_warmup` == true, this I/O request is from a warm up task + bool is_warmup {false}; }; } // namespace io diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 9c92d48e9a8148..f8182d14c07e48 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -114,7 +114,7 @@ class BaseTablet : public std::enable_shared_from_this { virtual Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, - bool skip_missing_version) = 0; + const CaptureRowsetOps& opts) = 0; virtual size_t tablet_footprint() = 0; @@ -305,11 +305,16 @@ class BaseTablet : public std::enable_shared_from_this { void traverse_rowsets(std::function visitor, bool include_stale = false) { std::shared_lock rlock(_meta_lock); - for (auto& [v, rs] : _rs_version_map) { + traverse_rowsets_unlocked(visitor, include_stale); + } + + void traverse_rowsets_unlocked(std::function visitor, + bool include_stale = false) const { + for (const auto& [v, rs] : _rs_version_map) { visitor(rs); } if (!include_stale) return; - for (auto& [v, rs] : _stale_rs_version_map) { + for (const auto& [v, rs] : _stale_rs_version_map) { visitor(rs); } } @@ -333,7 +338,7 @@ class BaseTablet : public std::enable_shared_from_this { [[nodiscard]] Result capture_consistent_rowsets_unlocked( const Version& version_range, const CaptureRowsetOps& options) const; - [[nodiscard]] Result> capture_consistent_versions_unlocked( + [[nodiscard]] virtual Result> capture_consistent_versions_unlocked( const Version& version_range, const CaptureRowsetOps& options) const; [[nodiscard]] Result> capture_rs_readers_unlocked( @@ -409,6 +414,21 @@ struct CaptureRowsetOps { bool quiet = false; bool include_stale_rowsets = true; bool enable_fetch_rowsets_from_peers = false; + + // ======== only take effect in cloud mode ======== + + // Enable preference for cached/warmed-up rowsets when building version paths. + // When enabled, the capture process will prioritize already cached rowsets + // to avoid cold data reads and improve query performance. + bool enable_prefer_cached_rowset {false}; + + // Query freshness tolerance in milliseconds. + // Defines the time window for considering data as "fresh enough". + // Rowsets that became visible within this time range can be skipped if not warmed up, + // but older rowsets (before current_time - query_freshness_tolerance_ms) that are + // not warmed up will trigger fallback to normal capture. + // Set to -1 to disable freshness tolerance checking. + int64_t query_freshness_tolerance_ms {-1}; }; struct CaptureRowsetResult { diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index f18ec4b6bceb61..7fb3a0030497ae 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -225,4 +225,7 @@ int64_t Rowset::approximate_cache_index_size() { return total_cache_size; } +std::chrono::time_point Rowset::visible_timestamp() const { + return _rowset_meta->visible_timestamp(); +} } // namespace doris diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index e1324f49396b7e..29866df47a0e88 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -327,6 +327,8 @@ class Rowset : public std::enable_shared_from_this, public MetadataAdder int64_t approximate_cache_index_size(); + std::chrono::time_point visible_timestamp() const; + protected: friend class RowsetFactory; diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 887659e61c761f..6a96b8959fc6ff 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -359,6 +360,22 @@ class RowsetMeta : public MetadataAdder { int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); } + // for cloud only + bool has_visible_ts_ms() const { return _rowset_meta_pb.has_visible_ts_ms(); } + int64_t visible_ts_ms() const { return _rowset_meta_pb.visible_ts_ms(); } + std::chrono::time_point visible_timestamp() const { + using namespace std::chrono; + if (has_visible_ts_ms()) { + return time_point(milliseconds(visible_ts_ms())); + } + return system_clock::from_time_t(newest_write_timestamp()); + } +#ifdef BE_TEST + void set_visible_ts_ms(int64_t visible_ts_ms) { + _rowset_meta_pb.set_visible_ts_ms(visible_ts_ms); + } +#endif + void set_tablet_schema(const TabletSchemaSPtr& tablet_schema); void set_tablet_schema(const TabletSchemaPB& tablet_schema); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index a79de2964620b8..373c2d2963c048 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -945,11 +945,11 @@ void Tablet::acquire_version_and_rowsets( } Status Tablet::capture_rs_readers(const Version& spec_version, std::vector* rs_splits, - bool skip_missing_version) { + const CaptureRowsetOps& opts) { std::shared_lock rlock(_meta_lock); std::vector version_path; *rs_splits = DORIS_TRY(capture_rs_readers_unlocked( - spec_version, CaptureRowsetOps {.skip_missing_versions = skip_missing_version})); + spec_version, CaptureRowsetOps {.skip_missing_versions = opts.skip_missing_versions})); return Status::OK(); } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index a8c9df89ff0889..a9230a838532fa 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -188,7 +188,7 @@ class Tablet final : public BaseTablet { // If skip_missing_version is true, skip versions if they are missing. Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, - bool skip_missing_version) override; + const CaptureRowsetOps& opts) override; // Find the missed versions until the spec_version. // diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 68f6d323bbc5ba..6acfcb5785eac3 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -249,7 +249,11 @@ class TabletMeta : public MetadataAdder { void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version); bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } - +#ifdef BE_TEST + void set_enable_unique_key_merge_on_write(bool value) { + _enable_unique_key_merge_on_write = value; + } +#endif // TODO(Drogon): thread safety const BinlogConfig& binlog_config() const { return _binlog_config; } void set_binlog_config(BinlogConfig binlog_config) { diff --git a/be/src/olap/version_graph.cpp b/be/src/olap/version_graph.cpp index c5f8aff9d47def..6c9a8072d66948 100644 --- a/be/src/olap/version_graph.cpp +++ b/be/src/olap/version_graph.cpp @@ -25,6 +25,7 @@ #include // IWYU pragma: keep #include #include +#include #include #include @@ -329,6 +330,27 @@ Status TimestampedVersionTracker::capture_consistent_versions( return _version_graph.capture_consistent_versions(spec_version, version_path); } +Status TimestampedVersionTracker::capture_consistent_versions_with_validator( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + return _version_graph.capture_consistent_versions_with_validator(spec_version, version_path, + validator); +} + +Status TimestampedVersionTracker::capture_consistent_versions_prefer_cache( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + return _version_graph.capture_consistent_versions_prefer_cache(spec_version, version_path, + validator); +} + +Status TimestampedVersionTracker::capture_consistent_versions_with_validator_mow( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + return _version_graph.capture_consistent_versions_with_validator_mow(spec_version, version_path, + validator); +} + void TimestampedVersionTracker::capture_expired_paths( int64_t stale_sweep_endtime, std::vector* path_version_vec) const { std::map::const_iterator iter = @@ -406,6 +428,10 @@ double TimestampedVersionTracker::get_orphan_vertex_ratio() { return _version_graph.get_orphan_vertex_ratio(); } +std::string TimestampedVersionTracker::debug_string() const { + return _version_graph.debug_string(); +} + void TimestampedVersionPathContainer::add_timestamped_version(TimestampedVersionSharedPtr version) { // Compare and refresh `_max_create_time`. if (version->get_create_time() > _max_create_time) { @@ -628,6 +654,172 @@ Status VersionGraph::capture_consistent_versions(const Version& spec_version, return Status::OK(); } +Status VersionGraph::capture_consistent_versions_prefer_cache( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + if (spec_version.first > spec_version.second) { + return Status::Error( + "invalid specified version. spec_version={}-{}", spec_version.first, + spec_version.second); + } + + int64_t cur_idx = -1; + for (size_t i = 0; i < _version_graph.size(); i++) { + if (_version_graph[i].value == spec_version.first) { + cur_idx = i; + break; + } + } + + if (cur_idx < 0) { + return Status::InternalError("failed to find path in version_graph. spec_version={}", + spec_version.to_string()); + } + + int64_t end_value = spec_version.second + 1; + while (_version_graph[cur_idx].value < end_value) { + int64_t next_idx = -1; + int64_t first_idx = -1; + for (const auto& it : _version_graph[cur_idx].edges) { + // Only consider incremental versions. + if (_version_graph[it].value < _version_graph[cur_idx].value) { + break; + } + if (first_idx == -1) { + first_idx = it; + } + + if (!validator(_version_graph[cur_idx].value, _version_graph[it].value - 1)) { + continue; + } + + next_idx = it; + break; + } + + if (next_idx > -1) { + version_path.emplace_back(_version_graph[cur_idx].value, + _version_graph[next_idx].value - 1); + + cur_idx = next_idx; + } else if (first_idx != -1) { + // if all edges are not in cache, use the first edge if possible + version_path.emplace_back(_version_graph[cur_idx].value, + _version_graph[first_idx].value - 1); + cur_idx = first_idx; + } else { + return Status::OK(); + } + } + return Status::OK(); +} + +Status VersionGraph::capture_consistent_versions_with_validator( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + if (spec_version.first > spec_version.second) { + return Status::Error( + "invalid specified version. spec_version={}-{}", spec_version.first, + spec_version.second); + } + + int64_t cur_idx = -1; + for (size_t i = 0; i < _version_graph.size(); i++) { + if (_version_graph[i].value == spec_version.first) { + cur_idx = i; + break; + } + } + + if (cur_idx < 0) { + return Status::InternalError("failed to find path in version_graph. spec_version={}", + spec_version.to_string()); + } + + int64_t end_value = spec_version.second + 1; + while (_version_graph[cur_idx].value < end_value) { + int64_t next_idx = -1; + for (const auto& it : _version_graph[cur_idx].edges) { + // Only consider incremental versions. + if (_version_graph[it].value < _version_graph[cur_idx].value) { + break; + } + + if (!validator(_version_graph[cur_idx].value, _version_graph[it].value - 1)) { + continue; + } + + next_idx = it; + break; + } + + if (next_idx > -1) { + version_path.emplace_back(_version_graph[cur_idx].value, + _version_graph[next_idx].value - 1); + + cur_idx = next_idx; + } else { + return Status::OK(); + } + } + return Status::OK(); +} + +Status VersionGraph::capture_consistent_versions_with_validator_mow( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const { + if (spec_version.first > spec_version.second) { + return Status::Error( + "invalid specified version. spec_version={}-{}", spec_version.first, + spec_version.second); + } + + int64_t cur_idx = -1; + for (size_t i = 0; i < _version_graph.size(); i++) { + if (_version_graph[i].value == spec_version.first) { + cur_idx = i; + break; + } + } + + if (cur_idx < 0) { + return Status::InternalError("failed to find path in version_graph. spec_version={}", + spec_version.to_string()); + } + + int64_t end_value = spec_version.second + 1; + while (_version_graph[cur_idx].value < end_value) { + int64_t next_idx = -1; + for (const auto& it : _version_graph[cur_idx].edges) { + // Only consider incremental versions. + if (_version_graph[it].value < _version_graph[cur_idx].value) { + break; + } + + if (!validator(_version_graph[cur_idx].value, _version_graph[it].value - 1)) { + if (_version_graph[cur_idx].value + 1 == _version_graph[it].value) { + break; + } + end_value = std::min(_version_graph[it].value, end_value); + continue; + } + + next_idx = it; + break; + } + + if (next_idx > -1) { + version_path.emplace_back(_version_graph[cur_idx].value, + _version_graph[next_idx].value - 1); + + cur_idx = next_idx; + } else { + return Status::OK(); + } + } + return Status::OK(); +} + double VersionGraph::get_orphan_vertex_ratio() { int64_t vertex_num = _version_graph.size(); int64_t orphan_vertex_num = 0; @@ -639,4 +831,19 @@ double VersionGraph::get_orphan_vertex_ratio() { return orphan_vertex_num / (double)vertex_num; } +std::string VersionGraph::debug_string() const { + std::stringstream ss; + ss << "VersionGraph: ["; + for (size_t i = 0; i < _version_graph.size(); ++i) { + ss << "{value: " << _version_graph[i].value << ", edges: ["; + for (const auto& edge : _version_graph[i].edges) { + if (_version_graph[edge].value > _version_graph[i].value) { + ss << _version_graph[edge].value << ", "; + } + } + ss << "]}, "; + } + ss << "]"; + return ss.str(); +} } // namespace doris diff --git a/be/src/olap/version_graph.h b/be/src/olap/version_graph.h index 56d07a52871ae7..4c65d9208614c1 100644 --- a/be/src/olap/version_graph.h +++ b/be/src/olap/version_graph.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -55,9 +56,40 @@ class VersionGraph { Status capture_consistent_versions(const Version& spec_version, std::vector* version_path) const; + Status capture_consistent_versions_prefer_cache( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + + // Given a start, this method can find a version path which satisfy the following conditions: + // 1. all edges satisfy the conditions specified by `validator` in the graph. + // 2. the destination version is as far as possible. + // 3. the path is the shortest path. + // The version paths are added to version_path as return info. + // If this version not in main version, version_path can be included expired rowset. + // NOTE: this method may return edges which is in stale path + // + // @param validator: Function that takes (start_version, end_version) representing a rowset + // and returns true if the rowset should be included in the path, false to skip it + Status capture_consistent_versions_with_validator( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + + // Capture consistent versions with validator for merge-on-write (MOW) tables. + // Similar to capture_consistent_versions_with_validator but with special handling for MOW tables. + // For MOW tables, newly generated delete bitmap marks will be on the rowsets which are in newest layout. + // So we can only capture rowsets which are in newest data layout to ensure data correctness. + // + // @param validator: Function that takes (start_version, end_version) representing a rowset + // and returns true if the rowset is warmed up, false if not warmed up + Status capture_consistent_versions_with_validator_mow( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + // See comment of TimestampedVersionTracker's get_orphan_vertex_ratio(); double get_orphan_vertex_ratio(); + std::string debug_string() const; + private: /// Private method add a version to graph. void _add_vertex_to_graph(int64_t vertex_value); @@ -168,6 +200,35 @@ class TimestampedVersionTracker { Status capture_consistent_versions(const Version& spec_version, std::vector* version_path) const; + Status capture_consistent_versions_prefer_cache( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + + // Given a start, this method can find a version path which satisfy the following conditions: + // 1. all edges satisfy the conditions specified by `validator` in the graph. + // 2. the destination version is as far as possible. + // 3. the path is the shortest path. + // The version paths are added to version_path as return info. + // If this version not in main version, version_path can be included expired rowset. + // NOTE: this method may return edges which is in stale path + // + // @param validator: Function that takes (start_version, end_version) representing a rowset + // and returns true if the rowset should be included in the path, false to skip it + Status capture_consistent_versions_with_validator( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + + // Capture consistent versions with validator for merge-on-write (MOW) tables. + // Similar to capture_consistent_versions_with_validator but with special handling for MOW tables. + // For MOW tables, newly generated delete bitmap marks will be on the rowsets which are in newest layout. + // So we can only capture rowsets which are in newest data layout to ensure data correctness. + // + // @param validator: Function that takes (start_version, end_version) representing a rowset + // and returns true if the rowset is warmed up, false if not warmed up + Status capture_consistent_versions_with_validator_mow( + const Version& spec_version, std::vector& version_path, + const std::function& validator) const; + /// Capture all expired path version. /// When the last rowset create time of a path greater than expired time which can be expressed /// "now() - tablet_rowset_stale_sweep_time_sec" , this path will be remained. @@ -193,6 +254,8 @@ class TimestampedVersionTracker { // If a vertex is no longer the starting point of any edge, then this vertex is defined as orphan vertex double get_orphan_vertex_ratio(); + std::string debug_string() const; + private: /// Construct rowsets version tracker with main path rowset meta. void _construct_versioned_tracker(const std::vector& rs_metas); diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp index 64437641d4e8b1..9f38f77b3d9a6d 100644 --- a/be/src/pipeline/exec/olap_scan_operator.cpp +++ b/be/src/pipeline/exec/olap_scan_operator.cpp @@ -561,12 +561,20 @@ Status OlapScanLocalState::prepare(RuntimeState* state) { } } + CaptureRowsetOps opts { + .skip_missing_versions = PipelineXLocalState<>::_state->skip_missing_version(), + .enable_fetch_rowsets_from_peers = config::enable_fetch_rowsets_from_peer_replicas, + .enable_prefer_cached_rowset = + config::is_cloud_mode() + ? PipelineXLocalState<>::_state->enable_prefer_cached_rowset() + : false, + .query_freshness_tolerance_ms = + config::is_cloud_mode() + ? PipelineXLocalState<>::_state->query_freshness_tolerance_ms() + : -1}; for (size_t i = 0; i < _scan_ranges.size(); i++) { - _read_sources[i] = DORIS_TRY(_tablets[i].tablet->capture_read_source( - {0, _tablets[i].version}, - {.skip_missing_versions = RuntimeFilterConsumer::_state->skip_missing_version(), - .enable_fetch_rowsets_from_peers = - config::enable_fetch_rowsets_from_peer_replicas})); + _read_sources[i] = + DORIS_TRY(_tablets[i].tablet->capture_read_source({0, _tablets[i].version}, opts)); if (!PipelineXLocalState<>::_state->skip_delete_predicate()) { _read_sources[i].fill_delete_predicates(); } diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 14320be48e8b77..44cc3508e44491 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -433,6 +433,20 @@ class RuntimeState { return _query_options.partitioned_hash_agg_rows_threshold; } + bool enable_prefer_cached_rowset() const { + return _query_options.__isset.enable_prefer_cached_rowset && + _query_options.enable_prefer_cached_rowset; + } + + int64_t query_freshness_tolerance_ms() const { + return _query_options.query_freshness_tolerance_ms; + } + + bool enable_query_freshness_tolerance() const { + return _query_options.__isset.query_freshness_tolerance_ms && + _query_options.query_freshness_tolerance_ms > 0; + } + std::vector tablet_commit_infos() const { std::lock_guard lock(_tablet_infos_mutex); return _tablet_commit_infos; diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index 707defa16902db..ddb17471e087fb 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -200,11 +200,16 @@ Status NewOlapScanner::init() { ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_hotspot().count(*tablet); } - auto maybe_read_source = tablet->capture_read_source( - _tablet_reader_params.version, - {.skip_missing_versions = _state->skip_missing_version(), - .enable_fetch_rowsets_from_peers = - config::enable_fetch_rowsets_from_peer_replicas}); + CaptureRowsetOps opts { + .skip_missing_versions = _state->skip_missing_version(), + .enable_fetch_rowsets_from_peers = + config::enable_fetch_rowsets_from_peer_replicas, + .enable_prefer_cached_rowset = + config::is_cloud_mode() ? _state->enable_prefer_cached_rowset() : false, + .query_freshness_tolerance_ms = + config::is_cloud_mode() ? _state->query_freshness_tolerance_ms() : -1}; + auto maybe_read_source = + tablet->capture_read_source(_tablet_reader_params.version, opts); if (!maybe_read_source) { LOG(WARNING) << "fail to init reader. res=" << maybe_read_source.error(); return maybe_read_source.error(); diff --git a/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp b/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp new file mode 100644 index 00000000000000..928701dae39357 --- /dev/null +++ b/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp @@ -0,0 +1,804 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include + +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "olap/base_tablet.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_factory.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/tablet_meta.h" +#include "util/uid_util.h" + +namespace doris { + +using namespace std::chrono; + +class TestQueryPreferCache : public testing::Test { +public: + TestQueryPreferCache() : _engine(CloudStorageEngine(EngineOptions {})) {} + + void SetUp() override { + config::read_cluster_cache_opt_verbose_log = true; + _tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F)); + } + void TearDown() override { config::read_cluster_cache_opt_verbose_log = false; } + + RowsetSharedPtr create_rowset_without_visible_time(Version version) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version(version); + rs_meta->set_rowset_id(_engine.next_rowset_id()); + RowsetSharedPtr rowset; + Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset); + if (!st.ok()) { + return nullptr; + } + return rowset; + } + + RowsetSharedPtr create_rowset(Version version, + time_point visible_timestamp = system_clock::now() - + seconds(100)) { + auto rs = create_rowset_without_visible_time(version); + if (!rs) { + return nullptr; + } + rs->rowset_meta()->set_visible_ts_ms( + duration_cast(visible_timestamp.time_since_epoch()).count()); + return rs; + } + + CloudTabletSPtr create_tablet_with_initial_rowsets(int max_version, bool is_mow = false, + bool warmup = true) { + CloudTabletSPtr tablet = + std::make_shared(_engine, std::make_shared(*_tablet_meta)); + tablet->tablet_meta()->set_enable_unique_key_merge_on_write(is_mow); + std::vector rowsets; + auto rs1 = create_rowset(Version {0, 1}); + rowsets.emplace_back(rs1); + tablet->add_warmed_up_rowset(rs1->rowset_id()); + for (int ver = 2; ver <= max_version; ver++) { + auto rs = create_rowset(Version {ver, ver}); + if (warmup) { + tablet->add_warmed_up_rowset(rs->rowset_id()); + } + rowsets.emplace_back(rs); + } + { + std::unique_lock wlock {tablet->get_header_lock()}; + tablet->add_rowsets(rowsets, false, wlock, false); + } + return tablet; + } + + void add_new_version_rowset(CloudTabletSPtr tablet, int64_t version, bool warmed_up, + time_point visible_timestamp) { + auto rowset = create_rowset(Version {version, version}, visible_timestamp); + if (warmed_up) { + tablet->add_warmed_up_rowset(rowset->rowset_id()); + } + std::unique_lock wlock {tablet->get_header_lock()}; + tablet->add_rowsets({rowset}, false, wlock, false); + } + + void do_cumu_compaction(CloudTabletSPtr tablet, int64_t start_version, int64_t end_version, + bool warmed_up, time_point visible_timestamp) { + std::unique_lock wrlock {tablet->get_header_lock()}; + std::vector input_rowsets; + auto output_rowset = create_rowset(Version {start_version, end_version}, visible_timestamp); + if (warmed_up) { + tablet->add_warmed_up_rowset(output_rowset->rowset_id()); + } + std::ranges::copy_if(std::views::values(tablet->rowset_map()), + std::back_inserter(input_rowsets), [=](const RowsetSharedPtr& rowset) { + return rowset->version().first >= start_version && + rowset->version().first <= end_version; + }); + if (input_rowsets.size() == 1) { + tablet->add_rowsets({output_rowset}, true, wrlock); + } else { + tablet->delete_rowsets(input_rowsets, wrlock); + tablet->add_rowsets({output_rowset}, false, wrlock); + } + } + + void check_capture_result(CloudTabletSPtr tablet, Version spec_version, + const std::vector& expected_versions) { + CaptureRowsetOps opts {.skip_missing_versions = false, + .enable_prefer_cached_rowset = true, + .query_freshness_tolerance_ms = -1}; + auto res = tablet->capture_read_source(spec_version, opts); + ASSERT_TRUE(res.has_value()); + std::vector rs_splits = std::move(res.value().rs_splits); + auto dump_versions = [](const std::vector& expected_versions, + const std::vector& splits) { + std::vector expected_str; + for (const auto& version : expected_versions) { + expected_str.push_back(version.to_string()); + } + std::vector versions; + for (const auto& split : splits) { + versions.push_back(split.rs_reader->rowset()->version().to_string()); + } + return fmt::format("expected_versions: {}, actual_versions: {}", + fmt::join(expected_str, ", "), fmt::join(versions, ", ")); + }; + ASSERT_EQ(rs_splits.size(), expected_versions.size()) + << dump_versions(expected_versions, rs_splits); + for (size_t i = 0; i < rs_splits.size(); i++) { + ASSERT_EQ(rs_splits[i].rs_reader->rowset()->version(), expected_versions[i]) + << dump_versions(expected_versions, rs_splits); + } + } + +protected: + std::string _json_rowset_meta; + TabletMetaSharedPtr _tablet_meta; + +private: + CloudStorageEngine _engine; +}; + +TEST_F(TestQueryPreferCache, testCapture_1_1) { + /* + be startup time now-10s now + now - 30s + ā”‚ ā”‚ 10s ā”‚ + ā”‚ ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚incacheā”‚ ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ ā”‚ + now-40s ā”‚ now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: We only care about rowsets that are created after startup time point. For other historical rowsets, + we just assume that they are warmuped up. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(30)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, false, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_1_2) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_1_3) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-16] ā”‚ ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-13s ā”‚ now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: should not capture [2-16], otherwise we will meet cache miss +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_1_4) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-16] ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_2_1) { + /* + be startup time now-10s now + now - 30s + ā”‚ ā”‚ 10s ā”‚ + ā”‚ ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ ā”‚ + now-40s ā”‚ now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: We only care about rowsets that are created after startup time point. For other historical rowsets, + we just assume that they are warmuped up. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(30)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, false, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_2_2) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_2_3) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ [2-16] ā”‚ ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-13s ā”‚ now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: should not capture [2-16], otherwise we will meet cache miss +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_2_4) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-16] ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_3_1) { + /* + be startup time now-10s now + now - 30s + ā”‚ ā”‚ 10s ā”‚ + ā”‚ ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ ā”‚ + now-40s ā”‚ now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: We only care about rowsets that are created after startup time point. For other historical rowsets, + we just assume that they are warmuped up. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(30)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, false, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_3_2) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_3_3) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ [2-16] ā”‚ ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-13s ā”‚ now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: should not capture [2-16], otherwise we will meet cache miss +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_3_4) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-16] ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +TEST_F(TestQueryPreferCache, testCapture_4_1) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-16]ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-16],[17-17],[18-18] + note: when there are no warmed up rowset at some vertex, choose the latest edge +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, false, false); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, false, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, false, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, expected_versions); +} + +} // namespace doris \ No newline at end of file diff --git a/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp b/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp new file mode 100644 index 00000000000000..1a24ea275be007 --- /dev/null +++ b/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp @@ -0,0 +1,1074 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include + +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "olap/base_tablet.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_factory.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/tablet_meta.h" +#include "util/uid_util.h" + +namespace doris { + +using namespace std::chrono; + +class TestFreshnessTolerance : public testing::Test { +public: + TestFreshnessTolerance() : _engine(CloudStorageEngine(EngineOptions {})) {} + + void SetUp() override { + config::read_cluster_cache_opt_verbose_log = true; + _tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F)); + } + void TearDown() override { config::read_cluster_cache_opt_verbose_log = false; } + + RowsetSharedPtr create_rowset_without_visible_time(Version version) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version(version); + rs_meta->set_rowset_id(_engine.next_rowset_id()); + RowsetSharedPtr rowset; + Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset); + if (!st.ok()) { + return nullptr; + } + return rowset; + } + + RowsetSharedPtr create_rowset(Version version, + time_point visible_timestamp = system_clock::now() - + seconds(100)) { + auto rs = create_rowset_without_visible_time(version); + if (!rs) { + return nullptr; + } + rs->rowset_meta()->set_visible_ts_ms( + duration_cast(visible_timestamp.time_since_epoch()).count()); + return rs; + } + + CloudTabletSPtr create_tablet_with_initial_rowsets(int max_version, bool is_mow = false) { + CloudTabletSPtr tablet = + std::make_shared(_engine, std::make_shared(*_tablet_meta)); + tablet->tablet_meta()->set_enable_unique_key_merge_on_write(is_mow); + std::vector rowsets; + auto rs1 = create_rowset(Version {0, 1}); + rowsets.emplace_back(rs1); + tablet->add_warmed_up_rowset(rs1->rowset_id()); + for (int ver = 2; ver <= max_version; ver++) { + auto rs = create_rowset(Version {ver, ver}); + tablet->add_warmed_up_rowset(rs->rowset_id()); + rowsets.emplace_back(rs); + } + { + std::unique_lock wlock {tablet->get_header_lock()}; + tablet->add_rowsets(rowsets, false, wlock, false); + } + return tablet; + } + + void add_new_version_rowset(CloudTabletSPtr tablet, int64_t version, bool warmed_up, + time_point visible_timestamp) { + auto rowset = create_rowset(Version {version, version}, visible_timestamp); + if (warmed_up) { + tablet->add_warmed_up_rowset(rowset->rowset_id()); + } + std::unique_lock wlock {tablet->get_header_lock()}; + tablet->add_rowsets({rowset}, false, wlock, false); + } + + void do_cumu_compaction(CloudTabletSPtr tablet, int64_t start_version, int64_t end_version, + bool warmed_up, time_point visible_timestamp) { + std::unique_lock wrlock {tablet->get_header_lock()}; + std::vector input_rowsets; + auto output_rowset = create_rowset(Version {start_version, end_version}, visible_timestamp); + if (warmed_up) { + tablet->add_warmed_up_rowset(output_rowset->rowset_id()); + } + std::ranges::copy_if(std::views::values(tablet->rowset_map()), + std::back_inserter(input_rowsets), [=](const RowsetSharedPtr& rowset) { + return rowset->version().first >= start_version && + rowset->version().first <= end_version; + }); + if (input_rowsets.size() == 1) { + tablet->add_rowsets({output_rowset}, true, wrlock); + } else { + tablet->delete_rowsets(input_rowsets, wrlock); + tablet->add_rowsets({output_rowset}, false, wrlock); + } + } + + void check_capture_result(CloudTabletSPtr tablet, Version spec_version, + int64_t query_freshness_tolerance_ms, + const std::vector& expected_versions) { + CaptureRowsetOps opts {.skip_missing_versions = false, + .enable_prefer_cached_rowset = false, + .query_freshness_tolerance_ms = query_freshness_tolerance_ms}; + auto res = tablet->capture_read_source(spec_version, opts); + ASSERT_TRUE(res.has_value()); + std::vector rs_splits = std::move(res.value().rs_splits); + auto dump_versions = [](const std::vector& expected_versions, + const std::vector& splits) { + std::vector expected_str; + for (const auto& version : expected_versions) { + expected_str.push_back(version.to_string()); + } + std::vector versions; + for (const auto& split : splits) { + versions.push_back(split.rs_reader->rowset()->version().to_string()); + } + return fmt::format("expected_versions: {}, actual_versions: {}", + fmt::join(expected_str, ", "), fmt::join(versions, ", ")); + }; + ASSERT_EQ(rs_splits.size(), expected_versions.size()) + << dump_versions(expected_versions, rs_splits); + for (size_t i = 0; i < rs_splits.size(); i++) { + ASSERT_EQ(rs_splits[i].rs_reader->rowset()->version(), expected_versions[i]) + << dump_versions(expected_versions, rs_splits); + } + } + +protected: + std::string _json_rowset_meta; + TabletMetaSharedPtr _tablet_meta; + +private: + CloudStorageEngine _engine; +}; + +TEST_F(TestFreshnessTolerance, testVisibleTimestamp) { + { + // for historical rowset, visible time is not set, RowsetMeta::visible_timestamp() uses + // newest_write_timestamp + auto tp1 = system_clock::now() - seconds(100); + auto rs = create_rowset_without_visible_time({2, 2}); + auto d = duration_cast(tp1.time_since_epoch()).count(); + rs->rowset_meta()->set_newest_write_timestamp(d); + ASSERT_EQ(rs->rowset_meta()->visible_timestamp(), system_clock::from_time_t(d)); + } + + { + // when visible_ts_ms is set, RowsetMeta::visible_timestamp() uses visible_ts_ms which is more precise + auto tp1 = system_clock::now() - seconds(100); + auto tp2 = system_clock::now() - seconds(50); + auto rs = create_rowset_without_visible_time({2, 2}); + auto d1 = duration_cast(tp1.time_since_epoch()).count(); + auto d2 = duration_cast(tp2.time_since_epoch()).count(); + rs->rowset_meta()->set_newest_write_timestamp(d1); + rs->rowset_meta()->set_visible_ts_ms(d2); + ASSERT_EQ(rs->rowset_meta()->visible_timestamp(), + time_point(milliseconds(d2))); + } +} + +TEST_F(TestFreshnessTolerance, testCapture_1_1) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_1_2) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + NOTE: rowset[16-16] should be visible becasue it's within the query freshness tolerance time limit. + However, since the data files of rowset[16-16] is not in the cache, there is no difference between + capturing up to version 16 and capturing up to version 18. So we capture up to version 18. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, false, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_1_3) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_1_4) { + /* + be startup time now-10s now + now - 30s + ā”‚ ā”‚ 10s ā”‚ + ā”‚ ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ ā”‚ + now-40s ā”‚ now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17] + note: We only care about rowsets that are created after startup time point. For other historical rowsets, + we just assume that they are warmuped up. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(30)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, false, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_2_1) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_2_2) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_2_3) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_2_4) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-16] ā”‚ ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-13s ā”‚ now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + note: should not capture [2-16], otherwise we will meet cache miss +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCapture_3_1) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-17],[18-18] + note: should fallback +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, false, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_1_1) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_1_2) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17],[18-18] + NOTE: rowset[16-16] must be visible becasue it's within the query freshness tolerance time limit. + However, since the data files of rowset[16-16] is not in the cache, there is no difference between + capturing up to version 16 and capturing up to version 18 +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, false, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, + {16, 16}, {17, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_1_3) { + /* + now-10s now + + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_1_4) { + /* + be startup time now-10s now + now - 30s + ā”‚ ā”‚ 10s ā”‚ + ā”‚ ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ [2-10] ā”‚ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ā”‚ ā”‚ [17-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ ā”‚ + now-40s ā”‚ now-20s now-15s ā”‚ now-7s now-3s ā”‚ + ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17] + note: We only care about rowsets that are created after startup time point. For other historical rowsets, + we just assume that they are warmuped up. +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(30)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, false, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_1) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_2) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-15],[16-16],[17-17] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_3) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16],[17-17] + note: due to the existence of rowset [11-17], we can only capture up to version 17 + because newly rowsets may generate delete bitmap marks on [11-17]. If we capture [18-18], + we may meet data correctness issue if [18-18] has duplicate rows with [11-17] + */ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}, {17, 17}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_4) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ [2-16] ā”‚ ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + now-13s ā”‚ now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16] + note: due to the existence of rowset [2-16], we can only capture up to version 16 + because newly rowsets may generate delete bitmap marks on [2-16]. If we capture [17-17], + we may meet data correctness issue if [17-17] has duplicate rows with [2-16] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_5) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-16] ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ in cacheā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-10],[11-15],[16-16] +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 15}, {16, 16}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_2_6) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ + ā”‚ ā”‚ ā”‚ ā”‚in cacheā”‚ā”‚ + ā”‚ ā”‚ [2-17] ā”‚ ā”‚[18-18] ā”‚ā”‚ + ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ + ā”‚ ā”‚ + ā”‚ now-1s now-3s ā”‚ + ā”‚ ā”‚ + ā”‚ ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-16] ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-13s ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚in cacheā”‚ ā”‚ ā”‚ ā”‚in cache ā”‚ ā”‚ā”‚in cacheā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [2-10] ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-40s now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + return: [2-17],[18-18] + note: because rowset [11-15] is not warmed up, we can only choose a path whose max verion is below 15 + but rowset version 16 is within the query freshness tolerance time limit. So we should fallback to + capture rowsets with tablet's max version +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, false, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, true, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, true, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, true, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 2, 16, false, system_clock::now() - seconds(13)); + do_cumu_compaction(tablet, 2, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} + +TEST_F(TestFreshnessTolerance, testCaptureMow_3_1) { + /* + now-10s now + ā”‚ 10s ā”‚ + ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ + ā”‚in cacheā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ + ā”‚ [2-10] ā”‚ ā”‚ā”‚ [11-17]ā”‚ ā”‚[18-18]ā”‚ ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ + ā”‚ ā”‚ + now-40s ā”‚ now-1s now-3s ā”‚ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ stale rowsets ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ in cacheā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ ā”‚ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ [11-15] ā”‚ ā”‚ [16-16] ā”‚ ā”‚ā”‚ [17-17]ā”‚ ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ now-20s now-15s ā”‚ now-7s ā”‚ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + + return: [2-10],[11-17],[18-18] + note: should fallback +*/ + _engine.set_startup_timepoint(system_clock::now() - seconds(200)); + auto tablet = create_tablet_with_initial_rowsets(15, true); + do_cumu_compaction(tablet, 2, 10, true, system_clock::now() - seconds(40)); + do_cumu_compaction(tablet, 11, 15, true, system_clock::now() - seconds(20)); + add_new_version_rowset(tablet, 16, false, system_clock::now() - seconds(15)); + add_new_version_rowset(tablet, 17, false, system_clock::now() - seconds(7)); + add_new_version_rowset(tablet, 18, false, system_clock::now() - seconds(3)); + do_cumu_compaction(tablet, 11, 17, false, system_clock::now() - seconds(1)); + + std::string compaction_status; + tablet->get_compaction_status(&compaction_status); + std::cout << compaction_status << std::endl; + + int64_t query_freshness_tolerance_ms = 10000; // 10s + std::vector expected_versions = {{0, 1}, {2, 10}, {11, 17}, {18, 18}}; + check_capture_result(tablet, Version {0, 18}, query_freshness_tolerance_ms, expected_versions); +} +} // namespace doris diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index 5ec3df0417591c..fe9751ff7bfbc9 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -135,7 +135,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent) { auto non_existent_id = _engine.next_rowset_id(); - WarmUpState result = _tablet->complete_rowset_segment_warmup(non_existent_id, Status::OK()); + WarmUpState result = + _tablet->complete_rowset_segment_warmup(non_existent_id, Status::OK(), 1, 0); EXPECT_EQ(result, WarmUpState::NONE); } @@ -151,12 +152,12 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { // Complete one segment, should still be in TRIGGERED_BY_JOB state WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); // Complete second segment, should still be in TRIGGERED_BY_JOB state WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB); // Verify current state is still TRIGGERED_BY_JOB @@ -176,12 +177,67 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { // Complete first segment WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); // Complete second segment, should transition to DONE state WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result2, WarmUpState::DONE); + + // Verify final state is DONE + WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(final_state, WarmUpState::DONE); +} + +// Test complete_rowset_segment_warmup with inverted index file, partial completion +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIndexPartial) { + auto rowset = create_rowset(Version(6, 6), 1); + ASSERT_NE(rowset, nullptr); + + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); + _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); + + // Complete one segment file + WarmUpState result1 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + + // Complete inverted index file, should still be in TRIGGERED_BY_JOB state + WarmUpState result2 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 0, 1); + EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB); + + // Verify current state is still TRIGGERED_BY_JOB + WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB); +} + +// Test complete_rowset_segment_warmup with inverted index file, full completion +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIndexFull) { + auto rowset = create_rowset(Version(6, 6), 1); + ASSERT_NE(rowset, nullptr); + + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); + + // Complete segment file + WarmUpState result1 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + + // Complete inverted index file + WarmUpState result2 = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 0, 1); EXPECT_EQ(result2, WarmUpState::DONE); // Verify final state is DONE @@ -201,7 +257,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { // Complete with error status, should still transition to DONE when all segments complete Status error_status = Status::InternalError("Test error"); - WarmUpState result = _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), error_status); + WarmUpState result = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), error_status, 1, 0); EXPECT_EQ(result, WarmUpState::DONE); // Verify final state is DONE even with error @@ -235,13 +292,13 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { WarmUpState::TRIGGERED_BY_JOB); // Complete rowset1 (2 segments) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), WarmUpState::TRIGGERED_BY_JOB); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), WarmUpState::DONE); // Complete rowset3 (1 segment) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset3->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset3->rowset_id(), Status::OK(), 1, 0), WarmUpState::DONE); // Verify states after completion @@ -266,7 +323,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); // Any completion call should handle the edge case gracefully - WarmUpState result = _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK()); + WarmUpState result = + _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); // With 0 segments, the counter should already be 0, so this should transition to DONE EXPECT_EQ(result, WarmUpState::DONE); } @@ -285,11 +343,11 @@ TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); // Interleaved completion operations - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), WarmUpState::TRIGGERED_BY_JOB); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset2->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset2->rowset_id(), Status::OK(), 1, 0), WarmUpState::TRIGGERED_BY_SYNC_ROWSET); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK()), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), WarmUpState::TRIGGERED_BY_JOB); // Check states are maintained correctly diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp index b1eb148e8a304b..d0fe006edad465 100644 --- a/be/test/olap/tablet_test.cpp +++ b/be/test/olap/tablet_test.cpp @@ -297,11 +297,11 @@ TEST_F(TestTablet, pad_rowset) { Version version(5, 5); std::vector splits; - ASSERT_FALSE(_tablet->capture_rs_readers(version, &splits, false).ok()); + ASSERT_FALSE(_tablet->capture_rs_readers(version, &splits, {}).ok()); splits.clear(); static_cast(PadRowsetAction::_pad_rowset(_tablet.get(), version)); - ASSERT_TRUE(_tablet->capture_rs_readers(version, &splits, false).ok()); + ASSERT_TRUE(_tablet->capture_rs_readers(version, &splits, {}).ok()); } TEST_F(TestTablet, cooldown_policy) { diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 501dc4f43b3a05..c740a8720cfbb1 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -1048,7 +1048,6 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string return; } - // We don't actually need to parse the rowset meta doris::RowsetMetaCloudPB rs_meta; rs_meta.ParseFromString(tmp_rowset_val); if (rs_meta.txn_id() <= 0) { @@ -1063,9 +1062,22 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string INSTANCE_LOG(INFO) << "remove tmp rowset meta, tablet_id=" << tablet_id << " tmp_rowset_key=" << hex(tmp_rowset_key); + using namespace std::chrono; + auto rowset_visible_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + rs_meta.set_visible_ts_ms(rowset_visible_time); + std::string rowset_val; + if (!rs_meta.SerializeToString(&rowset_val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + SS << "failed to serialize rowset meta, tablet_id=" << tablet_id + << " rowset_id=" << rowset_id; + msg = ss.str(); + return; + } + int64_t version = compaction.output_versions(0); auto rowset_key = meta_rowset_key({instance_id, tablet_id, version}); - txn->put(rowset_key, tmp_rowset_val); + txn->put(rowset_key, rowset_val); INSTANCE_LOG(INFO) << "put rowset meta, tablet_id=" << tablet_id << " rowset_key=" << hex(rowset_key); @@ -1450,9 +1462,31 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str : cast_as(err); return; } + + RowsetMetaCloudPB tmp_rowset_meta; + if (!tmp_rowset_meta.ParseFromString(tmp_rowset_val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + SS << "malformed tmp rowset meta, unable to deserialize, tablet_id=" << new_tablet_id + << " key=" << hex(tmp_rowset_key); + msg = ss.str(); + return; + } + using namespace std::chrono; + auto rowset_visible_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + tmp_rowset_meta.set_visible_ts_ms(rowset_visible_time); + std::string rowset_val; + if (!tmp_rowset_meta.SerializeToString(&rowset_val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + SS << "failed to serialize rowset meta, tablet_id=" << new_tablet_id + << " rowset_id=" << tmp_rowset_meta.rowset_id_v2(); + msg = ss.str(); + return; + } + auto rowset_key = meta_rowset_key( {instance_id, new_tablet_id, schema_change.output_versions().at(i)}); - txn->put(rowset_key, tmp_rowset_val); + txn->put(rowset_key, rowset_val); txn->remove(tmp_rowset_key); } diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 229a4e205fa869..4ed3c2a1c3d749 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -1158,6 +1158,10 @@ void commit_txn_immediately( std::vector> rowsets; std::unordered_map tablet_stats; // tablet_id -> stats rowsets.reserve(tmp_rowsets_meta.size()); + + int64_t rowsets_visible_ts_ms = + duration_cast(system_clock::now().time_since_epoch()).count(); + for (auto& [_, i] : tmp_rowsets_meta) { int64_t tablet_id = i.tablet_id(); int64_t table_id = tablet_ids[tablet_id].table_id(); @@ -1179,6 +1183,7 @@ void commit_txn_immediately( int64_t new_version = new_versions[ver_key]; i.set_start_version(new_version); i.set_end_version(new_version); + i.set_visible_ts_ms(rowsets_visible_ts_ms); std::string key = meta_rowset_key({instance_id, tablet_id, i.end_version()}); std::string val; @@ -2327,6 +2332,9 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* continue; } + int64_t rowsets_visible_ts_ms = + duration_cast(system_clock::now().time_since_epoch()).count(); + std::vector> rowsets; std::unordered_map tablet_stats; // tablet_id -> stats for (const auto& sub_txn_info : sub_txn_infos) { @@ -2360,6 +2368,7 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* } i.set_start_version(new_version); i.set_end_version(new_version); + i.set_visible_ts_ms(rowsets_visible_ts_ms); LOG(INFO) << "xxx update rowset version, txn_id=" << txn_id << ", sub_txn_id=" << sub_txn_id << ", table_id=" << table_id << ", partition_id=" << partition_id << ", tablet_id=" << tablet_id diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 1de69f59d08a81..5be472ffc8ed54 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -62,6 +62,9 @@ void convert_tmp_rowsets( // tablet_id -> stats std::unordered_map tablet_stats; + int64_t rowsets_visible_ts_ms = + duration_cast(system_clock::now().time_since_epoch()).count(); + for (auto& [tmp_rowset_key, tmp_rowset_pb] : tmp_rowsets_meta) { std::string tmp_rowst_data; err = txn->get(tmp_rowset_key, &tmp_rowst_data); @@ -171,6 +174,7 @@ void convert_tmp_rowsets( tmp_rowset_pb.set_start_version(version); tmp_rowset_pb.set_end_version(version); + tmp_rowset_pb.set_visible_ts_ms(rowsets_visible_ts_ms); rowset_val.clear(); if (!tmp_rowset_pb.SerializeToString(&rowset_val)) { diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp index 89bf7dcb54fb75..2df9ef629820a5 100644 --- a/cloud/test/meta_service_job_test.cpp +++ b/cloud/test/meta_service_job_test.cpp @@ -1094,6 +1094,14 @@ TEST(MetaServiceJobTest, CompactionJobTest) { auto rowset_key = meta_rowset_key({instance_id, tablet_id, input_version_end}); std::string rowset_val; EXPECT_EQ(txn->get(rowset_key, &rowset_val), TxnErrorCode::TXN_OK) << hex(rowset_key); + doris::RowsetMetaCloudPB rowset_meta; + ASSERT_TRUE(rowset_meta.ParseFromString(rowset_val)); + ASSERT_TRUE(rowset_meta.has_visible_ts_ms() && rowset_meta.visible_ts_ms() > 0); + using namespace std::chrono; + auto visible_tp = time_point(milliseconds(rowset_meta.visible_ts_ms())); + std::time_t visible_time = system_clock::to_time_t(visible_tp); + std::cout << "visible time: " + << std::put_time(std::localtime(&visible_time), "%Y%m%d %H:%M:%S") << "\n"; }; auto test_abort_compaction_job = [&](int64_t table_id, int64_t index_id, int64_t partition_id, @@ -3205,6 +3213,12 @@ TEST(MetaServiceJobTest, SchemaChangeJobTest) { EXPECT_EQ(saved_rowset.start_version(), rs.start_version()); EXPECT_EQ(saved_rowset.end_version(), rs.end_version()); EXPECT_EQ(saved_rowset.rowset_id_v2(), rs.rowset_id_v2()); + ASSERT_TRUE(saved_rowset.has_visible_ts_ms() && saved_rowset.visible_ts_ms() > 0); + using namespace std::chrono; + auto visible_tp = time_point(milliseconds(saved_rowset.visible_ts_ms())); + std::time_t visible_time = system_clock::to_time_t(visible_tp); + std::cout << "visible time: " + << std::put_time(std::localtime(&visible_time), "%Y%m%d %H:%M:%S") << "\n"; } for (int i = 3; i < 5; ++i) { // [14-14][15-15] auto [k, v] = it->next(); diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 9f2bc6b11a22fd..fe1bd9e2ccb93d 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -10999,4 +11000,66 @@ TEST(MetaServiceTest, CreateTabletIdempotentAndHandlingError) { ASSERT_EQ(res.status().code(), MetaServiceCode::KV_TXN_GET_ERR); } +TEST(MetaServiceTest, RowsetVisibleTimeTest) { + auto meta_service = get_meta_service(); + using namespace std::chrono; + int64_t txn_id = -1; + // begin txn + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(666); + txn_info_pb.set_label("test_label"); + txn_info_pb.add_table_ids(1234); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + } + + // mock rowset and tablet + int64_t tablet_id_base = 1103; + for (int i = 0; i < 5; ++i) { + create_tablet(meta_service.get(), 1234, 1235, 1236, tablet_id_base + i); + auto tmp_rowset = create_rowset(txn_id, tablet_id_base + i); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + { + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(666); + req.set_txn_id(txn_id); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + for (int i = 0; i < 5; ++i) { + int64_t tablet_id = tablet_id_base + i; + int64_t ver = 2; + std::string rowset_key = meta_rowset_key({mock_instance, tablet_id, ver}); + std::string val; + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(rowset_key, &val), TxnErrorCode::TXN_OK); + RowsetMetaCloudPB rowset_pb; + ASSERT_TRUE(rowset_pb.ParseFromString(val)); + ASSERT_TRUE(rowset_pb.has_visible_ts_ms()); + std::cout << rowset_pb.visible_ts_ms() << "\n"; + ASSERT_GT(rowset_pb.visible_ts_ms(), 0); + auto visible_tp = time_point(milliseconds(rowset_pb.visible_ts_ms())); + std::time_t visible_time = system_clock::to_time_t(visible_tp); + std::cout << "visible time: " + << std::put_time(std::localtime(&visible_time), "%Y%m%d %H:%M:%S") << "\n"; + } +} } // namespace doris::cloud diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index e1b0a972a7f539..1c70ce07566696 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -1248,6 +1248,24 @@ public Pair isWorkloadGroupInUse(String groupName) { } } + public boolean getEnablePreferCachedRowset(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getEnablePreferCachedRowset(qualifiedUser); + } finally { + readUnlock(); + } + } + + public long getQueryFreshnessToleranceMs(String qualifiedUser) { + readLock(); + try { + return propertyMgr.getQueryFreshnessToleranceMs(qualifiedUser); + } finally { + readUnlock(); + } + } + public void getAllDomains(Set allDomains) { readLock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index 67f3c7859e26f0..71789cd4ff1bef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -73,6 +73,12 @@ public class CommonUserProperties implements Writable, GsonPostProcessable { @SerializedName(value = "wg", alternate = {"workloadGroup"}) private String workloadGroup = WorkloadGroupMgr.DEFAULT_GROUP_NAME; + @SerializedName(value = "epcr", alternate = {"enablePreferCachedRowset"}) + private boolean enablePreferCachedRowset = false; + + @SerializedName(value = "qft", alternate = {"queryFreshnessTolerance"}) + private long queryFreshnessToleranceMs = -1; + private String[] sqlBlockRulesSplit = {}; long getMaxConn() { @@ -186,6 +192,22 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, json); } + public long getQueryFreshnessToleranceMs() { + return queryFreshnessToleranceMs; + } + + public void setQueryFreshnessToleranceMs(long queryFreshnessToleranceMs) { + this.queryFreshnessToleranceMs = queryFreshnessToleranceMs; + } + + public boolean getEnablePreferCachedRowset() { + return enablePreferCachedRowset; + } + + public void setEnablePreferCachedRowset(boolean enablePreferCachedRowset) { + this.enablePreferCachedRowset = enablePreferCachedRowset; + } + @Override public void gsonPostProcess() throws IOException { if (!Strings.isNullOrEmpty(sqlBlockRules)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java index b79530af6f5d49..84d4af1c8c8555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java @@ -85,6 +85,9 @@ public class UserProperty implements Writable { public static final String DEFAULT_CLOUD_CLUSTER = "default_cloud_cluster"; public static final String DEFAULT_COMPUTE_GROUP = "default_compute_group"; + public static final String PROP_ENABLE_PREFER_CACHED_ROWSET = "enable_prefer_cached_rowset"; + public static final String PROP_QUERY_FRESHNESS_TOLERANCE = "query_freshness_tolerance_ms"; + // for system user public static final Set ADVANCED_PROPERTIES = Sets.newHashSet(); // for normal user @@ -132,6 +135,8 @@ public class UserProperty implements Writable { COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_WORKLOAD_GROUP + "$", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + DEFAULT_CLOUD_CLUSTER + "$", Pattern.CASE_INSENSITIVE)); COMMON_PROPERTIES.add(Pattern.compile("^" + DEFAULT_COMPUTE_GROUP + "$", Pattern.CASE_INSENSITIVE)); + COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUERY_FRESHNESS_TOLERANCE + "$", Pattern.CASE_INSENSITIVE)); + COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_ENABLE_PREFER_CACHED_ROWSET + "$", Pattern.CASE_INSENSITIVE)); } public UserProperty() { @@ -194,6 +199,14 @@ public long getExecMemLimit() { return commonProperties.getExecMemLimit(); } + public long getQueryFreshnessToleranceMs() { + return commonProperties.getQueryFreshnessToleranceMs(); + } + + public boolean getEnablePreferCachedRowset() { + return commonProperties.getEnablePreferCachedRowset(); + } + public void update(List> properties) throws UserException { update(properties, false); } @@ -211,6 +224,8 @@ public void update(List> properties, boolean isReplay) thro int insertTimeout = this.commonProperties.getInsertTimeout(); String initCatalog = this.commonProperties.getInitCatalog(); String workloadGroup = this.commonProperties.getWorkloadGroup(); + long queryFreshnessToleranceMs = this.commonProperties.getQueryFreshnessToleranceMs(); + boolean enablePreferCachedRowset = this.commonProperties.getEnablePreferCachedRowset(); String newDefaultCloudCluster = defaultCloudCluster; @@ -343,6 +358,21 @@ public void update(List> properties, boolean isReplay) thro throw new DdlException("workload group " + value + " not exists"); } workloadGroup = value; + } else if (keyArr[0].equalsIgnoreCase(PROP_QUERY_FRESHNESS_TOLERANCE)) { + // set property "query_freshness_tolerance" = "1000"; + if (keyArr.length != 1) { + throw new DdlException(PROP_QUERY_FRESHNESS_TOLERANCE + " format error"); + } + queryFreshnessToleranceMs = getLongProperty(key, value, keyArr, PROP_QUERY_FRESHNESS_TOLERANCE); + } else if (keyArr[0].equalsIgnoreCase(PROP_ENABLE_PREFER_CACHED_ROWSET)) { + if (keyArr.length != 1) { + throw new DdlException(PROP_ENABLE_PREFER_CACHED_ROWSET + " format error"); + } + try { + enablePreferCachedRowset = Boolean.parseBoolean(value); + } catch (NumberFormatException e) { + throw new DdlException(PROP_ENABLE_PREFER_CACHED_ROWSET + " is not boolean"); + } } else { if (isReplay) { // After using SET PROPERTY to modify the user property, if FE rolls back to a version without @@ -367,6 +397,8 @@ public void update(List> properties, boolean isReplay) thro this.commonProperties.setInsertTimeout(insertTimeout); this.commonProperties.setInitCatalog(initCatalog); this.commonProperties.setWorkloadGroup(workloadGroup); + this.commonProperties.setQueryFreshnessToleranceMs(queryFreshnessToleranceMs); + this.commonProperties.setEnablePreferCachedRowset(enablePreferCachedRowset); defaultCloudCluster = newDefaultCloudCluster; } @@ -464,6 +496,11 @@ public List> fetchProperty() { result.add(Lists.newArrayList(PROP_WORKLOAD_GROUP, String.valueOf(commonProperties.getWorkloadGroup()))); + result.add(Lists.newArrayList(PROP_ENABLE_PREFER_CACHED_ROWSET, + String.valueOf(commonProperties.getEnablePreferCachedRowset()))); + result.add(Lists.newArrayList(PROP_QUERY_FRESHNESS_TOLERANCE, + String.valueOf(commonProperties.getQueryFreshnessToleranceMs()))); + // default cloud cluster if (defaultCloudCluster != null) { result.add(Lists.newArrayList(DEFAULT_CLOUD_CLUSTER, defaultCloudCluster)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index d34dbb9aeaef81..ea77f1a78bac70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -241,6 +241,24 @@ public Pair isWorkloadGroupInUse(String groupName) { return Pair.of(false, ""); } + public boolean getEnablePreferCachedRowset(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + existProperty = getPropertyIfNull(qualifiedUser, existProperty); + if (existProperty == null) { + return false; + } + return existProperty.getEnablePreferCachedRowset(); + } + + public long getQueryFreshnessToleranceMs(String qualifiedUser) { + UserProperty existProperty = propertyMap.get(qualifiedUser); + existProperty = getPropertyIfNull(qualifiedUser, existProperty); + if (existProperty == null) { + return -1; + } + return existProperty.getQueryFreshnessToleranceMs(); + } + /** * The method determines which user property to return based on the existProperty parameter * and system configuration: diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5b959c8b9814ff..859877f374f1db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -751,6 +751,9 @@ public class SessionVariable implements Serializable, Writable { public static final String DEFAULT_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE = "default_variant_max_sparse_column_statistics_size"; + public static final String ENABLE_PREFER_CACHED_ROWSET = "enable_prefer_cached_rowset"; + public static final String QUERY_FRESHNESS_TOLERANCE_MS = "query_freshness_tolerance_ms"; + /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. */ @@ -2258,6 +2261,14 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { needForward = true) public boolean enableExternalTableBatchMode = true; + @VariableMgr.VarAttr(name = ENABLE_PREFER_CACHED_ROWSET, needForward = false, + description = {"ę˜Æ否åÆē”Ø prefer cached rowset åŠŸčƒ½", + "Whether to enable prefer cached rowset feature"}) + public boolean enablePreferCachedRowset = false; + + @VariableMgr.VarAttr(name = QUERY_FRESHNESS_TOLERANCE_MS, needForward = false) + public long queryFreshnessToleranceMs = -1; + public Set getIgnoredRuntimeFilterIds() { Set ids = Sets.newLinkedHashSet(); if (ignoreRuntimeFilterIds.isEmpty()) { @@ -3295,6 +3306,30 @@ public int getParallelExecInstanceNum() { } } + public boolean getEnablePreferCachedRowset() { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getEnv() != null && connectContext.getEnv().getAuth() != null) { + boolean userEnablePreferCachedRowset = connectContext.getEnv().getAuth() + .getEnablePreferCachedRowset(connectContext.getQualifiedUser()); + if (userEnablePreferCachedRowset) { + return userEnablePreferCachedRowset; + } + } + return enablePreferCachedRowset; + } + + public long getQueryFreshnessToleranceMs() { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getEnv() != null && connectContext.getEnv().getAuth() != null) { + long userQueryFreshnessToleranceMs = connectContext.getEnv().getAuth() + .getQueryFreshnessToleranceMs(connectContext.getQualifiedUser()); + if (userQueryFreshnessToleranceMs > 0) { + return userQueryFreshnessToleranceMs; + } + } + return queryFreshnessToleranceMs; + } + public int getExchangeInstanceParallel() { return exchangeInstanceParallel; } @@ -4295,6 +4330,10 @@ public TQueryOptions toThrift() { tResult.setEnableJoinSpill(enableJoinSpill); tResult.setEnableSortSpill(enableSortSpill); tResult.setEnableAggSpill(enableAggSpill); + + tResult.setEnablePreferCachedRowset(getEnablePreferCachedRowset()); + tResult.setQueryFreshnessToleranceMs(getQueryFreshnessToleranceMs()); + tResult.setEnableForceSpill(enableForceSpill); tResult.setMinRevocableMem(minRevocableMem); tResult.setDataQueueMaxBlocks(dataQueueMaxBlocks); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java index 6762f6bbad0380..143cbfaa2fdbc2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java @@ -109,6 +109,8 @@ public void testUpdate() throws UserException { properties.add(Pair.of("sql_block_rules", "rule1,rule2")); properties.add(Pair.of("cpu_resource_limit", "2")); properties.add(Pair.of("query_timeout", "500")); + properties.add(Pair.of("enable_prefer_cached_rowset", "true")); + properties.add(Pair.of("query_freshness_tolerance_ms", "4500")); UserProperty userProperty = new UserProperty(); userProperty.update(properties); @@ -119,6 +121,8 @@ public void testUpdate() throws UserException { Assert.assertEquals(2, userProperty.getCpuResourceLimit()); Assert.assertEquals(500, userProperty.getQueryTimeout()); Assert.assertEquals(Sets.newHashSet(), userProperty.getCopiedResourceTags()); + Assert.assertEquals(true, userProperty.getEnablePreferCachedRowset()); + Assert.assertEquals(4500, userProperty.getQueryFreshnessToleranceMs()); // fetch property List> rows = userProperty.fetchProperty(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java index 850d8b27b062af..e9ae96128468bf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java @@ -280,7 +280,7 @@ public void test() throws Exception { Assert.assertEquals(1000000, execMemLimit); List> userProps = Env.getCurrentEnv().getAuth().getUserProperties(Auth.ROOT_USER); - Assert.assertEquals(13, userProps.size()); + Assert.assertEquals(15, userProps.size()); // now : // be1 be2 be3 ==>tag1; diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 1e97d5ad476cb1..776072576988fd 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -140,6 +140,8 @@ message RowsetMetaPB { optional bool enable_inverted_index_file_info = 1006; repeated InvertedIndexFileInfo inverted_index_file_info = 1007; + + optional int64 visible_ts_ms = 1010; } message SchemaDictKeyList { @@ -233,6 +235,8 @@ message RowsetMetaCloudPB { optional bool enable_inverted_index_file_info = 106; repeated InvertedIndexFileInfo inverted_index_file_info = 107; + + optional int64 visible_ts_ms = 109; } message SegmentStatisticsPB { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 508d64f772ae28..2dfab470dd0160 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -366,6 +366,9 @@ struct TQueryOptions { // upgrade options. keep them same in every branch. 200: optional bool new_is_ip_address_in_range = false; + 172: optional bool enable_prefer_cached_rowset + 173: optional i64 query_freshness_tolerance_ms + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.out b/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.out new file mode 100644 index 00000000000000..04cf3be33e8192 --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster1 -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} + +-- !cluster2_0 -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} + +-- !cluster1_new_data -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} +6 {"a":1111.11111} + +-- !cluster2_1 -- +1 \N 1 3 +3 {"a":"11111"} 0 4 +4 {"a":1111111111} 0 5 +5 {"a":1111.11111} 0 6 +6 {"a":1111.11111} 0 7 + +-- !cluster2_2 -- +1 {"a":1} 0 2 +1 \N 1 3 +3 {"a":"11111"} 0 4 +4 {"a":1111111111} 0 5 +5 {"a":1111.11111} 0 6 +6 {"a":1111.11111} 0 7 + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.out b/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.out new file mode 100644 index 00000000000000..b99240d21e24ff --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.out @@ -0,0 +1,24 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster1 -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} + +-- !cluster2_0 -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} + +-- !cluster1_new_data -- +3 {"a":"11111"} +4 {"a":1111111111} +5 {"a":1111.11111} +6 {"a":1111.11111} + +-- !cluster2_1 -- +1 {"a":1} 0 2 +1 \N 1 3 +3 {"a":"11111"} 0 4 +4 {"a":1111111111} 0 5 +5 {"a":1111.11111} 0 6 + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.out b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.out new file mode 100644 index 00000000000000..7cefab58718a8e --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster2 -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.out b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.out new file mode 100644 index 00000000000000..8191de7859e2eb --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster2_0 -- +3 3 +4 4 +5 5 +6 6 + +-- !cluster2_1 -- +1 2 1 3 +3 3 0 4 +4 4 0 5 +5 5 0 6 +6 6 0 7 +9 9 0 8 + +-- !cluster2_2 -- +1 1 0 2 +1 2 1 3 +3 3 0 4 +4 4 0 5 +5 5 0 6 +6 6 0 7 + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.out b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.out new file mode 100644 index 00000000000000..e7fffb3bcb7ec1 --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster2_0 -- +2 2 + +-- !cluster2_1 -- +1 \N 1 3 +2 2 0 2 +9 9 0 4 + +-- !cluster2_2 -- + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.out b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.out new file mode 100644 index 00000000000000..b915a1b6c9449a --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster2_0 -- +3 3 +4 4 +5 5 +6 6 + +-- !cluster2_1 -- +1 2 1 3 +3 3 0 4 +4 4 0 5 +5 5 0 6 +6 6 0 7 +9 9 0 8 + +-- !cluster2 -- +1 1 0 2 +1 2 1 3 +3 3 0 4 +4 4 0 5 +5 5 0 6 +6 6 0 7 + diff --git a/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.out b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.out new file mode 100644 index 00000000000000..59c500c665d402 --- /dev/null +++ b/regression-test/data/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cluster2 -- +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +9 9 + diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.groovy new file mode 100644 index 00000000000000..34ca1d7e8a4b4e --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_enable_prefer_cached_rowset.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_enable_prefer_cached_rowset', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'block_file_cache_monitor_interval_sec=1', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(2000) + } + + def updateBeConf = {cluster, key, value -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + def (code, out, err) = update_be_config(ip, port, key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + def injectCompactionRowsetDownloadSlow = {cluster, sleep_s -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + def injectName = 'CloudTablet::add_rowsets.download_data.callback.block_compaction_rowset' + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + GetDebugPoint().enableDebugPoint(ip, port as int, NodeType.BE, injectName, [sleep:sleep_s]) + } + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + updateBeConf(clusterName2, "enable_warmup_immediately_on_new_rowset", "true") + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + sql """ + create table test ( + col0 int not null, + col1 variant NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false"); + """ + + clearFileCacheOnAllBackends() + + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test(col0,__DORIS_DELETE_SIGN__) values (1, 1);""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + + sql """use @${clusterName1}""" + qt_cluster1 """select * from test""" + + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + qt_cluster2_0 """select * from test""" + + // switch to source cluster and trigger compaction + sql """use @${clusterName1}""" + trigger_and_wait_compaction("test", "cumulative") + // load new data to increase the version + sql """insert into test values (6, '{"a" : 1111.11111}')""" + qt_cluster1_new_data "select * from test;" + + // inject to let cluster2 read compaction rowset data slowly + injectCompactionRowsetDownloadSlow(clusterName2, 10) + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + sql "set enable_profile=true;" + sql "set profile_level=2;" + + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + + // when enable_prefer_cached_rowset = false, need to read all data including compaction rowsets + qt_cluster2_1 "select * from test order by col0, __DORIS_VERSION_COL__;" + + sql "set enable_prefer_cached_rowset = true" + // when enable_prefer_cached_rowset = true, only need to read newly load data, compaction rowsets data will be skipped + def t1 = System.currentTimeMillis() + def capturePreferCacheCount = getBrpcMetricsByCluster(clusterName2, "capture_prefer_cache_count") + qt_cluster2_2 "select * from test order by col0, __DORIS_VERSION_COL__;" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 2000 + assert getBrpcMetricsByCluster(clusterName2, "capture_prefer_cache_count") == capturePreferCacheCount + 1 + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.groovy new file mode 100644 index 00000000000000..215e588137ed8e --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/no_warmup/test_query_freshness_tolerance.groovy @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_query_freshness_tolerance', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'block_file_cache_monitor_interval_sec=1', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(2000) + } + + def updateBeConf = {cluster, key, value -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + def (code, out, err) = update_be_config(ip, port, key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + def ret = matcher[0][1] as long + logger.info("getBrpcMetrics, ${url}, name:${name}, value:${ret}") + return ret + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + def injectS3FileReadSlow = {cluster, sleep_s -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + def injectName = 'S3FileReader::read_at_impl.io_slow' + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + GetDebugPoint().enableDebugPoint(ip, port as int, NodeType.BE, injectName, [sleep:sleep_s]) + } + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + updateBeConf(clusterName2, "enable_warmup_immediately_on_new_rowset", "true") + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + sql """ + create table test ( + col0 int not null, + col1 variant NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false"); + """ + + clearFileCacheOnAllBackends() + + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test(col0,__DORIS_DELETE_SIGN__) values (1, 1);""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + + sql """use @${clusterName1}""" + qt_cluster1 """select * from test""" + + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + qt_cluster2_0 """select * from test""" + + // sleep for 5s to let these rowsets meet the requirement of query freshness tolerance + sleep(5000) + + // switch to source cluster and trigger compaction + sql """use @${clusterName1}""" + trigger_and_wait_compaction("test", "cumulative") + // load new data to increase the version + sql """insert into test values (6, '{"a" : 1111.11111}')""" + qt_cluster1_new_data "select * from test;" + + // inject to let cluster2 read compaction rowset data slowly + injectS3FileReadSlow(clusterName2, 10) + // switch to read cluster, trigger a sync rowset + sql """use @${clusterName2}""" + sql "set enable_profile=true;" + sql "set profile_level=2;" + + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + + sql "set query_freshness_tolerance_ms = 5000" + def t1 = System.currentTimeMillis() + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + // when query_freshness_tolerance_ms is set, newly load data and compaction rowsets data will be skipped + qt_cluster2_1 "select * from test order by col0, __DORIS_VERSION_COL__;" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 3000 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + // query with freshness tolerance should not fallback + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.groovy new file mode 100644 index 00000000000000..3ce4ee58f4b771 --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_compaction_query_tolerance.groovy @@ -0,0 +1,316 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_warmup_delay_compaction_query_tolerance', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warm_up_rowset_slow_log_ms=1', + 'enable_compaction_delay_commit_for_warm_up=true', + 'warm_up_rowset_sync_wait_min_timeout_ms=20000', + 'warm_up_rowset_sync_wait_max_timeout_ms=20000', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(1000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + logger.info("Metric ${name} on ${ip}:${port} is ${matcher[0][1]}") + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBeIpAndPort = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + if (cluster_bes.isEmpty()) { + throw new RuntimeException("No BE found for cluster: ${cluster}") + } + + def firstBe = cluster_bes[0] + return [ip: firstBe[1], http_port:firstBe[4], rpc_port: firstBe[5]] + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def compaction_sync_wait = getBrpcMetrics(ip, port, "file_cache_warm_up_rowset_wait_for_compaction_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}" + + ", compaction_sync_wait=${compaction_sync_wait}") + } + } + + def waitForBrpcMetricValue = { ip, port, metricName, targetValue, timeoutMs -> + def delta_time = 100 + def useTime = 0 + + for(int t = delta_time; t <= timeoutMs; t += delta_time){ + try { + def currentValue = getBrpcMetrics(ip, port, metricName) + + if (currentValue == targetValue) { + logger.info("BE ${ip}:${port} metric ${metricName} reached target value: ${targetValue}") + return true + } + + logger.info("BE ${ip}:${port} metric ${metricName} current value: ${currentValue}, target: ${targetValue}") + + } catch (Exception e) { + logger.warn("Failed to get metric ${metricName} from BE ${ip}:${port}: ${e.message}") + } + + useTime = t + sleep(delta_time) + } + + assertTrue(useTime <= timeoutMs, "waitForBrpcMetricValue timeout") + } + + def getTabletStatus = { ip, port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def do_cumu_compaction = { def be, def tbl, def tablet_id, int start, int end -> + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", [tablet_id: "${tablet_id}", start_version: "${start}", end_version: "${end}"]) + trigger_and_wait_compaction(tbl, "cumulative") + GetDebugPoint().disableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets") + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 int NOT NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true"); + """ + + clearFileCacheOnAllBackends() + sleep(5000) + + sql """use @${clusterName1}""" + // load data + sql """insert into test values (1, 1)""" + sql """insert into test values (2, 2)""" + sql """insert into test values (3, 3)""" + sql """insert into test values (4, 4)""" + sql """insert into test values (5, 5)""" + sql """insert into test values (6, 6)""" + sleep(3000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + def be = getBeIpAndPort(clusterName2) + def src_be = getBeIpAndPort(clusterName1) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + def num_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + def num_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert num_submitted >= 6 + assert num_finished == num_submitted + + // inject sleep when read cluster warm up rowset for compaction and load + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudInternalServiceImpl::warm_up_rowset.download_segment", [sleep:10]) + + // trigger and wait compaction async + def future = thread { + sql """use @${clusterName1}""" + do_cumu_compaction(src_be, "test", tablet_id, 2, 5) + } + // wait until the warmup for compaction started + waitForBrpcMetricValue(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_num", 1, /*timeout*/10000) + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + assertEquals(num_submitted + 1, getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num")) + assertEquals(num_finished, getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num")) + + + // a new insert will trigger the sync rowset operation in the following query + sql """insert into test values (9, 9)""" + + + // in this moment, compaction has completed, but not commited, it's waiting for warm up + // trigger a query on read cluster, can't read the compaction data + sql """use @${clusterName2}""" + sql "select * from test" + def tablet_status = getTabletStatus(be.ip, be.http_port, tablet_id) + def rowsets = tablet_status ["rowsets"] + assert rowsets[1].contains("[2-2]") + assert rowsets[2].contains("[3-3]") + assert rowsets[3].contains("[4-4]") + assert rowsets[4].contains("[5-5]") + assert rowsets[5].contains("[6-6]") + assert rowsets[6].contains("[7-7]") + assert rowsets[7].contains("[8-8]") + + sql "set enable_profile=true;" + sql "set profile_level=2;" + + sql "set query_freshness_tolerance_ms = 5000" + def t1 = System.currentTimeMillis() + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + // should not contains (9,9) + qt_cluster2 """select * from test""" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 3000 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + + future.get() + assert num_finished + 2 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert 0 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_timeout_num") + + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.groovy new file mode 100644 index 00000000000000..de4887624e4945 --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_idx_query_tolerance.groovy @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_warmup_delay_idx_query_tolerance', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warm_up_rowset_slow_log_ms=1', + 'enable_compaction_delay_commit_for_warm_up=true', + 'warm_up_rowset_sync_wait_min_timeout_ms=100', // to cauase timeout + 'warm_up_rowset_sync_wait_max_timeout_ms=100', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + logger.info("Metric ${name} on ${ip}:${port} is ${matcher[0][1]}") + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBeIpAndPort = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + if (cluster_bes.isEmpty()) { + throw new RuntimeException("No BE found for cluster: ${cluster}") + } + + def firstBe = cluster_bes[0] + return [ip: firstBe[1], http_port:firstBe[4], rpc_port: firstBe[5]] + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def compaction_sync_wait = getBrpcMetrics(ip, port, "file_cache_warm_up_rowset_wait_for_compaction_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}" + + ", compaction_sync_wait=${compaction_sync_wait}") + } + } + + def waitForBrpcMetricValue = { ip, port, metricName, targetValue, timeoutMs -> + def delta_time = 100 + def useTime = 0 + + for(int t = delta_time; t <= timeoutMs; t += delta_time){ + try { + def currentValue = getBrpcMetrics(ip, port, metricName) + + if (currentValue == targetValue) { + logger.info("BE ${ip}:${port} metric ${metricName} reached target value: ${targetValue}") + return true + } + + logger.info("BE ${ip}:${port} metric ${metricName} current value: ${currentValue}, target: ${targetValue}") + + } catch (Exception e) { + logger.warn("Failed to get metric ${metricName} from BE ${ip}:${port}: ${e.message}") + } + + useTime = t + sleep(delta_time) + } + + assertTrue(useTime <= timeoutMs, "waitForBrpcMetricValue timeout") + } + + def getTabletStatus = { ip, port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def do_cumu_compaction = { def be, def tbl, def tablet_id, int start, int end -> + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", [tablet_id: "${tablet_id}", start_version: "${start}", end_version: "${end}"]) + trigger_and_wait_compaction(tbl, "cumulative") + GetDebugPoint().disableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets") + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 int NOT NULL, + INDEX idx1(col1) USING INVERTED + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "false"); + """ + + clearFileCacheOnAllBackends() + + sql """use @${clusterName1}""" + // load data + sql """insert into test values (1, 1)""" + sql """insert into test(col0,col1,__DORIS_DELETE_SIGN__) values (1, 2, 1)""" + sql """insert into test values (3, 3)""" + sql """insert into test values (4, 4)""" + sql """insert into test values (5, 5)""" + sql """insert into test values (6, 6)""" + sleep(5000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + def be = getBeIpAndPort(clusterName2) + def src_be = getBeIpAndPort(clusterName1) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + def num_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + def num_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + def num_idx_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_index_num") + def num_idx_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_index_num") + assert num_submitted >= 6 + assert num_finished == num_submitted + assert num_idx_submitted >= 6 + assert num_idx_finished == num_idx_submitted + + sql """use @${clusterName2}""" + // ensure that base rowsets' meta are loaded on target cluster + qt_cluster2_0 "select * from test order by col0, __DORIS_VERSION_COL__;" + sql """use @${clusterName1}""" + + // inject sleep when read cluster warm up rowset for compaction and load + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudInternalServiceImpl::warm_up_rowset.download_inverted_idx", [sleep:10]) + + // trigger and wait compaction async + def future = thread { + sql """use @${clusterName1}""" + do_cumu_compaction(src_be, "test", tablet_id, 2, 5) + } + sleep(500) + // wait until the warmup for compaction started + waitForBrpcMetricValue(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_num", 1, /*timeout*/10000) + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + assert num_submitted + 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + assert num_finished + 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert num_idx_submitted + 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_index_num") + assert num_idx_finished == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_index_num") + + + // a new insert will trigger the sync rowset operation in the following query + sql """insert into test values (9, 9)""" + + // trigger a query on read cluster without query tolerance, read the origin data + sql """use @${clusterName2}""" + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + qt_cluster2_1 "select * from test order by col0, __DORIS_VERSION_COL__;" + def tablet_status = getTabletStatus(be.ip, be.http_port, tablet_id) + def rowsets = tablet_status ["rowsets"] + assert rowsets[1].contains("[2-5]") + assert rowsets[2].contains("[6-6]") + assert rowsets[3].contains("[7-7]") + assert rowsets[4].contains("[8-8]") + + sql "set enable_profile=true;" + sql "set profile_level=2;" + + // trigger a query on read cluster without query tolerance, read the compacted data + sql "set query_freshness_tolerance_ms = 5000" + def t1 = System.currentTimeMillis() + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + // should not contains (9,9) + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + qt_cluster2_2 "select * from test order by col0, __DORIS_VERSION_COL__;" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 3000 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + + sleep(10000) + // assert num_finished + 2 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_timeout_num") + + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_index_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_index_num") + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.groovy new file mode 100644 index 00000000000000..688aa5e4446a57 --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_sc_query_tolerance.groovy @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_warmup_delay_sc_query_tolerance', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warm_up_rowset_slow_log_ms=1', + 'enable_compaction_delay_commit_for_warm_up=true', + 'warm_up_rowset_sync_wait_min_timeout_ms=100', + 'warm_up_rowset_sync_wait_max_timeout_ms=100', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(1000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + logger.info("Metric ${name} on ${ip}:${port} is ${matcher[0][1]}") + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBeIpAndPort = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + if (cluster_bes.isEmpty()) { + throw new RuntimeException("No BE found for cluster: ${cluster}") + } + + def firstBe = cluster_bes[0] + return [ip: firstBe[1], http_port:firstBe[4], rpc_port: firstBe[5]] + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def compaction_sync_wait = getBrpcMetrics(ip, port, "file_cache_warm_up_rowset_wait_for_compaction_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}" + + ", compaction_sync_wait=${compaction_sync_wait}") + } + } + + def waitForBrpcMetricValue = { ip, port, metricName, targetValue, timeoutMs -> + def delta_time = 100 + def useTime = 0 + + for(int t = delta_time; t <= timeoutMs; t += delta_time){ + try { + def currentValue = getBrpcMetrics(ip, port, metricName) + + if (currentValue == targetValue) { + logger.info("BE ${ip}:${port} metric ${metricName} reached target value: ${targetValue}") + return true + } + + logger.info("BE ${ip}:${port} metric ${metricName} current value: ${currentValue}, target: ${targetValue}") + + } catch (Exception e) { + logger.warn("Failed to get metric ${metricName} from BE ${ip}:${port}: ${e.message}") + } + + useTime = t + sleep(delta_time) + } + + assertTrue(useTime <= timeoutMs, "waitForBrpcMetricValue timeout") + } + + def getTabletStatus = { ip, port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + def do_cumu_compaction = { def be, def tbl, def tablet_id, int start, int end -> + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", [tablet_id: "${tablet_id}", start_version: "${start}", end_version: "${end}"]) + trigger_and_wait_compaction(tbl, "cumulative") + GetDebugPoint().disableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets") + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 int NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false"); + """ + + clearFileCacheOnAllBackends() + + sql """use @${clusterName1}""" + // load data + sql """insert into test values (1, 1),(2,2);""" + sql """insert into test(col0,__DORIS_DELETE_SIGN__) values (1, 1);""" + sleep(5000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + def be = getBeIpAndPort(clusterName2) + def src_be = getBeIpAndPort(clusterName1) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + def num_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + def num_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert num_submitted >= 1 + assert num_finished == num_submitted + + sql """use @${clusterName2}""" + // ensure that base rowsets' meta are loaded on target cluster + qt_cluster2_0 "select * from test order by col0, __DORIS_VERSION_COL__;" + sql """use @${clusterName1}""" + + // inject sleep when read cluster warm up rowset for compaction and load + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudInternalServiceImpl::warm_up_rowset.download_segment", [sleep:10]) + + sql """insert into test values (9, 9)""" + + do_cumu_compaction(src_be, "test", tablet_id, 2, 4) + + // trigger a heavy SC + sql "alter table test modify column col1 varchar(1000);" + + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='test' ORDER BY createtime DESC LIMIT 1 """ + time 1000 + } + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + // assert num_submitted + 2 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + // assert num_finished == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + + + sql """use @${clusterName2}""" + + sql "set enable_profile=true;" + sql "set profile_level=2;" + + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + + qt_cluster2_1 "select * from test order by col0, __DORIS_VERSION_COL__;" + + sql "set query_freshness_tolerance_ms = 5000" + def t1 = System.currentTimeMillis() + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + qt_cluster2_2 "select * from test order by col0, __DORIS_VERSION_COL__;" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 3000 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + + sleep(10000) + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.groovy new file mode 100644 index 00000000000000..b13609ed42e1e8 --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_delay_timeout_compaction_query_tolerance.groovy @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_warmup_delay_timeout_compaction_query_tolerance', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warm_up_rowset_slow_log_ms=1', + 'enable_compaction_delay_commit_for_warm_up=true', + 'read_cluster_cache_opt_verbose_log=true', + 'warm_up_rowset_sync_wait_min_timeout_ms=100', + 'warm_up_rowset_sync_wait_max_timeout_ms=100', // to cause timeout + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(2000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + logger.info("Metric ${name} on ${ip}:${port} is ${matcher[0][1]}") + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBeIpAndPort = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + if (cluster_bes.isEmpty()) { + throw new RuntimeException("No BE found for cluster: ${cluster}") + } + + def firstBe = cluster_bes[0] + return [ip: firstBe[1], http_port:firstBe[4], rpc_port: firstBe[5]] + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def compaction_sync_wait = getBrpcMetrics(ip, port, "file_cache_warm_up_rowset_wait_for_compaction_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}" + + ", compaction_sync_wait=${compaction_sync_wait}") + } + } + + def waitForBrpcMetricValue = { ip, port, metricName, targetValue, timeoutMs -> + def delta_time = 100 + def useTime = 0 + + for(int t = delta_time; t <= timeoutMs; t += delta_time){ + try { + def currentValue = getBrpcMetrics(ip, port, metricName) + + if (currentValue == targetValue) { + logger.info("BE ${ip}:${port} metric ${metricName} reached target value: ${targetValue}") + return true + } + + logger.info("BE ${ip}:${port} metric ${metricName} current value: ${currentValue}, target: ${targetValue}") + + } catch (Exception e) { + logger.warn("Failed to get metric ${metricName} from BE ${ip}:${port}: ${e.message}") + } + + useTime = t + sleep(delta_time) + } + + assertTrue(useTime <= timeoutMs, "waitForBrpcMetricValue timeout") + } + + def getTabletStatus = { ip, port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def do_cumu_compaction = { def be, def tbl, def tablet_id, int start, int end -> + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", [tablet_id: "${tablet_id}", start_version: "${start}", end_version: "${end}"]) + trigger_and_wait_compaction(tbl, "cumulative") + GetDebugPoint().disableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets") + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 int NOT NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false"); + """ + + clearFileCacheOnAllBackends() + + sql """use @${clusterName1}""" + // load data + sql """insert into test values (1, 1)""" + sql """insert into test(col0,col1,__DORIS_DELETE_SIGN__) values (1, 2, 1)""" + sql """insert into test values (3, 3)""" + sql """insert into test values (4, 4)""" + sql """insert into test values (5, 5)""" + sql """insert into test values (6, 6)""" + sleep(5000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + def be = getBeIpAndPort(clusterName2) + def src_be = getBeIpAndPort(clusterName1) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + def num_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + def num_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + def num_requested = getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + assert num_submitted >= 6 + assert num_finished == num_submitted + assert num_requested == num_finished + + sql """use @${clusterName2}""" + // ensure that base rowsets' meta are loaded on target cluster + qt_cluster2_0 "select * from test order by col0, __DORIS_VERSION_COL__;" + sql """use @${clusterName1}""" + + // inject sleep when read cluster warm up rowset for compaction and load + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudInternalServiceImpl::warm_up_rowset.download_segment", [sleep:10]) + + // trigger and wait compaction async + def future = thread { + sql """use @${clusterName1}""" + do_cumu_compaction(src_be, "test", tablet_id, 2, 5) + } + // wait until the warmup for compaction started + waitForBrpcMetricValue(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_num", 1, /*timeout*/10000) + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + assert num_submitted + 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + assert num_finished == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + + + // a new insert will trigger the sync rowset operation in the following query + sql """insert into test values (9, 9)""" + sleep(500) + assert num_submitted + 2 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + + + // trigger a query on read cluster without query tolerance, read the origin data + sql """use @${clusterName2}""" + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + qt_cluster2_1 "select * from test order by col0, __DORIS_VERSION_COL__;" + def tablet_status = getTabletStatus(be.ip, be.http_port, tablet_id) + def rowsets = tablet_status ["rowsets"] + assert rowsets[1].contains("[2-5]") + assert rowsets[2].contains("[6-6]") + assert rowsets[3].contains("[7-7]") + assert rowsets[4].contains("[8-8]") + + // this query will trigger sync_rowsets, due to compaction cnts changes, version_overlap will be true, so that compaction rowset + // and new load rowset's warmup task will be triggered. However, these rowsets' warmup tasks have been triggered in passive warmup + // we check that they will not be triggered again + assert num_submitted + 2 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + + sql "set enable_profile=true;" + sql "set profile_level=2;" + + // trigger a query on read cluster without query tolerance, read the compacted data + sql "set query_freshness_tolerance_ms = 5000" + def t1 = System.currentTimeMillis() + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + // should not contains (9,9) + sql "set skip_delete_sign=true;" + sql "set show_hidden_columns=true;" + sql "set skip_storage_engine_merge=true;" + qt_cluster2 "select * from test order by col0, __DORIS_VERSION_COL__;" + def t2 = System.currentTimeMillis() + logger.info("query in cluster2 cost=${t2 - t1} ms") + assert t2 - t1 < 3000 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + + future.get() + assert num_finished == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + assert 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_warm_up_rowset_wait_for_compaction_timeout_num") + + sleep(10000) + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + } +} diff --git a/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.groovy b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.groovy new file mode 100644 index 00000000000000..ea1aafbc44cf37 --- /dev/null +++ b/regression-test/suites/cloud_p0/read_cluster_cache/warmup/test_warmup_download_fail.groovy @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import groovy.json.JsonSlurper + +suite('test_warmup_download_fail', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warm_up_rowset_slow_log_ms=1', + 'enable_compaction_delay_commit_for_warm_up=true', + 'warm_up_rowset_sync_wait_min_timeout_ms=20000', + 'warm_up_rowset_sync_wait_max_timeout_ms=20000', + ] + options.enableDebugPoints() + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(1000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + logger.info("Metric ${name} on ${ip}:${port} is ${matcher[0][1]}") + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def getBeIpAndPort = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + if (cluster_bes.isEmpty()) { + throw new RuntimeException("No BE found for cluster: ${cluster}") + } + + def firstBe = cluster_bes[0] + return [ip: firstBe[1], http_port:firstBe[4], rpc_port: firstBe[5]] + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def compaction_sync_wait = getBrpcMetrics(ip, port, "file_cache_warm_up_rowset_wait_for_compaction_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}" + + ", compaction_sync_wait=${compaction_sync_wait}") + } + } + + def getTabletStatus = { ip, port, tablet_id -> + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${ip}:${port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def getBrpcMetricsByCluster = {cluster, name-> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + assert cluster_bes.size() > 0, "No backend found for cluster ${cluster}" + def be = cluster_bes[0] + def ip = be[1] + def port = be[5] + return getBrpcMetrics(ip, port, name) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 int NOT NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true"); + """ + + clearFileCacheOnAllBackends() + sleep(1000) + + sql """use @${clusterName1}""" + // load data + sql """insert into test values (1, 1)""" + sql """insert into test values (2, 2)""" + sql """insert into test values (3, 3)""" + sql """insert into test values (4, 4)""" + sql """insert into test values (5, 5)""" + sql """insert into test values (6, 6)""" + sleep(5000) + + def tablets = sql_return_maparray """ show tablets from test; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + def be = getBeIpAndPort(clusterName2) + def src_be = getBeIpAndPort(clusterName1) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + def num_submitted = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_submitted_segment_num") + def num_finished = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + def num_failed = getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_failed_segment_num") + assert num_submitted >= 6 + assert num_finished == num_submitted + assert num_failed == 0 + + GetDebugPoint().enableDebugPoint(be.ip, be.http_port as int, NodeType.BE, "CloudInternalServiceImpl::warm_up_rowset.download_segment.inject_error") + + // a new insert will trigger the sync rowset operation in the following query + sql """insert into test values (9, 9)""" + sleep(1000) + + assert num_failed + 1 == getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_failed_segment_num") + + + sql """use @${clusterName2}""" + sql "set enable_profile=true;" + sql "set profile_level=2;" + + // although download failed, the query should still read the newly inserted data + sql "set query_freshness_tolerance_ms = 5000" + def queryFreshnessToleranceCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") + def fallbackCount = getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") + qt_cluster2 """select * from test""" + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_count") == queryFreshnessToleranceCount + 1 + assert getBrpcMetricsByCluster(clusterName2, "capture_with_freshness_tolerance_fallback_count") == fallbackCount + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + + assert getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_finished_segment_num") + getBrpcMetrics(be.ip, be.rpc_port, "file_cache_event_driven_warm_up_failed_segment_num") + == getBrpcMetrics(src_be.ip, src_be.rpc_port, "file_cache_event_driven_warm_up_requested_segment_num") + } +} diff --git a/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy b/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy new file mode 100644 index 00000000000000..3305d4ce5dbfdb --- /dev/null +++ b/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_read_cluster_var_property') { + if (!isCloudMode()) { + return + } + String userName = "test_read_cluster_var_property_user" + String pwd = '123456' + sql """drop user if exists ${userName}""" + sql """CREATE USER '${userName}' IDENTIFIED BY '${pwd}'""" + sql """GRANT ADMIN_PRIV ON *.*.* TO ${userName}""" + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + def ret = matcher[0][1] as long + logger.info("getBrpcMetrics, ${url}, name:${name}, value:${ret}") + return ret + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + connect(userName, "${pwd}", context.config.jdbcUrl) { + // test non-mow table + try { + def tableName = "test_read_cluster_var_property" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} + (k int, v1 int, v2 int ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH (k) + BUCKETS 1 PROPERTIES( + "replication_num" = "1", + "disable_auto_compaction" = "true"); + """ + + (1..20).each{ id -> + sql """insert into ${tableName} select number, number, number from numbers("number"="10");""" + } + + sql "select * from ${tableName};" + + def backends = sql_return_maparray('show backends') + def tabletStats = sql_return_maparray("show tablets from ${tableName};") + assert tabletStats.size() == 1 + def tabletId = tabletStats[0].TabletId + def tabletBackendId = tabletStats[0].BackendId + def tabletBackend + for (def be : backends) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + try { + // 1. test enable_prefer_cached_rowset + sql "set enable_prefer_cached_rowset=true;" + def preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount + 1 + + sql "set enable_prefer_cached_rowset=false;" + preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount + + // user property has higher prioroty than session variable + sql "set property for '${userName}' enable_prefer_cached_rowset=true;" + preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == 1 + preferCachedRowsetCount + } finally { + sql "set enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + } + + try { + // 2. test query_freshness_tolerance_ms + sql "set query_freshness_tolerance_ms=1000;" + def queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance + 1 + + sql "set query_freshness_tolerance_ms=-1;" + queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance + + // user property has higher prioroty than session variable + sql "set property for '${userName}' query_freshness_tolerance_ms=2000;" + queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == 1 + queryFreshnessTolerance + } finally { + sql "set query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + } + } catch (Exception e) { + logger.error("Error occurred while testing query_freshness_tolerance_ms: ${e.message}") + } finally { + sql "set enable_prefer_cached_rowset=false;" + sql "set query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + } + + // test mow table + try { + def tableName = "test_read_cluster_var_property_mow" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} + (k int, v1 int, v2 int ) + UNIQUE KEY(k) DISTRIBUTED BY HASH (k) + BUCKETS 1 PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true"); + """ + + (1..20).each{ id -> + sql """insert into ${tableName} select number, number, number from numbers("number"="10");""" + } + + sql "select * from ${tableName};" + + def backends = sql_return_maparray('show backends') + def tabletStats = sql_return_maparray("show tablets from ${tableName};") + assert tabletStats.size() == 1 + def tabletId = tabletStats[0].TabletId + def tabletBackendId = tabletStats[0].BackendId + def tabletBackend + for (def be : backends) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + try { + // 1. test enable_prefer_cached_rowset + // enable_prefer_cached_rowset should not take effect on mow table + sql "set enable_prefer_cached_rowset=true;" + def preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount + + sql "set enable_prefer_cached_rowset=false;" + preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount + + // user property has higher prioroty than session variable + sql "set property for '${userName}' enable_prefer_cached_rowset=true;" + preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount + } finally { + sql "set enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + } + + try { + // 2. test query_freshness_tolerance_ms + sql "set query_freshness_tolerance_ms=1000;" + def queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance + 1 + + sql "set query_freshness_tolerance_ms=-1;" + queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance + + // user property has higher prioroty than session variable + sql "set property for '${userName}' query_freshness_tolerance_ms=2000;" + queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") + sql "select * from ${tableName};" + assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == 1 + queryFreshnessTolerance + } finally { + sql "set query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + } + } catch (Exception e) { + logger.error("Error occurred while testing query_freshness_tolerance_ms: ${e.message}") + throw e + } finally { + sql "set enable_prefer_cached_rowset=false;" + sql "set query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + } + } +} \ No newline at end of file From aece3813cc19fbe013a5186e2203bf0ebb7a3baf Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Sun, 28 Sep 2025 18:30:38 +0800 Subject: [PATCH 3/3] fix --- .../test_read_cluster_var_property.groovy | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy b/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy index 3305d4ce5dbfdb..12d9c682001d6d 100644 --- a/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy +++ b/regression-test/suites/cloud_p0/test_read_cluster_var_property.groovy @@ -85,13 +85,13 @@ suite('test_read_cluster_var_property') { assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount // user property has higher prioroty than session variable - sql "set property for '${userName}' enable_prefer_cached_rowset=true;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='true';" preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") sql "select * from ${tableName};" assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == 1 + preferCachedRowsetCount } finally { sql "set enable_prefer_cached_rowset=false;" - sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='false';" } try { @@ -107,21 +107,21 @@ suite('test_read_cluster_var_property') { assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance // user property has higher prioroty than session variable - sql "set property for '${userName}' query_freshness_tolerance_ms=2000;" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='2000';" queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") sql "select * from ${tableName};" assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == 1 + queryFreshnessTolerance } finally { sql "set query_freshness_tolerance_ms=-1;" - sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='-1';" } } catch (Exception e) { logger.error("Error occurred while testing query_freshness_tolerance_ms: ${e.message}") } finally { sql "set enable_prefer_cached_rowset=false;" sql "set query_freshness_tolerance_ms=-1;" - sql "set property for '${userName}' enable_prefer_cached_rowset=false;" - sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='false';" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='-1';" } // test mow table @@ -171,13 +171,13 @@ suite('test_read_cluster_var_property') { assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount // user property has higher prioroty than session variable - sql "set property for '${userName}' enable_prefer_cached_rowset=true;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='true';" preferCachedRowsetCount = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") sql "select * from ${tableName};" assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_prefer_cache_count") == preferCachedRowsetCount } finally { sql "set enable_prefer_cached_rowset=false;" - sql "set property for '${userName}' enable_prefer_cached_rowset=false;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='false';" } try { @@ -193,13 +193,13 @@ suite('test_read_cluster_var_property') { assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == queryFreshnessTolerance // user property has higher prioroty than session variable - sql "set property for '${userName}' query_freshness_tolerance_ms=2000;" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='2000';" queryFreshnessTolerance = getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") sql "select * from ${tableName};" assert getBrpcMetrics(tabletBackend.Host, tabletBackend.BrpcPort, "capture_with_freshness_tolerance_count") == 1 + queryFreshnessTolerance } finally { sql "set query_freshness_tolerance_ms=-1;" - sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='-1';" } } catch (Exception e) { logger.error("Error occurred while testing query_freshness_tolerance_ms: ${e.message}") @@ -207,8 +207,8 @@ suite('test_read_cluster_var_property') { } finally { sql "set enable_prefer_cached_rowset=false;" sql "set query_freshness_tolerance_ms=-1;" - sql "set property for '${userName}' enable_prefer_cached_rowset=false;" - sql "set property for '${userName}' query_freshness_tolerance_ms=-1;" + sql "set property for '${userName}' 'enable_prefer_cached_rowset'='false';" + sql "set property for '${userName}' 'query_freshness_tolerance_ms'='-1';" } } } \ No newline at end of file