Skip to content

Commit 9c57d4b

Browse files
committed
[Improve](segment) Put Segment footer into index page cache
This commit modified the caching logic for the Segment footer. Previously, the Segment footer was cached in the `DATA_PAGE` cache. This was incorrect as the footer is metadata, not data. This commit changes it to use the more appropriate `INDEX_PAGE` cache. Additionally, the calculation for `_meta_mem_usage` has been adjusted for better accuracy. The footer size is no longer included in the calculation since it's now managed by the `StoragePageCache`. The memory estimation for column readers is also updated to use `config::max_segment_partial_column_cache_size` for a more realistic value.
1 parent 35a0650 commit 9c57d4b

File tree

1 file changed

+12
-4
lines changed

1 file changed

+12
-4
lines changed

be/src/olap/rowset/segment_v2/segment.cpp

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,13 +197,16 @@ Status Segment::_open(OlapReaderStatistics* stats) {
197197
_num_rows = footer_pb_shared->num_rows();
198198

199199
// An estimated memory usage of a segment
200-
_meta_mem_usage += footer_pb_shared->ByteSizeLong();
200+
// Footer is seperated to StoragePageCache so we don't need to add it to _meta_mem_usage
201+
// _meta_mem_usage += footer_pb_shared->ByteSizeLong();
201202
if (_pk_index_meta != nullptr) {
202203
_meta_mem_usage += _pk_index_meta->ByteSizeLong();
203204
}
204205

205206
_meta_mem_usage += sizeof(*this);
206-
_meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
207+
_meta_mem_usage += std::min(static_cast<int>(_tablet_schema->num_columns()),
208+
config::max_segment_partial_column_cache_size) *
209+
config::estimated_mem_per_column_reader;
207210

208211
// 1024 comes from SegmentWriterOptions
209212
_meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
@@ -1064,11 +1067,16 @@ Status Segment::_get_segment_footer(std::shared_ptr<SegmentFooterPB>& footer_pb,
10641067

10651068
PageCacheHandle cache_handle;
10661069

1070+
// Put segment footer into index page cache.
1071+
// Rationale:
1072+
// - Footer is metadata (small, parsed with indexes), not data page payload.
1073+
// - Using PageTypePB::INDEX_PAGE keeps it under the same eviction policy/shards
1074+
// as other index/metadata pages and avoids competing with DATA_PAGE budget.
10671075
if (!segment_footer_cache->lookup(cache_key, &cache_handle,
1068-
segment_v2::PageTypePB::DATA_PAGE)) {
1076+
segment_v2::PageTypePB::INDEX_PAGE)) {
10691077
RETURN_IF_ERROR(_parse_footer(footer_pb_shared, stats));
10701078
segment_footer_cache->insert(cache_key, footer_pb_shared, footer_pb_shared->ByteSizeLong(),
1071-
&cache_handle, segment_v2::PageTypePB::DATA_PAGE);
1079+
&cache_handle, segment_v2::PageTypePB::INDEX_PAGE);
10721080
} else {
10731081
VLOG_DEBUG << fmt::format("Segment footer of {}:{}:{} is found in cache",
10741082
_file_reader->path().native(), _file_reader->size(),

0 commit comments

Comments
 (0)