Skip to content

Commit 145fea1

Browse files
committed
[Improve](segment) Put Segment footer into index page cache
This commit modified the caching logic for the Segment footer. Previously, the Segment footer was cached in the `DATA_PAGE` cache. This was incorrect as the footer is metadata, not data. This commit changes it to use the more appropriate `INDEX_PAGE` cache. Additionally, the calculation for `_meta_mem_usage` has been adjusted for better accuracy. The footer size is no longer included in the calculation since it's now managed by the `StoragePageCache`. The memory estimation for column readers is also updated to use `config::max_segment_partial_column_cache_size` for a more realistic value.
1 parent 35a0650 commit 145fea1

File tree

1 file changed

+11
-4
lines changed

1 file changed

+11
-4
lines changed

be/src/olap/rowset/segment_v2/segment.cpp

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,13 +197,15 @@ Status Segment::_open(OlapReaderStatistics* stats) {
197197
_num_rows = footer_pb_shared->num_rows();
198198

199199
// An estimated memory usage of a segment
200-
_meta_mem_usage += footer_pb_shared->ByteSizeLong();
200+
// Footer is seperated to StoragePageCache so we don't need to add it to _meta_mem_usage
201+
// _meta_mem_usage += footer_pb_shared->ByteSizeLong();
201202
if (_pk_index_meta != nullptr) {
202203
_meta_mem_usage += _pk_index_meta->ByteSizeLong();
203204
}
204205

205206
_meta_mem_usage += sizeof(*this);
206-
_meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
207+
_meta_mem_usage +=
208+
config::max_segment_partial_column_cache_size * config::estimated_mem_per_column_reader;
207209

208210
// 1024 comes from SegmentWriterOptions
209211
_meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
@@ -1064,11 +1066,16 @@ Status Segment::_get_segment_footer(std::shared_ptr<SegmentFooterPB>& footer_pb,
10641066

10651067
PageCacheHandle cache_handle;
10661068

1069+
// Put segment footer into index page cache.
1070+
// Rationale:
1071+
// - Footer is metadata (small, parsed with indexes), not data page payload.
1072+
// - Using PageTypePB::INDEX_PAGE keeps it under the same eviction policy/shards
1073+
// as other index/metadata pages and avoids competing with DATA_PAGE budget.
10671074
if (!segment_footer_cache->lookup(cache_key, &cache_handle,
1068-
segment_v2::PageTypePB::DATA_PAGE)) {
1075+
segment_v2::PageTypePB::INDEX_PAGE)) {
10691076
RETURN_IF_ERROR(_parse_footer(footer_pb_shared, stats));
10701077
segment_footer_cache->insert(cache_key, footer_pb_shared, footer_pb_shared->ByteSizeLong(),
1071-
&cache_handle, segment_v2::PageTypePB::DATA_PAGE);
1078+
&cache_handle, segment_v2::PageTypePB::INDEX_PAGE);
10721079
} else {
10731080
VLOG_DEBUG << fmt::format("Segment footer of {}:{}:{} is found in cache",
10741081
_file_reader->path().native(), _file_reader->size(),

0 commit comments

Comments
 (0)