Adding chromosome base file splitting

AdityaPandeyCN · AdityaPandeyCN · commit 632d4b64516d · 2025-10-05T01:36:16.000+05:30
Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> clang format Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> code organization Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> clang changes Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> Add region query benchmark (compiler-research#8) * query performance Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> * clang format Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> * code organization Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> * clang changes Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> --------- Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> add chromosome based file splitting Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> delete example sam file Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> clang changes Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> test file changes Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com> clang changes Signed-off-by: AdityaPandeyCN <adityapand3y666@gmail.com>
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -9,8 +9,19 @@ target_include_directories(sam_generator PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
-target_link_libraries(sam_generator PRIVATE benchmark::benchmark)
-add_dependencies(sam_generator benchmark::benchmark)
+target_link_libraries(sam_generator PRIVATE 
+    benchmark::benchmark
+)
+
+add_library(ramtools_views STATIC
+    ${CMAKE_SOURCE_DIR}/tools/ramview.cxx
+    ${CMAKE_SOURCE_DIR}/tools/ramntupleview.cxx
+)
+
+target_link_libraries(ramtools_views PRIVATE 
+    ramcore 
+    ROOT::TreePlayer
+)
 
 ROOT_EXECUTABLE(sam_to_ram_benchmark
     sam_to_ram_benchmark.cxx
@@ -28,14 +39,30 @@ ROOT_EXECUTABLE(conversion_time_benchmark
         sam_generator
 )
 
-install(TARGETS sam_to_ram_benchmark conversion_time_benchmark
+ROOT_EXECUTABLE(region_query_benchmark
+    region_query_benchmark.cxx
+    LIBRARIES
+        benchmark::benchmark
+        ramcore
+        sam_generator
+        ramtools_views
+)
+
+install(TARGETS sam_to_ram_benchmark conversion_time_benchmark region_query_benchmark
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
 
 add_custom_target(benchmark
+    COMMAND ${CMAKE_COMMAND} -E echo "=== SAM to RAM Benchmark ==="
     COMMAND sam_to_ram_benchmark
+    COMMAND ${CMAKE_COMMAND} -E echo ""
+    COMMAND ${CMAKE_COMMAND} -E echo "=== Conversion Time Benchmark ==="
     COMMAND conversion_time_benchmark
-    DEPENDS sam_to_ram_benchmark conversion_time_benchmark
+    COMMAND ${CMAKE_COMMAND} -E echo ""
+    COMMAND ${CMAKE_COMMAND} -E echo "=== Region Query Benchmark ==="
+    COMMAND region_query_benchmark
+    DEPENDS sam_to_ram_benchmark conversion_time_benchmark region_query_benchmark
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Running all RAM tools benchmarks"
 )
 
diff --git a/benchmark/benchmark_utils.h b/benchmark/benchmark_utils.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#ifdef _WIN32
+#define NULL_DEVICE "NUL"
+#else
+#define NULL_DEVICE "/dev/null"
+#endif
diff --git a/benchmark/region_query_benchmark.cxx b/benchmark/region_query_benchmark.cxx
@@ -0,0 +1,86 @@
+#include <benchmark/benchmark.h>
+#include "generate_sam_benchmark.h"
+#include "benchmark_utils.h"
+#include "ramcore/SamToTTree.h"
+#include "ramcore/SamToNTuple.h"
+#include <string>
+#include <cstdio>
+
+void ramview(const char *file, const char *query, bool cache = true, bool perfstats = false,
+             const char *perfstatsfilename = "perf.root");
+void ramntupleview(const char *file, const char *query, bool cache = true, bool perfstats = false,
+                   const char *perfstatsfilename = "perf.root");
+
+class RegionQueryFixture : public benchmark::Fixture {
+public:
+   void SetUp(const benchmark::State &state) override
+   {
+      num_reads_ = static_cast<int>(state.range(0));
+      sam_file_ = "region_query_test_" + std::to_string(num_reads_) + ".sam";
+
+      GenerateSAMFile(sam_file_, num_reads_);
+   }
+
+   void TearDown(const benchmark::State &) override { std::remove(sam_file_.c_str()); }
+
+protected:
+   int num_reads_;
+   std::string sam_file_;
+   static constexpr const char *region_ = "chr1:1-100000000";
+
+   void suppress_output() { freopen(NULL_DEVICE, "w", stdout); }
+
+   void restore_output() { freopen("/dev/tty", "w", stdout); }
+};
+
+BENCHMARK_DEFINE_F(RegionQueryFixture, TTree)(benchmark::State &state)
+{
+   std::string root_file = "ttree_" + std::to_string(num_reads_) + ".root";
+
+   suppress_output();
+   samtoram(sam_file_.c_str(), root_file.c_str(), true, true, true, 1, 0);
+   restore_output();
+
+   for (auto _ : state) {
+      suppress_output();
+      ramview(root_file.c_str(), region_, true, false, "perf.root");
+      restore_output();
+   }
+
+   std::remove(root_file.c_str());
+
+   state.counters["reads_per_sec"] = benchmark::Counter(num_reads_, benchmark::Counter::kIsRate);
+}
+
+BENCHMARK_DEFINE_F(RegionQueryFixture, RNTuple)(benchmark::State &state)
+{
+   std::string root_file = "rntuple_" + std::to_string(num_reads_) + ".root";
+
+   suppress_output();
+   samtoramntuple(sam_file_.c_str(), root_file.c_str(), true, true, true, 505, 0);
+   restore_output();
+
+   for (auto _ : state) {
+      suppress_output();
+      ramntupleview(root_file.c_str(), region_, true, false, "perf.root");
+      restore_output();
+   }
+
+   std::remove(root_file.c_str());
+
+   state.counters["reads_per_sec"] = benchmark::Counter(num_reads_, benchmark::Counter::kIsRate);
+}
+
+BENCHMARK_REGISTER_F(RegionQueryFixture, TTree)
+   ->Args({1000})
+   ->Args({10000})
+   ->Args({100000})
+   ->Unit(benchmark::kMillisecond);
+
+BENCHMARK_REGISTER_F(RegionQueryFixture, RNTuple)
+   ->Args({1000})
+   ->Args({10000})
+   ->Args({100000})
+   ->Unit(benchmark::kMillisecond);
+
+BENCHMARK_MAIN();
diff --git a/benchmark/sam_to_ram_benchmark.cxx b/benchmark/sam_to_ram_benchmark.cxx
@@ -1,18 +1,13 @@
 #include <benchmark/benchmark.h>
 #include "generate_sam_benchmark.h"
+#include "benchmark_utils.h"
 #include "ramcore/SamToTTree.h"
 #include "ramcore/SamToNTuple.h"
 #include <filesystem>
 #include <iostream>
 #include <cstdio>
 #include <cstring>
 
-#ifdef _WIN32
-#define NULL_DEVICE "NUL"
-#else
-#define NULL_DEVICE "/dev/null"
-#endif
-
 static void BM_SamToRamComparison(benchmark::State &state)
 {
    int num_reads = state.range(0);
diff --git a/inc/ramcore/SamToNTuple.h b/inc/ramcore/SamToNTuple.h
@@ -1,13 +1,16 @@
 #ifndef RAMCORE_SAMTONTUPLE_H
 #define RAMCORE_SAMTONTUPLE_H
 
-#include <cstdint> 
+#include <cstdint>
+#include <string>
 
 void samtoramntuple(const char *datafile,
                     const char *treefile,
                     bool index, bool split, bool cache,
                     int compression_algorithm,
                     uint32_t quality_policy);
 
-#endif // RAMCORE_SAMTONTUPLE_H 
+void samtoramntuple_split_by_chromosome(const char *datafile, const char *output_prefix, int compression_algorithm,
+                                        uint32_t quality_policy);
 
+#endif
diff --git a/src/ramcore/SamToNTuple.cxx b/src/ramcore/SamToNTuple.cxx
@@ -10,6 +10,12 @@
 #include <TNamed.h>
 #include <TFile.h>
 
+#include <map>
+#include <memory>
+#include <iostream>
+#include <fstream>
+#include <cstdio>
+
 void samtoramntuple(const char *datafile,
                     const char *treefile,
                     bool index, bool split, bool cache,
@@ -120,3 +126,66 @@ void samtoramntuple(const char *datafile,
     stopwatch.Print();
 }
 
+void samtoramntuple_split_by_chromosome(const char *datafile, const char *output_prefix, int compression_algorithm,
+                                        uint32_t quality_policy)
+{
+   std::ifstream input(datafile);
+   if (!input) {
+      std::cerr << "Error: Cannot open " << datafile << std::endl;
+      return;
+   }
+
+   std::vector<std::string> headers;
+   std::map<std::string, std::unique_ptr<std::ofstream>> chr_files;
+   std::map<std::string, std::string> chr_temp_filenames;
+   std::string line;
+
+   while (std::getline(input, line)) {
+      if (line.empty())
+         continue;
+
+      if (line[0] == '@') {
+         headers.push_back(line);
+         continue;
+      }
+
+      size_t pos = line.find('\t');
+      if (pos == std::string::npos)
+         continue;
+      pos = line.find('\t', pos + 1);
+      if (pos == std::string::npos)
+         continue;
+
+      size_t end_pos = line.find('\t', pos + 1);
+      if (end_pos == std::string::npos)
+         continue;
+
+      std::string rname = line.substr(pos + 1, end_pos - pos - 1);
+      if (rname == "*")
+         continue;
+
+      if (chr_files.find(rname) == chr_files.end()) {
+         std::string temp_filename = std::string(output_prefix) + "_" + rname + ".tmp.sam";
+         chr_temp_filenames[rname] = temp_filename;
+         chr_files[rname] = std::make_unique<std::ofstream>(temp_filename);
+
+         for (const auto &header : headers) {
+            *(chr_files[rname]) << header << "\n";
+         }
+      }
+
+      *(chr_files[rname]) << line << "\n";
+   }
+
+   input.close();
+   for (auto &[chr, file] : chr_files) {
+      file->close();
+   }
+
+   for (const auto &[chr, temp_filename] : chr_temp_filenames) {
+      std::string output_filename = std::string(output_prefix) + "_" + chr + ".root";
+      samtoramntuple(temp_filename.c_str(), output_filename.c_str(), false, false, false, compression_algorithm,
+                     quality_policy);
+      std::remove(temp_filename.c_str());
+   }
+}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -24,8 +24,9 @@ function(add_ramcore_test test_name)
 endfunction()
 
 add_ramcore_test(ramcoretests ramcoretests.cxx)
+add_ramcore_test(chromosome_split_test chromosome_split_test.cxx)
 
-install(TARGETS ramcoretests
+install(TARGETS ramcoretests chromosome_split_test
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
 
diff --git a/test/chromosome_split_test.cxx b/test/chromosome_split_test.cxx
@@ -0,0 +1,100 @@
+#include <gtest/gtest.h>
+#include "ramcore/SamToNTuple.h"
+#include "rntuple/RAMNTupleRecord.h"
+#include "generate_sam_benchmark.h"
+#include <ROOT/RNTupleReader.hxx>
+#include <cstdio>
+#include <filesystem>
+
+class ChromosomeSplitTest : public ::testing::Test {
+protected:
+   void SetUp() override
+   {
+      if (!std::filesystem::exists("test.sam")) {
+         GenerateSAMFile("test.sam", 100);
+      }
+      CleanupTestFiles();
+   }
+
+   void TearDown() override { CleanupTestFiles(); }
+
+   void CleanupTestFiles()
+   {
+      std::remove("test_regular.root");
+      for (const auto &entry : std::filesystem::directory_iterator(".")) {
+         std::string filename = entry.path().filename().string();
+         if (filename.find("test_split_") == 0 && filename.find(".root") != std::string::npos) {
+            std::remove(filename.c_str());
+         }
+      }
+   }
+};
+
+TEST_F(ChromosomeSplitTest, NoDataLoss)
+{
+   samtoramntuple("test.sam", "test_regular.root", false, true, true, 505, 1);
+   auto regularReader = ROOT::Experimental::RNTupleReader::Open("RAM", "test_regular.root");
+   Long64_t totalEntries = regularReader->GetNEntries();
+
+   samtoramntuple_split_by_chromosome("test.sam", "test_split", 505, 1);
+
+   Long64_t splitEntriesSum = 0;
+   for (const auto &entry : std::filesystem::directory_iterator(".")) {
+      std::string filename = entry.path().filename().string();
+      if (filename.find("test_split_") == 0 && filename.find(".root") != std::string::npos) {
+         auto reader = ROOT::Experimental::RNTupleReader::Open("RAM", filename);
+         if (reader) {
+            splitEntriesSum += reader->GetNEntries();
+         }
+      }
+   }
+
+   EXPECT_EQ(totalEntries, splitEntriesSum);
+}
+
+TEST_F(ChromosomeSplitTest, CorrectChromosomeAssignment)
+{
+   samtoramntuple_split_by_chromosome("test.sam", "test_split", 505, 1);
+
+   for (const auto &entry : std::filesystem::directory_iterator(".")) {
+      std::string filename = entry.path().filename().string();
+      if (filename.find("test_split_") == 0 && filename.find(".root") != std::string::npos) {
+         size_t pos = filename.find("test_split_");
+         size_t end = filename.find(".root");
+         std::string expectedChr = filename.substr(pos + 11, end - pos - 11);
+
+         auto reader = ROOT::Experimental::RNTupleReader::Open("RAM", filename);
+         ASSERT_NE(reader, nullptr);
+
+         auto viewRecord = reader->GetView<RAMNTupleRecord>("record");
+
+         for (auto i : reader->GetEntryRange()) {
+            const auto &record = viewRecord(i);
+            std::string actualChr = record.GetRNAME();
+            EXPECT_EQ(expectedChr, actualChr) << "Wrong chromosome in " << filename << " at entry " << i;
+         }
+      }
+   }
+}
+
+TEST_F(ChromosomeSplitTest, MetadataPresent)
+{
+   samtoramntuple_split_by_chromosome("test.sam", "test_split", 505, 1);
+
+   int filesChecked = 0;
+   for (const auto &entry : std::filesystem::directory_iterator(".")) {
+      std::string filename = entry.path().filename().string();
+      if (filename.find("test_split_") == 0 && filename.find(".root") != std::string::npos) {
+         auto metaReader = ROOT::Experimental::RNTupleReader::Open("METADATA", filename);
+         EXPECT_NE(metaReader, nullptr) << "Missing METADATA in " << filename;
+
+         if (metaReader) {
+            EXPECT_GT(metaReader->GetNEntries(), 0);
+         }
+
+         filesChecked++;
+      }
+   }
+
+   EXPECT_GT(filesChecked, 0);
+}
diff --git a/tools/samtoramntuple.cxx b/tools/samtoramntuple.cxx

Original file line number	Diff line number	Diff line change
`@@ -24,8 +24,9 @@ function(add_ramcore_test test_name)`
`24`	`24`	`endfunction()`
`25`	`25`
`26`	`26`	`add_ramcore_test(ramcoretests ramcoretests.cxx)`
	`27`	`+add_ramcore_test(chromosome_split_test chromosome_split_test.cxx)`
`27`	`28`
`28`		`-install(TARGETS ramcoretests`
	`29`	`+install(TARGETS ramcoretests chromosome_split_test`
`29`	`30`	`RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}`
`30`	`31`	`)`
`31`	`32`