Changes required to work with htsjdk 2.22.0

Remove CRAMContainerStreamWriter2 entirely
Use htsjdk snapshot for testing
Register VCFAltHeaderLine
Register more header line classes

Co-authored-by: Tom White <[email protected]>
Co-authored-by: Louis Bergelson <[email protected]>

Author: tomwhite

pom.xml

@@ -34,7 +34,7 @@
         <java.version>1.8</java.version>
         <maven.compiler.target>${java.version}</maven.compiler.target>
         <maven.compiler.source>${java.version}</maven.compiler.source>
-        <htsjdk.version>2.19.0</htsjdk.version>
+        <htsjdk.version>2.20.0</htsjdk.version>
         <hadoop.version>2.7.5</hadoop.version>
         <mockito.version>2.22.0</mockito.version>
         <kryo-serializers.version>0.42</kryo-serializers.version>

src/htsjdk/java/htsjdk/samtools/CRAMContainerStreamWriter2.java

@@ -11,6 +11,7 @@ public CachingBAMFileIndexOptimized(SeekableStream stream, SAMSequenceDictionary
     super(stream, dictionary);
   }
 
+  @Override
   protected BAMIndexContent query(final int referenceSequence, final int startPos, final int endPos) {
     seek(4);
 
@@ -31,7 +32,8 @@ protected BAMIndexContent query(final int referenceSequence, final int startPos,
     return super.query(referenceSequence, startPos, endPos);
   }
 
-  private void skipToSequence(final int sequenceIndex) {
+  @Override
+  protected void skipToSequence(final int sequenceIndex) {
     //Use sequence position cache if available
     if(sequenceIndexes[sequenceIndex] != -1){
       seek(sequenceIndexes[sequenceIndex]);

src/htsjdk/java/htsjdk/samtools/CachingBAMFileIndexOptimized.java

@@ -26,7 +26,7 @@ public void processIndex(CRAIIndex index, long partLength) {
   }
 
   private static CRAIEntry shift(CRAIEntry entry, long offset) {
-    return new CRAIEntry(entry.getSequenceId(), entry.getAlignmentStart(), entry.getAlignmentSpan(), entry.getContainerStartByteOffset() + offset, entry.getSliceByteOffset(), entry.getSliceByteSize());
+    return new CRAIEntry(entry.getSequenceId(), entry.getAlignmentStart(), entry.getAlignmentSpan(), entry.getContainerStartByteOffset() + offset, entry.getSliceByteOffsetFromCompressionHeaderStart(), entry.getSliceByteSize());
   }
 
   @Override

src/htsjdk/java/htsjdk/samtools/cram/CRAIIndexMerger.java

@@ -25,7 +25,8 @@
  */
 package org.disq_bio.disq.impl.formats.cram;
 
-import htsjdk.samtools.CRAMContainerStreamWriter2;
+import htsjdk.samtools.CRAMCRAIIndexer;
+import htsjdk.samtools.CRAMContainerStreamWriter;
 import htsjdk.samtools.SAMFileHeader;
 import htsjdk.samtools.SAMRecord;
 import htsjdk.samtools.cram.CRAIIndex;
@@ -83,7 +84,7 @@ public RecordWriter<Void, SAMRecord> getRecordWriter(TaskAttemptContext taskAtte
   static class CramRecordWriter extends RecordWriter<Void, SAMRecord> {
 
     private final OutputStream out;
-    private final CRAMContainerStreamWriter2 cramWriter;
+    private final CRAMContainerStreamWriter cramWriter;
 
     public CramRecordWriter(
         Configuration conf,
@@ -103,8 +104,12 @@ public CramRecordWriter(
         indexStream = null;
       }
       cramWriter =
-          new CRAMContainerStreamWriter2(
-              out, indexStream, referenceSource, header, file.toString());
+          new CRAMContainerStreamWriter(
+              out,
+              referenceSource,
+              header,
+              file.toString(),
+              indexStream == null ? null : new CRAMCRAIIndexer(indexStream, header));
     }
 
     @Override

src/main/java/org/disq_bio/disq/impl/formats/cram/CramOutputFormat.java

@@ -25,7 +25,7 @@
  */
 package org.disq_bio.disq.impl.formats.cram;
 
-import htsjdk.samtools.CRAMContainerStreamWriter2;
+import htsjdk.samtools.CRAMContainerStreamWriter;
 import htsjdk.samtools.SAMFileHeader;
 import htsjdk.samtools.SAMRecord;
 import htsjdk.samtools.cram.CRAIIndex;
@@ -134,8 +134,8 @@ private void writeHeader(
       OutputStream out,
       String headerFile,
       CRAMReferenceSource referenceSource) {
-    CRAMContainerStreamWriter2 cramWriter =
-        new CRAMContainerStreamWriter2(out, null, referenceSource, header, headerFile);
+    CRAMContainerStreamWriter cramWriter =
+        new CRAMContainerStreamWriter(out, null, referenceSource, header, headerFile);
     cramWriter.writeHeader(header);
     cramWriter.finish(false); // don't write terminator
   }

src/main/java/org/disq_bio/disq/impl/formats/cram/CramSink.java

@@ -166,7 +166,7 @@ private NavigableSet<Long> getContainerOffsetsFromFile(
       NavigableSet<Long> containerOffsets = new TreeSet<>();
       while (it.hasNext()) {
         Container container = it.next();
-        containerOffsets.add(container.offset);
+        containerOffsets.add(container.byteOffset);
       }
       containerOffsets.add(cramFileLength);
       return containerOffsets;

src/main/java/org/disq_bio/disq/impl/formats/cram/CramSource.java

@@ -125,6 +125,7 @@ public void registerClasses(final Kryo kryo) {
     kryo.register(htsjdk.variant.variantcontext.VariantContext.Type.class);
 
     // htsjdk.variant.vcf
+    kryo.register(htsjdk.variant.vcf.VCFAltHeaderLine.class);
     kryo.register(htsjdk.variant.vcf.VCFCompoundHeaderLine.SupportedHeaderLineType.class);
     kryo.register(htsjdk.variant.vcf.VCFContigHeaderLine.class);
     kryo.register(htsjdk.variant.vcf.VCFFilterHeaderLine.class);
@@ -133,7 +134,11 @@ public void registerClasses(final Kryo kryo) {
     kryo.register(htsjdk.variant.vcf.VCFHeaderLine.class);
     kryo.register(htsjdk.variant.vcf.VCFHeaderLineCount.class);
     kryo.register(htsjdk.variant.vcf.VCFHeaderLineType.class);
+    kryo.register(htsjdk.variant.vcf.VCFHeaderVersion.class);
     kryo.register(htsjdk.variant.vcf.VCFInfoHeaderLine.class);
+    kryo.register(htsjdk.variant.vcf.VCFMetaHeaderLine.class);
+    kryo.register(htsjdk.variant.vcf.VCFPedigreeHeaderLine.class);
+    kryo.register(htsjdk.variant.vcf.VCFSampleHeaderLine.class);
     kryo.register(htsjdk.variant.vcf.VCFSimpleHeaderLine.class);
 
     // java.io