Merge pull request #1756 from dib-lab/fix/document-cqf

[MRG] Document QFCounttable

Author: standage

khmer/__init__.py

@@ -86,6 +86,8 @@
 
 
 _buckets_per_byte = {
+    # calculated by hand from settings in third-part/cqf/gqf.h
+    'qfcounttable': 1/1.26,
     'countgraph': 1,
     'smallcountgraph': 2,
     'nodegraph': 8,

khmer/_oxli/graphs.pxd

@@ -37,9 +37,9 @@ cdef extern from "khmer/_cpy_khmer.hh":
 cdef extern from "oxli/storage.hh":
     cdef cppclass CpStorage "oxli::Storage":
         CpStorage()
-        
+
         vector[uint64_t] get_tablesizes()
-        const size_t n_tables() 
+        const size_t n_tables()
         void save(string, WordLength)
         void load(string, WordLength&)
         const uint64_t n_occupied()
@@ -85,11 +85,11 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli":
         void consume_seqfile_banding[SeqIO](shared_ptr[CpReadParser[SeqIO]]&,
                                     uint32_t, uint32_t, uint32_t &, uint64_t &) except +oxli_raise_py_error
 
-        void consume_seqfile_banding_with_mask[SeqIO](const string &, uint32_t, uint32_t, 
-                                                      CpHashtable *, uint32_t, uint32_t &, 
+        void consume_seqfile_banding_with_mask[SeqIO](const string &, uint32_t, uint32_t,
+                                                      CpHashtable *, uint32_t, uint32_t &,
                                                       uint64_t &) except +oxli_raise_py_error
         void consume_seqfile_banding_with_mask[SeqIO](shared_ptr[CpReadParser[SeqIO]]&,
-                                                      uint32_t, uint32_t, 
+                                                      uint32_t, uint32_t,
                                                       CpHashtable *, uint32_t,
                                                       uint32_t &, uint64_t &) except +oxli_raise_py_error
 
@@ -132,7 +132,7 @@ cdef extern from "oxli/hashtable.hh" namespace "oxli":
         CpNodetable(WordLength, vector[uint64_t])
 
     cdef cppclass CpQFCounttable "oxli::QFCounttable" (CpHashtable):
-        CpQFCounttable(WordLength, int) except +oxli_raise_py_error
+        CpQFCounttable(WordLength, uint64_t) except +oxli_raise_py_error
 
 
 cdef extern from "oxli/hashgraph.hh" namespace "oxli":

khmer/_oxli/graphs.pyx

@@ -45,7 +45,7 @@ cdef CpHashtable * hashtable_arg_shim(object table,
                                                SmallCounttable, QFCounttable)):
     cdef CPyHashtable_Object* cpyhashtable
     cdef CpHashtable * hashtable
-                                          
+
     if isinstance(table, allowed):
         if isinstance(table, CYTHON_TABLES):
             hashtable = (<Hashtable>table).c_table.get()
@@ -226,7 +226,7 @@ cdef class Hashtable:
                                                                      total_reads,
                                                                      n_consumed)
         return total_reads, n_consumed
-                                                                     
+
     def consume_seqfile_banding(self, file_name, num_bands, band):
         """Count all k-mers from file_name."""
         cdef unsigned long long n_consumed = 0
@@ -270,7 +270,7 @@ cdef class Hashtable:
         """Calculate the k-mer abundance distribution over reads."""
         cdef CpHashtable * cptracking = hashtable_arg_shim(tracking,
                                                       allowed=(PyNodegraph, Nodetable))
- 
+
         cdef CPyReadParser_Object* parser
         parser = <CPyReadParser_Object*>read_parser
         cdef uint64_t * x = deref(self.c_table).abundance_distribution[CpFastxReader](
@@ -311,14 +311,36 @@ cdef class Hashtable:
 
 
 cdef class QFCounttable(Hashtable):
-    def __cinit__(self, int k, int starting_size):
-        # starting size has to be a power of two
-        power_of_two = ((starting_size & (starting_size - 1) == 0) and
-                        (starting_size != 0))
+    """Count kmers using a counting quotient filter.
+
+    The counting quotient filter (CQF) is an extension of the quotient filter
+    that supports counting in addition to simple membership testing. A CQF has
+    better cache locality compared to (Small)Counttable which increases
+    performance.
+
+    Each new k-mer uses one slot, and the number of slots used per k-mer
+    increases the more often the same k-mer is entered into the CQF. As a result
+    the CQF can be "full" and will stop accepting calls to `add` and `count`.
+
+    Parameters
+    ----------
+    k : integer
+        k-mer size
+
+    size : integer
+        Set the number of slots used by the counting quotient filter. This
+        determines the amount of memory used and how many k-mers can be entered
+        into the datastructure. Each slot uses roughly 1.3 bytes.
+    """
+    def __cinit__(self, int k, uint64_t size):
+        # size has to be a power of two
+        power_of_two = ((size & (size - 1) == 0) and
+                        (size != 0))
         if not power_of_two:
-            raise ValueError("starting_size has to be a power of two.")
+            raise ValueError("size has to be a power of two, not"
+                             " {}.".format(size))
         if type(self) is QFCounttable:
-            self.c_table.reset(<CpHashtable*>new CpQFCounttable(k, int(log(starting_size, 2))))
+            self.c_table.reset(<CpHashtable*>new CpQFCounttable(k, int(log(size, 2))))
 
     @classmethod
     def load(cls, file_name):
@@ -346,5 +368,3 @@ cdef class Nodetable(Hashtable):
         if type(self) is Nodetable:
             primes = get_n_primes_near_x(n_tables, starting_size)
             self.c_table.reset(<CpHashtable*>new CpNodetable(k, primes))
-
-

tests/test_script_arguments.py

@@ -109,6 +109,7 @@ def test_check_tablespace(graph_type, buckets_per_byte):
 
 
 @pytest.mark.parametrize('graph_type,exp_buckets', [
+    ('qfcounttable', '2.4 million buckets'),
     ('countgraph', '3.0 million buckets'),
     ('smallcountgraph', '6.0 million buckets'),
     ('nodegraph', '24.0 million buckets'),