bloom

Author: adienes

lazy_iterate.jl

@@ -0,0 +1,66 @@
+using BitIntegers
+
+using Base: count_ones
+import Base: in!, iterate, IteratorSize
+
+function _hash_bloom(x, seeds, T)
+    out = zero(T)
+    for h in seeds
+        out |= one(T)<<(hash(x, h) % (8sizeof(T)))
+    end
+    return out
+end
+
+mutable struct BloomFilter{T,K}
+    memory::T
+    const seeds::NTuple{K, UInt64}
+    const p::Float64 # max tolerable false positive rate
+    function BloomFilter{T,K}(p, seeds) where {T<:Unsigned,K}
+        new{T,K}(zero(T), seeds, p)
+    end
+end
+
+function BloomFilter(; k=5, p=1/16)
+    return BloomFilter{UInt64, k}(p, ntuple(_ -> rand(UInt), k))
+end
+
+function in!(x, f::BloomFilter{T}) where {T}
+    h = _hash_bloom(x, f.seeds, T)
+    r = (f.memory & h) == h
+    f.memory |= h
+    return r
+end
+
+function false_positive_rate(f::BloomFilter{T,K}) where {T,K}
+    pop = count_ones(f.memory)
+    bits = 8sizeof(T)
+    return (pop/bits)^K
+end
+
+struct Unique
+    f::Set
+    g
+end
+
+function iterate(u::Unique)
+    x, next = iterate(u.g)
+    push!(u.f, x)
+    return (x, next)
+end
+
+function iterate(u::Unique, state)
+    it = iterate(u.g, state)
+    if isnothing(it)
+        return nothing
+    else
+        x, next = it
+        p = in!(x, u.f)
+        if !p # definitely unique
+            return (x, next)
+        else
+            return iterate(u, next)
+        end
+    end
+end
+
+IteratorSize(::Unique) = Base.SizeUnknown()