Merge pull request #55 from alan-turing-institute/hackmat

Hackmat (addresses #49)

Author: ablaom

src/ScientificTypes.jl

@@ -10,6 +10,8 @@ export autotype
 
 using Tables, CategoricalArrays, ColorTypes
 
+const CategoricalElement{U} = Union{CategoricalValue{<:Any,U},CategoricalString{U}}
+
 # ## FOR DEFINING SCITYPES ON OBJECTS DETECTED USING TRAITS
 
 # We define a "dynamically" extended function `trait`:

src/conventions/mlj/finite.jl

@@ -22,6 +22,19 @@ function _finalize_finite_coerce(v, verbosity, T2)
     return categorical(v, true, ordered=T2<:Union{Missing,OrderedFactor})
 end
 
+# HACK: The following method (9 lines) can be removed after resolution
+# of https://github.com/JuliaData/CategoricalArrays.jl/issues/226:
+function _finalize_finite_coerce(v::CategoricalArray, verbosity, T2)
+    su = scitype_union(v)
+    if su >: Missing && !(T2 >: Missing)
+        verbosity > 0 && _coerce_missing_warn(T2)
+    end
+    if su <: T2
+        return v
+    end
+    return ordered!(compress(v), T2<:Union{Missing,OrderedFactor})
+end
+
 # if v is not a CategoricalArray:
 function coerce(v::AbstractArray,
                 ::Type{T2}; verbosity=1) where T2<:Union{Missing,Finite}
@@ -35,14 +48,9 @@ coerce(v::CategoricalArray,
        ::Type{T2}; verbosity=1) where T2<:Union{Missing,Finite} =
            _finalize_finite_coerce(v, verbosity, T2)
 
-# if v is a CategoricalArray{Any}
+# if v is a CategoricalArray{Any} (a bit of a hack):
 function coerce(v::CategoricalArray{Any},
                 ::Type{T2}; verbosity=1)  where T2<:Union{Missing,Finite}
-
-    # AFTER CategoricalArrays 0.7.2 IS RELEASED:
-    # return _finalize_finite_coerce(broadcast(identity, v), verbosity, T2)
-
-    # TEMPORARY HACK:
     levels_ = levels(v)
     isordered_ = isordered(v)
     vraw = broadcast(get_, v)

src/conventions/mlj/mlj.jl

@@ -1,10 +1,10 @@
 scitype(::AbstractFloat, ::Val{:mlj}) = Continuous
 scitype(::Integer, ::Val{:mlj}) = Count
 
-_coerce_missing_warn(T) =
-    @warn "Missing values encountered coercing scitype to $T.\n"*
-          "Coerced to Union{Missing,$T} instead. "
-
+function _coerce_missing_warn(::Type{T}) where T
+    T >: Missing || @warn "Missing values encountered coercing scitype to $T.\n"*
+                          "Coerced to Union{Missing,$T} instead. "
+end
 
 # ## IMPLEMENT PERFORMANCE BOOSTING FOR ARRAYS
 
@@ -27,6 +27,9 @@ function coerce(y::AbstractArray{<:Union{Missing,Real}},
     return float(y)
 end
 
+_float(y::CategoricalElement) = float(_int(y))
+_float(y) = float(y)
+
 # NOTE: case where the data may have been badly encoded and resulted
 # in an Any[] array a user should proceed with caution here in
 # particular: - if at one point it encounters a type for which there
@@ -39,14 +42,15 @@ function coerce(y::AbstractArray, T::Type{<:Union{Missing,Continuous}}; verbosit
     has_chars    = findfirst(e->isa(e,Char), y) !== nothing
     has_chars && verbosity > 0 && @warn "Char values will be coerced to " *
                                         "AbstractFloat (e.g. 'A' to 65.0)."
-    return float.(y)
+    return _float.(y)
 end
 
 
 ## COERCE ARRAY TO COUNT
 
 _int(::Missing)  = missing
 _int(x::Integer) = x
+_int(x::CategoricalElement) = CategoricalArrays.order(x.pool)[x.level]
 _int(x) = Int(x) # may throw InexactError
 
 # no-op case

src/tables.jl

@@ -36,6 +36,11 @@ function _coerce_col(X, name, types; args...)
     Xcol = Tables.columntable(X)
     y = getproperty(X, name)
     if haskey(types, name)
+        # HACK y isa LazyArrays.ApplyArray, see issue #49
+        props = propertynames(y)
+        if :f in props && :args in props
+            y = convert(Vector, y)
+        end
         return coerce(y, types[name]; args...)
     else
         return y

test/runtests.jl

@@ -169,7 +169,16 @@ end
     @test scitype_union(coerce([:x, :y], Finite)) === Multiclass{2}
     @test scitype_union(@test_logs((:warn, r"Missing values encountered"),
                                 coerce([:x, :y, missing], Finite))) ===
-                                       Union{Missing, Multiclass{2}}
+                                    Union{Missing, Multiclass{2}}
+
+    # More finite conversions (to check resolution of #48):
+    y = categorical([1, 2, 3, missing]) # unordered
+    yc = coerce(y, OrderedFactor)
+    @test isordered(yc)
+    @test yc[1].pool.ordered
+    scitype(y) == AbstractVector{OrderedFactor{2}}
+    scitype_union(y) == OrderedFactor{2}
+
 end
 
 @testset "coercion works for arrays too" begin
@@ -215,4 +224,25 @@ end
     @test eltype(v2c) <: CategoricalValue{Char}
 end
 
+@testset "Cat->Count,Continuous (mlj)" begin
+    a = categorical(["a","b","a","b",missing])
+    a1 = coerce(a, Union{Count,Missing})
+    @test scitype_union(a1) == Union{Missing,Count}
+    @test all(skipmissing(a1 .== [1, 2, 1, 2, missing]))
+    a1 = coerce(a, Union{Continuous,Missing})
+    @test scitype_union(a1) == Union{Missing,Continuous}
+    @test all(skipmissing(a1 .== [1., 2., 1., 2., missing]))
+
+    # XXX
+
+    y = categorical(1:10, ordered=true)
+    new_order = [4, 10, 9, 7, 6, 2, 8, 3, 1, 5]
+    levels!(y, new_order)
+    @test all(coerce(y, Count) .== sortperm(new_order))
+    @test all(coerce(y, Count) .== [9, 6, 8, 1, 10, 5, 4, 7, 3, 2])
+
+    y = categorical([1:10..., missing, 11], ordered=true)
+    @test all(skipmissing(coerce(y, Union{Continuous, Missing}) .== float.([1:10...,missing,11])))
+end
+
 include("autotype.jl")