Merge pull request #177 from JuliaAI/dev

For a 3.0.0 release

Author: ablaom

Project.toml

@@ -20,9 +20,9 @@ ColorTypes = "0.9, 0.10, 0.11"
 Distributions = "0.25.1"
 PrettyTables = "1"
 Reexport = "1.2"
-ScientificTypesBase = "2.2"
-StatisticalTraits = "2"
-Tables = "1"
+ScientificTypesBase = "3.0"
+StatisticalTraits = "3.0"
+Tables = "1.6.1"
 julia = "1"
 
 [extras]

README.md

@@ -45,9 +45,8 @@ The module `ScientificTypes` defined in this repo rexports the
 scientific types and associated methods defined in [ScientificTypesBase.jl](https://github.com/JuliaAI/ScientificTypesBase.jl)
 and provides:
 
-- a collection of `ScientificTypes.scitype` definitions that
-  articulate a default convention, importing the module automatically
-  activating the convention
+- a collection of `scitype` definitions that
+  articulate a default convention.
 
 - a `coerce` function, for changing machine types to reflect a specified
   scientific interpretation (scientific type)
@@ -75,17 +74,15 @@ sch = schema(X)
 will print
 
 ```
-_.table =
-┌─────────┬─────────────────────────┬────────────────────────────┐
-│ _.names │ _.types                 │ _.scitypes                 │
-├─────────┼─────────────────────────┼────────────────────────────┤
-│ a       │ Float64                 │ Continuous                 │
-│ b       │ Union{Missing, Float64} │ Union{Missing, Continuous} │
-│ c       │ Int64                   │ Count                      │
-│ d       │ Int64                   │ Count                      │
-│ e       │ Union{Missing, Char}    │ Union{Missing, Unknown}    │
-└─────────┴─────────────────────────┴────────────────────────────┘
-_.nrows = 5
+┌───────┬────────────────────────────┬─────────────────────────┐
+│ names │ scitypes                   │ types                   │
+├───────┼────────────────────────────┼─────────────────────────┤
+│ a     │ Continuous                 │ Float64                 │
+│ b     │ Union{Missing, Continuous} │ Union{Missing, Float64} │
+│ c     │ Count                      │ Int64                   │
+│ d     │ Count                      │ Int64                   │
+│ e     │ Union{Missing, Unknown}    │ Union{Missing, Char}    │
+└───────┴────────────────────────────┴─────────────────────────┘
 ```
 
 Detail is obtained in the obvious way; for example:
@@ -105,17 +102,15 @@ schema(Xc)
 which prints
 
 ```
-_.table =
-┌─────────┬──────────────────────────────────────────────┬───────────────────────────────┐
-│ _.names │ _.types                                      │ _.scitypes                    │
-├─────────┼──────────────────────────────────────────────┼───────────────────────────────┤
-│ a       │ Float64                                      │ Continuous                    │
-│ b       │ Union{Missing, Int64}                        │ Union{Missing, Count}         │
-│ c       │ Int64                                        │ Count                         │
-│ d       │ CategoricalValue{Int64,UInt32}               │ Multiclass{2}                 │
-│ e       │ Union{Missing, CategoricalValue{Char,UInt32}}│ Union{Missing, Multiclass{2}} │
-└─────────┴──────────────────────────────────────────────┴───────────────────────────────┘
-_.nrows = 5
+┌───────┬───────────────────────────────┬────────────────────────────────────────────────┐
+│ names │ scitypes                      │ types                                          │
+├───────┼───────────────────────────────┼────────────────────────────────────────────────┤
+│ a     │ Continuous                    │ Float64                                        │
+│ b     │ Union{Missing, Count}         │ Union{Missing, Int64}                          │
+│ c     │ Count                         │ Int64                                          │
+│ d     │ Multiclass{2}                 │ CategoricalValue{Int64, UInt32}                │
+│ e     │ Union{Missing, Multiclass{2}} │ Union{Missing, CategoricalValue{Char, UInt32}} │
+└───────┴───────────────────────────────┴────────────────────────────────────────────────┘
 
 ```
 

docs/Project.toml

@@ -7,4 +7,4 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
 Documenter = "0.25"
-ScientificTypesBase = "2.1"
+ScientificTypesBase = "3"

docs/src/index.md

@@ -463,8 +463,6 @@ before passing to `coerce`.
 
 ```@docs
 ScientificTypes.scitype
-ScientificTypes.elscitype
-ScientificTypes.scitype_union
 coerce
 autotype
 ```

src/ScientificTypes.jl

@@ -3,33 +3,35 @@ module ScientificTypes
 # Dependencies
 using Reexport
 @reexport using ScientificTypesBase
+export scitype, elscitype, scitype_union
 using Tables
 using CategoricalArrays
 using ColorTypes
 using PrettyTables
 using Dates
 import Distributions
 
-import StatisticalTraits: info
-
-# re-export from StatisticalTraits
-export info
-
 # exports
 export coerce, coerce!, autotype, schema, levels, levels!
 
-# -------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 # Abbreviations
 
-const ST   = ScientificTypesBase
-const Arr  = AbstractArray
+const ST  = ScientificTypesBase
+const Arr = AbstractArray
 const CArr = CategoricalArray
-const Cat  = CategoricalValue
+const Cat = CategoricalValue
+const COLS_SPECIALIZATION_THRESHOLD = 30
+const ROWS_SPECIALIZATION_THRESHOLD = 10000
+const SCHEMA_SPECIALIZATION_THRESHOLD = Tables.SCHEMA_SPECIALIZATION_THRESHOLD
 
-# Indicate the convention, see init.jl where it is set.
+#---------------------------------------------------------------------------------------
+# Define convention
 struct DefaultConvention <: Convention end
-
-include("init.jl")
+const CONV = DefaultConvention()
+# -------------------------------------------------------------
+# vtrait function, returns either `Val{:table}()` or `Val{:other}()`
+vtrait(X) = Val{ifelse(Tables.istable(X), :table, :other)}()
 
 # -------------------------------------------------------------
 # Includes

src/autotype.jl

@@ -12,11 +12,13 @@ an array based on rules
                               autotype, `only_changes` should be true.
 * `rules=(:few_to_finite,)`: the set of rules to apply.
 """
-autotype(X; kw...) = _autotype(X, Val(ST.trait(X)); kw...)
+autotype(X; kw...) = _autotype(X, vtrait(X); kw...)
 
 # For an array object (trait:other)
-function _autotype(X::Arr, ::Val{:other};
-                   rules::NTuple{N,Symbol} where N=(:few_to_finite,))
+function _autotype(
+    X::Arr, ::Val{:other};
+    rules::NTuple{N,Symbol} where N = (:few_to_finite,)
+)
     # check that the rules are recognised
     _check_rules(rules)
     # inspect the current element scitype
@@ -36,8 +38,11 @@ function _autotype(X::Arr, ::Val{:other};
 end
 
 # For a table object (trait:table)
-function _autotype(X, ::Val{:table}; only_changes::Bool=true,
-                  rules::NTuple{N,Symbol} where N=(:few_to_finite,))
+function _autotype(
+    X, ::Val{:table}; 
+    only_changes::Bool=true,
+    rules::NTuple{N,Symbol} where N = (:few_to_finite,)
+)
     # check that the rules are recognised
     _check_rules(rules)
     # recuperate the schema of `X`
@@ -59,7 +64,7 @@ function _autotype(X, ::Val{:table}; only_changes::Bool=true,
         # doesn't really matter, there are few rules and sugg is fast
         sugg_type = stype
         for rule in rules
-            sugg_type = eval(:($rule($sugg_type, $col, $sch.nrows)))
+            sugg_type = eval(:($rule($sugg_type, $col, $(nrows(X)))))
         end
         # store the suggested type
         suggested_types[name] = sugg_type
@@ -76,16 +81,49 @@ function _autotype(X, ::Val{:table}; only_changes::Bool=true,
 end
 
 # convenience functions to pass a single rule at the time
-autotype(X, rule::Symbol; args...) =
-    autotype(X; rules=(rule,), args...)
+autotype(X, rule::Symbol; args...) = autotype(X; rules=(rule,), args...)
 # convenience function to splat rules
-autotype(X, rules::NTuple{N,Symbol} where N; args...) =
-    autotype(X; rules=rules, args...)
+autotype(X, rules::NTuple{N,Symbol} where N; args...) = autotype(X; rules=rules, args...)
+
+"""
+    nrows(X)
+
+Helper method to return the number of rows a table `X` has.
+
+**Note**
+A more general version of this method is defined in `MLJModelInterface.jl`.
+This method is needed here in order for `auto_type` method to run. 
+"""
+function nrows(X)
+    if !Tables.istable(X)
+        throw(ArgumentError("input argument must be a Tables.jl compatible table"))
+    end
+    if Tables.rowaccess(X)
+        rows = Tables.rows(X)
+        return _nrows_rat(Base.IteratorSize(typeof(rows)), rows)
+        
+    else
+        cols = Tables.columns(X)
+        return _nrows_cat(cols)
+    end
+end
+
+# number of rows for columnaccessed table
+function _nrows_cat(cols)
+    names = Tables.columnnames(cols)
+    !isempty(names) || return 0
+    return length(Tables.getcolumn(cols, names[1]))
+end
+
+# number of rows for rowaccessed table
+_nrows_rat(::Base.HasShape, rows) = size(rows, 1)
+_nrows_rat(::Base.HasLength, rows) = length(rows)
+_nrows_rat(iter_size, rows) = length(collect(rows))
 
 # -----------------------------------------------------------------
 # rules
 
-function _check_rules(rules::NTuple{N,Symbol} where N)
+function _check_rules(rules::NTuple{N, Symbol} where N)
     for rule in rules
         rule in (:few_to_finite,
                  :discrete_to_continuous,
@@ -161,15 +199,14 @@ end
 Helper function to suggest a finite type corresponding to `T` when there are
 few unique values.
 """
-function sugg_finite(::Type{<:Union{Missing,T}}) where T
-    T <: Real && return OrderedFactor
-    return Multiclass
+function sugg_finite(type::Type)
+    return ifelse(nonmissing(type) <: Real, OrderedFactor, Multiclass)
 end
 
 """
     T_or_Union_Missing_T(type, T)
 
-Helper function to return either `T` or `Union{Missing,T}`.
+Helper function to return either `T` or `Union{Missing, T}`.
 """
 function T_or_Union_Missing_T(type::Type, T::Type)
     return ifelse(type >: Missing, Union{Missing, T}, T)

src/coerce.jl

@@ -1,20 +1,35 @@
-const ColKey = Union{Symbol,AbstractString}
+const ColKey = Union{Symbol, AbstractString}
 
 """
-    coerce(A, specs...; tight=false, verbosity=1)
+    coerce(A, S)
 
-Given a table `A`, return a copy of `A`, ensuring that the element
+Return new version of the array `A` whose scientific element type is `S`.
+
+```
+julia> v = coerce([3, 7, 5], Continuous)
+3-element Vector{Float64}:
+ 3.0
+ 7.0
+ 5.0
+
+julia> scitype(v)
+AbstractVector{Continuous}
+
+```
+    coerce(X, specs...; tight=false, verbosity=1)
+
+Given a table `X`, return a copy of `X`, ensuring that the element
 scitypes of the columns match the new specification, `specs`. There
-are three valid specifiations:
+are three valid specifications:
 
 (i) one or more `column_name=>Scitype` pairs:
 
-    coerce(X, col1=>Sciyype1, col2=>Scitype2, ... ; verbosity=1)
+    coerce(X, col1=>Scitype1, col2=>Scitype2, ... ; verbosity=1)
 
 (ii) one or more `OldScitype=>NewScitype` pairs (`OldScitype` covering
 both the `OldScitype` and `Union{Missing,OldScitype}` cases):
 
-    coerce(X, OldScitype1=>NewSciyype1, OldScitype2=>NewScitype2, ... ; verbosity=1)
+    coerce(X, OldScitype1=>NewScitype1, OldScitype2=>NewScitype2, ... ; verbosity=1)
 
 (iii) a dictionary of scientific types keyed on column names:
 
@@ -24,7 +39,7 @@ where `ColKey = Union{Symbol,AbstractString}`.
 
 ### Examples
 
-Specifiying  `column_name=>Scitype` pairs:
+Specifying  `column_name=>Scitype` pairs:
 
 ```
 using CategoricalArrays, DataFrames, Tables
@@ -45,7 +60,7 @@ Xc = coerce(X, Count=>Continuous)
 schema(Xfixed).scitypes # (Continuous, Continuous, Continuous)
 ```
 """
-coerce(X, a...; kw...) = coerce(Val(ST.trait(X)), X, a...; kw...)
+coerce(X, a...; kw...) = coerce(vtrait(X), X, a...; kw...)
 
 # Non tabular data is not supported
 coerce(::Val{:other}, X, a...; kw...) =
@@ -156,14 +171,14 @@ end
 """
 coerce!(X, ...)
 
-Same as [`ScientificTypes.coerce`](@ref) except it does the modification in
-place provided `X` supports in-place modification (at the moment, only the
-DataFrame! does). An error is thrown otherwise. The arguments are the same as
-`coerce`.
+Same as [`ScientificTypes.coerce`](@ref) except it does the
+modification in place provided `X` supports in-place modification (eg,
+DataFrames). An error is thrown otherwise. The arguments are the same
+as `coerce`.
 
 """
 coerce!(X, a...;  kw...) = begin
-    coerce!(Val(ST.trait(X)), X, a...; kw...)
+    coerce!(vtrait(X), X, a...; kw...)
 end
 
 coerce!(::Val{:other}, X, a...; kw...) =

src/convention/coerce.jl

@@ -223,6 +223,35 @@ F = Real
 _2d{C} =AbstractArray{C} where C<:Union{ColorTypes.AbstractRGB, ColorTypes.Gray}
 
 # Single Image
+
+"""
+    coerce(image::AbstractArray{<:Real, N}, I)
+
+Given a an array called `image` representing one or more images,
+return a transformed version of the data so as to enforce an
+appropriate scientific interpretation `I`:
+
+single or collection ? | N | I                  | `scitype` of result
+-----------------------|---|--------------------|----------------------------
+single                 | 2 | `GrayImage`        | `GrayImage{W,H}`
+single                 | 3 | `ColorImage`       | `ColorImage{W,H}`
+collection             | 3 | `GrayImage`        | `AbstractVector{<:GrayImage}`
+collection             | 4 (W x H x {1} x C)| `GrayImage` | `AbstractVector{<:GrayImage}`
+collection             | 4 | `ColorImage`       | `AbstractVector{<:ColorImage}`
+
+```
+imgs = rand(10, 10, 3, 5)
+v = coerce(imgs, ColorImage)
+
+julia> typeof(v)
+Vector{Matrix{ColorTypes.RGB{Float64}}}
+
+julia> scitype(v)
+AbstractVector{ColorImage{10, 10}}
+
+```
+
+"""
 function coerce(y::Arr{<:F, 2}, T2::Type{GrayImage})
     return ColorTypes.Gray.(y)
 end
@@ -249,4 +278,4 @@ end
 
 function coerce(y::_4Dcollection, T2::Type{ColorImage})
     return [broadcast(ColorTypes.RGB, y[:,:,1, idx], y[:,:,2,idx], y[:,:,3, idx]) for idx=1:size(y,4)]
-end
+end