Merge pull request #794 from JuliaAI/dev

For a 0.20.7 release

Author: ablaom

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJBase"
 uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.20.6"
+version = "0.20.7"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"

src/measures/finite.jl

@@ -966,12 +966,7 @@ function _mtn(m::CM, return_type::Type{LittleDict})
     return LittleDict(m.labels, vec(_sum))
 end
 
-@inline function _mean(x::Arr{<:Real})
-    for i in eachindex(x)
-        @inbounds x[i] = ifelse(isnan(x[i]), zero(eltype(x)), x[i])
-    end
-    return mean(x)
-end
+@inline _mean(x::Arr{<:Real}) = mean(skipnan(x)) # defined in src/data/data.jl
 
 @inline function _class_w(level_m::Arr{<:String},
                           class_w::AbstractDict{<:Any, <:Real})

src/resampling.jl

@@ -827,11 +827,11 @@ measure or vector.
 
 Do `subtypes(MLJ.ResamplingStrategy)` to obtain a list of available
 resampling strategies. If `resampling` is not an object of type
-`MLJ.ResamplingStrategy`, then a vector of pairs (of the form
+`MLJ.ResamplingStrategy`, then a vector of tuples (of the form
 `(train_rows, test_rows)` is expected. For example, setting
 
-    resampling = [(1:100), (101:200)),
-                   (101:200), (1:100)]
+    resampling = [((1:100), (101:200)),
+                   ((101:200), (1:100))]
 
 gives two-fold cross-validation using the first 200 rows of data.
 
@@ -1164,7 +1164,7 @@ function evaluate!(mach::Machine, resampling, weights,
 
     if !(resampling isa TrainTestPairs)
         error("`resampling` must be an "*
-              "`MLJ.ResamplingStrategy` or tuple of pairs "*
+              "`MLJ.ResamplingStrategy` or tuple of rows "*
               "of the form `(train_rows, test_rows)`")
     end
 

test/measures/finite.jl

@@ -284,6 +284,28 @@ end
     2 ./ ( 1 ./ ( sum(cm_tp) ./ sum(cm_fp.+cm_tp) ) + 1 ./ ( sum(cm_tp) ./ sum(cm_fn.+cm_tp) ) )
 end
 
+@testset "issue #630" begin
+    # multiclass fscore corner case of absent class
+
+    y = coerce([1, 2, 2, 2, 3], OrderedFactor)[1:4]
+    # [1, 2, 2, 2] # but 3 is in the pool
+    yhat = reverse(y)
+    # [2, 2, 2, 1]
+
+    # In this case, assigning "3" as "positive" gives all true negative,
+    # and so NaN for that class's contribution to the average F1Score,
+    # which should accordingly be skipped.
+
+    # postive class | TP | FP | FN | score for that class
+    # --------------|----|----|----|---------------------
+    #  1            | 0  | 1  | 2  | 0
+    #  2            | 2  | 1  | 1  | 2/3
+    #  3            | 0  | 0  | 0  | NaN
+
+    # mean score with skippin NaN is 1/3
+    @test MulticlassFScore()(yhat, y) ≈ 1/3
+end
+
 @testset "Metadata binary" begin
     for m in (accuracy, recall, Precision(), f1score, specificity)
         e = info(m)

test/resampling.jl

@@ -486,24 +486,43 @@ end
 end
 
 @testset_accelerated "class weights in evaluation" accel begin
-    x = [1,2,3,4,5,6,7]
-    X, y = Tables.table([x x x x x x]), coerce([1,2,1,3,1,2,2], Multiclass)
-    model = Models.DeterministicConstantClassifier()
-    mach = machine(model, X, y)
+    X, y = make_blobs(rng=rng)
     cv=CV(nfolds = 2)
+    fold1, fold2 = partition(eachindex(y), 0.5)
+    m = MLJBase.MulticlassFScore()
     class_w = Dict(1=>1, 2=>2, 3=>3)
-    e = evaluate!(mach,
-                  resampling=cv,
-                  measure=MLJBase.MulticlassFScore(return_type=Vector),
-                  class_weights=class_w,
-                  verbosity=verb,
-                  acceleration=accel).measurement[1]
-    @test round(e, digits=3) ≈ 0.217
+
+    model = Models.DeterministicConstantClassifier()
+    mach = machine(model, X, y)
+
+    # fscore by hand:
+    fit!(mach, rows=fold1, verbosity=0)
+    score1 = m(predict(mach, rows=fold2), y[fold2], class_w)
+    fit!(mach, rows=fold2, verbosity=0)
+    score2 = m(predict(mach, rows=fold1), y[fold1], class_w)
+    score_by_hand = mean([score1, score2])
+
+    # fscore by evaluate!:
+    score = evaluate!(
+        mach,
+        resampling=cv,
+        measure=m,
+        class_weights=class_w,
+        verbosity=verb,
+        acceleration=accel,
+    ).measurement[1]
+
+    @test score ≈ score_by_hand
 
     # if class weights in `evaluate!` isn't specified:
-    e = evaluate!(mach, resampling=cv, measure=multiclass_f1score,
-                  verbosity=verb, acceleration=accel).measurement[1]
-    @test e ≈ 0.15
+    plain_score = evaluate!(
+        mach,
+        resampling=cv,
+        measure=m,
+        verbosity=verb,
+        acceleration=accel,
+    ).measurement[1]
+    @test !(score ≈ plain_score)
 end
 
 @testset_accelerated "resampler as machine" accel begin
@@ -794,4 +813,3 @@ end
 
 #end
 true
-