Evaluating Model Performance

MLJ allows quick evaluation of a supervised model's performance against a battery of selected losses or scores. For more on available performance measures, see Performance Measures.

In addition to hold-out and cross-validation, the user can specify their own list of train/test pairs of row indices for resampling, or define their own re-usable resampling strategies.

For simultaneously evaluating multiple models and/or data sets, see Benchmarking.

Evaluating against a single measure

julia> using MLJ

julia> X = (a=rand(12), b=rand(12), c=rand(12));

julia> y = X.a + 2X.b + 0.05*rand(12);

julia> model = @load RidgeRegressor pkg=MultivariateStats
RidgeRegressor(
    lambda = 1.0) @ 8…84

julia> cv=CV(nfolds=3)
CV(
    nfolds = 3,
    shuffle = false,
    rng = MersenneTwister(UInt32[0x5e195684, 0x67952e4a, 0x3888593c, 0x4fe704ab]) @ 379) @ 1…97

julia> evaluate(model, X, y, resampling=cv, measure=l2, verbosity=0)
┌───────────┬───────────────┬────────────────────────┐
│ _.measure │ _.measurement │ _.per_fold             │
├───────────┼───────────────┼────────────────────────┤
│ l2        │ 0.277         │ [0.0722, 0.536, 0.223] │
└───────────┴───────────────┴────────────────────────┘
_.per_observation = [[[0.00026, 0.0324, ..., 0.042], [0.69, 0.212, ..., 1.09], [0.134, 0.22, ..., 0.534]]]

Alternatively, instead of applying evaluate to a model + data, one may call evaluate! on an existing machine wrapping the model in data:

julia> mach = machine(model, X, y)
Machine{RidgeRegressor} @ 1…28

julia> evaluate!(mach, resampling=cv, measure=l2, verbosity=0)
┌───────────┬───────────────┬────────────────────────┐
│ _.measure │ _.measurement │ _.per_fold             │
├───────────┼───────────────┼────────────────────────┤
│ l2        │ 0.277         │ [0.0722, 0.536, 0.223] │
└───────────┴───────────────┴────────────────────────┘
_.per_observation = [[[0.00026, 0.0324, ..., 0.042], [0.69, 0.212, ..., 1.09], [0.134, 0.22, ..., 0.534]]]

(The latter call is a mutating call as the learned parameters stored in the machine potentially change. )

Multiple measures

julia> evaluate!(mach,
                 resampling=cv,
                 measure=[l1, rms, rmslp1], verbosity=0)
┌───────────┬───────────────┬───────────────────────┐
│ _.measure │ _.measurement │ _.per_fold            │
├───────────┼───────────────┼───────────────────────┤
│ l1        │ 0.436         │ [0.216, 0.681, 0.41]  │
│ rms       │ 0.527         │ [0.269, 0.732, 0.473] │
│ rmslp1    │ 0.25          │ [0.139, 0.376, 0.165] │
└───────────┴───────────────┴───────────────────────┘
_.per_observation = [[[0.0161, 0.18, ..., 0.205], [0.83, 0.461, ..., 1.04], [0.366, 0.469, ..., 0.731]], missing, missing]

Custom measures and weighted measures

julia> my_loss(yhat, y) = maximum((yhat - y).^2);

julia> my_per_observation_loss(yhat, y) = abs.(yhat - y);

julia> MLJ.reports_each_observation(::typeof(my_per_observation_loss)) = true;

julia> my_weighted_score(yhat, y) = 1/mean(abs.(yhat - y));

julia> my_weighted_score(yhat, y, w) = 1/mean(abs.((yhat - y).^w));

julia> MLJ.supports_weights(::typeof(my_weighted_score)) = true;

julia> MLJ.orientation(::typeof(my_weighted_score)) = :score;

julia> holdout = Holdout(fraction_train=0.8)
Holdout(
    fraction_train = 0.8,
    shuffle = false,
    rng = MersenneTwister(UInt32[0x5e195684, 0x67952e4a, 0x3888593c, 0x4fe704ab]) @ 379) @ 1…63

julia> weights = [1, 1, 2, 1, 1, 2, 3, 1, 1, 2, 3, 1];

julia> evaluate!(mach,
                 resampling=CV(nfolds=3),
                 measure=[my_loss, my_per_observation_loss, my_weighted_score, l1],
                 weights=weights, verbosity=0)
┌ Warning: Sample weights ignored in evaluations of the following measures, as unsupported: 
│ my_loss, my_per_observation_loss 
└ @ MLJBase ~/.julia/packages/MLJBase/8HOpr/src/resampling.jl:543
┌─────────────────────────┬───────────────┬───────────────────────┐
│ _.measure               │ _.measurement │ _.per_fold            │
├─────────────────────────┼───────────────┼───────────────────────┤
│ my_loss                 │ 0.613         │ [0.214, 1.09, 0.534]  │
│ my_per_observation_loss │ 0.436         │ [0.216, 0.681, 0.41]  │
│ my_weighted_score       │ 3.8           │ [6.5, 1.86, 3.04]     │
│ l1                      │ 0.629         │ [0.332, 0.992, 0.565] │
└─────────────────────────┴───────────────┴───────────────────────┘
_.per_observation = [missing, [[0.0161, 0.18, ..., 0.205], [0.83, 0.461, ..., 1.04], [0.366, 0.469, ..., 0.731]], missing, [[0.0161, 0.18, ..., 0.205], [0.83, 0.922, ..., 1.04], [0.366, 0.939, ..., 0.731]]]

User-specified train/test sets

Users can either provide their own list of train/test pairs of row indices for resampling, as in this example:

julia> fold1 = 1:6; fold2 = 7:12;

julia> evaluate!(mach,
                 resampling = [(fold1, fold2), (fold2, fold1)],
                 measure=[l1, l2], verbosity=0)
┌───────────┬───────────────┬────────────────┐
│ _.measure │ _.measurement │ _.per_fold     │
├───────────┼───────────────┼────────────────┤
│ l1        │ 0.404         │ [0.45, 0.358]  │
│ l2        │ 0.218         │ [0.254, 0.183] │
└───────────┴───────────────┴────────────────┘
_.per_observation = [[[0.304, 0.843, ..., 0.672], [0.017, 0.143, ..., 0.4]], [[0.0925, 0.711, ..., 0.451], [0.000288, 0.0205, ..., 0.16]]]

Or define their own re-usable ResamplingStrategy objects, - see Custom resampling strategies below.

Built-in resampling strategies

holdout = Holdout(; fraction_train=0.7,
                     shuffle=nothing,
                     rng=nothing)

train_test_pairs(holdout, rows)

cv = CV(; nfolds=6,  shuffle=nothing, rng=nothing)

train_test_pairs(cv, rows)

stratified_cv = StratifiedCV(; nfolds=6,
                               shuffle=false,
                               rng=Random.GLOBAL_RNG)

train_test_pairs(stratified_cv, rows, y)

Custom resampling strategies

To define your own resampling strategy, make relevant parameters of your strategy the fields of a new type MyResamplingStrategy <: MLJ.ResamplingStrategy, and implement one of the following methods:

MLJ.train_test_pairs(my_strategy::MyResamplingStrategy, rows)
MLJ.train_test_pairs(my_strategy::MyResamplingStrategy, rows, y)
MLJ.train_test_pairs(my_strategy::MyResamplingStrategy, rows, X, y)

Each method takes a vector of indices rows and return a vector [(t1, e1), (t2, e2), ... (tk, ek)] of train/test pairs of row indices selected from rows. Here X, y are the input and target data (ignored in simple strategies, such as Holdout and CV).

Here is the code for the Holdout strategy as an example:

struct Holdout <: ResamplingStrategy
    fraction_train::Float64
    shuffle::Bool
    rng::Union{Int,AbstractRNG}

    function Holdout(fraction_train, shuffle, rng)
        0 < fraction_train < 1 ||
            error("`fraction_train` must be between 0 and 1.")
        return new(fraction_train, shuffle, rng)
    end
end

# Keyword Constructor
function Holdout(; fraction_train::Float64=0.7, shuffle=nothing, rng=nothing)
    if rng isa Integer
        rng = MersenneTwister(rng)
    end
    if shuffle === nothing
        shuffle = ifelse(rng===nothing, false, true)
    end
    if rng === nothing
        rng = Random.GLOBAL_RNG
    end
    return Holdout(fraction_train, shuffle, rng)
end

function train_test_pairs(holdout::Holdout, rows)
    train, test = partition(rows, holdout.fraction_train,
                          shuffle=holdout.shuffle, rng=holdout.rng)
    return [(train, test),]
end

API

evaluate!(mach,
          resampling=CV(),
          measure=nothing,
          weights=nothing,
          operation=predict,
          repeats = 1,
          acceleration=default_resource(),
          force=false,
          verbosity=1,
          check_measure=true)

resampling = [(1:100), (101:200)),
               (101:200), (1:100)]