Common MLJ Workflows

Data ingestion

import RDatasets
channing = RDatasets.dataset("boot", "channing")
first(channing, 4)

4 rows × 5 columns

	Sex	Entry	Exit	Time	Cens
	Cat…	Int32	Int32	Int32	Int32
1	Male	782	909	127	1
2	Male	1020	1128	108	1
3	Male	856	969	113	1
4	Male	915	957	42	1

Inspecting metadata, including column scientific types:

schema(channing)

┌─────────┬────────────────────────────────┬───────────────┐
│ _.names │ _.types                        │ _.scitypes    │
├─────────┼────────────────────────────────┼───────────────┤
│ Sex     │ CategoricalValue{String,UInt8} │ Multiclass{2} │
│ Entry   │ Int32                          │ Count         │
│ Exit    │ Int32                          │ Count         │
│ Time    │ Int32                          │ Count         │
│ Cens    │ Int32                          │ Count         │
└─────────┴────────────────────────────────┴───────────────┘
_.nrows = 462

Unpacking data and correcting for wrong scitypes:

y, X =  unpack(channing,
               ==(:Exit),            # y is the :Exit column
               !=(:Time);            # X is the rest, except :Time
               :Exit=>Continuous,
               :Entry=>Continuous,
               :Cens=>Multiclass)
first(X, 4)

4 rows × 3 columns

	Sex	Entry	Cens
	Cat…	Float64	Cat…
1	Male	782.0	1
2	Male	1020.0	1
3	Male	856.0	1
4	Male	915.0	1

Note: Before julia 1.2, replace !=(:Time) with col -> col != :Time.

y[1:4]

4-element Array{Float64,1}:
  909.0
 1128.0
  969.0
  957.0

Loading a built-in supervised dataset:

X, y = @load_iris;
selectrows(X, 1:4) # selectrows works for any Tables.jl table

(sepal_length = [5.1, 4.9, 4.7, 4.6],
 sepal_width = [3.5, 3.0, 3.2, 3.1],
 petal_length = [1.4, 1.4, 1.3, 1.5],
 petal_width = [0.2, 0.2, 0.2, 0.2],)

y[1:4]

4-element CategoricalArray{String,1,UInt32}:
 "setosa"
 "setosa"
 "setosa"
 "setosa"

Model search

Reference: Model Search

Searching for a supervised model:

X, y = @load_boston
models(matching(X, y))

57-element Array{NamedTuple{(:name, :package_name, :is_supervised, :docstring, :hyperparameter_ranges, :hyperparameter_types, :hyperparameters, :implemented_methods, :is_pure_julia, :is_wrapper, :load_path, :package_license, :package_url, :package_uuid, :prediction_type, :supports_online, :supports_weights, :input_scitype, :target_scitype, :output_scitype),T} where T<:Tuple,1}:
 (name = ARDRegressor, package_name = ScikitLearn, ... )                
 (name = AdaBoostRegressor, package_name = ScikitLearn, ... )           
 (name = BaggingRegressor, package_name = ScikitLearn, ... )            
 (name = BayesianRidgeRegressor, package_name = ScikitLearn, ... )      
 (name = ConstantRegressor, package_name = MLJModels, ... )             
 (name = DecisionTreeRegressor, package_name = DecisionTree, ... )      
 (name = DeterministicConstantRegressor, package_name = MLJModels, ... )
 (name = DummyRegressor, package_name = ScikitLearn, ... )              
 (name = ElasticNetCVRegressor, package_name = ScikitLearn, ... )       
 (name = ElasticNetRegressor, package_name = MLJLinearModels, ... )     
 ⋮                                                                      
 (name = RidgeRegressor, package_name = MultivariateStats, ... )        
 (name = RidgeRegressor, package_name = ScikitLearn, ... )              
 (name = RobustRegressor, package_name = MLJLinearModels, ... )         
 (name = SGDRegressor, package_name = ScikitLearn, ... )                
 (name = SVMLinearRegressor, package_name = ScikitLearn, ... )          
 (name = SVMNuRegressor, package_name = ScikitLearn, ... )              
 (name = SVMRegressor, package_name = ScikitLearn, ... )                
 (name = TheilSenRegressor, package_name = ScikitLearn, ... )           
 (name = XGBoostRegressor, package_name = XGBoost, ... )

models(matching(X, y))[6]

CART decision tree regressor.
→ based on [DecisionTree](https://github.com/bensadeghi/DecisionTree.jl).
→ do `@load DecisionTreeRegressor pkg="DecisionTree"` to use the model.
→ do `?DecisionTreeRegressor` for documentation.
(name = "DecisionTreeRegressor",
 package_name = "DecisionTree",
 is_supervised = true,
 docstring = "CART decision tree regressor.\n→ based on [DecisionTree](https://github.com/bensadeghi/DecisionTree.jl).\n→ do `@load DecisionTreeRegressor pkg=\"DecisionTree\"` to use the model.\n→ do `?DecisionTreeRegressor` for documentation.",
 hyperparameter_ranges = (nothing, nothing, nothing, nothing, nothing, nothing, nothing),
 hyperparameter_types = ("Int64", "Int64", "Int64", "Float64", "Int64", "Bool", "Float64"),
 hyperparameters = (:max_depth, :min_samples_leaf, :min_samples_split, :min_purity_increase, :n_subfeatures, :post_prune, :merge_purity_threshold),
 implemented_methods = Symbol[:predict, :clean!, :fit, :fitted_params],
 is_pure_julia = true,
 is_wrapper = false,
 load_path = "MLJModels.DecisionTree_.DecisionTreeRegressor",
 package_license = "MIT",
 package_url = "https://github.com/bensadeghi/DecisionTree.jl",
 package_uuid = "7806a523-6efd-50cb-b5f6-3fa6f1930dbb",
 prediction_type = :deterministic,
 supports_online = false,
 supports_weights = false,
 input_scitype = Table{_s24} where _s24<:Union{AbstractArray{_s23,1} where _s23<:Continuous, AbstractArray{_s23,1} where _s23<:Count, AbstractArray{_s23,1} where _s23<:OrderedFactor},
 target_scitype = AbstractArray{Continuous,1},
 output_scitype = Unknown,)

Instantiating a model

Reference: Getting Started

@load DecisionTreeClassifier
model = DecisionTreeClassifier(min_samples_split=5, max_depth=4)

DecisionTreeClassifier(
    max_depth = 4,
    min_samples_leaf = 1,
    min_samples_split = 5,
    min_purity_increase = 0.0,
    n_subfeatures = 0,
    post_prune = false,
    merge_purity_threshold = 1.0,
    pdf_smoothing = 0.0,
    display_depth = 5) @135

model = @load DecisionTreeClassifier
model.min_samples_split = 5
model.max_depth = 4

Evaluating a model

Reference: Evaluating Model Performance

X, y = @load_boston
model = @load KNNRegressor
evaluate(model, X, y, resampling=CV(nfolds=5), measure=[rms, mav])

┌───────────┬───────────────┬───────────────────────────────┐
│ _.measure │ _.measurement │ _.per_fold                    │
├───────────┼───────────────┼───────────────────────────────┤
│ rms       │ 8.77          │ [8.53, 8.8, 10.7, 9.43, 5.59] │
│ mae       │ 6.02          │ [6.52, 5.7, 7.65, 6.09, 4.11] │
└───────────┴───────────────┴───────────────────────────────┘
_.per_observation = [missing, missing]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Basic fit/evaluate/predict by hand:

Reference: Getting Started, Machines, Evaluating Model Performance, Performance Measures

import RDatasets
vaso = RDatasets.dataset("robustbase", "vaso"); # a DataFrame
first(vaso, 3)

3 rows × 3 columns

	Volume	Rate	Y
	Float64	Float64	Int64
1	3.7	0.825	1
2	3.5	1.09	1
3	1.25	2.5	1

y, X = unpack(vaso, ==(:Y), c -> true; :Y => Multiclass)

tree_model = @load DecisionTreeClassifier

Bind the model and data together in a machine , which will additionally store the learned parameters (fitresults) when fit:

tree = machine(tree_model, X, y)

Machine{DecisionTreeClassifier} @572 trained 0 times.
  args: 
    1:	Source @791 ⏎ `Table{AbstractArray{Continuous,1}}`
    2:	Source @310 ⏎ `AbstractArray{Multiclass{2},1}`

Split row indices into training and evaluation rows:

train, test = partition(eachindex(y), 0.7, shuffle=true, rng=1234); # 70:30 split

([27, 28, 30, 31, 32, 18, 21, 9, 26, 14  …  7, 39, 2, 37, 1, 8, 19, 25, 35, 34], [22, 13, 11, 4, 10, 16, 3, 20, 29, 23, 12, 24])

Fit on train and evaluate on test:

fit!(tree, rows=train)
yhat = predict(tree, X[test,:])
mean(cross_entropy(yhat, y[test]))

6.5216583816514975

Predict on new data:

Xnew = (Volume=3*rand(3), Rate=3*rand(3))
predict(tree, Xnew)      # a vector of distributions

3-element MLJBase.UnivariateFiniteArray{Multiclass{2},Int64,UInt32,Float64,1}:
 UnivariateFinite{Multiclass{2}}(0=>0.273, 1=>0.727)
 UnivariateFinite{Multiclass{2}}(0=>0.0, 1=>1.0)    
 UnivariateFinite{Multiclass{2}}(0=>0.273, 1=>0.727)

predict_mode(tree, Xnew) # a vector of point-predictions

3-element CategoricalArray{Int64,1,UInt32}:
 1
 1
 1

More performance evaluation examples

import LossFunctions.ZeroOneLoss

Evaluating model + data directly:

evaluate(tree_model, X, y,
         resampling=Holdout(fraction_train=0.7, shuffle=true, rng=1234),
         measure=[cross_entropy, ZeroOneLoss()])

┌───────────────┬───────────────┬────────────┐
│ _.measure     │ _.measurement │ _.per_fold │
├───────────────┼───────────────┼────────────┤
│ cross_entropy │ 6.52          │ [6.52]     │
│ ZeroOneLoss   │ 0.417         │ [0.417]    │
└───────────────┴───────────────┴────────────┘
_.per_observation = [[[0.105, 36.0, ..., 1.3]], [[0.0, 1.0, ..., 1.0]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

If a machine is already defined, as above:

evaluate!(tree,
          resampling=Holdout(fraction_train=0.7, shuffle=true, rng=1234),
          measure=[cross_entropy, ZeroOneLoss()])

┌───────────────┬───────────────┬────────────┐
│ _.measure     │ _.measurement │ _.per_fold │
├───────────────┼───────────────┼────────────┤
│ cross_entropy │ 6.52          │ [6.52]     │
│ ZeroOneLoss   │ 0.417         │ [0.417]    │
└───────────────┴───────────────┴────────────┘
_.per_observation = [[[0.105, 36.0, ..., 1.3]], [[0.0, 1.0, ..., 1.0]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Using cross-validation:

evaluate!(tree, resampling=CV(nfolds=5, shuffle=true, rng=1234),
          measure=[cross_entropy, ZeroOneLoss()])

┌───────────────┬───────────────┬──────────────────────────────────┐
│ _.measure     │ _.measurement │ _.per_fold                       │
├───────────────┼───────────────┼──────────────────────────────────┤
│ cross_entropy │ 3.28          │ [9.25, 0.598, 4.93, 1.07, 0.546] │
│ ZeroOneLoss   │ 0.407         │ [0.5, 0.375, 0.375, 0.5, 0.286]  │
└───────────────┴───────────────┴──────────────────────────────────┘
_.per_observation = [[[2.22e-16, 0.944, ..., 2.22e-16], [0.847, 0.56, ..., 0.56], [0.799, 0.598, ..., 36.0], [2.01, 2.01, ..., 0.143], [0.405, 0.405, ..., 1.1]], [[0.0, 1.0, ..., 0.0], [1.0, 0.0, ..., 0.0], [1.0, 0.0, ..., 1.0], [1.0, 1.0, ..., 0.0], [0.0, 0.0, ..., 1.0]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

With user-specified train/test pairs of row indices:

f1, f2, f3 = 1:13, 14:26, 27:36
pairs = [(f1, vcat(f2, f3)), (f2, vcat(f3, f1)), (f3, vcat(f1, f2))];
evaluate!(tree,
          resampling=pairs,
          measure=[cross_entropy, ZeroOneLoss()])

┌───────────────┬───────────────┬───────────────────────┐
│ _.measure     │ _.measurement │ _.per_fold            │
├───────────────┼───────────────┼───────────────────────┤
│ cross_entropy │ 5.88          │ [2.16, 11.0, 4.51]    │
│ ZeroOneLoss   │ 0.241         │ [0.304, 0.304, 0.115] │
└───────────────┴───────────────┴───────────────────────┘
_.per_observation = [[[0.154, 0.154, ..., 0.154], [2.22e-16, 36.0, ..., 2.22e-16], [2.22e-16, 2.22e-16, ..., 0.693]], [[0.0, 0.0, ..., 0.0], [0.0, 1.0, ..., 0.0], [0.0, 0.0, ..., 0.0]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Changing a hyperparameter and re-evaluating:

tree_model.max_depth = 3
evaluate!(tree,
          resampling=CV(nfolds=5, shuffle=true, rng=1234),
          measure=[cross_entropy, ZeroOneLoss()])

┌───────────────┬───────────────┬────────────────────────────────────┐
│ _.measure     │ _.measurement │ _.per_fold                         │
├───────────────┼───────────────┼────────────────────────────────────┤
│ cross_entropy │ 3.11          │ [9.18, 0.484, 4.86, 0.564, 0.488]  │
│ ZeroOneLoss   │ 0.332         │ [0.375, 0.25, 0.375, 0.375, 0.286] │
└───────────────┴───────────────┴────────────────────────────────────┘
_.per_observation = [[[2.22e-16, 1.32, ..., 2.22e-16], [2.22e-16, 0.318, ..., 0.318], [0.575, 2.22e-16, ..., 36.0], [1.5, 1.5, ..., 2.22e-16], [0.636, 2.22e-16, ..., 0.754]], [[0.0, 1.0, ..., 0.0], [0.0, 0.0, ..., 0.0], [0.0, 0.0, ..., 1.0], [1.0, 1.0, ..., 0.0], [0.0, 0.0, ..., 1.0]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Inspecting training results

Fit a ordinary least square model to some synthetic data:

x1 = rand(100)
x2 = rand(100)

X = (x1=x1, x2=x2)
y = x1 - 2x2 + 0.1*rand(100);

ols_model = @load LinearRegressor pkg=GLM
ols =  machine(ols_model, X, y)
fit!(ols)

Machine{LinearRegressor} @030 trained 1 time.
  args: 
    1:	Source @223 ⏎ `Table{AbstractArray{Continuous,1}}`
    2:	Source @325 ⏎ `AbstractArray{Continuous,1}`

Get a named tuple representing the learned parameters, human-readable if appropriate:

fitted_params(ols)

(coef = [0.9863299659039038, -1.9831253672137887],
 intercept = 0.04619969177488885,)

Get other training-related information:

report(ols)

(deviance = 0.07717394585833826,
 dof_residual = 97.0,
 stderror = [0.00948601320153524, 0.009982246480066854, 0.007652131807118573],
 vcov = [8.998444645970084e-5 -6.015998122589539e-8 -4.718861250724212e-5; -6.015998122589539e-8 9.96452447888071e-5 -5.07239515115565e-5; -4.718861250724212e-5 -5.07239515115565e-5 5.855512119351575e-5],)

Basic fit/transform for unsupervised models

Load data:

X, y = @load_iris
train, test = partition(eachindex(y), 0.97, shuffle=true, rng=123)

([125, 100, 130, 9, 70, 148, 39, 64, 6, 107  …  110, 59, 139, 21, 112, 144, 140, 72, 109, 41], [106, 147, 47, 5])

Instantiate and fit the model/machine:

@load PCA
pca_model = PCA(maxoutdim=2)
pca = machine(pca_model, X)
fit!(pca, rows=train)

Machine{PCA} @301 trained 1 time.
  args: 
    1:	Source @769 ⏎ `Table{AbstractArray{Continuous,1}}`

Transform selected data bound to the machine:

transform(pca, rows=test);

(x1 = [-3.3942826854483243, -1.5219827578765068, 2.538247455185219, 2.7299639893931373],
 x2 = [0.5472450223745241, -0.36842368617126214, 0.5199299511335698, 0.3448466122232363],)

Transform new data:

Xnew = (sepal_length=rand(3), sepal_width=rand(3),
        petal_length=rand(3), petal_width=rand(3));
transform(pca, Xnew)

(x1 = [5.195277587498025, 4.51620289754621, 4.207989874700417],
 x2 = [-4.502928258114949, -4.665066733597676, -5.0788155703668245],)

Inverting learned transformations

y = rand(100);
stand_model = UnivariateStandardizer()
stand = machine(stand_model, y)
fit!(stand)
z = transform(stand, y);
@assert inverse_transform(stand, z) ≈ y # true

[ Info: Training Machine{UnivariateStandardizer} @829.

Nested hyperparameter tuning

Reference: Tuning Models

Define a model with nested hyperparameters:

tree_model = @load DecisionTreeClassifier
forest_model = EnsembleModel(atom=tree_model, n=300)

ProbabilisticEnsembleModel(
    atom = DecisionTreeClassifier(
            max_depth = -1,
            min_samples_leaf = 1,
            min_samples_split = 2,
            min_purity_increase = 0.0,
            n_subfeatures = 0,
            post_prune = false,
            merge_purity_threshold = 1.0,
            pdf_smoothing = 0.0,
            display_depth = 5),
    atomic_weights = Float64[],
    bagging_fraction = 0.8,
    rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 66,
    n = 300,
    acceleration = CPU1{Nothing}(nothing),
    out_of_bag_measure = Any[]) @506

Inspect all hyperparameters, even nested ones (returns nested named tuple):

params(forest_model)

(atom = (max_depth = -1,
         min_samples_leaf = 1,
         min_samples_split = 2,
         min_purity_increase = 0.0,
         n_subfeatures = 0,
         post_prune = false,
         merge_purity_threshold = 1.0,
         pdf_smoothing = 0.0,
         display_depth = 5,),
 atomic_weights = Float64[],
 bagging_fraction = 0.8,
 rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 66,
 n = 300,
 acceleration = CPU1{Nothing}(nothing),
 out_of_bag_measure = Any[],)

Define ranges for hyperparameters to be tuned:

r1 = range(forest_model, :bagging_fraction, lower=0.5, upper=1.0, scale=:log10)

MLJBase.NumericRange(Float64, :bagging_fraction, ... )

r2 = range(forest_model, :(atom.n_subfeatures), lower=1, upper=4) # nested

MLJBase.NumericRange(Int64, :(atom.n_subfeatures), ... )

Wrap the model in a tuning strategy:

tuned_forest = TunedModel(model=forest_model,
                          tuning=Grid(resolution=12),
                          resampling=CV(nfolds=6),
                          ranges=[r1, r2],
                          measure=cross_entropy)

ProbabilisticTunedModel(
    model = ProbabilisticEnsembleModel(
            atom = DecisionTreeClassifier @753,
            atomic_weights = Float64[],
            bagging_fraction = 0.8,
            rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 66,
            n = 300,
            acceleration = CPU1{Nothing}(nothing),
            out_of_bag_measure = Any[]),
    tuning = Grid(
            goal = nothing,
            resolution = 12,
            shuffle = true,
            rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 66),
    resampling = CV(
            nfolds = 6,
            shuffle = false,
            rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 66),
    measure = cross_entropy(
            eps = 2.220446049250313e-16),
    weights = nothing,
    operation = MLJModelInterface.predict,
    range = MLJBase.NumericRange{T,MLJBase.Bounded,Symbol} where T[NumericRange{Float64,…} @139, NumericRange{Int64,…} @997],
    selection_heuristic = MLJTuning.NaiveSelection(nothing),
    train_best = true,
    repeats = 1,
    n = nothing,
    acceleration = CPU1{Nothing}(nothing),
    acceleration_resampling = CPU1{Nothing}(nothing),
    check_measure = true) @753

Bound the wrapped model to data:

tuned = machine(tuned_forest, X, y)

Machine{ProbabilisticTunedModel{Grid,…}} @856 trained 0 times.
  args: 
    1:	Source @918 ⏎ `Table{AbstractArray{Continuous,1}}`
    2:	Source @086 ⏎ `AbstractArray{Multiclass{3},1}`

Fitting the resultant machine optimizes the hyperparameters specified in range, using the specified tuning and resampling strategies and performance measure (possibly a vector of measures), and retrains on all data bound to the machine:

fit!(tuned)

Machine{ProbabilisticTunedModel{Grid,…}} @856 trained 1 time.
  args: 
    1:	Source @918 ⏎ `Table{AbstractArray{Continuous,1}}`
    2:	Source @086 ⏎ `AbstractArray{Multiclass{3},1}`

Inspecting the optimal model:

F = fitted_params(tuned)

(best_model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @803,
 best_fitted_params = (fitresult = WrappedEnsemble{Tuple{Node{Float64,…},…},…} @008,),)

F.best_model

ProbabilisticEnsembleModel(
    atom = DecisionTreeClassifier(
            max_depth = -1,
            min_samples_leaf = 1,
            min_samples_split = 2,
            min_purity_increase = 0.0,
            n_subfeatures = 3,
            post_prune = false,
            merge_purity_threshold = 1.0,
            pdf_smoothing = 0.0,
            display_depth = 5),
    atomic_weights = Float64[],
    bagging_fraction = 0.5,
    rng = MersenneTwister(UInt32[0xf52cdeeb, 0xb7b9fdb7, 0x56985e7f, 0xaa6313fc]) @ 94,
    n = 300,
    acceleration = CPU1{Nothing}(nothing),
    out_of_bag_measure = Any[]) @803

Inspecting details of tuning procedure:

report(tuned)

(best_model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @803,
 best_history_entry = (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @803,
                       measure = MLJBase.CrossEntropy{Float64}[cross_entropy],
                       measurement = [0.1532362925865559],
                       per_fold = Array{Float64,1}[[3.663735981263026e-15, 0.000400668151865417, 0.2032523278751231, 0.2512194400560992, 0.21467305048007884, 0.24987226895616516]],),
 history = NamedTuple{(:model, :measure, :measurement, :per_fold),Tuple{MLJ.ProbabilisticEnsembleModel{MLJModels.DecisionTree_.DecisionTreeClassifier},Array{MLJBase.CrossEntropy{Float64},1},Array{Float64,1},Array{Array{Float64,1},1}}}[(model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @644, measure = [cross_entropy], measurement = [0.19110960524738227], per_fold = [[0.030130535711461605, 0.007071036131917015, 0.2770357589204575, 0.2568641367893142, 0.31729942011465534, 0.25825674381648794]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @532, measure = [cross_entropy], measurement = [0.17074537484195726], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.21860403528956085, 0.2775778433553792, 0.25490875300818394, 0.2733816173986121]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @033, measure = [cross_entropy], measurement = [0.17499501374785661], per_fold = [[0.035037300456911204, 0.00734559856740453, 0.24931987916573764, 0.22335845684853303, 0.2970991787464979, 0.23780966870205547]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @555, measure = [cross_entropy], measurement = [0.4208407355714085], per_fold = [[0.028420236073152427, 0.0046158088505033265, 0.3278811122929353, 0.33016648181579605, 1.5559049900525568, 0.2780557843435068]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @163, measure = [cross_entropy], measurement = [0.16280949031563094], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.2280539128781533, 0.2684017350751886, 0.23864289580466735, 0.24175839813576908]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @428, measure = [cross_entropy], measurement = [0.20590762244227268], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.27588856739645623, 0.38234455526439637, 0.30280963971711716, 0.2744029722756588]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @649, measure = [cross_entropy], measurement = [0.6440218908588728], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.37117199401860435, 1.6203521606785092, 1.5675667253518184, 0.3050404651042979]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @001, measure = [cross_entropy], measurement = [0.4121239007869524], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.2832075417158895, 1.6023794125389452, 0.31680371368820387, 0.2703527367786683]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @023, measure = [cross_entropy], measurement = [0.15951296122299546], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.22858795079978866, 0.2682017186190386, 0.23278731802071223, 0.22750077989842596]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @464, measure = [cross_entropy], measurement = [0.17847052846930037], per_fold = [[0.03754509626634034, 0.006935916395612418, 0.2326254473528447, 0.24696078147183104, 0.31752673926124203, 0.22922919006793163]])  …  (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @803, measure = [cross_entropy], measurement = [0.1532362925865559], per_fold = [[3.663735981263026e-15, 0.000400668151865417, 0.2032523278751231, 0.2512194400560992, 0.21467305048007884, 0.24987226895616516]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @623, measure = [cross_entropy], measurement = [1.1120514433837096], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 1.6920306582567002, 2.884026942756648, 1.6274195023790787, 0.4688315569098245]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @404, measure = [cross_entropy], measurement = [0.43070842324762976], per_fold = [[0.02943719131206908, 0.00381779606870095, 0.3389948537396552, 0.36759721388675837, 1.563700717869984, 0.28070276660861104]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @344, measure = [cross_entropy], measurement = [0.20151623987729086], per_fold = [[0.05613401072292118, 0.01600091306283926, 0.24521070561687025, 0.2506177722983447, 0.31539942663948584, 0.3257346109232841]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @977, measure = [cross_entropy], measurement = [0.19225596406866177], per_fold = [[0.027017242638620157, 0.00463346535104786, 0.2799379480375072, 0.29279360569396484, 0.28983800572704044, 0.2593155169637901]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @109, measure = [cross_entropy], measurement = [0.2135748315057913], per_fold = [[0.06008123336250985, 0.023160956917179266, 0.31753297581837736, 0.2677862792961699, 0.30603990131273096, 0.30684764232778045]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @289, measure = [cross_entropy], measurement = [0.2149905466102604], per_fold = [[0.06553346885764669, 0.03962148506746178, 0.3203053374085624, 0.25195991357962594, 0.31463247768062547, 0.2978905970676403]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @742, measure = [cross_entropy], measurement = [0.20004658174118184], per_fold = [[0.05916494010070639, 0.020227117723511122, 0.29868832353697927, 0.224389597580039, 0.32054275326356163, 0.27726675824229363]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @911, measure = [cross_entropy], measurement = [0.4252541955623274], per_fold = [[3.663735981263026e-15, 3.663735981263026e-15, 0.30382828064217293, 1.6138019781901587, 0.3469628291621348, 0.2869320853794904]]), (model = ProbabilisticEnsembleModel{DecisionTreeClassifier} @181, measure = [cross_entropy], measurement = [0.17262950854995698], per_fold = [[0.031844335124697865, 0.0064707628856672085, 0.23096039637009447, 0.26153688140469183, 0.27447190169384805, 0.2304927738207425]])],
 best_report = (measures = Any[],
                oob_measurements = missing,),
 plotting = (parameter_names = ["bagging_fraction", "atom.n_subfeatures"],
             parameter_scales = Symbol[:log10, :linear],
             parameter_values = Any[0.6851754923600618 2; 0.5671562610977313 4; … ; 0.8277532798848107 3; 0.5325205447199813 2],
             measurements = [0.19110960524738227, 0.17074537484195726, 0.17499501374785661, 0.4208407355714085, 0.16280949031563094, 0.20590762244227268, 0.6440218908588728, 0.4121239007869524, 0.15951296122299546, 0.17847052846930037  …  0.1532362925865559, 1.1120514433837096, 0.43070842324762976, 0.20151623987729086, 0.19225596406866177, 0.2135748315057913, 0.2149905466102604, 0.20004658174118184, 0.4252541955623274, 0.17262950854995698],),)

Visualizing these results:

using Plots
plot(tuned)

Predicting on new data using the optimized model:

predict(tuned, Xnew)

3-element Array{UnivariateFinite{Multiclass{3},String,UInt32,Float64},1}:
 UnivariateFinite{Multiclass{3}}(versicolor=>0.0, virginica=>0.0, setosa=>1.0)      
 UnivariateFinite{Multiclass{3}}(versicolor=>0.44, virginica=>0.0333, setosa=>0.527)
 UnivariateFinite{Multiclass{3}}(versicolor=>0.44, virginica=>0.0333, setosa=>0.527)

Constructing a linear pipeline

Reference: Composing Models

Constructing a linear (unbranching) pipeline with a learned target transformation/inverse transformation:

X, y = @load_reduced_ames
@load KNNRegressor
pipe = @pipeline(X -> coerce(X, :age=>Continuous),
                 OneHotEncoder,
                 KNNRegressor(K=3),
                 target = UnivariateStandardizer)

Pipeline377(
    one_hot_encoder = OneHotEncoder(
            features = Symbol[],
            drop_last = false,
            ordered_factor = true,
            ignore = false),
    knn_regressor = KNNRegressor(
            K = 3,
            algorithm = :kdtree,
            metric = Distances.Euclidean(0.0),
            leafsize = 10,
            reorder = true,
            weights = :uniform),
    target = UnivariateStandardizer()) @008

Evaluating the pipeline (just as you would any other model):

pipe.knn_regressor.K = 2
pipe.one_hot_encoder.drop_last = true
evaluate(pipe, X, y, resampling=Holdout(), measure=rms, verbosity=2)

┌───────────┬───────────────┬────────────┐
│ _.measure │ _.measurement │ _.per_fold │
├───────────┼───────────────┼────────────┤
│ rms       │ 53100.0       │ [53100.0]  │
└───────────┴───────────────┴────────────┘
_.per_observation = [missing]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Inspecting the learned parameters in a pipeline:

mach = machine(pipe, X, y) |> fit!
F = fitted_params(mach)
F.one_hot_encoder

(fitresult = OneHotEncoderResult @212,)

Constructing a linear (unbranching) pipeline with a static (unlearned) target transformation/inverse transformation:

@load DecisionTreeRegressor
pipe2 = @pipeline(X -> coerce(X, :age=>Continuous),
                  OneHotEncoder,
                  DecisionTreeRegressor(max_depth=4),
                  target = y -> log.(y),
                  inverse = z -> exp.(z))

Pipeline388(
    one_hot_encoder = OneHotEncoder(
            features = Symbol[],
            drop_last = false,
            ordered_factor = true,
            ignore = false),
    decision_tree_regressor = DecisionTreeRegressor(
            max_depth = 4,
            min_samples_leaf = 5,
            min_samples_split = 2,
            min_purity_increase = 0.0,
            n_subfeatures = 0,
            post_prune = false,
            merge_purity_threshold = 1.0),
    target = WrappedFunction(
            f = getfield(Main.ex-workflows, Symbol("##28#29"))()),
    inverse = WrappedFunction(
            f = getfield(Main.ex-workflows, Symbol("##30#31"))())) @276

Creating a homogeneous ensemble of models

Reference: Homogeneous Ensembles

X, y = @load_iris
tree_model = @load DecisionTreeClassifier
forest_model = EnsembleModel(atom=tree_model, bagging_fraction=0.8, n=300)
forest = machine(forest_model, X, y)
evaluate!(forest, measure=cross_entropy)

┌───────────────┬───────────────┬────────────────────────────────────────────────┐
│ _.measure     │ _.measurement │ _.per_fold                                     │
├───────────────┼───────────────┼────────────────────────────────────────────────┤
│ cross_entropy │ 0.411         │ [3.66e-15, 3.66e-15, 0.273, 1.6, 0.301, 0.292] │
└───────────────┴───────────────┴────────────────────────────────────────────────┘
_.per_observation = [[[3.66e-15, 3.66e-15, ..., 3.66e-15], [3.66e-15, 3.66e-15, ..., 3.66e-15], [0.0202, 0.00669, ..., 3.66e-15], [3.66e-15, 0.128, ..., 3.66e-15], [3.66e-15, 0.0236, ..., 3.66e-15], [0.0168, 0.452, ..., 0.0408]]]
_.fitted_params_per_fold = [ … ]
_.report_per_fold = [ … ]

Performance curves

Generate a plot of performance, as a function of some hyperparameter (building on the preceding example)

Single performance curve:

r = range(forest_model, :n, lower=1, upper=1000, scale=:log10)
curve = learning_curve(forest,
                            range=r,
                            resampling=Holdout(),
                            resolution=50,
                            measure=cross_entropy,
                            verbosity=0)

(parameter_name = "n",
 parameter_scale = :log10,
 parameter_values = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11  …  281, 324, 373, 429, 494, 569, 655, 754, 869, 1000],
 measurements = [12.014551129705719, 12.014551129705719, 12.014551129705719, 12.014551129705719, 12.014551129705719, 12.014551129705719, 12.014551129705719, 12.014551129705719, 6.791695298580701, 6.8040645777860735  …  1.3900008106631785, 1.3612206907525481, 1.3375605378456865, 1.3234292620201193, 1.3101327238333627, 1.306812596552505, 1.3000382045119372, 1.2987672626491396, 1.297341518760223, 1.2936597809282755],)

using Plots
plot(curve.parameter_values, curve.measurements, xlab=curve.parameter_name, xscale=curve.parameter_scale)

Multiple curves:

curve = learning_curve(forest,
                       range=r,
                       resampling=Holdout(),
                       measure=cross_entropy,
                       resolution=50,
                       rng_name=:rng,
                       rngs=4,
                       verbosity=0)

(parameter_name = "n",
 parameter_scale = :log10,
 parameter_values = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11  …  281, 324, 373, 429, 494, 569, 655, 754, 869, 1000],
 measurements = [8.009700753137146 4.004850376568572 15.218431430960575 4.004850376568572; 8.009700753137146 4.004850376568572 15.218431430960575 4.004850376568572; … ; 1.1789270348047116 1.2111535876355835 1.255377125622927 1.2484454017608544; 1.1846797728333145 1.2125794930060028 1.2545682750219258 1.2490244891719904],)

plot(curve.parameter_values, curve.measurements,
xlab=curve.parameter_name, xscale=curve.parameter_scale)