How are errors handled in nadir?
We try to handle errors gracefully. We understand that specifying complicated models may not always go smoothly, and models can be finnicky.
When a candidate learner throws an error during the
super_learner() training process, we set its weight to 0
and collect the thrown errors for the user in the verbose output.
For example, lnr_lmer will throw an error if the formula
does not use random effects. The below code snippet shows how we collect
the errors and ensure that super learning can continue even if one or
more learners fail.
library(nadir)
# train a super_learner() model
#
# lnr_lmer will error because we didn't use random effects
sl_model <- super_learner(
mtcars,
formula = mpg ~ cyl,
learners = list(lnr_mean, lnr_lmer)
)
#> Warning: package 'future' was built under R version 4.5.2
# observe, prediction falls back to relying on the other learners we specified:
# in this case, just lnr_mean
sl_model$predict(mtcars)
#> [1] 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062
#> [9] 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062
#> [17] 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062
#> [25] 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062 20.09062
# if we specify a more complicated super learner, we can see that as long as we
# specify random effects, lnr_lmer doesn't fail.
#
# also, you can see how {nadir} collects errors in its output.
sl_model2 <- super_learner(
mtcars,
formulas = list(
.default = mpg ~ cyl,
lmer_2 = mpg ~ (1 | cyl) + hp
),
learners = list(lnr_mean, lnr_lmer, lnr_lmer)
)
sl_model2 |> str()
#> List of 20
#> $ predict :function (newdata)
#> ..- attr(*, "srcref")= 'srcref' int [1:8] 1022 41 1038 5 41 5 8228 8244
#> .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x108d88888>
#> $ y_variable : chr "mpg"
#> $ fit_learners :List of 2
#> ..$ mean :function (newdata)
#> .. ..- attr(*, "srcref")= 'srcref' int [1:8] 27 19 29 3 19 3 4474 4476
#> .. .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x1307719c8>
#> ..$ lmer_2:function (newdata)
#> .. ..- attr(*, "srcref")= 'srcref' int [1:8] 448 10 450 3 10 3 4895 4897
#> .. .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x1307719c8>
#> $ train_on_whole_dataset : logi TRUE
#> $ outcome_type : chr "continuous"
#> $ learner_weights : Named num [1:2] 0.192 0.808
#> ..- attr(*, "names")= chr [1:2] "mean" "lmer_2"
#> $ holdout_predictions : tibble [32 × 5] (S3: tbl_df/tbl/data.frame)
#> ..$ .sl_fold : int [1:32] 1 1 1 1 1 1 1 2 2 2 ...
#> ..$ mean : num [1:32] 20.3 20.3 20.3 20.3 20.3 ...
#> ..$ lmer_2 : Named num [1:32] 20.5 20.5 20.7 16.6 27.1 ...
#> .. ..- attr(*, "names")= chr [1:32] "Mazda RX4" "Mazda RX4 Wag" "Valiant" "Merc 450SLC" ...
#> ..$ mpg : num [1:32] 21 21 18.1 15.2 30.4 15.5 13.3 22.8 24.4 17.8 ...
#> ..$ .sl_rowid: int [1:32] 1 2 6 14 19 22 24 3 8 11 ...
#> $ formulas :List of 2
#> ..$ mean :Class 'formula' language mpg ~ cyl
#> .. .. ..- attr(*, ".Environment")=<environment: R_GlobalEnv>
#> ..$ lmer_2:Class 'formula' language mpg ~ (1 | cyl) + hp
#> .. .. ..- attr(*, ".Environment")=<environment: R_GlobalEnv>
#> $ n_folds : int 5
#> $ n_obs : int 32
#> $ oof_predictions : num [1:32] 20.5 20.5 25.4 20.4 16.7 ...
#> $ oof_predict :function (newdata = NULL, rowids = NULL)
#> ..- attr(*, "srcref")= 'srcref' int [1:8] 1210 18 1275 3 18 3 8416 8481
#> .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x108d88888>
#> $ oof_predict_fold :function (newdata_list = NULL, modify = NULL)
#> ..- attr(*, "srcref")= 'srcref' int [1:8] 1138 23 1160 3 23 3 8344 8366
#> .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x108d88888>
#> $ oof_predict_modified :function (modify = NULL)
#> ..- attr(*, "srcref")= 'srcref' int [1:8] 1284 27 1294 3 27 3 8490 8500
#> .. ..- attr(*, "srcfile")=Classes 'srcfilealias', 'srcfile' <environment: 0x108d88888>
#> $ fold_assignments : int [1:32] 1 1 2 5 3 1 3 2 4 4 ...
#> $ rowids : int [1:32] 1 2 3 4 5 6 7 8 9 10 ...
#> $ training_data :'data.frame': 32 obs. of 11 variables:
#> ..$ mpg : num [1:32] 21 21 22.8 21.4 18.7 18.1 14.3 24.4 22.8 19.2 ...
#> ..$ cyl : num [1:32] 6 6 4 6 8 6 8 4 4 6 ...
#> ..$ disp: num [1:32] 160 160 108 258 360 ...
#> ..$ hp : num [1:32] 110 110 93 110 175 105 245 62 95 123 ...
#> ..$ drat: num [1:32] 3.9 3.9 3.85 3.08 3.15 2.76 3.21 3.69 3.92 3.92 ...
#> ..$ wt : num [1:32] 2.62 2.88 2.32 3.21 3.44 ...
#> ..$ qsec: num [1:32] 16.5 17 18.6 19.4 17 ...
#> ..$ vs : num [1:32] 0 0 1 1 0 1 0 1 1 1 ...
#> ..$ am : num [1:32] 1 1 1 0 0 0 0 0 0 0 ...
#> ..$ gear: num [1:32] 4 4 4 3 3 3 3 4 4 4 ...
#> ..$ carb: num [1:32] 4 4 1 1 2 1 4 2 2 4 ...
#> $ errors_from_training_cv_stage1 :List of 5
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "No random effects terms specified in formula"
#> .. ..$ call : language lnr_lmer_1(training_data[[1L]], formula = mpg ~ cyl, NULL)
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "No random effects terms specified in formula"
#> .. ..$ call : language lnr_lmer_1(training_data[[2L]], formula = mpg ~ cyl, NULL)
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "No random effects terms specified in formula"
#> .. ..$ call : language lnr_lmer_1(training_data[[3L]], formula = mpg ~ cyl, NULL)
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "No random effects terms specified in formula"
#> .. ..$ call : language lnr_lmer_1(training_data[[4L]], formula = mpg ~ cyl, NULL)
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "No random effects terms specified in formula"
#> .. ..$ call : language lnr_lmer_1(training_data[[5L]], formula = mpg ~ cyl, NULL)
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> $ errors_from_predicting_cv_stage2:List of 5
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "attempt to apply non-function"
#> .. ..$ call : language trained_learners[["lmer_1"]][[1L]](validation_data[[1L]])
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "attempt to apply non-function"
#> .. ..$ call : language trained_learners[["lmer_1"]][[2L]](validation_data[[2L]])
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "attempt to apply non-function"
#> .. ..$ call : language trained_learners[["lmer_1"]][[3L]](validation_data[[3L]])
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "attempt to apply non-function"
#> .. ..$ call : language trained_learners[["lmer_1"]][[4L]](validation_data[[4L]])
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> ..$ lmer_1:List of 2
#> .. ..$ message: chr "attempt to apply non-function"
#> .. ..$ call : language trained_learners[["lmer_1"]][[5L]](validation_data[[5L]])
#> .. ..- attr(*, "class")= chr [1:3] "simpleError" "error" "condition"
#> $ erring_learners : chr "lmer_1"
#> - attr(*, "class")= chr "nadir_sl_model"Note that the language objects in the error calls are modified inside
super_learner() to make them more user-friendly so that
they contain the names of the learners, and information about what
formula, fold, and extra learner arguments they were called with rather
than obscure errors that refer to things like learners[[i]]
and use do.call() to programmatically pass the extra
arguments.