Running `super_learner()` in Parallel

Parallelization is supported in super_learner() and cv_super_learner(), and is implemented through the future package.

If you’d like to use the parallel versions of super_learner() or cv_super_learner() it is as simple as library(future) (as long as you have it installed) and declaring a plan like plan(multicore).

Note that you can only use plan(multicore) on Linux or outside-of-RStudio on macOS. If you want to take advantage of parallelization on macOS in RStudio or on Windows, you should use plan(multisession). See the future website for more details.

library(nadir)
library(future)
#> Warning: package 'future' was built under R version 4.5.2
library(tidytuesdayR)
#> Warning: package 'tidytuesdayR' was built under R version 4.5.2
library(dplyr)
#> 
#> Attaching package: 'dplyr'
#> The following objects are masked from 'package:stats':
#> 
#>     filter, lag
#> The following objects are masked from 'package:base':
#> 
#>     intersect, setdiff, setequal, union
library(microbenchmark)


plan(multicore) # or similar, see https://future.futureverse.org/
# we recommend you to use a multicore setup on a Unix/Linux machine if you
# actually want to see a speed gain from parallelizing your code.
#
# note that plan(multicore) does not run in RStudio or Windows but multisession
# does.

# we also recommend, if your data is of moderate or large size, then set
# the following to be able to work with large data objects in parallel.
# this may be useful for larger datasets with parallel processing.
options(future.globals.maxSize = +Inf)

data("Boston", package = "MASS")
data <- Boston

Speed gains are most obvious in `cv_super_learner()`

Let’s run a timing test to see if we can tell if there’s an improvement in performance from using a multicore vs. a sequential plan:

# sequential version:
plan(sequential)

microbenchmark(
  {
    cv_super_learner(
      data,
      formula = medv ~ .,
      learners = list(rf = lnr_rf, lm = lnr_lm, mean = lnr_mean)
    )
  },
  times = 3
)
#> Warning in microbenchmark({: less accurate nanosecond times to avoid potential
#> integer overflows
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> Unit: seconds
#>                                                                                                                     expr
#>  {     cv_super_learner(data, formula = medv ~ ., learners = list(rf = lnr_rf,          lm = lnr_lm, mean = lnr_mean)) }
#>       min       lq    mean   median       uq      max neval
#>  9.540046 9.602019 9.74192 9.663993 9.842858 10.02172     3

# multicore version:
plan(multicore, workers = 10)

microbenchmark(
  {
    cv_super_learner(
      data,
      formula = medv ~ .,
      learners = list(rf = lnr_rf, lm = lnr_lm, mean = lnr_mean)
    )
  },
  times = 3
)
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> Unit: seconds
#>                                                                                                                     expr
#>  {     cv_super_learner(data, formula = medv ~ ., learners = list(rf = lnr_rf,          lm = lnr_lm, mean = lnr_mean)) }
#>       min       lq     mean   median       uq      max neval
#>  2.498552 2.532292 2.573058 2.566033 2.610311 2.654589     3

learners <- list(
  mean = lnr_mean,
  lm = lnr_lm,
  rf = lnr_rf,
  earth = lnr_earth,
  xgboost = lnr_xgboost,
  glmnet0 = lnr_glmnet,
  glmnet1 = lnr_glmnet,
  glmnet2 = lnr_glmnet,
  glmnet3 = lnr_glmnet
)

extra_args <- list(
  glmnet0 = list(lambda = 0.01),
  glmnet1 = list(lambda = 0.2),
  glmnet2 = list(lambda = 0.4),
  glmnet3 = list(lambda = 0.6)
)

plan(sequential)

microbenchmark(
  {
    cv_out <- cv_super_learner(
      data = mtcars,
      formulas = mpg ~ .,
      learners = learners,
      extra_learner_args = extra_args
    )
  },
  times = 3
)
#> Warning in microbenchmark({: less accurate nanosecond times to avoid potential
#> integer overflows
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> Unit: seconds
#>                                                                                                                                  expr
#>  {     cv_out <- cv_super_learner(data = mtcars, formulas = mpg ~          ., learners = learners, extra_learner_args = extra_args) }
#>       min       lq     mean   median       uq      max neval
#>  4.517461 4.569724 4.697515 4.621987 4.787543 4.953098     3

plan(multicore)

microbenchmark(
  {
    cv_out <- cv_super_learner(
      data = mtcars,
      formulas = mpg ~ .,
      learners = learners,
      extra_learner_args = extra_args
    )
  },
  times = 3
)
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> The loss_metric is being inferred based on the outcome_type=continuous -> using CV-MSE
#> Unit: seconds
#>                                                                                                                                  expr
#>  {     cv_out <- cv_super_learner(data = mtcars, formulas = mpg ~          ., learners = learners, extra_learner_args = extra_args) }
#>      min      lq     mean   median       uq      max neval
#>  1.66915 1.68448 1.742346 1.699809 1.778944 1.858079     3

But why is it not so obvious for just `super_learner()`?

Because cv_super_learner() involves an additional layer of cross-validation, the effect of parallelization is more obvious in cv_super_learner() than compared to super_learner(). However, to make it more obvious that parallelization is working in super_learner() as well, if the number of cv folds we want to run is higher, this increases the relative payoff of using the parallel option.

plan(sequential)

microbenchmark(
  {
    sl_out <- nadir::super_learner(
      data = Boston,
      formulas = medv ~ .,
      learners = learners,
      n_folds = 20,
      extra_learner_args = extra_args
    )
  },
  times = 3
)
#> Unit: seconds
#>                                                                                                                                                     expr
#>  {     sl_out <- nadir::super_learner(data = Boston, formulas = medv ~          ., learners = learners, n_folds = 20, extra_learner_args = extra_args) }
#>       min       lq     mean  median       uq      max neval
#>  12.19756 12.26733 12.43542 12.3371 12.55434 12.77159     3

plan(multicore)

microbenchmark(
  {
    sl_out <- nadir::super_learner(
      data = Boston,
      formulas = medv ~ .,
      learners = learners,
      n_folds = 20,
      extra_learner_args = extra_args
    )
  },
  times = 3
)
#> Unit: seconds
#>                                                                                                                                                     expr
#>  {     sl_out <- nadir::super_learner(data = Boston, formulas = medv ~          ., learners = learners, n_folds = 20, extra_learner_args = extra_args) }
#>       min       lq     mean   median       uq      max neval
#>  9.353273 9.446765 9.522549 9.540258 9.607187 9.674116     3

Speed gains are most obvious in cv_super_learner()

But why is it not so obvious for just super_learner()?

Speed gains are most obvious in `cv_super_learner()`

But why is it not so obvious for just `super_learner()`?