I have a use case where a bmr is aggregated using many measures, and it may be easier to parallelize over measures to speed up overall aggregating.
Here's an example including an experimental combination function I wrote for testing:
library(mlr3)
library(mlr3proba)
library(mlr3benchmark)
lgr::get_logger("mlr3")$set_threshold("warn")
measures = list(
msr("surv.cindex", id = "harrell_c", label = "Harrell's C"),
msr("surv.cindex", id = "uno_c", weight_meth = "G2", label = "Uno's C"),
msr("surv.rcll", id = "rcll", ERV = FALSE, label = "Right-Censored Log Loss"),
msr("surv.rcll", id = "rcll_erv", ERV = TRUE, label = "Right-Censored Log Loss (ERV)"),
msr("surv.logloss", id = "logloss", ERV = FALSE, label = "Log Loss"),
msr("surv.logloss", id = "logloss_erv", ERV = TRUE, label = "Log Loss (ERV)"),
msr("surv.intlogloss", id = "intlogloss_proper", ERV = FALSE, proper = TRUE, label = "Integrated Log Loss (Proper)"),
msr("surv.intlogloss", id = "intlogloss_proper_erv", ERV = TRUE, proper = TRUE, label = "Integrated Log Loss (Proper, ERV)"),
msr("surv.calib_alpha", id = "caliba", label = "Van Houwelingen's Alpha"),
msr("surv.dcalib", id = "dcalib", truncate = 10, label = "D-Calibration (truncated)"),
msr("surv.graf", id = "graf_proper", proper = TRUE, ERV = FALSE, label = "Graf Score (Proper)"),
msr("surv.graf", id = "graf_proper_erv", proper = TRUE, ERV = TRUE, label = "Graf Score (Proper, ERV)"),
msr("surv.graf", id = "graf_improper", proper = FALSE, ERV = FALSE, label = "Graf Score (Improper)"),
msr("surv.graf", id = "graf_improper_erv", proper = FALSE, ERV = TRUE, label = "Graf Score (Improper, ERV)")
)
names(measures) = mlr3misc::ids(measures)
# Example bmr
bmr = benchmark(benchmark_grid(
tasks = tsks(c("rats", "gbcs", "grace")),
learners = lrns(c("surv.kaplan", "surv.coxph")),
resamplings = rsmp("holdout")
), store_backends = TRUE)
bmr
#> <BenchmarkResult> of 6 rows with 6 resampling runs
#> nr task_id learner_id resampling_id iters warnings errors
#> 1 rats surv.kaplan holdout 1 0 0
#> 2 rats surv.coxph holdout 1 0 0
#> 3 gbcs surv.kaplan holdout 1 0 0
#> 4 gbcs surv.coxph holdout 1 0 0
#> 5 grace surv.kaplan holdout 1 0 0
#> 6 grace surv.coxph holdout 1 0 0
# Score separately, hold resulting bmas in list
bmalist = mlr3misc::named_list(mlr3misc::ids(measures))
for (measure in measures) {
bmalist[[measure$id]] = mlr3benchmark::as_benchmark_aggr(bmr, measures = measure)
}
bmalist[1:3]
#> $harrell_c
#> <BenchmarkAggr> of 6 rows with 3 tasks, 2 learners and 1 measure
#> task_id learner_id harrell_c
#> 1: rats kaplan 0.5000000
#> 2: rats coxph 0.8183594
#> 3: gbcs kaplan 0.5000000
#> 4: gbcs coxph 0.7394313
#> 5: grace kaplan 0.5000000
#> 6: grace coxph 0.8564613
#>
#> $uno_c
#> <BenchmarkAggr> of 6 rows with 3 tasks, 2 learners and 1 measure
#> task_id learner_id uno_c
#> 1: rats kaplan 0.5000000
#> 2: rats coxph 0.8372653
#> 3: gbcs kaplan 0.5000000
#> 4: gbcs coxph 0.7217345
#> 5: grace kaplan 0.5000000
#> 6: grace coxph 0.8505907
#>
#> $rcll
#> <BenchmarkAggr> of 6 rows with 3 tasks, 2 learners and 1 measure
#> task_id learner_id rcll
#> 1: rats kaplan 3.180493
#> 2: rats coxph 3.124062
#> 3: gbcs kaplan 7.258260
#> 4: gbcs coxph 7.199384
#> 5: grace kaplan 3.770636
#> 6: grace coxph 3.445030My experimental function works on a list with n > 1 BenchmarkAggr objects, which I assume to be more useful than e.g a $combine() method that would only join 2 objects, but I'm not sure about the API.
#' Combine a list of bma objects to a single bma
#'
#' Joins respective $data fields on “learner_id”, “task_id”.
#' @param bmalist List of `BenchmarkAggr` objects.
#' @return An object of class `BenchmarkAggr`.
combine_bma = function(bmalist) {
sapply(bmalist, \(x) checkmate::assert_class(x, classes = "BenchmarkAggr"))
# Initial dt for consecutive joins, needs factor vars
dt = data.table::data.table(task_id = factor(1), learner_id = factor(1))
# Joining $data fields containing scores as dt
for (aggr in seq_along(bmalist)) {
# print(names(bmalist[aggr]))
dt = dt[bmalist[[aggr]]$data, on = c("learner_id", "task_id")]
}
# Converting to bma again
mlr3benchmark::as_benchmark_aggr(dt)
}
combine_bma(bmalist)
#> <BenchmarkAggr> of 6 rows with 3 tasks, 2 learners and 14 measures
#> task_id learner_id harrell_c uno_c rcll rcll_erv logloss
#> 1: rats kaplan 0.5000000 0.5000000 3.180493 0.000000000 21.752574
#> 2: rats coxph 0.8183594 0.8372653 3.124062 0.017742667 21.377039
#> 3: gbcs kaplan 0.5000000 0.5000000 7.258260 0.000000000 32.781586
#> 4: gbcs coxph 0.7394313 0.7217345 7.199384 0.008111614 32.787356
#> 5: grace kaplan 0.5000000 0.5000000 3.770636 0.000000000 9.926067
#> 6: grace coxph 0.8564613 0.8505907 3.445030 0.086353110 9.055365
#> logloss_erv intlogloss_proper intlogloss_proper_erv caliba dcalib
#> 1: 0.0000000000 0.1525563 0.00000000 1.0072800 0.01037591
#> 2: 0.0172639721 0.1224677 0.19722946 1.0026337 0.43158202
#> 3: 0.0000000000 0.2403609 0.00000000 0.7743133 3.84231309
#> 4: -0.0001759957 0.2238057 0.06887646 0.9429916 0.71061424
#> 5: 0.0000000000 0.4261547 0.00000000 1.1657555 5.85295235
#> 6: 0.0877187299 0.2423848 0.43122818 1.1925903 4.79665437
#> graf_proper graf_proper_erv graf_improper graf_improper_erv
#> 1: 0.04949192 0.0000000 0.05400789 0.0000000
#> 2: 0.04157216 0.1600213 0.04723149 0.1254706
#> 3: 0.08935573 0.0000000 0.13500880 0.0000000
#> 4: 0.08019706 0.1024968 0.10572968 0.2168683
#> 5: 0.16694804 0.0000000 0.21295790 0.0000000
#> 6: 0.08578223 0.4861741 0.09824500 0.5386647
combine_bma(c(bmalist$harrell_c, bmalist$uno_c))
#> <BenchmarkAggr> of 6 rows with 3 tasks, 2 learners and 2 measures
#> task_id learner_id harrell_c uno_c
#> 1: rats kaplan 0.5000000 0.5000000
#> 2: rats coxph 0.8183594 0.8372653
#> 3: gbcs kaplan 0.5000000 0.5000000
#> 4: gbcs coxph 0.7394313 0.7217345
#> 5: grace kaplan 0.5000000 0.5000000
#> 6: grace coxph 0.8564613 0.8505907Created on 2024-01-24 with reprex v2.1.0
Session info
sessioninfo::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#> setting value
#> version R version 4.3.2 (2023-10-31)
#> os macOS Sonoma 14.2.1
#> system aarch64, darwin20
#> ui X11
#> language (EN)
#> collate en_US.UTF-8
#> ctype en_US.UTF-8
#> tz Europe/Berlin
#> date 2024-01-24
#> pandoc 3.1.1 @ /System/Volumes/Data/Applications/RStudio.app/Contents/Resources/app/quarto/bin/tools/ (via rmarkdown)
#>
#> ─ Packages ───────────────────────────────────────────────────────────────────
#> package * version date (UTC) lib source
#> backports 1.4.1 2021-12-13 [1] CRAN (R 4.3.0)
#> checkmate 2.3.1 2023-12-04 [1] CRAN (R 4.3.1)
#> cli 3.6.2 2023-12-11 [1] CRAN (R 4.3.1)
#> codetools 0.2-19 2023-02-01 [2] CRAN (R 4.3.2)
#> colorspace 2.1-0 2023-01-23 [1] CRAN (R 4.3.0)
#> crayon 1.5.2 2022-09-29 [1] CRAN (R 4.3.0)
#> data.table 1.14.10 2023-12-08 [1] CRAN (R 4.3.1)
#> dictionar6 0.1.3 2021-09-13 [1] CRAN (R 4.3.0)
#> digest 0.6.34 2024-01-11 [1] CRAN (R 4.3.1)
#> distr6 1.8.4 2023-11-13 [1] Github (xoopR/distr6@1854b22)
#> dplyr 1.1.4 2023-11-17 [1] CRAN (R 4.3.1)
#> evaluate 0.23 2023-11-01 [1] CRAN (R 4.3.1)
#> fansi 1.0.6 2023-12-08 [1] CRAN (R 4.3.1)
#> fastmap 1.1.1 2023-02-24 [1] CRAN (R 4.3.0)
#> fs 1.6.3 2023-07-20 [1] CRAN (R 4.3.0)
#> future 1.33.1 2023-12-22 [1] CRAN (R 4.3.1)
#> future.apply 1.11.1 2023-12-21 [1] CRAN (R 4.3.1)
#> generics 0.1.3 2022-07-05 [1] CRAN (R 4.3.0)
#> ggplot2 3.4.4 2023-10-12 [1] CRAN (R 4.3.1)
#> globals 0.16.2 2022-11-21 [1] CRAN (R 4.3.0)
#> glue 1.7.0 2024-01-09 [1] CRAN (R 4.3.1)
#> gtable 0.3.4 2023-08-21 [1] CRAN (R 4.3.0)
#> htmltools 0.5.7 2023-11-03 [1] CRAN (R 4.3.1)
#> knitr 1.45 2023-10-30 [1] CRAN (R 4.3.1)
#> lattice 0.21-9 2023-10-01 [1] CRAN (R 4.3.1)
#> lgr 0.4.4 2022-09-05 [1] CRAN (R 4.3.0)
#> lifecycle 1.0.4 2023-11-07 [1] CRAN (R 4.3.1)
#> listenv 0.9.0 2022-12-16 [1] CRAN (R 4.3.0)
#> magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.3.0)
#> Matrix 1.6-1.1 2023-09-18 [1] CRAN (R 4.3.1)
#> mlr3 * 0.17.2 2024-01-09 [1] CRAN (R 4.3.1)
#> mlr3benchmark * 0.1.6 2023-05-30 [1] CRAN (R 4.3.0)
#> mlr3misc 0.13.0-9000 2023-10-17 [1] Github (mlr-org/mlr3misc@43c0ffa)
#> mlr3pipelines 0.5.0-2 2023-12-08 [1] CRAN (R 4.3.1)
#> mlr3proba * 0.5.8 2024-01-24 [1] Github (mlr-org/mlr3proba@930d8b0)
#> mlr3viz 0.7.0.9000 2023-12-22 [1] Github (mlr-org/mlr3viz@73aa4ec)
#> munsell 0.5.0 2018-06-12 [1] CRAN (R 4.3.0)
#> ooplah 0.2.0 2022-01-21 [1] CRAN (R 4.3.0)
#> palmerpenguins 0.1.1 2022-08-15 [1] CRAN (R 4.3.0)
#> paradox 0.11.1 2023-03-17 [1] CRAN (R 4.3.0)
#> parallelly 1.36.0 2023-05-26 [1] CRAN (R 4.3.0)
#> param6 0.2.4 2023-07-24 [1] Github (xoopR/param6@0fa3577)
#> pillar 1.9.0 2023-03-22 [1] CRAN (R 4.3.0)
#> pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.3.0)
#> purrr 1.0.2 2023-08-10 [1] CRAN (R 4.3.0)
#> R.cache 0.16.0 2022-07-21 [1] CRAN (R 4.3.0)
#> R.methodsS3 1.8.2 2022-06-13 [1] CRAN (R 4.3.0)
#> R.oo 1.26.0 2024-01-24 [1] CRAN (R 4.3.1)
#> R.utils 2.12.3 2023-11-18 [1] CRAN (R 4.3.1)
#> R6 2.5.1 2021-08-19 [1] CRAN (R 4.3.0)
#> Rcpp 1.0.12 2024-01-09 [1] CRAN (R 4.3.1)
#> reprex 2.1.0 2024-01-11 [1] CRAN (R 4.3.1)
#> RhpcBLASctl 0.23-42 2023-02-11 [1] CRAN (R 4.3.0)
#> rlang 1.1.3 2024-01-10 [1] CRAN (R 4.3.1)
#> rmarkdown 2.25 2023-09-18 [1] CRAN (R 4.3.1)
#> rstudioapi 0.15.0 2023-07-07 [1] CRAN (R 4.3.0)
#> scales 1.3.0 2023-11-28 [1] CRAN (R 4.3.1)
#> sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.0)
#> set6 0.2.6 2023-10-16 [1] Github (xoopR/set6@a901255)
#> styler 1.10.2.9000 2024-01-24 [1] Github (r-lib/styler@ffe79c1)
#> survival 3.5-7 2023-08-14 [1] CRAN (R 4.3.0)
#> survivalmodels 0.1.18 2023-11-21 [1] Github (RaphaelS1/survivalmodels@79b913f)
#> tibble 3.2.1 2023-03-20 [1] CRAN (R 4.3.0)
#> tidyselect 1.2.0 2022-10-10 [1] CRAN (R 4.3.0)
#> utf8 1.2.4 2023-10-22 [1] CRAN (R 4.3.1)
#> uuid 1.2-0 2024-01-14 [1] CRAN (R 4.3.1)
#> vctrs 0.6.5 2023-12-01 [1] CRAN (R 4.3.1)
#> withr 3.0.0 2024-01-16 [1] CRAN (R 4.3.1)
#> xfun 0.41 2023-11-01 [1] CRAN (R 4.3.1)
#> yaml 2.3.8 2023-12-11 [1] CRAN (R 4.3.1)
#>
#> [1] /Users/Lukas/Library/R/arm64/4.3/library
#> [2] /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library
#>
#> ──────────────────────────────────────────────────────────────────────────────
I have a use case where a
bmris aggregated using many measures, and it may be easier to parallelize over measures to speed up overall aggregating.Here's an example including an experimental combination function I wrote for testing:
My experimental function works on a list with n > 1
BenchmarkAggrobjects, which I assume to be more useful than e.g a$combine()method that would only join 2 objects, but I'm not sure about the API.Created on 2024-01-24 with reprex v2.1.0
Session info