added outer fold score to msg in raschka, added stroke to bars

Author: ercbk

README.Rmd

@@ -65,7 +65,7 @@ runs <- runs_raw %>%
 raschka <- runs %>% 
    filter(method == "raschka") %>%
    ggplot(aes(y = duration, x = implementation, label = duration)) + 
-   geom_bar(stat = "identity", width = 0.50, fill = "#195198") +
+   geom_bar(aes(color = after_scale(prismatic::clr_darken(rep("#195198",4), 0.3))), fill = "#195198", stat = "identity", width = 0.50) +
    coord_flip() +
    scale_x_reordered() +
    geom_text(hjust = 1.3,  size = 3.5, color = "white") +
@@ -77,7 +77,7 @@ raschka <- runs %>%
 kj <- runs %>% 
    filter(method == "kj") %>%
    ggplot(aes(y = duration, x = implementation, label = duration)) + 
-   geom_bar(stat = "identity", width = 0.50, fill = "#BD9865") +
+   geom_bar(aes(color = after_scale(prismatic::clr_darken(rep("#BD9865",5), 0.3))), stat = "identity", width = 0.50, fill = "#BD9865") +
    coord_flip() +
    scale_x_reordered() +
    geom_text(hjust = 1.3,  size = 3.5, color = "white") +

README_files/figure-gfm/unnamed-chunk-1-1.png

@@ -295,6 +295,11 @@ if (!is.data.frame(chosen_preds)) {
 }
 test_error <- round(error_FUN(test_dat$y, chosen_preds), 5)
 
+# avg error across outer-loop folds in ncv
+outer_kfold_error <- algorithm_comparison %>% 
+   filter(model == chosen_alg) %>%
+   mutate(mean_error = round(mean_error, 5)) %>% 
+   pull(mean_error)
 
 # Create output message and text me the results
 best_hyper_vals <- best_hyper_vals %>% 
@@ -303,6 +308,7 @@ best_hyper_vals <- best_hyper_vals %>%
       glue_collapse(sep = ",", last = " and ")
 
 msg <- glue("Avg K-Fold CV error: {avg_kfold_error[[1]]}
+     Outer-Fold Avg Error: {outer_kfold_error}
      Test Error: {test_error}
      Best Parameters for {chosen_alg}:
      {best_hyper_vals}")

duration-experiment/raschka/nested-cv-kj-raschka.R

@@ -241,8 +241,15 @@ best_params <- as.data.frame(chosen_train$tuning_result$params) %>%
       glue_data("{param} = {value}") %>% 
       glue_collapse(sep = ",", last = " and ")
 
+# avg error across outer-loop folds in ncv
+outer_kfold_error <- algorithm_comparison %>% 
+   filter(model == chosen_alg) %>%
+   mutate(mean_error = round(mean_error, 5)) %>% 
+   pull(mean_error)
+
 
 msg <- glue("Average of K-fold CV test folds: {kfold_error}
+     Outer-Fold Avg Error: {outer_kfold_error}
      Training Error: {train_error}
      Test Error: {test_error}
      Best Parameters for {chosen_alg}:

duration-experiment/raschka/nested-cv-mlr3-raschka.R

@@ -267,15 +267,16 @@
 
 # Average error across tuning folds and the parameter values chosen during tuning
 k_fold_score = round(-1 * gcv_model_select.best_score_, 5)
+outer_kfold_score = round(results[results.model == chosen_alg]['mean_error'][0], 5)
 best_hyper_vals = gcv_model_select.best_params_
 
 # best_hyper_vals is a dict. Use it to create the df, then add the errors.
 model_stats = pd.DataFrame(data = best_hyper_vals, index = [1])
-model_stats = model_stats.assign(kfold_error = k_fold_score, train_error = train_error, test_error = test_error)
+model_stats = model_stats.assign(kfold_error = k_fold_score, outer_fold_error = outer_kfold_score, train_error = train_error, test_error = test_error)
 
 
 # evidently has to be written into a paragraph because print outputs can't be saved and there's no glue in python.
-msg = f'Python script finished in {time_elapsed} seconds. The chosen algorithm was {chosen_alg} with parameters, {best_hyper_vals}. Avg score over cv\'s test folds was {k_fold_score}. Training Error: {train_error}, Test Error: {test_error}'
+msg = f'Python script finished in {time_elapsed} seconds. The chosen algorithm was {chosen_alg} with parameters, {best_hyper_vals}. Avg score over cv\'s test folds was {k_fold_score}. Outer fold avg score was {outer_kfold_score}. Training Error: {train_error}, Test Error: {test_error}'
 
 # text me the results
 pb.push_note("Nested CV script finished", msg)

duration-experiment/raschka/nested-cv-py-raschka.py

@@ -301,8 +301,14 @@ best_params <- as.data.frame(chosen_tuned$best_params_) %>%
    glue_data("{param} = {value}") %>% 
    glue_collapse(sep = ",", last = " and ")
 
+outer_kfold_error <- purrr::map_dfr(algorithm_comparison, ~select(., mean_error), .id = "model") %>% 
+   filter(mean_error == min(mean_error)) %>%
+   mutate(mean_error = round(mean_error, 5))
+   pull(mean_error)
+
 
 msg <- glue("Average of K-fold CV test folds: {kfold_error}
+     Outer-Fold Avg Error: {outer_kfold_error}
      Training Error: {train_error}
      Test Error: {test_error}
      Best Parameters for {chosen_alg}: