From ceb77f3cbe6e3459a5beafc9d034109ffd161df9 Mon Sep 17 00:00:00 2001 From: johannes wasmer <johannes.wasmer@gmail.com> Date: Wed, 10 Jul 2024 09:39:26 +0100 Subject: [PATCH] add mlfpo24 presentation figures (jij-prediction) --- .../AutoML_perf_3/README.md | 59 +++++++++++++++++++ .../AutoML_perf_3/ldb_performance_boxplot.png | 3 + .../AutoML_perf_3/leaderboard.csv | 41 +++++++++++++ .../test3.png | 3 + 4 files changed, 106 insertions(+) create mode 100644 fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/README.md create mode 100644 fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/ldb_performance_boxplot.png create mode 100644 fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/leaderboard.csv create mode 100644 fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/test3.png diff --git a/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/README.md b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/README.md new file mode 100644 index 0000000..4731be0 --- /dev/null +++ b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/README.md @@ -0,0 +1,59 @@ +# AutoML Leaderboard + +| Best model | name | model_type | metric_type | metric_value | train_time | single_prediction_time | +|:-------------|:---------------------------------------------------------------------------------------------------------------|:---------------|:--------------|---------------:|-------------:|-------------------------:| +| | [1_Linear](1_Linear/README.md) | Linear | rmse | 2.52361 | 52.21 | 0.4477 | +| | [2_Default_CatBoost](2_Default_CatBoost/README.md) | CatBoost | rmse | 1.23681 | 181.06 | 0.7293 | +| | [3_Default_NeuralNetwork](3_Default_NeuralNetwork/README.md) | Neural Network | rmse | 1.18171 | 79.17 | 0.474 | +| | [4_Default_RandomForest](4_Default_RandomForest/README.md) | Random Forest | rmse | 2.14166 | 66.75 | 1.0023 | +| | [5_CatBoost](5_CatBoost/README.md) | CatBoost | rmse | 1.24372 | 771.64 | 0.9002 | +| | [9_RandomForest](9_RandomForest/README.md) | Random Forest | rmse | 2.0701 | 83.89 | 1.7152 | +| | [13_NeuralNetwork](13_NeuralNetwork/README.md) | Neural Network | rmse | 2.4491 | 153.41 | 0.7696 | +| | [6_CatBoost](6_CatBoost/README.md) | CatBoost | rmse | 1.2727 | 267.36 | 0.5813 | +| | [10_RandomForest](10_RandomForest/README.md) | Random Forest | rmse | 2.36807 | 96.47 | 2.2326 | +| | [14_NeuralNetwork](14_NeuralNetwork/README.md) | Neural Network | rmse | 1.06647 | 208.17 | 0.5224 | +| | [7_CatBoost](7_CatBoost/README.md) | CatBoost | rmse | 1.24338 | 184.28 | 0.5238 | +| | [11_RandomForest](11_RandomForest/README.md) | Random Forest | rmse | 2.21546 | 133.72 | 1.1149 | +| | [15_NeuralNetwork](15_NeuralNetwork/README.md) | Neural Network | rmse | 1.42765 | 225.97 | 0.7887 | +| | [8_CatBoost](8_CatBoost/README.md) | CatBoost | rmse | 1.27917 | 228.33 | 0.9027 | +| | [12_RandomForest](12_RandomForest/README.md) | Random Forest | rmse | 2.25663 | 96.36 | 1.0009 | +| **the best** | [16_NeuralNetwork](16_NeuralNetwork/README.md) | Neural Network | rmse | 1.06008 | 35.81 | 0.4092 | +| | [16_NeuralNetwork_GoldenFeatures](16_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.03367 | 423.36 | 1.5723 | +| | [14_NeuralNetwork_GoldenFeatures](14_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 0.965365 | 135.92 | 0.7047 | +| | [3_Default_NeuralNetwork_GoldenFeatures](3_Default_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.20543 | 69.6 | 2.2865 | +| | [14_NeuralNetwork_GoldenFeatures_RandomFeature](14_NeuralNetwork_GoldenFeatures_RandomFeature/README.md) | Neural Network | rmse | 1.06208 | 162.46 | 0.8336 | +| | [14_NeuralNetwork_GoldenFeatures_SelectedFeatures](14_NeuralNetwork_GoldenFeatures_SelectedFeatures/README.md) | Neural Network | rmse | 1.78175 | 12.31 | 0.305 | +| | [2_Default_CatBoost_SelectedFeatures](2_Default_CatBoost_SelectedFeatures/README.md) | CatBoost | rmse | 1.84165 | 17.34 | 0.1177 | +| | [9_RandomForest_SelectedFeatures](9_RandomForest_SelectedFeatures/README.md) | Random Forest | rmse | 2.42095 | 16.14 | 0.5727 | +| | [17_NeuralNetwork_GoldenFeatures](17_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.54396 | 145.95 | 0.8348 | +| | [18_NeuralNetwork_GoldenFeatures](18_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.01251 | 101.97 | 0.8213 | +| | [19_NeuralNetwork_GoldenFeatures](19_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.19927 | 118.08 | 0.7489 | +| | [20_CatBoost](20_CatBoost/README.md) | CatBoost | rmse | 1.254 | 198.05 | 0.8427 | +| | [21_CatBoost](21_CatBoost/README.md) | CatBoost | rmse | 1.26172 | 87.8 | 0.6903 | +| | [22_CatBoost](22_CatBoost/README.md) | CatBoost | rmse | 1.24025 | 183.23 | 0.8749 | +| | [23_CatBoost](23_CatBoost/README.md) | CatBoost | rmse | 1.17609 | 135.53 | 0.6779 | +| | [24_RandomForest](24_RandomForest/README.md) | Random Forest | rmse | 2.26038 | 62 | 1.0414 | +| | [25_RandomForest](25_RandomForest/README.md) | Random Forest | rmse | 1.99645 | 85.16 | 1.0301 | +| | [26_RandomForest](26_RandomForest/README.md) | Random Forest | rmse | 2.29906 | 83.98 | 1.1382 | +| | [27_RandomForest](27_RandomForest/README.md) | Random Forest | rmse | 2.08971 | 102.79 | 0.818 | +| | [28_NeuralNetwork_GoldenFeatures](28_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.4392 | 67 | 0.8453 | +| | [29_NeuralNetwork_GoldenFeatures](29_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.07526 | 39.25 | 0.7892 | +| | [30_NeuralNetwork_GoldenFeatures](30_NeuralNetwork_GoldenFeatures/README.md) | Neural Network | rmse | 1.04295 | 68.3 | 0.7141 | +| | [31_RandomForest](31_RandomForest/README.md) | Random Forest | rmse | 1.97433 | 109.2 | 0.9585 | +| | [32_RandomForest](32_RandomForest/README.md) | Random Forest | rmse | 2.07303 | 117.31 | 1.0869 | +| | [Ensemble](Ensemble/README.md) | Ensemble | rmse | 1.06008 | 0.53 | 0.6805 | + +### AutoML Performance + + +### AutoML Performance Boxplot + + +### Features Importance + + + + +### Spearman Correlation of Models + + diff --git a/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/ldb_performance_boxplot.png b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/ldb_performance_boxplot.png new file mode 100644 index 0000000..e777b7e --- /dev/null +++ b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/ldb_performance_boxplot.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17464cf5c775ed7621a3f5ba2042f5162b2e35a8ab3b83e5ac4291f6fed74faa +size 27295 diff --git a/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/leaderboard.csv b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/leaderboard.csv new file mode 100644 index 0000000..d05a845 --- /dev/null +++ b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/AutoML_perf_3/leaderboard.csv @@ -0,0 +1,41 @@ +name,model_type,metric_type,metric_value,train_time,single_prediction_time +1_Linear,Linear,rmse,2.5236072339279745,52.21,0.4477 +2_Default_CatBoost,CatBoost,rmse,1.236805535613667,181.06,0.7293 +3_Default_NeuralNetwork,Neural Network,rmse,1.1817094321988502,79.17,0.474 +4_Default_RandomForest,Random Forest,rmse,2.141655333956899,66.75,1.0023 +5_CatBoost,CatBoost,rmse,1.243717278194893,771.64,0.9002 +9_RandomForest,Random Forest,rmse,2.0700985836899877,83.89,1.7152 +13_NeuralNetwork,Neural Network,rmse,2.449095032741127,153.41,0.7696 +6_CatBoost,CatBoost,rmse,1.272702544093295,267.36,0.5813 +10_RandomForest,Random Forest,rmse,2.3680734931221514,96.47,2.2326 +14_NeuralNetwork,Neural Network,rmse,1.0664748741456416,208.17,0.5224 +7_CatBoost,CatBoost,rmse,1.2433845807411108,184.28,0.5238 +11_RandomForest,Random Forest,rmse,2.2154627643729228,133.72,1.1149 +15_NeuralNetwork,Neural Network,rmse,1.4276468884977862,225.97,0.7887 +8_CatBoost,CatBoost,rmse,1.2791693218897453,228.33,0.9027 +12_RandomForest,Random Forest,rmse,2.256625660315601,96.36,1.0009 +16_NeuralNetwork,Neural Network,rmse,1.060075990845332,35.81,0.4092 +16_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.0336705943023028,423.36,1.5723 +14_NeuralNetwork_GoldenFeatures,Neural Network,rmse,0.965365408199375,135.92,0.7047 +3_Default_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.205427067284829,69.6,2.2865 +14_NeuralNetwork_GoldenFeatures_RandomFeature,Neural Network,rmse,1.062081810198114,162.46,0.8336 +14_NeuralNetwork_GoldenFeatures_SelectedFeatures,Neural Network,rmse,1.7817537923259728,12.31,0.305 +2_Default_CatBoost_SelectedFeatures,CatBoost,rmse,1.8416549784796679,17.34,0.1177 +9_RandomForest_SelectedFeatures,Random Forest,rmse,2.4209528538702165,16.14,0.5727 +17_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.543957299099634,145.95,0.8348 +18_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.012511402724327,101.97,0.8213 +19_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.1992749062961559,118.08,0.7489 +20_CatBoost,CatBoost,rmse,1.2539967055980967,198.05,0.8427 +21_CatBoost,CatBoost,rmse,1.2617175094726218,87.8,0.6903 +22_CatBoost,CatBoost,rmse,1.2402538729248644,183.23,0.8749 +23_CatBoost,CatBoost,rmse,1.1760915799803964,135.53,0.6779 +24_RandomForest,Random Forest,rmse,2.260383562606155,62.0,1.0414 +25_RandomForest,Random Forest,rmse,1.996449381673639,85.16,1.0301 +26_RandomForest,Random Forest,rmse,2.2990612093994462,83.98,1.1382 +27_RandomForest,Random Forest,rmse,2.0897081587274693,102.79,0.818 +28_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.439197707899377,67.0,0.8453 +29_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.0752647694988577,39.25,0.7892 +30_NeuralNetwork_GoldenFeatures,Neural Network,rmse,1.042952029809796,68.3,0.7141 +31_RandomForest,Random Forest,rmse,1.9743307167305604,109.2,0.9585 +32_RandomForest,Random Forest,rmse,2.073029746230502,117.31,1.0869 +Ensemble,Ensemble,rmse,1.060075990845332,0.53,0.6805 diff --git a/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/test3.png b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/test3.png new file mode 100644 index 0000000..75c6b6a --- /dev/null +++ b/fig/presentation-2024-07/ruess-automl/autoML_analysis_train_test_split/test3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28499fa4ee960890ecc557cac951905fe9168f864bfa77088ab1bff1d5622a76 +size 43944 -- GitLab