Hyperparameters bruteforce rework

Threads
Variables parameters
Plot graph

src/training/decision_tree/main.py

@@ -1,13 +1,60 @@
 import os
+import time
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 
-from sklearn.ensemble import HistGradientBoostingRegressor, GradientBoostingRegressor, RandomForestRegressor
-from sklearn.model_selection import train_test_split, GroupShuffleSplit
-from sklearn.tree import DecisionTreeRegressor
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.model_selection import GroupShuffleSplit
 
 from src.training.decision_tree import preprocessor
 from src.training.decision_tree.model import Model
 from src.training.decision_tree.visualizer import Visualizer
 
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def get_axis_limits(values):
+    minimum = min(values)
+    maximum = max(values)
+
+    if minimum == maximum:
+        padding = abs(minimum) * 0.05 or 0.5
+    else:
+        padding = (maximum - minimum) * 0.05
+
+    return minimum - padding, maximum + padding
+
+
+def train_model(parameter_1, parameter_2, parameter_1_name, parameter_2_name,
+                base_model_parameters, split):
+    start_time = time.perf_counter()
+    model_parameters = {
+        **base_model_parameters,
+        parameter_1_name: parameter_1,
+        parameter_2_name: parameter_2
+    }
+
+    model = Model(
+        tree=RandomForestRegressor(**model_parameters),
+        feature_names=preprocessor.get_feature_names(),
+        train_test_split=split
+    )
+
+    model.train()
+    train_mae, train_mse, test_mae, test_mse = model.evaluate()
+    model.release_training_data()
+
+    return {
+        "model": model,
+        "parameter_1": parameter_1,
+        "parameter_2": parameter_2,
+        "train_mae": train_mae,
+        "train_mse": train_mse,
+        "test_mae": test_mae,
+        "test_mse": test_mse,
+        "duration": time.perf_counter() - start_time
+    }
+
 
 def create_split(x, y, game_ids):
 
@@ -47,50 +94,138 @@ if __name__ == "__main__":
         x, y, game_ids = preprocessor.run("./dataset/output.csv")
 
         split = create_split(x, y, game_ids)
-        
-        trained_model = []
-        
-        is_hyperparameter_tuning = True
-        for depth in range(1, 10):
-            for leafs in range(1, int(depth * 5)):
-                print(f"Training model with depth={depth} and leafs={leafs}")
+        del x, y, game_ids
 
-                model = Model(
-                    tree=RandomForestRegressor(
-                        n_estimators=25,
-                        max_depth=depth,
-                        max_features=0.35,
-                        min_samples_leaf=leafs,
-                        random_state=42
-                    ),
-                    feature_names=preprocessor.get_feature_names(),
-                    train_test_split=split
+        parameter_1 = {
+            "name": "min_samples_leaf",
+            "label": "Leafs",
+            "values": list(range(16, 19))
+        }
+        parameter_2 = {
+            "name": "max_features",
+            "label": "Max features",
+            "values": [
+                round(value, 2)
+                for value in np.arange(0.2, 0.5, 0.05)
+            ]
+        }
+        base_model_parameters = {
+            "max_depth": 39,
+            "min_samples_leaf": 17,
+            "n_estimators": 26,
+            "max_features": 0.45,
+            "random_state": 42,
+            "n_jobs": 1
+        }
+
+        parameter_sets = [
+            (parameter_1_value, parameter_2_value)
+            for parameter_1_value in parameter_1["values"]
+            for parameter_2_value in parameter_2["values"]
+        ]
+        completed_parameter_1 = []
+        completed_parameter_2 = []
+        completed_maes = []
+
+        # Plot graph
+        fig, ax = plt.subplots()
+        scatter = ax.scatter([], [], c=[], cmap="RdYlGn_r", vmin=0, vmax=2)
+        colorbar = fig.colorbar(scatter, ax=ax)
+        colorbar.set_label("Test MAE")
+        ax.set_xlabel(parameter_1["label"])
+        ax.set_ylabel(parameter_2["label"])
+        ax.set_title(
+            "Random forest hyperparameter search "
+            f"({parameter_1['label']}, {parameter_2['label']})"
+        )
+        ax.set_xlim(get_axis_limits(parameter_1["values"]))
+        ax.set_ylim(get_axis_limits(parameter_2["values"]))
+        plt.ion()
+        plt.show(block=False)
+
+        cpu_count = os.cpu_count() or 1
+        worker_count = max(1, min(cpu_count - 2, 8))
+        print(
+            f"Training {len(parameter_sets)} models with "
+            f"{worker_count} concurrent models..."
+        )
+
+        with ThreadPoolExecutor(max_workers=worker_count) as executor:
+            parameter_iterator = iter(parameter_sets)
+            futures = {}
+
+            def submit_next():
+                try:
+                    parameter_1_value, parameter_2_value = next(parameter_iterator)
+                except StopIteration:
+                    return
+
+                future = executor.submit(
+                    train_model,
+                    parameter_1_value,
+                    parameter_2_value,
+                    parameter_1["name"],
+                    parameter_2["name"],
+                    base_model_parameters,
+                    split
+                )
+                futures[future] = (parameter_1_value, parameter_2_value)
+
+            for _ in range(worker_count):
+                submit_next()
+
+            best_result = None
+            while futures:
+                completed_futures, _ = wait(
+                    futures,
+                    return_when=FIRST_COMPLETED
                 )
 
-                model.train()
-                [train_mae, train_mse, test_mae, test_mse] = model.evaluate()
-                
-                # Verify it's not overfitting
-                if test_mae > 1.5 * train_mae:
-                    print("Model is overfitting, skipping saving...")
-                else:
-                    trained_model.append({
-                        "depth": depth,
-                        "leafs": leafs,
-                        "train_mae": train_mae,
-                        "train_mse": train_mse,
-                        "test_mae": test_mae,
-                        "test_mse": test_mse
-                    })
-                    
-        best_model = min(trained_model, key=lambda x: x["test_mae"])
-        
-        print("Best model so far:")
-        print("MAE:", best_model["test_mae"])
-        print("MSE:", best_model["test_mse"])
-                    
-                    
-        #model.save_model(model_path)
+                for future in completed_futures:
+                    futures.pop(future)
+                    result = future.result()
+                    completed_parameter_1.append(result["parameter_1"])
+                    completed_parameter_2.append(result["parameter_2"])
+                    completed_maes.append(result["test_mae"])
+
+                    scatter.set_offsets(np.column_stack((
+                        completed_parameter_1,
+                        completed_parameter_2
+                    )))
+                    scatter.set_array(np.asarray(completed_maes))
+                    fig.canvas.draw_idle()
+                    fig.canvas.flush_events()
+                    plt.pause(0.001)
+
+                    if (best_result is None or
+                            result["test_mae"] < best_result["test_mae"]):
+                        old_best_result = best_result
+                        best_result = result
+                        if old_best_result is not None:
+                            del old_best_result["model"]
+                    else:
+                        del result["model"]
+
+                    print(
+                        f"Completed {parameter_1['name']}={result['parameter_1']}, "
+                        f"{parameter_2['name']}={result['parameter_2']}. "
+                        f"Best so far: {parameter_1['name']}="
+                        f"{best_result['parameter_1']}, "
+                        f"{parameter_2['name']}={best_result['parameter_2']}, "
+                        f"MAE={best_result['test_mae']:.6f}, "
+                        f"duration={result['duration']:.1f}s"
+                    )
+
+                    submit_next()
+
+        model = best_result["model"]
+        plt.ioff()
+        graph_path = f"doc/hyperparametersOptimisation/{parameter_1['name']}_{parameter_2['name']}.png"
+        os.makedirs(os.path.dirname(graph_path), exist_ok=True)
+        fig.savefig(graph_path, dpi=200, bbox_inches="tight")
+        print(f"Saved hyperparameter search graph to {graph_path}")
+        print("Close the graph window to finish.")
+        plt.show(block=True)
 
     visualizer = Visualizer(model=model)
     visualizer.show_feature_importance(15)

src/training/decision_tree/model.py

@@ -67,6 +67,12 @@ class Model:
         
         return [train_mae, train_mse, test_mae, test_mse]
 
+    def release_training_data(self):
+        self._x_train = None
+        self._x_test = None
+        self._y_train = None
+        self._y_test = None
+
 
     def save_model(self, path):
         joblib.dump({