diff --git a/doc/hyperparametersOptimisation/Depth_Leafs.png b/doc/hyperparametersOptimisation/Depth_Leafs.png new file mode 100644 index 0000000..0095251 Binary files /dev/null and b/doc/hyperparametersOptimisation/Depth_Leafs.png differ diff --git a/doc/hyperparametersOptimisation/max_depth_n_estimators.png b/doc/hyperparametersOptimisation/max_depth_n_estimators.png new file mode 100644 index 0000000..d7e1736 Binary files /dev/null and b/doc/hyperparametersOptimisation/max_depth_n_estimators.png differ diff --git a/doc/hyperparametersOptimisation/min_samples_leaf_max_features.png b/doc/hyperparametersOptimisation/min_samples_leaf_max_features.png new file mode 100644 index 0000000..83119a5 Binary files /dev/null and b/doc/hyperparametersOptimisation/min_samples_leaf_max_features.png differ diff --git a/src/training/decision_tree/main.py b/src/training/decision_tree/main.py index 37c82d2..0d30666 100644 --- a/src/training/decision_tree/main.py +++ b/src/training/decision_tree/main.py @@ -1,13 +1,60 @@ import os +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait -from sklearn.ensemble import HistGradientBoostingRegressor, GradientBoostingRegressor, RandomForestRegressor -from sklearn.model_selection import train_test_split, GroupShuffleSplit -from sklearn.tree import DecisionTreeRegressor +from sklearn.ensemble import RandomForestRegressor +from sklearn.model_selection import GroupShuffleSplit from src.training.decision_tree import preprocessor from src.training.decision_tree.model import Model from src.training.decision_tree.visualizer import Visualizer +import matplotlib.pyplot as plt +import numpy as np + + +def get_axis_limits(values): + minimum = min(values) + maximum = max(values) + + if minimum == maximum: + padding = abs(minimum) * 0.05 or 0.5 + else: + padding = (maximum - minimum) * 0.05 + + return minimum - padding, maximum + padding + + +def train_model(parameter_1, parameter_2, parameter_1_name, parameter_2_name, + base_model_parameters, split): + start_time = time.perf_counter() + model_parameters = { + **base_model_parameters, + parameter_1_name: parameter_1, + parameter_2_name: parameter_2 + } + + model = Model( + tree=RandomForestRegressor(**model_parameters), + feature_names=preprocessor.get_feature_names(), + train_test_split=split + ) + + model.train() + train_mae, train_mse, test_mae, test_mse = model.evaluate() + model.release_training_data() + + return { + "model": model, + "parameter_1": parameter_1, + "parameter_2": parameter_2, + "train_mae": train_mae, + "train_mse": train_mse, + "test_mae": test_mae, + "test_mse": test_mse, + "duration": time.perf_counter() - start_time + } + def create_split(x, y, game_ids): @@ -47,50 +94,138 @@ if __name__ == "__main__": x, y, game_ids = preprocessor.run("./dataset/output.csv") split = create_split(x, y, game_ids) - - trained_model = [] - - is_hyperparameter_tuning = True - for depth in range(1, 10): - for leafs in range(1, int(depth * 5)): - print(f"Training model with depth={depth} and leafs={leafs}") + del x, y, game_ids - model = Model( - tree=RandomForestRegressor( - n_estimators=25, - max_depth=depth, - max_features=0.35, - min_samples_leaf=leafs, - random_state=42 - ), - feature_names=preprocessor.get_feature_names(), - train_test_split=split + parameter_1 = { + "name": "min_samples_leaf", + "label": "Leafs", + "values": list(range(16, 19)) + } + parameter_2 = { + "name": "max_features", + "label": "Max features", + "values": [ + round(value, 2) + for value in np.arange(0.2, 0.5, 0.05) + ] + } + base_model_parameters = { + "max_depth": 39, + "min_samples_leaf": 17, + "n_estimators": 26, + "max_features": 0.45, + "random_state": 42, + "n_jobs": 1 + } + + parameter_sets = [ + (parameter_1_value, parameter_2_value) + for parameter_1_value in parameter_1["values"] + for parameter_2_value in parameter_2["values"] + ] + completed_parameter_1 = [] + completed_parameter_2 = [] + completed_maes = [] + + # Plot graph + fig, ax = plt.subplots() + scatter = ax.scatter([], [], c=[], cmap="RdYlGn_r", vmin=0, vmax=2) + colorbar = fig.colorbar(scatter, ax=ax) + colorbar.set_label("Test MAE") + ax.set_xlabel(parameter_1["label"]) + ax.set_ylabel(parameter_2["label"]) + ax.set_title( + "Random forest hyperparameter search " + f"({parameter_1['label']}, {parameter_2['label']})" + ) + ax.set_xlim(get_axis_limits(parameter_1["values"])) + ax.set_ylim(get_axis_limits(parameter_2["values"])) + plt.ion() + plt.show(block=False) + + cpu_count = os.cpu_count() or 1 + worker_count = max(1, min(cpu_count - 2, 8)) + print( + f"Training {len(parameter_sets)} models with " + f"{worker_count} concurrent models..." + ) + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + parameter_iterator = iter(parameter_sets) + futures = {} + + def submit_next(): + try: + parameter_1_value, parameter_2_value = next(parameter_iterator) + except StopIteration: + return + + future = executor.submit( + train_model, + parameter_1_value, + parameter_2_value, + parameter_1["name"], + parameter_2["name"], + base_model_parameters, + split + ) + futures[future] = (parameter_1_value, parameter_2_value) + + for _ in range(worker_count): + submit_next() + + best_result = None + while futures: + completed_futures, _ = wait( + futures, + return_when=FIRST_COMPLETED ) - model.train() - [train_mae, train_mse, test_mae, test_mse] = model.evaluate() - - # Verify it's not overfitting - if test_mae > 1.5 * train_mae: - print("Model is overfitting, skipping saving...") - else: - trained_model.append({ - "depth": depth, - "leafs": leafs, - "train_mae": train_mae, - "train_mse": train_mse, - "test_mae": test_mae, - "test_mse": test_mse - }) - - best_model = min(trained_model, key=lambda x: x["test_mae"]) - - print("Best model so far:") - print("MAE:", best_model["test_mae"]) - print("MSE:", best_model["test_mse"]) - - - #model.save_model(model_path) + for future in completed_futures: + futures.pop(future) + result = future.result() + completed_parameter_1.append(result["parameter_1"]) + completed_parameter_2.append(result["parameter_2"]) + completed_maes.append(result["test_mae"]) + + scatter.set_offsets(np.column_stack(( + completed_parameter_1, + completed_parameter_2 + ))) + scatter.set_array(np.asarray(completed_maes)) + fig.canvas.draw_idle() + fig.canvas.flush_events() + plt.pause(0.001) + + if (best_result is None or + result["test_mae"] < best_result["test_mae"]): + old_best_result = best_result + best_result = result + if old_best_result is not None: + del old_best_result["model"] + else: + del result["model"] + + print( + f"Completed {parameter_1['name']}={result['parameter_1']}, " + f"{parameter_2['name']}={result['parameter_2']}. " + f"Best so far: {parameter_1['name']}=" + f"{best_result['parameter_1']}, " + f"{parameter_2['name']}={best_result['parameter_2']}, " + f"MAE={best_result['test_mae']:.6f}, " + f"duration={result['duration']:.1f}s" + ) + + submit_next() + + model = best_result["model"] + plt.ioff() + graph_path = f"doc/hyperparametersOptimisation/{parameter_1['name']}_{parameter_2['name']}.png" + os.makedirs(os.path.dirname(graph_path), exist_ok=True) + fig.savefig(graph_path, dpi=200, bbox_inches="tight") + print(f"Saved hyperparameter search graph to {graph_path}") + print("Close the graph window to finish.") + plt.show(block=True) visualizer = Visualizer(model=model) visualizer.show_feature_importance(15) diff --git a/src/training/decision_tree/model.py b/src/training/decision_tree/model.py index 3cfc24e..0e5ca3f 100644 --- a/src/training/decision_tree/model.py +++ b/src/training/decision_tree/model.py @@ -67,6 +67,12 @@ class Model: return [train_mae, train_mse, test_mae, test_mse] + def release_training_data(self): + self._x_train = None + self._x_test = None + self._y_train = None + self._y_test = None + def save_model(self, path): joblib.dump({