Add stub

src/data_processing/dataset_loader.py

@@ -0,0 +1,97 @@
+import pandas as pd
+import torch
+
+INPUT_CSV = "../../dataset/output.csv"
+OUTPUT_FILE = "../../dataset/output.pt"
+
+SQUARES = [
+    f"{file}{rank}"
+    for rank in range(8, 0, -1)
+    for file in "abcdefgh"
+]
+
+PIECE_TO_PLANE = {
+    "P": 0,
+    "N": 1,
+    "B": 2,
+    "R": 3,
+    "Q": 4,
+    "K": 5,
+    "p": 6,
+    "n": 7,
+    "b": 8,
+    "r": 9,
+    "q": 10,
+    "k": 11,
+}
+
+def board_to_planes(row):
+    planes = torch.zeros((12, 8, 8), dtype=torch.float32)
+
+    for index, square in enumerate(SQUARES):
+        piece = row[square]
+
+        if pd.isna(piece):
+            continue
+
+        plane = PIECE_TO_PLANE.get(piece)
+
+        if plane is None:
+            continue
+
+        row_index = index // 8
+        col_index = index % 8
+
+        planes[plane, row_index, col_index] = 1.0
+
+    return planes
+
+def parse_eval(value):
+    value = str(value).strip()
+
+    if value.startswith("#"):
+        mate_distance = int(value[1:])
+
+        if mate_distance > 0:
+            return 10.0
+        else:
+            return -10.0
+
+    return float(value)
+
+def main():
+    df = pd.read_csv(INPUT_CSV)
+
+    positions = []
+    evaluations = []
+    turns = []
+
+    for _, row in df.iterrows():
+        positions.append(board_to_planes(row))
+
+        evaluations.append(parse_eval(row["eval"]))
+
+        turns.append(1 if row["turn"] == "white" else 0)
+
+
+    X = torch.stack(positions)
+    y = torch.tensor(evaluations, dtype=torch.float32)
+    turn = torch.tensor(turns, dtype=torch.float32)
+
+    torch.save(
+        {
+            "X": X,
+            "y": y,
+            "turn": turn,
+        },
+        OUTPUT_FILE,
+    )
+
+    print(f"Loaded {len(df):,} positions")
+    print(f"X shape:    {X.shape}")
+    print(f"y shape:    {y.shape}")
+    print(f"turn shape: {turn.shape}")
+    print(f"Saved to:   {OUTPUT_FILE}")
+
+if __name__ == "__main__":
+    main()

src/data_processing/extract.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""
+Extract the first N lines from a huge file without loading it into RAM.
+
+Works because Python file objects are iterators: reading line-by-line
+only ever keeps one line (plus a small internal read-ahead buffer) in
+memory at a time, regardless of the file's total size.
+
+Usage:
+    python extract_lines.py input.txt output.txt --lines 500
+
+    # Start from a specific line (e.g. skip a header, or resume mid-file)
+    python extract_lines.py input.txt output.txt --lines 500 --start 1000
+"""
+
+import argparse
+from itertools import islice
+
+
+def extract_lines(input_path: str, output_path: str, start: int, count: int) -> int:
+    written = 0
+    with open(input_path, "r", encoding="utf-8", errors="replace", buffering=1024 * 1024) as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+        for line in islice(fin, start, start + count):
+            fout.write(line)
+            written += 1
+    return written
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract a slice of lines from a huge file, streaming (low RAM).")
+    parser.add_argument("input", help="Path to the big input file")
+    parser.add_argument("output", help="Path to write the extracted lines to")
+    parser.add_argument("--lines", type=int, default=500, help="Number of lines to extract (default: 500)")
+    parser.add_argument("--start", type=int, default=0, help="Line number to start at, 0-indexed (default: 0)")
+    args = parser.parse_args()
+
+    n = extract_lines(args.input, args.output, args.start, args.lines)
+    print(f"Wrote {n} lines (lines {args.start}-{args.start + n - 1}) to {args.output}")
+
+if __name__ == "__main__":
+    main()

src/data_processing/format.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+"""
+Turn refined PGN games into a CSV where each row is the board state
+after one evaluated move.
+
+Row layout:
+    game_id, a8, b8, ..., h1, eval, turn
+
+  - game_id : Lichess game ID extracted from the Site header.
+              Example:
+                  [Site "https://lichess.org/qE0J8lxe"]
+              becomes:
+                  qE0J8lxe
+
+  - a8..h1 : the 64 squares after the move.
+              Uppercase = white, lowercase = black,
+              empty string = empty square.
+
+  - eval    : the %eval attached to the move.
+
+  - turn    : "white" or "black", meaning whose turn it is NEXT.
+
+Moves without an evaluation are skipped.
+
+Streams game-by-game, so memory usage stays low.
+
+Requires:
+    pip install chess
+
+Usage:
+    python pgn_to_csv.py refined.pgn out.csv
+    python pgn_to_csv.py refined.pgn out.csv --no-game-id
+"""
+
+import argparse
+import csv
+import re
+import sys
+
+import chess
+
+TAG_RE = re.compile(r'^\[(\w+)\s+"(.*)"\]\s*$')
+EVAL_RE = re.compile(r'\[%eval\s+([^\]]+)\]')
+MOVE_NUM_RE = re.compile(r'^\d+\.+$')
+RESULT_RE = re.compile(r'^(1-0|0-1|1/2-1/2|\*)$')
+SUFFIX_ANNOTATION_RE = re.compile(r'[!?]+$')
+
+SQUARE_COLUMNS = [
+    f + str(r)
+    for r in range(8, 0, -1)
+    for f in "abcdefgh"
+]
+
+
+def parse_headers(header_lines):
+    """
+    Convert PGN header lines into a dictionary.
+
+    Example:
+        [Site "https://lichess.org/qE0J8lxe"]
+
+    becomes:
+        {
+            "Site": "https://lichess.org/qE0J8lxe"
+        }
+    """
+    headers = {}
+
+    for line in header_lines:
+        match = TAG_RE.match(line)
+
+        if match:
+            key, value = match.groups()
+            headers[key] = value
+
+    return headers
+
+
+def extract_game_id(site):
+    """
+    Extract the Lichess game ID from the Site URL.
+
+    Example:
+        https://lichess.org/qE0J8lxe
+        -> qE0J8lxe
+    """
+    if not site:
+        return None
+
+    return site.rstrip("/").split("/")[-1]
+
+
+def tokenize_movetext(movetext):
+    """
+    Yield (san_move, eval_str_or_None) pairs from a movetext blob.
+    """
+    tokens = re.findall(r'\{[^}]*\}|\S+', movetext)
+
+    i = 0
+    n = len(tokens)
+
+    while i < n:
+        tok = tokens[i]
+
+        if MOVE_NUM_RE.match(tok) or RESULT_RE.match(tok):
+            i += 1
+            continue
+
+        move_san = tok
+        eval_val = None
+
+        if i + 1 < n and tokens[i + 1].startswith('{'):
+            match = EVAL_RE.search(tokens[i + 1])
+
+            if match:
+                eval_val = match.group(1)
+
+            i += 2
+
+        else:
+            i += 1
+
+        yield move_san, eval_val
+
+
+def board_row(board):
+    """
+    Return the 64 square values for the current board,
+    in a8..h1 order.
+    """
+    row = []
+
+    for name in SQUARE_COLUMNS:
+        square = chess.parse_square(name)
+        piece = board.piece_at(square)
+
+        row.append(piece.symbol() if piece else "")
+
+    return row
+
+
+def process_game(header_lines, movetext_lines, writer, include_game_id):
+    """
+    Process one game and write its evaluated positions.
+
+    Returns:
+        number of rows written
+    """
+
+    if not movetext_lines:
+        return 0
+
+    headers = parse_headers(header_lines)
+
+    site = headers.get("Site")
+    game_id = extract_game_id(site)
+
+    if game_id is None:
+        print(
+            "  [warning] Game has no valid Site header, skipping.",
+            file=sys.stderr
+        )
+        return 0
+
+    movetext = " ".join(
+        line.strip()
+        for line in movetext_lines
+        if line.strip()
+    )
+
+    board = chess.Board()
+    rows_written = 0
+
+    for move_san, eval_val in tokenize_movetext(movetext):
+
+        # No evaluation means this position has no target value.
+        # Do not put it into the regression dataset.
+        if eval_val is None:
+            continue
+
+        clean_san = SUFFIX_ANNOTATION_RE.sub("", move_san)
+
+        try:
+            board.push_san(clean_san)
+
+        except (
+            chess.IllegalMoveError,
+            chess.InvalidMoveError,
+            chess.AmbiguousMoveError,
+            ValueError,
+        ) as e:
+
+            print(
+                f"  [game {game_id}] stopping early: "
+                f"could not parse move '{move_san}' ({e})",
+                file=sys.stderr,
+            )
+
+            break
+
+        turn = (
+            "white"
+            if board.turn == chess.WHITE
+            else "black"
+        )
+
+        row = board_row(board)
+
+        row += [
+            eval_val,
+            turn,
+        ]
+
+        if include_game_id:
+            row = [game_id] + row
+
+        writer.writerow(row)
+
+        rows_written += 1
+
+    return rows_written
+
+
+def refine(input_path, output_path, include_game_id):
+    header_lines = []
+    movetext_lines = []
+
+    state = "seek_header"
+
+    games = 0
+    total_rows = 0
+
+    header_row = SQUARE_COLUMNS + ["eval", "turn"]
+
+    if include_game_id:
+        header_row = ["game_id"] + header_row
+
+    with (
+        open(
+            input_path,
+            "r",
+            encoding="utf-8",
+            errors="replace",
+            buffering=1024 * 1024,
+        ) as fin,
+
+        open(
+            output_path,
+            "w",
+            encoding="utf-8",
+            newline="",
+        ) as fout
+    ):
+
+        writer = csv.writer(fout)
+        writer.writerow(header_row)
+
+        for raw_line in fin:
+            line = raw_line.rstrip("\n")
+
+            if state == "seek_header":
+
+                if line.startswith("["):
+                    header_lines.append(line)
+
+                elif line.strip() == "" and header_lines:
+                    state = "seek_movetext"
+
+            elif state == "seek_movetext":
+
+                if line.strip() == "":
+                    rows = process_game(
+                        header_lines,
+                        movetext_lines,
+                        writer,
+                        include_game_id,
+                    )
+
+                    total_rows += rows
+                    games += 1
+
+                    header_lines = []
+                    movetext_lines = []
+
+                    state = "seek_header"
+
+                else:
+                    movetext_lines.append(line)
+
+        # Handle final game if the file doesn't end with a blank line.
+        if header_lines and movetext_lines:
+            rows = process_game(
+                header_lines,
+                movetext_lines,
+                writer,
+                include_game_id,
+            )
+
+            total_rows += rows
+            games += 1
+
+    return games, total_rows
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Convert refined PGN games into a "
+            "per-move board-state CSV."
+        )
+    )
+
+    parser.add_argument(
+        "input",
+        help="Path to the refined PGN file",
+    )
+
+    parser.add_argument(
+        "output",
+        help="Path to write the CSV file",
+    )
+
+    parser.add_argument(
+        "--no-game-id",
+        action="store_true",
+        help="Omit the leading game_id column.",
+    )
+
+    args = parser.parse_args()
+
+    games, rows = refine(
+        args.input,
+        args.output,
+        include_game_id=not args.no_game_id,
+    )
+
+    print(
+        f"Processed {games:,} games "
+        f"-> {rows:,} evaluated positions written to "
+        f"{args.output}"
+    )
+
+
+if __name__ == "__main__":
+    main()

src/data_processing/refine.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""
+Refine a huge Lichess-style PGN file, streaming one game at a time so RAM
+stays flat regardless of file size (works fine on a 200GB file).
+
+Rules applied to every game:
+  1. If the game has no %eval annotations at all, drop it entirely.
+  2. Keep only these header tags: Site, WhiteElo, BlackElo.
+  3. In the movetext, remove every comment/annotation except %eval.
+  4. Keep only the %eval annotation from comments containing multiple
+     attributes, e.g. { [%eval 0.18] [%clk 0:01:00] } -> { [%eval 0.18] }.
+  5. Remove non-%eval textual comments, e.g. { Black offers draw }.
+  6. Remove NAGs such as !, ?, !?, ?!, etc.
+
+Games are separated by a blank line after the movetext.
+
+Usage:
+    python refine_pgn.py input.pgn output.pgn
+"""
+
+import argparse
+import re
+
+# Only these header tags survive
+KEEP_TAGS = {"Site", "WhiteElo", "BlackElo"}
+
+# Matches a header tag line, e.g. [WhiteElo "1500"]
+TAG_RE = re.compile(r'^\[(\w+)\s+"(.*)"\]\s*$')
+
+# Matches a %eval annotation inside a PGN comment.
+#
+# Examples:
+#   [%eval 0.18]
+#   [%eval -1.25]
+#   [%eval #26]
+#
+# The value is captured without the surrounding brackets.
+EVAL_RE = re.compile(r'\[%eval\s+([^\]]+)\]')
+
+# Matches PGN comments enclosed in { ... }.
+COMMENT_RE = re.compile(r'\{([^}]*)\}')
+
+# Detect presence of an %eval annotation anywhere in the movetext.
+EVAL_PRESENT_RE = re.compile(r'\[%eval\b')
+
+
+def clean_comment(match: re.Match) -> str:
+    """
+    Replace a PGN comment with only its %eval annotation.
+
+    Examples:
+
+        { [%eval 0.18] }
+            -> { [%eval 0.18] }
+
+        { [%eval 0.18] [%clk 0:01:00] }
+            -> { [%eval 0.18] }
+
+        { Black offers draw }
+            -> ""
+
+        { [%clk 0:01:00] }
+            -> ""
+    """
+    comment = match.group(1)
+
+    eval_match = EVAL_RE.search(comment)
+
+    if eval_match:
+        return "{ [%eval " + eval_match.group(1).strip() + "] }"
+
+    return ""
+
+
+def clean_movetext(movetext: str) -> str:
+    """
+    Remove every PGN comment except %eval annotations.
+
+    NAGs such as !, ?, !?, ?! are also removed.
+    """
+
+    # Keep only %eval from {...} comments.
+    text = COMMENT_RE.sub(clean_comment, movetext)
+
+    # Remove NAGs / move annotations:
+    #
+    #   !
+    #   ?
+    #   !!
+    #   ??
+    #   !?
+    #   ?!
+    #
+    # These are annotations, not %eval attributes.
+    text = re.sub(r'(?<=[a-zA-Z0-9+#=])(?:!!|\?\?|!\?|\?!|!|\?)(?=\s|$)', '', text)
+
+    # Collapse whitespace.
+    text = re.sub(r'[ \t]+', ' ', text)
+
+    # Remove excessive whitespace around eval comments.
+    text = re.sub(r'\s+\{', ' {', text)
+    text = re.sub(r'\}\s+', '} ', text)
+
+    return text.strip()
+
+
+def process_game(header_lines, movetext_lines, fout) -> bool:
+    """
+    Filter/transform one game and write it out if it qualifies.
+
+    Returns True if the game was written.
+    """
+
+    if not movetext_lines:
+        return False
+
+    movetext = ' '.join(
+        line.strip()
+        for line in movetext_lines
+        if line.strip()
+    )
+
+    # Drop games with no %eval at all.
+    if not EVAL_PRESENT_RE.search(movetext):
+        return False
+
+    # Keep only whitelisted header tags.
+    kept_headers = []
+
+    for line in header_lines:
+        m = TAG_RE.match(line)
+
+        if m and m.group(1) in KEEP_TAGS:
+            kept_headers.append(line.rstrip('\n'))
+
+    # Remove everything except %eval.
+    cleaned_movetext = clean_movetext(movetext)
+
+    for h in kept_headers:
+        fout.write(h + "\n")
+
+    fout.write("\n")
+    fout.write(cleaned_movetext + "\n")
+    fout.write("\n")
+
+    return True
+
+
+def refine(input_path: str, output_path: str):
+    games_in = 0
+    games_out = 0
+
+    header_lines = []
+    movetext_lines = []
+
+    # State:
+    #   "seek_header"    -> collecting [Tag "..."] lines
+    #   "seek_movetext"  -> collecting movetext until blank line
+    state = "seek_header"
+
+    with open(
+        input_path,
+        "r",
+        encoding="utf-8",
+        errors="replace",
+        buffering=1024 * 1024
+    ) as fin, open(
+        output_path,
+        "w",
+        encoding="utf-8",
+        buffering=1024 * 1024
+    ) as fout:
+
+        for raw_line in fin:
+            line = raw_line.rstrip("\n")
+
+            if state == "seek_header":
+
+                if line.startswith("["):
+                    header_lines.append(line)
+
+                elif line.strip() == "":
+                    # Blank line between headers and movetext.
+                    if header_lines:
+                        state = "seek_movetext"
+
+                # Anything else while no headers are collected:
+                # ignore it.
+
+            elif state == "seek_movetext":
+
+                if line.strip() == "":
+                    # End of this game's movetext.
+                    games_in += 1
+
+                    if process_game(
+                        header_lines,
+                        movetext_lines,
+                        fout
+                    ):
+                        games_out += 1
+
+                    header_lines = []
+                    movetext_lines = []
+                    state = "seek_header"
+
+                else:
+                    movetext_lines.append(line)
+
+        # Handle a file without a trailing blank line.
+        if header_lines and movetext_lines:
+            games_in += 1
+
+            if process_game(
+                header_lines,
+                movetext_lines,
+                fout
+            ):
+                games_out += 1
+
+    return games_in, games_out
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Refine a large PGN file: keep only selected headers "
+            "and %eval annotations."
+        )
+    )
+
+    parser.add_argument(
+        "input",
+        help="Path to the input PGN file"
+    )
+
+    parser.add_argument(
+        "output",
+        help="Path to write the refined PGN file"
+    )
+
+    args = parser.parse_args()
+
+    total_in, total_out = refine(
+        args.input,
+        args.output
+    )
+
+    print(
+        f"Read {total_in} games, "
+        f"kept {total_out} (had %eval), "
+        f"dropped {total_in - total_out} (no %eval)."
+    )
+
+    print(f"Output written to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

src/training/decision_tree/main.py

@@ -0,0 +1,71 @@
+import os
+
+from sklearn.ensemble import HistGradientBoostingRegressor, GradientBoostingRegressor, RandomForestRegressor
+from sklearn.model_selection import train_test_split, GroupShuffleSplit
+from sklearn.tree import DecisionTreeRegressor
+
+from src.training.decision_tree import preprocessor
+from src.training.decision_tree.model import Model
+from src.training.decision_tree.visualizer import Visualizer
+
+
+def create_split(x, y, game_ids):
+
+    splitter = GroupShuffleSplit(
+        n_splits=1,
+        test_size=0.10,
+        random_state=42
+    )
+
+    train_indices, test_indices = next(splitter.split(x, y, groups=game_ids))
+
+    x_train = x[train_indices]
+    x_test = x[test_indices]
+
+    y_train = y[train_indices]
+    y_test = y[test_indices]
+
+    train_games = set(game_ids[train_indices])
+    test_games = set(game_ids[test_indices])
+
+    print("Train positions:", len(x_train))
+    print("Test positions:", len(x_test))
+    print("Train games:", len(train_games))
+    print("Test games:", len(test_games))
+    print("Overlapping games:", len(train_games & test_games))
+
+    return x_train, x_test, y_train, y_test
+
+if __name__ == "__main__":
+
+    model_path = "C:/Users/Laurent/Desktop/chess-engine/src/training/decision_tree/cache/model.joblib"
+
+    if os.path.exists(model_path):
+        model = Model.load(model_path)
+
+    else:
+        x, y, game_ids = preprocessor.run("C:/Users/Laurent/Desktop/chess-engine/dataset/v1/output.csv")
+
+        split = create_split(x, y, game_ids)
+
+        depth = 35
+        leafs = int(depth * 3.5)
+
+        model = Model(
+            tree=RandomForestRegressor(
+                n_estimators=25,
+                max_depth=depth,
+                max_features=0.35,
+                min_samples_leaf=leafs,
+                random_state=42
+            ),
+            feature_names=preprocessor.get_feature_names(),
+            train_test_split=split
+        )
+
+        model.train()
+        model.evaluate()
+        #model.save_model(model_path)
+
+    visualizer = Visualizer(model=model)
+    visualizer.show_feature_importance(15)

src/training/decision_tree/model.py

@@ -0,0 +1,75 @@
+import os
+import joblib
+from sklearn.metrics import mean_absolute_error, mean_squared_error
+
+
+class Model:
+    def __init__(self, tree=None, feature_names=None, train_test_split=None):
+        self._model = tree
+        self._feature_names = feature_names
+        self._x_train = self._x_test = None
+        self._y_train = self._y_test = None
+
+        if train_test_split is not None:
+            self._x_train, self._x_test, \
+            self._y_train, self._y_test = train_test_split
+
+    @classmethod
+    def load(cls, path):
+        data = joblib.load(path)
+
+        model = cls()
+        model._model = data["model"]
+        model._feature_names = data["feature_names"]
+
+        print(f"Loaded model from {path}")
+
+        return model
+
+    def get_feature_names(self):
+        return self._feature_names
+
+    def get_feature_importance(self):
+        importances = self._model.feature_importances_
+
+        return sorted(
+            zip(self._feature_names, importances),
+            key=lambda x: x[1],
+            reverse=True
+        )
+
+    def train(self):
+        print("Start training...")
+        self._model.fit(self._x_train, self._y_train)
+        print("Training complete.")
+
+    def evaluate(self):
+        print("Evaluating model...")
+
+        train_predictions = self._model.predict(self._x_train)
+        test_predictions = self._model.predict(self._x_test)
+
+        train_mae = mean_absolute_error(self._y_train, train_predictions)
+        train_mse = mean_squared_error(self._y_train, train_predictions)
+
+        test_mae = mean_absolute_error(self._y_test, test_predictions)
+        test_mse = mean_squared_error(self._y_test, test_predictions)
+
+        print()
+        print("Training:")
+        print("MAE:", train_mae)
+        print("MSE:", train_mse)
+
+        print()
+        print("Test:")
+        print("MAE:", test_mae)
+        print("MSE:", test_mse)
+
+
+    def save_model(self, path):
+        joblib.dump({
+            "model": self._model,
+            "feature_names": self._feature_names
+        }, path)
+
+        print(f"Saved model to {path}")

src/training/decision_tree/visualizer.py

@@ -0,0 +1,50 @@
+import chess
+import numpy as np
+from matplotlib import pyplot as plt
+
+
+class Visualizer:
+    def __init__(self, model):
+        self._model = model
+
+    def show_feature_importance(self, top_n=15):
+        importances = self._model.get_feature_importance()
+        print("Feature importance")
+        print("-------------------------")
+
+        for feature_name, importance in importances[:top_n]:
+            print(f"{feature_name:<35} {importance:.6f}")
+
+    def show_piece_heatmaps(self):
+        importances = dict(self._model.get_feature_importance())
+
+        pieces = [
+            "P", "N", "B", "R", "Q", "K",
+            "p", "n", "b", "r", "q", "k"
+        ]
+
+        fig, axes = plt.subplots(3, 4, figsize=(16, 12))
+
+        for ax, piece in zip(axes.flat, pieces):
+            board = np.zeros((8, 8))
+
+            for row, rank in enumerate(range(7, -1, -1)):
+                for col, file in enumerate(range(8)):
+                    square = chess.square_name(chess.square(file, rank))
+                    feature_name = f"{piece}_{square}"
+                    board[row, col] = importances.get(feature_name, 0)
+
+            image = ax.imshow(board)
+
+            ax.set_xticks(range(8))
+            ax.set_xticklabels("abcdefgh")
+            ax.set_yticks(range(8))
+            ax.set_yticklabels("87654321")
+            ax.set_title(piece)
+
+            fig.colorbar(image, ax=ax, fraction=0.046, pad=0.04)
+
+        fig.suptitle("Piece-square feature importance", fontsize=16)
+        plt.tight_layout()
+        plt.show()
+        input()