Naaturel/chess-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Laurent
2026-09-18 19:48:19 +02:00
parent 41bcfd6f01
commit b27d3a0f64
6 changed files with 2779364 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
cleaning/extract.py Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""
Extract the first N lines from a huge file without loading it into RAM.
Works because Python file objects are iterators: reading line-by-line
only ever keeps one line (plus a small internal read-ahead buffer) in
memory at a time, regardless of the file's total size.
Usage:
python extract_lines.py input.txt output.txt --lines 500
# Start from a specific line (e.g. skip a header, or resume mid-file)
python extract_lines.py input.txt output.txt --lines 500 --start 1000
"""
import argparse
from itertools import islice
def extract_lines(input_path: str, output_path: str, start: int, count: int) -> int:
written = 0
with open(input_path, "r", encoding="utf-8", errors="replace", buffering=1024 * 1024) as fin, \
open(output_path, "w", encoding="utf-8") as fout:
for line in islice(fin, start, start + count):
fout.write(line)
written += 1
return written
def main():
parser = argparse.ArgumentParser(description="Extract a slice of lines from a huge file, streaming (low RAM).")
parser.add_argument("input", help="Path to the big input file")
parser.add_argument("output", help="Path to write the extracted lines to")
parser.add_argument("--lines", type=int, default=500, help="Number of lines to extract (default: 500)")
parser.add_argument("--start", type=int, default=0, help="Line number to start at, 0-indexed (default: 0)")
args = parser.parse_args()
n = extract_lines(args.input, args.output, args.start, args.lines)
print(f"Wrote {n} lines (lines {args.start}-{args.start + n - 1}) to {args.output}")
if __name__ == "__main__":
main()

168
cleaning/format.py Normal file
View File

@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""
Turn refined PGN games (Site/WhiteElo/BlackElo headers + movetext with
%eval comments) into a CSV where each row is the board state *after* one
move.
Row layout (67 columns):
game_id, a8, b8, c8, d8, e8, f8, g8, h8, a7, ... , h1, eval, turn
- game_id : 0-based index of the game in the input file. Not asked for
explicitly, but with many games concatenated into one CSV
there is no other way to tell which rows belong to which
game, so I added it as column 1. Drop it with --no-game-id
if you don't want it.
- a8..h1 : the 64 squares (rank 8 down to rank 1, file a to h), each
holding the FEN piece letter on that square after the move
(uppercase = white, lowercase = black) or an empty string
if the square is empty.
- eval : the %eval attached to that move (blank if that particular
move had none).
- turn : "white" or "black" -- whose turn it is to move NEXT.
Streams game-by-game (like the two earlier scripts) so it stays cheap on
RAM no matter how large the input file is. Requires `pip install chess`.
Usage:
python pgn_to_csv.py refined.pgn out.csv
python pgn_to_csv.py refined.pgn out.csv --no-game-id
"""
import argparse
import csv
import re
import sys
import chess
TAG_RE = re.compile(r'^\[(\w+)\s+"(.*)"\]\s*$')
EVAL_RE = re.compile(r'\[%eval\s+([^\]]+)\]')
MOVE_NUM_RE = re.compile(r'^\d+\.+$')
RESULT_RE = re.compile(r'^(1-0|0-1|1/2-1/2|\*)$')
# Strip trailing annotation glyphs python-chess's SAN parser won't accept
# (e.g. "e5?!", "Nc3!!", "Qxd1+!") while keeping the "+"/"#" that ARE part
# of legal SAN.
SUFFIX_ANNOTATION_RE = re.compile(r'[!?]+$')
SQUARE_COLUMNS = [f + str(r) for r in range(8, 0, -1) for f in "abcdefgh"]
def tokenize_movetext(movetext: str):
"""Yield (san_move, eval_str_or_None) pairs from a movetext blob."""
tokens = re.findall(r'\{[^}]*\}|\S+', movetext)
i = 0
n = len(tokens)
while i < n:
tok = tokens[i]
if MOVE_NUM_RE.match(tok) or RESULT_RE.match(tok):
i += 1
continue
move_san = tok
eval_val = None
if i + 1 < n and tokens[i + 1].startswith('{'):
m = EVAL_RE.search(tokens[i + 1])
if m:
eval_val = m.group(1)
i += 2
else:
i += 1
yield move_san, eval_val
def board_row(board: chess.Board):
"""Return the 64 square values for the current board, a8..h1 order."""
row = []
for name in SQUARE_COLUMNS:
sq = chess.parse_square(name)
piece = board.piece_at(sq)
row.append(piece.symbol() if piece else "")
return row
def process_game(game_id, header_lines, movetext_lines, writer, include_game_id):
if not movetext_lines:
return 0
movetext = ' '.join(line.strip() for line in movetext_lines if line.strip())
board = chess.Board()
rows_written = 0
for move_san, eval_val in tokenize_movetext(movetext):
clean_san = SUFFIX_ANNOTATION_RE.sub('', move_san)
try:
board.push_san(clean_san)
except (chess.IllegalMoveError, chess.InvalidMoveError, chess.AmbiguousMoveError, ValueError) as e:
print(f" [game {game_id}] stopping early: could not parse move "
f"'{move_san}' ({e})", file=sys.stderr)
break
turn = "white" if board.turn == chess.WHITE else "black"
row = (board_row(board) + [eval_val if eval_val is not None else "", turn])
if include_game_id:
row = [game_id] + row
writer.writerow(row)
rows_written += 1
return rows_written
def refine(input_path: str, output_path: str, include_game_id: bool):
header_lines = []
movetext_lines = []
state = "seek_header"
game_id = 0
total_rows = 0
header_row = SQUARE_COLUMNS + ["eval", "turn"]
if include_game_id:
header_row = ["game_id"] + header_row
with open(input_path, "r", encoding="utf-8", errors="replace", buffering=1024 * 1024) as fin, \
open(output_path, "w", encoding="utf-8", newline="") as fout:
writer = csv.writer(fout)
writer.writerow(header_row)
for raw_line in fin:
line = raw_line.rstrip("\n")
if state == "seek_header":
if line.startswith("["):
header_lines.append(line)
elif line.strip() == "" and header_lines:
state = "seek_movetext"
elif state == "seek_movetext":
if line.strip() == "":
total_rows += process_game(game_id, header_lines, movetext_lines,
writer, include_game_id)
game_id += 1
header_lines = []
movetext_lines = []
state = "seek_header"
else:
movetext_lines.append(line)
if header_lines and movetext_lines:
total_rows += process_game(game_id, header_lines, movetext_lines,
writer, include_game_id)
game_id += 1
return game_id, total_rows
def main():
parser = argparse.ArgumentParser(
description="Convert refined PGN games into a per-move board-state CSV.")
parser.add_argument("input", help="Path to the refined PGN file")
parser.add_argument("output", help="Path to write the CSV file")
parser.add_argument("--no-game-id", action="store_true",
help="Omit the leading game_id column (66-col strict layout).")
args = parser.parse_args()
games, rows = refine(args.input, args.output, include_game_id=not args.no_game_id)
print(f"Processed {games} games -> {rows} rows written to {args.output}")
if __name__ == "__main__":
main()

259
cleaning/refine.py Normal file
View File

@@ -0,0 +1,259 @@
#!/usr/bin/env python3
"""
Refine a huge Lichess-style PGN file, streaming one game at a time so RAM
stays flat regardless of file size (works fine on a 200GB file).
Rules applied to every game:
1. If the game has no %eval annotations at all, drop it entirely.
2. Keep only these header tags: Site, WhiteElo, BlackElo.
3. In the movetext, remove every comment/annotation except %eval.
4. Keep only the %eval annotation from comments containing multiple
attributes, e.g. { [%eval 0.18] [%clk 0:01:00] } -> { [%eval 0.18] }.
5. Remove non-%eval textual comments, e.g. { Black offers draw }.
6. Remove NAGs such as !, ?, !?, ?!, etc.
Games are separated by a blank line after the movetext.
Usage:
python refine_pgn.py input.pgn output.pgn
"""
import argparse
import re
# Only these header tags survive
KEEP_TAGS = {"Site", "WhiteElo", "BlackElo"}
# Matches a header tag line, e.g. [WhiteElo "1500"]
TAG_RE = re.compile(r'^\[(\w+)\s+"(.*)"\]\s*$')
# Matches a %eval annotation inside a PGN comment.
#
# Examples:
# [%eval 0.18]
# [%eval -1.25]
# [%eval #26]
#
# The value is captured without the surrounding brackets.
EVAL_RE = re.compile(r'\[%eval\s+([^\]]+)\]')
# Matches PGN comments enclosed in { ... }.
COMMENT_RE = re.compile(r'\{([^}]*)\}')
# Detect presence of an %eval annotation anywhere in the movetext.
EVAL_PRESENT_RE = re.compile(r'\[%eval\b')
def clean_comment(match: re.Match) -> str:
"""
Replace a PGN comment with only its %eval annotation.
Examples:
{ [%eval 0.18] }
-> { [%eval 0.18] }
{ [%eval 0.18] [%clk 0:01:00] }
-> { [%eval 0.18] }
{ Black offers draw }
-> ""
{ [%clk 0:01:00] }
-> ""
"""
comment = match.group(1)
eval_match = EVAL_RE.search(comment)
if eval_match:
return "{ [%eval " + eval_match.group(1).strip() + "] }"
return ""
def clean_movetext(movetext: str) -> str:
"""
Remove every PGN comment except %eval annotations.
NAGs such as !, ?, !?, ?! are also removed.
"""
# Keep only %eval from {...} comments.
text = COMMENT_RE.sub(clean_comment, movetext)
# Remove NAGs / move annotations:
#
# !
# ?
# !!
# ??
# !?
# ?!
#
# These are annotations, not %eval attributes.
text = re.sub(r'(?<=[a-zA-Z0-9+#=])(?:!!|\?\?|!\?|\?!|!|\?)(?=\s|$)', '', text)
# Collapse whitespace.
text = re.sub(r'[ \t]+', ' ', text)
# Remove excessive whitespace around eval comments.
text = re.sub(r'\s+\{', ' {', text)
text = re.sub(r'\}\s+', '} ', text)
return text.strip()
def process_game(header_lines, movetext_lines, fout) -> bool:
"""
Filter/transform one game and write it out if it qualifies.
Returns True if the game was written.
"""
if not movetext_lines:
return False
movetext = ' '.join(
line.strip()
for line in movetext_lines
if line.strip()
)
# Drop games with no %eval at all.
if not EVAL_PRESENT_RE.search(movetext):
return False
# Keep only whitelisted header tags.
kept_headers = []
for line in header_lines:
m = TAG_RE.match(line)
if m and m.group(1) in KEEP_TAGS:
kept_headers.append(line.rstrip('\n'))
# Remove everything except %eval.
cleaned_movetext = clean_movetext(movetext)
for h in kept_headers:
fout.write(h + "\n")
fout.write("\n")
fout.write(cleaned_movetext + "\n")
fout.write("\n")
return True
def refine(input_path: str, output_path: str):
games_in = 0
games_out = 0
header_lines = []
movetext_lines = []
# State:
# "seek_header" -> collecting [Tag "..."] lines
# "seek_movetext" -> collecting movetext until blank line
state = "seek_header"
with open(
input_path,
"r",
encoding="utf-8",
errors="replace",
buffering=1024 * 1024
) as fin, open(
output_path,
"w",
encoding="utf-8",
buffering=1024 * 1024
) as fout:
for raw_line in fin:
line = raw_line.rstrip("\n")
if state == "seek_header":
if line.startswith("["):
header_lines.append(line)
elif line.strip() == "":
# Blank line between headers and movetext.
if header_lines:
state = "seek_movetext"
# Anything else while no headers are collected:
# ignore it.
elif state == "seek_movetext":
if line.strip() == "":
# End of this game's movetext.
games_in += 1
if process_game(
header_lines,
movetext_lines,
fout
):
games_out += 1
header_lines = []
movetext_lines = []
state = "seek_header"
else:
movetext_lines.append(line)
# Handle a file without a trailing blank line.
if header_lines and movetext_lines:
games_in += 1
if process_game(
header_lines,
movetext_lines,
fout
):
games_out += 1
return games_in, games_out
def main():
parser = argparse.ArgumentParser(
description=(
"Refine a large PGN file: keep only selected headers "
"and %eval annotations."
)
)
parser.add_argument(
"input",
help="Path to the input PGN file"
)
parser.add_argument(
"output",
help="Path to write the refined PGN file"
)
args = parser.parse_args()
total_in, total_out = refine(
args.input,
args.output
)
print(
f"Read {total_in} games, "
f"kept {total_out} (had %eval), "
f"dropped {total_in - total_out} (no %eval)."
)
print(f"Output written to {args.output}")
if __name__ == "__main__":
main()