Initial commit

cleaning/extract.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""
+Extract the first N lines from a huge file without loading it into RAM.
+
+Works because Python file objects are iterators: reading line-by-line
+only ever keeps one line (plus a small internal read-ahead buffer) in
+memory at a time, regardless of the file's total size.
+
+Usage:
+    python extract_lines.py input.txt output.txt --lines 500
+
+    # Start from a specific line (e.g. skip a header, or resume mid-file)
+    python extract_lines.py input.txt output.txt --lines 500 --start 1000
+"""
+
+import argparse
+from itertools import islice
+
+
+def extract_lines(input_path: str, output_path: str, start: int, count: int) -> int:
+    written = 0
+    with open(input_path, "r", encoding="utf-8", errors="replace", buffering=1024 * 1024) as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+        for line in islice(fin, start, start + count):
+            fout.write(line)
+            written += 1
+    return written
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract a slice of lines from a huge file, streaming (low RAM).")
+    parser.add_argument("input", help="Path to the big input file")
+    parser.add_argument("output", help="Path to write the extracted lines to")
+    parser.add_argument("--lines", type=int, default=500, help="Number of lines to extract (default: 500)")
+    parser.add_argument("--start", type=int, default=0, help="Line number to start at, 0-indexed (default: 0)")
+    args = parser.parse_args()
+
+    n = extract_lines(args.input, args.output, args.start, args.lines)
+    print(f"Wrote {n} lines (lines {args.start}-{args.start + n - 1}) to {args.output}")
+
+if __name__ == "__main__":
+    main()