#!/usr/bin/env python3
"""SQLite FTS5 full-text search over een directory. Stdlib only, 3.6+.

Run:
    python3 fts-search.py index ./docs
    python3 fts-search.py search "deploy checklist"
    python3 fts-search.py serve --port 8083

Spec (dit bestand is de spec):

    CLI:
    index <dir> [--db]    walk + (her)indexeer, incrementeel op mtime+size
    search <query>        BM25-ranking, snippets met <b>-highlights
    serve [--port]        HTTP UI + JSON API op /api/search

    JSON API:
    /api/search?q=&limit=&offset=   200  {"total","items":[{path,title,
                                        snippet,rank,mtime,size}]}
    /api/stats                       200  {"docs","bytes","indexed_at"}

    Ranking: SQLite bm25(fts), snippets via snippet() met <b>-markering.
    Query-taal: woorden worden AND-gecombineerd als frasen ("a" AND "b"),
    veilig ge-escaped, nooit rauwe MATCH-syntax van de gebruiker.

    Indexeer-regels: hidden files/dirs overgeslagen, binaire bestanden
    overgeslagen (extensie-lijst + NUL-byte sniff), alleen .txt/.md/.rst/
    .py/.js/.html/.css/.json/.toml/.yaml/.xml/.log/.csv en extensieloze
    tekstbestanden.

Caps: MAX_FILE 1 MiB, MAX_FILES 10000, MAX_QUERY 200 chars, MAX_LIMIT 50,
MAX_BODY-index 200 KiB per bestand.
"""

import argparse
import json
import os
import socketserver
import sqlite3
import threading
import time
from http.server import BaseHTTPRequestHandler, HTTPServer
from urllib.parse import parse_qs, urlparse

TEXT_EXTS = frozenset([
    ".txt", ".md", ".markdown", ".rst", ".py", ".js", ".ts", ".html",
    ".htm", ".css", ".json", ".toml", ".yaml", ".yml", ".xml", ".log",
    ".csv", ".tsv", ".ini", ".cfg", ".sh", ".sql", ".java", ".c", ".h",
    ".go", ".rs", "",
])
MAX_FILE = 1024 * 1024
MAX_FILES = 10000
MAX_INDEX_CHARS = 200 * 1024


class SearchError(Exception):
    pass


class SearchEngine(object):
    """SQLite + FTS5 index. Een verbinding, RLock, WAL."""

    SCHEMA = ("CREATE TABLE IF NOT EXISTS docs ("
              "path TEXT PRIMARY KEY,"
              "title TEXT NOT NULL,"
              "body TEXT NOT NULL,"
              "mtime REAL NOT NULL,"
              "size INTEGER NOT NULL,"
              "indexed_at TEXT NOT NULL)")

    def __init__(self, db_path):
        self.db_path = db_path
        self._lock = threading.RLock()
        self.db = sqlite3.connect(db_path, check_same_thread=False)
        self.db.row_factory = sqlite3.Row
        self.db.execute("PRAGMA journal_mode=WAL")
        self.db.execute("PRAGMA busy_timeout=5000")
        self.db.execute("PRAGMA synchronous=NORMAL")
        with self.db:
            self.db.execute(self.SCHEMA)
            try:
                self.db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS fts "
                                "USING fts5(path, title, body, "
                                "tokenize='porter unicode61')")
            except sqlite3.OperationalError:
                raise SearchError(
                    "deze SQLite heeft geen FTS5 (nodig voor ranking). "
                    "Python >= 3.9 bundelt het vrijwel altijd.")
            self.db.execute(
                "CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs "
                "BEGIN INSERT INTO fts(path, title, body) "
                "VALUES (new.path, new.title, new.body); END")
            self.db.execute(
                "CREATE TRIGGER IF NOT EXISTS docs_ad AFTER DELETE ON docs "
                "BEGIN DELETE FROM fts WHERE path=old.path; END")
            self.db.execute(
                "CREATE TRIGGER IF NOT EXISTS docs_au AFTER UPDATE ON docs "
                "BEGIN DELETE FROM fts WHERE path=old.path; "
                "INSERT INTO fts(path, title, body) "
                "VALUES (new.path, new.title, new.body); END")

    @staticmethod
    def match_query(query):
        """Bouw veilige MATCH-query: frasen met AND, alles ge-escaped."""
        words = [w for w in query.split() if w.strip("\"'")]
        terms = []
        for word in words[:10]:
            clean = word.strip("\"'()*:^").replace('"', "")
            if clean:
                terms.append('"%s"' % clean)
        return " AND ".join(terms)

    def search(self, query, limit=20, offset=0):
        query = (query or "")[:200].strip()
        if not query:
            return 0, []
        match = self.match_query(query)
        if not match:
            return 0, []
        limit = min(max(limit, 1), 50)
        offset = max(offset, 0)
        with self._lock:
            try:
                total = self.db.execute(
                    "SELECT COUNT(*) AS n FROM fts WHERE fts MATCH ?",
                    (match,)).fetchone()["n"]
                rows = self.db.execute(
                    "SELECT d.path, d.title, d.mtime, d.size,"
                    " bm25(fts) AS rank,"
                    " snippet(fts, 2, '<b>', '</b>', ' ... ', 32) AS snippet"
                    " FROM fts JOIN docs d ON d.path=fts.path"
                    " WHERE fts MATCH ? ORDER BY rank LIMIT ? OFFSET ?",
                    (match, limit, offset)).fetchall()
            except sqlite3.OperationalError:
                return 0, []
            return total, [{"path": r["path"], "title": r["title"],
                            "mtime": r["mtime"], "size": r["size"],
                            "rank": r["rank"], "snippet": r["snippet"]}
                           for r in rows]

    def stats(self):
        with self._lock:
            row = self.db.execute(
                "SELECT COUNT(*) AS docs, COALESCE(SUM(size),0) AS bytes,"
                " MAX(indexed_at) AS indexed_at FROM docs").fetchone()
            return {"docs": row["docs"], "bytes": row["bytes"],
                    "indexed_at": row["indexed_at"]}

    def upsert(self, path, title, body, mtime, size):
        with self._lock:
            now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
            self.db.execute(
                "INSERT OR REPLACE INTO docs"
                " (path, title, body, mtime, size, indexed_at)"
                " VALUES (?, ?, ?, ?, ?, ?)",
                (path, title, body[:MAX_INDEX_CHARS], mtime, size, now))

    def remove(self, path):
        with self._lock:
            self.db.execute("DELETE FROM docs WHERE path=?", (path,))

    def known(self):
        with self._lock:
            return {r["path"]: (r["mtime"], r["size"])
                    for r in self.db.execute("SELECT path, mtime, size"
                                            " FROM docs").fetchall()}

    def commit(self):
        with self._lock:
            self.db.commit()


class DocIndexer(object):
    """Walk + filter + incrementele indexering. Puur, testbaar."""

    def __init__(self, engine):
        self.engine = engine

    @staticmethod
    def is_indexable(name, full):
        base = os.path.basename(name)
        if base.startswith("."):
            return False
        _, ext = os.path.splitext(base)
        if ext.lower() not in TEXT_EXTS:
            return False
        try:
            if os.path.getsize(full) > MAX_FILE:
                return False
        except OSError:
            return False
        return True

    @staticmethod
    def read_text(full):
        try:
            with open(full, "rb") as fh:
                raw = fh.read(MAX_FILE + 1)
        except OSError:
            return None
        if b"\x00" in raw[:8192]:
            return None
        try:
            return raw.decode("utf-8")
        except UnicodeDecodeError:
            try:
                return raw.decode("latin-1")
            except UnicodeDecodeError:
                return None

    def index_dir(self, root):
        root = os.path.abspath(root)
        known = self.engine.known()
        seen = set()
        added = updated = skipped = 0
        for dirpath, dirnames, filenames in os.walk(root):
            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
            dirnames.sort()
            for name in sorted(filenames):
                full = os.path.join(dirpath, name)
                rel = os.path.relpath(full, root)
                if len(seen) >= MAX_FILES:
                    skipped += 1
                    continue
                if not self.is_indexable(name, full):
                    skipped += 1
                    continue
                try:
                    st = os.stat(full)
                except OSError:
                    skipped += 1
                    continue
                seen.add(rel)
                prev = known.get(rel)
                if prev == (st.st_mtime, st.st_size):
                    continue
                text = self.read_text(full)
                if text is None:
                    skipped += 1
                    continue
                title = os.path.basename(rel)
                self.engine.upsert(rel, title, text, st.st_mtime, st.st_size)
                if prev is None:
                    added += 1
                else:
                    updated += 1
        removed = 0
        for rel in known:
            if rel not in seen:
                self.engine.remove(rel)
                removed += 1
        self.engine.commit()
        return {"added": added, "updated": updated, "removed": removed,
                "skipped": skipped, "total": len(seen)}


class SearchHandler(BaseHTTPRequestHandler):
    server_version = "TaiSearch/1.0"
    protocol_version = "HTTP/1.1"

    def log_message(self, fmt, *args):
        pass

    def _send(self, code, body=b"", ctype="text/plain; charset=utf-8"):
        if isinstance(body, str):
            body = body.encode("utf-8")
        self.send_response(code)
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        if self.command != "HEAD" and body:
            self.wfile.write(body)

    def _json(self, code, obj):
        self._send(code, json.dumps(obj), "application/json")

    def do_GET(self):
        try:
            parsed = urlparse(self.path)
            if parsed.path == "/":
                self._send(200, SEARCH_HTML, "text/html; charset=utf-8")
                return
            if parsed.path == "/api/stats":
                self._json(200, self.server.engine.stats())
                return
            if parsed.path == "/api/search":
                qs = parse_qs(parsed.query)
                try:
                    limit = int(qs.get("limit", ["20"])[0])
                    offset = int(qs.get("offset", ["0"])[0])
                except ValueError:
                    limit, offset = 20, 0
                total, items = self.server.engine.search(
                    qs.get("q", [""])[0], limit=limit, offset=offset)
                self._json(200, {"total": total, "limit": limit,
                                 "offset": offset, "items": items})
                return
            self._send(404, "not found\n")
        except BrokenPipeError:
            pass


SEARCH_HTML = """<!DOCTYPE html>
<html lang="nl"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Zoeken</title>
<style>
*{box-sizing:border-box}
body{font-family:system-ui,-apple-system,"Segoe UI",sans-serif;margin:0;
color:#1a1a1a;padding:0 1em 3em}
main{max-width:44em;margin:0 auto}
h1{font-size:1.4em;margin:1em 0 .2em}
p.sub{color:#666;margin:.2em 0 1em;font-size:.9em}
form{display:flex;gap:.5em}
form input{flex:1;padding:.6em;font-size:1em;border:1px solid #bbb;
border-radius:6px}
form button{padding:.6em 1em;font-size:1em;border:1px solid #bbb;
background:#f4f4f4;border-radius:6px;cursor:pointer}
#meta{color:#666;font-size:.9em;margin:.8em 0}
ol{list-style:none;margin:0;padding:0}
li{padding:.7em 0;border-bottom:1px solid #e4e4e4}
li .path{font-weight:700;overflow-wrap:break-word}
li .path span{font-weight:400;color:#666;font-size:.85em}
li .snip{margin-top:.3em;color:#333;overflow-wrap:break-word}
li .snip b{background:#ffe58a;font-weight:700}
.pager{display:flex;gap:.5em;align-items:center;justify-content:center;
margin:1.2em 0;color:#666}
.pager button{padding:.5em .9em;font-size:.95em;border:1px solid #bbb;
background:#f4f4f4;border-radius:6px;cursor:pointer}
@media(max-width:30em){form{flex-direction:column}}
</style></head><body><main>
<h1>Zoeken</h1><p class="sub" id="stats"></p>
<form id="f"><input id="q" placeholder="zoekopdracht..." autocomplete="off">
<button type="submit">Zoek</button></form>
<p id="meta"></p><ol id="r"></ol>
<div class="pager"><button id="prev">Vorige</button><span id="count"></span>
<button id="next">Volgende</button></div>
<script>
"use strict";
var off=0,LIM=20,total=0;
function load(){
var q=document.getElementById("q").value;
fetch("api/search?q="+encodeURIComponent(q)+"&limit="+LIM+"&offset="+off)
.then(function(r){return r.json();}).then(function(d){
total=d.total;var ol=document.getElementById("r");ol.innerHTML="";
d.items.forEach(function(it){
var li=document.createElement("li");
// snippet bevat alleen <b>-tags van SQLite zelf; pad wel escapen
var div=document.createElement("div");div.textContent=it.path;
li.innerHTML='<div class="path"></div><div class="snip">'+it.snippet+"</div>";
li.querySelector(".path").appendChild(div.firstChild);
var s=document.createElement("span");
s.textContent="  ("+it.size+" B)";
li.querySelector(".path").appendChild(s);
ol.appendChild(li);});
document.getElementById("meta").textContent=
total+(total===1?" resultaat":" resultaten");
document.getElementById("count").textContent=
(total?off+1:0)+"-"+Math.min(off+LIM,total)+" van "+total;
document.getElementById("prev").disabled=off<=0;
document.getElementById("next").disabled=off+LIM>=total;});}
document.getElementById("f").addEventListener("submit",function(e){
e.preventDefault();off=0;load();});
document.getElementById("prev").addEventListener("click",function(){
off=Math.max(0,off-LIM);load();});
document.getElementById("next").addEventListener("click",function(){
off+=LIM;load();});
fetch("api/stats").then(function(r){return r.json();}).then(function(s){
document.getElementById("stats").textContent=
s.docs+" documenten geindexeerd"+(s.indexed_at?" ("+s.indexed_at+")":"");});
</script></main></body></html>
"""


class SearchServer(socketserver.ThreadingMixIn, HTTPServer):
    daemon_threads = True
    allow_reuse_address = True


def default_db():
    home = os.environ.get("TAI_HOME") or os.path.expanduser("~/.tai")
    try:
        os.makedirs(home, exist_ok=True)
        return os.path.join(home, "fts-example.db")
    except OSError:
        return os.path.abspath("./fts-example.db")


def cmd_index(args):
    if not os.path.isdir(args.dir):
        raise SearchError("geen directory: %s" % args.dir)
    engine = SearchEngine(args.db)
    result = DocIndexer(engine).index_dir(args.dir)
    print("index: %(added)d nieuw, %(updated)d bijgewerkt, "
          "%(removed)d weg, %(skipped)d overgeslagen "
          "(totaal %(total)d) -> %(db)s" % dict(result, db=args.db))


def cmd_search(args):
    if not os.path.exists(args.db):
        raise SearchError("geen index (%s). Run eerst: index <dir>" % args.db)
    engine = SearchEngine(args.db)
    total, items = engine.search(args.query, limit=args.limit,
                                 offset=args.offset)
    print("%d resultaat(en) voor %r:" % (total, args.query))
    for item in items:
        clean = item["snippet"].replace("<b>", "").replace("</b>", "")
        print("  [%.2f] %s\n        %s" % (item["rank"], item["path"], clean))


def cmd_serve(args):
    if not os.path.exists(args.db):
        raise SearchError("geen index (%s). Run eerst: index <dir>" % args.db)
    engine = SearchEngine(args.db)
    stats = engine.stats()
    server = SearchServer((args.host, args.port), SearchHandler)
    server.engine = engine
    print("search: http://%s:%d/ (%d docs)" % (args.host, args.port,
                                              stats["docs"]))
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass


def main(argv=None):
    parser = argparse.ArgumentParser(
        description="SQLite FTS5 full-text search over een directory.")
    parser.add_argument("--db", default=None, help="sqlite index file")
    sub = parser.add_subparsers(dest="cmd")  # 3.6: geen required-kwarg
    sub.required = True
    p_index = sub.add_parser("index", help="directory indexeren")
    p_index.add_argument("dir")
    p_search = sub.add_parser("search", help="zoeken op CLI")
    p_search.add_argument("query")
    p_search.add_argument("--limit", type=int, default=20)
    p_search.add_argument("--offset", type=int, default=0)
    p_serve = sub.add_parser("serve", help="HTTP UI + JSON API starten")
    p_serve.add_argument("--host", default="127.0.0.1")
    p_serve.add_argument("--port", type=int, default=8083)
    args = parser.parse_args(argv)
    if args.db is None:
        args.db = default_db()
    try:
        if args.cmd == "index":
            cmd_index(args)
        elif args.cmd == "search":
            cmd_search(args)
        elif args.cmd == "serve":
            cmd_serve(args)
    except SearchError as exc:
        parser.error(str(exc))


if __name__ == "__main__":
    main()