457 lines
17 KiB
Python
Executable File
457 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""SQLite FTS5 full-text search over een directory. Stdlib only, 3.6+.
|
|
|
|
Run:
|
|
python3 fts-search.py index ./docs
|
|
python3 fts-search.py search "deploy checklist"
|
|
python3 fts-search.py serve --port 8083
|
|
|
|
Spec (dit bestand is de spec):
|
|
|
|
CLI:
|
|
index <dir> [--db] walk + (her)indexeer, incrementeel op mtime+size
|
|
search <query> BM25-ranking, snippets met <b>-highlights
|
|
serve [--port] HTTP UI + JSON API op /api/search
|
|
|
|
JSON API:
|
|
/api/search?q=&limit=&offset= 200 {"total","items":[{path,title,
|
|
snippet,rank,mtime,size}]}
|
|
/api/stats 200 {"docs","bytes","indexed_at"}
|
|
|
|
Ranking: SQLite bm25(fts), snippets via snippet() met <b>-markering.
|
|
Query-taal: woorden worden AND-gecombineerd als frasen ("a" AND "b"),
|
|
veilig ge-escaped, nooit rauwe MATCH-syntax van de gebruiker.
|
|
|
|
Indexeer-regels: hidden files/dirs overgeslagen, binaire bestanden
|
|
overgeslagen (extensie-lijst + NUL-byte sniff), alleen .txt/.md/.rst/
|
|
.py/.js/.html/.css/.json/.toml/.yaml/.xml/.log/.csv en extensieloze
|
|
tekstbestanden.
|
|
|
|
Caps: MAX_FILE 1 MiB, MAX_FILES 10000, MAX_QUERY 200 chars, MAX_LIMIT 50,
|
|
MAX_BODY-index 200 KiB per bestand.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import socketserver
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
TEXT_EXTS = frozenset([
|
|
".txt", ".md", ".markdown", ".rst", ".py", ".js", ".ts", ".html",
|
|
".htm", ".css", ".json", ".toml", ".yaml", ".yml", ".xml", ".log",
|
|
".csv", ".tsv", ".ini", ".cfg", ".sh", ".sql", ".java", ".c", ".h",
|
|
".go", ".rs", "",
|
|
])
|
|
MAX_FILE = 1024 * 1024
|
|
MAX_FILES = 10000
|
|
MAX_INDEX_CHARS = 200 * 1024
|
|
|
|
|
|
class SearchError(Exception):
|
|
pass
|
|
|
|
|
|
class SearchEngine(object):
|
|
"""SQLite + FTS5 index. Een verbinding, RLock, WAL."""
|
|
|
|
SCHEMA = ("CREATE TABLE IF NOT EXISTS docs ("
|
|
"path TEXT PRIMARY KEY,"
|
|
"title TEXT NOT NULL,"
|
|
"body TEXT NOT NULL,"
|
|
"mtime REAL NOT NULL,"
|
|
"size INTEGER NOT NULL,"
|
|
"indexed_at TEXT NOT NULL)")
|
|
|
|
def __init__(self, db_path):
|
|
self.db_path = db_path
|
|
self._lock = threading.RLock()
|
|
self.db = sqlite3.connect(db_path, check_same_thread=False)
|
|
self.db.row_factory = sqlite3.Row
|
|
self.db.execute("PRAGMA journal_mode=WAL")
|
|
self.db.execute("PRAGMA busy_timeout=5000")
|
|
self.db.execute("PRAGMA synchronous=NORMAL")
|
|
with self.db:
|
|
self.db.execute(self.SCHEMA)
|
|
try:
|
|
self.db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS fts "
|
|
"USING fts5(path, title, body, "
|
|
"tokenize='porter unicode61')")
|
|
except sqlite3.OperationalError:
|
|
raise SearchError(
|
|
"deze SQLite heeft geen FTS5 (nodig voor ranking). "
|
|
"Python >= 3.9 bundelt het vrijwel altijd.")
|
|
self.db.execute(
|
|
"CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs "
|
|
"BEGIN INSERT INTO fts(path, title, body) "
|
|
"VALUES (new.path, new.title, new.body); END")
|
|
self.db.execute(
|
|
"CREATE TRIGGER IF NOT EXISTS docs_ad AFTER DELETE ON docs "
|
|
"BEGIN DELETE FROM fts WHERE path=old.path; END")
|
|
self.db.execute(
|
|
"CREATE TRIGGER IF NOT EXISTS docs_au AFTER UPDATE ON docs "
|
|
"BEGIN DELETE FROM fts WHERE path=old.path; "
|
|
"INSERT INTO fts(path, title, body) "
|
|
"VALUES (new.path, new.title, new.body); END")
|
|
|
|
@staticmethod
|
|
def match_query(query):
|
|
"""Bouw veilige MATCH-query: frasen met AND, alles ge-escaped."""
|
|
words = [w for w in query.split() if w.strip("\"'")]
|
|
terms = []
|
|
for word in words[:10]:
|
|
clean = word.strip("\"'()*:^").replace('"', "")
|
|
if clean:
|
|
terms.append('"%s"' % clean)
|
|
return " AND ".join(terms)
|
|
|
|
def search(self, query, limit=20, offset=0):
|
|
query = (query or "")[:200].strip()
|
|
if not query:
|
|
return 0, []
|
|
match = self.match_query(query)
|
|
if not match:
|
|
return 0, []
|
|
limit = min(max(limit, 1), 50)
|
|
offset = max(offset, 0)
|
|
with self._lock:
|
|
try:
|
|
total = self.db.execute(
|
|
"SELECT COUNT(*) AS n FROM fts WHERE fts MATCH ?",
|
|
(match,)).fetchone()["n"]
|
|
rows = self.db.execute(
|
|
"SELECT d.path, d.title, d.mtime, d.size,"
|
|
" bm25(fts) AS rank,"
|
|
" snippet(fts, 2, '<b>', '</b>', ' ... ', 32) AS snippet"
|
|
" FROM fts JOIN docs d ON d.path=fts.path"
|
|
" WHERE fts MATCH ? ORDER BY rank LIMIT ? OFFSET ?",
|
|
(match, limit, offset)).fetchall()
|
|
except sqlite3.OperationalError:
|
|
return 0, []
|
|
return total, [{"path": r["path"], "title": r["title"],
|
|
"mtime": r["mtime"], "size": r["size"],
|
|
"rank": r["rank"], "snippet": r["snippet"]}
|
|
for r in rows]
|
|
|
|
def stats(self):
|
|
with self._lock:
|
|
row = self.db.execute(
|
|
"SELECT COUNT(*) AS docs, COALESCE(SUM(size),0) AS bytes,"
|
|
" MAX(indexed_at) AS indexed_at FROM docs").fetchone()
|
|
return {"docs": row["docs"], "bytes": row["bytes"],
|
|
"indexed_at": row["indexed_at"]}
|
|
|
|
def upsert(self, path, title, body, mtime, size):
|
|
with self._lock:
|
|
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
self.db.execute(
|
|
"INSERT OR REPLACE INTO docs"
|
|
" (path, title, body, mtime, size, indexed_at)"
|
|
" VALUES (?, ?, ?, ?, ?, ?)",
|
|
(path, title, body[:MAX_INDEX_CHARS], mtime, size, now))
|
|
|
|
def remove(self, path):
|
|
with self._lock:
|
|
self.db.execute("DELETE FROM docs WHERE path=?", (path,))
|
|
|
|
def known(self):
|
|
with self._lock:
|
|
return {r["path"]: (r["mtime"], r["size"])
|
|
for r in self.db.execute("SELECT path, mtime, size"
|
|
" FROM docs").fetchall()}
|
|
|
|
def commit(self):
|
|
with self._lock:
|
|
self.db.commit()
|
|
|
|
|
|
class DocIndexer(object):
|
|
"""Walk + filter + incrementele indexering. Puur, testbaar."""
|
|
|
|
def __init__(self, engine):
|
|
self.engine = engine
|
|
|
|
@staticmethod
|
|
def is_indexable(name, full):
|
|
base = os.path.basename(name)
|
|
if base.startswith("."):
|
|
return False
|
|
_, ext = os.path.splitext(base)
|
|
if ext.lower() not in TEXT_EXTS:
|
|
return False
|
|
try:
|
|
if os.path.getsize(full) > MAX_FILE:
|
|
return False
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def read_text(full):
|
|
try:
|
|
with open(full, "rb") as fh:
|
|
raw = fh.read(MAX_FILE + 1)
|
|
except OSError:
|
|
return None
|
|
if b"\x00" in raw[:8192]:
|
|
return None
|
|
try:
|
|
return raw.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
try:
|
|
return raw.decode("latin-1")
|
|
except UnicodeDecodeError:
|
|
return None
|
|
|
|
def index_dir(self, root):
|
|
root = os.path.abspath(root)
|
|
known = self.engine.known()
|
|
seen = set()
|
|
added = updated = skipped = 0
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
|
|
dirnames.sort()
|
|
for name in sorted(filenames):
|
|
full = os.path.join(dirpath, name)
|
|
rel = os.path.relpath(full, root)
|
|
if len(seen) >= MAX_FILES:
|
|
skipped += 1
|
|
continue
|
|
if not self.is_indexable(name, full):
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
st = os.stat(full)
|
|
except OSError:
|
|
skipped += 1
|
|
continue
|
|
seen.add(rel)
|
|
prev = known.get(rel)
|
|
if prev == (st.st_mtime, st.st_size):
|
|
continue
|
|
text = self.read_text(full)
|
|
if text is None:
|
|
skipped += 1
|
|
continue
|
|
title = os.path.basename(rel)
|
|
self.engine.upsert(rel, title, text, st.st_mtime, st.st_size)
|
|
if prev is None:
|
|
added += 1
|
|
else:
|
|
updated += 1
|
|
removed = 0
|
|
for rel in known:
|
|
if rel not in seen:
|
|
self.engine.remove(rel)
|
|
removed += 1
|
|
self.engine.commit()
|
|
return {"added": added, "updated": updated, "removed": removed,
|
|
"skipped": skipped, "total": len(seen)}
|
|
|
|
|
|
class SearchHandler(BaseHTTPRequestHandler):
|
|
server_version = "TaiSearch/1.0"
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def log_message(self, fmt, *args):
|
|
pass
|
|
|
|
def _send(self, code, body=b"", ctype="text/plain; charset=utf-8"):
|
|
if isinstance(body, str):
|
|
body = body.encode("utf-8")
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
if self.command != "HEAD" and body:
|
|
self.wfile.write(body)
|
|
|
|
def _json(self, code, obj):
|
|
self._send(code, json.dumps(obj), "application/json")
|
|
|
|
def do_GET(self):
|
|
try:
|
|
parsed = urlparse(self.path)
|
|
if parsed.path == "/":
|
|
self._send(200, SEARCH_HTML, "text/html; charset=utf-8")
|
|
return
|
|
if parsed.path == "/api/stats":
|
|
self._json(200, self.server.engine.stats())
|
|
return
|
|
if parsed.path == "/api/search":
|
|
qs = parse_qs(parsed.query)
|
|
try:
|
|
limit = int(qs.get("limit", ["20"])[0])
|
|
offset = int(qs.get("offset", ["0"])[0])
|
|
except ValueError:
|
|
limit, offset = 20, 0
|
|
total, items = self.server.engine.search(
|
|
qs.get("q", [""])[0], limit=limit, offset=offset)
|
|
self._json(200, {"total": total, "limit": limit,
|
|
"offset": offset, "items": items})
|
|
return
|
|
self._send(404, "not found\n")
|
|
except BrokenPipeError:
|
|
pass
|
|
|
|
|
|
SEARCH_HTML = """<!DOCTYPE html>
|
|
<html lang="nl"><head><meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Zoeken</title>
|
|
<style>
|
|
*{box-sizing:border-box}
|
|
body{font-family:system-ui,-apple-system,"Segoe UI",sans-serif;margin:0;
|
|
color:#1a1a1a;padding:0 1em 3em}
|
|
main{max-width:44em;margin:0 auto}
|
|
h1{font-size:1.4em;margin:1em 0 .2em}
|
|
p.sub{color:#666;margin:.2em 0 1em;font-size:.9em}
|
|
form{display:flex;gap:.5em}
|
|
form input{flex:1;padding:.6em;font-size:1em;border:1px solid #bbb;
|
|
border-radius:6px}
|
|
form button{padding:.6em 1em;font-size:1em;border:1px solid #bbb;
|
|
background:#f4f4f4;border-radius:6px;cursor:pointer}
|
|
#meta{color:#666;font-size:.9em;margin:.8em 0}
|
|
ol{list-style:none;margin:0;padding:0}
|
|
li{padding:.7em 0;border-bottom:1px solid #e4e4e4}
|
|
li .path{font-weight:700;overflow-wrap:break-word}
|
|
li .path span{font-weight:400;color:#666;font-size:.85em}
|
|
li .snip{margin-top:.3em;color:#333;overflow-wrap:break-word}
|
|
li .snip b{background:#ffe58a;font-weight:700}
|
|
.pager{display:flex;gap:.5em;align-items:center;justify-content:center;
|
|
margin:1.2em 0;color:#666}
|
|
.pager button{padding:.5em .9em;font-size:.95em;border:1px solid #bbb;
|
|
background:#f4f4f4;border-radius:6px;cursor:pointer}
|
|
@media(max-width:30em){form{flex-direction:column}}
|
|
</style></head><body><main>
|
|
<h1>Zoeken</h1><p class="sub" id="stats"></p>
|
|
<form id="f"><input id="q" placeholder="zoekopdracht..." autocomplete="off">
|
|
<button type="submit">Zoek</button></form>
|
|
<p id="meta"></p><ol id="r"></ol>
|
|
<div class="pager"><button id="prev">Vorige</button><span id="count"></span>
|
|
<button id="next">Volgende</button></div>
|
|
<script>
|
|
"use strict";
|
|
var off=0,LIM=20,total=0;
|
|
function load(){
|
|
var q=document.getElementById("q").value;
|
|
fetch("api/search?q="+encodeURIComponent(q)+"&limit="+LIM+"&offset="+off)
|
|
.then(function(r){return r.json();}).then(function(d){
|
|
total=d.total;var ol=document.getElementById("r");ol.innerHTML="";
|
|
d.items.forEach(function(it){
|
|
var li=document.createElement("li");
|
|
// snippet bevat alleen <b>-tags van SQLite zelf; pad wel escapen
|
|
var div=document.createElement("div");div.textContent=it.path;
|
|
li.innerHTML='<div class="path"></div><div class="snip">'+it.snippet+"</div>";
|
|
li.querySelector(".path").appendChild(div.firstChild);
|
|
var s=document.createElement("span");
|
|
s.textContent=" ("+it.size+" B)";
|
|
li.querySelector(".path").appendChild(s);
|
|
ol.appendChild(li);});
|
|
document.getElementById("meta").textContent=
|
|
total+(total===1?" resultaat":" resultaten");
|
|
document.getElementById("count").textContent=
|
|
(total?off+1:0)+"-"+Math.min(off+LIM,total)+" van "+total;
|
|
document.getElementById("prev").disabled=off<=0;
|
|
document.getElementById("next").disabled=off+LIM>=total;});}
|
|
document.getElementById("f").addEventListener("submit",function(e){
|
|
e.preventDefault();off=0;load();});
|
|
document.getElementById("prev").addEventListener("click",function(){
|
|
off=Math.max(0,off-LIM);load();});
|
|
document.getElementById("next").addEventListener("click",function(){
|
|
off+=LIM;load();});
|
|
fetch("api/stats").then(function(r){return r.json();}).then(function(s){
|
|
document.getElementById("stats").textContent=
|
|
s.docs+" documenten geindexeerd"+(s.indexed_at?" ("+s.indexed_at+")":"");});
|
|
</script></main></body></html>
|
|
"""
|
|
|
|
|
|
class SearchServer(socketserver.ThreadingMixIn, HTTPServer):
|
|
daemon_threads = True
|
|
allow_reuse_address = True
|
|
|
|
|
|
def default_db():
|
|
home = os.environ.get("TAI_HOME") or os.path.expanduser("~/.tai")
|
|
try:
|
|
os.makedirs(home, exist_ok=True)
|
|
return os.path.join(home, "fts-example.db")
|
|
except OSError:
|
|
return os.path.abspath("./fts-example.db")
|
|
|
|
|
|
def cmd_index(args):
|
|
if not os.path.isdir(args.dir):
|
|
raise SearchError("geen directory: %s" % args.dir)
|
|
engine = SearchEngine(args.db)
|
|
result = DocIndexer(engine).index_dir(args.dir)
|
|
print("index: %(added)d nieuw, %(updated)d bijgewerkt, "
|
|
"%(removed)d weg, %(skipped)d overgeslagen "
|
|
"(totaal %(total)d) -> %(db)s" % dict(result, db=args.db))
|
|
|
|
|
|
def cmd_search(args):
|
|
if not os.path.exists(args.db):
|
|
raise SearchError("geen index (%s). Run eerst: index <dir>" % args.db)
|
|
engine = SearchEngine(args.db)
|
|
total, items = engine.search(args.query, limit=args.limit,
|
|
offset=args.offset)
|
|
print("%d resultaat(en) voor %r:" % (total, args.query))
|
|
for item in items:
|
|
clean = item["snippet"].replace("<b>", "").replace("</b>", "")
|
|
print(" [%.2f] %s\n %s" % (item["rank"], item["path"], clean))
|
|
|
|
|
|
def cmd_serve(args):
|
|
if not os.path.exists(args.db):
|
|
raise SearchError("geen index (%s). Run eerst: index <dir>" % args.db)
|
|
engine = SearchEngine(args.db)
|
|
stats = engine.stats()
|
|
server = SearchServer((args.host, args.port), SearchHandler)
|
|
server.engine = engine
|
|
print("search: http://%s:%d/ (%d docs)" % (args.host, args.port,
|
|
stats["docs"]))
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description="SQLite FTS5 full-text search over een directory.")
|
|
parser.add_argument("--db", default=None, help="sqlite index file")
|
|
sub = parser.add_subparsers(dest="cmd") # 3.6: geen required-kwarg
|
|
sub.required = True
|
|
p_index = sub.add_parser("index", help="directory indexeren")
|
|
p_index.add_argument("dir")
|
|
p_search = sub.add_parser("search", help="zoeken op CLI")
|
|
p_search.add_argument("query")
|
|
p_search.add_argument("--limit", type=int, default=20)
|
|
p_search.add_argument("--offset", type=int, default=0)
|
|
p_serve = sub.add_parser("serve", help="HTTP UI + JSON API starten")
|
|
p_serve.add_argument("--host", default="127.0.0.1")
|
|
p_serve.add_argument("--port", type=int, default=8083)
|
|
args = parser.parse_args(argv)
|
|
if args.db is None:
|
|
args.db = default_db()
|
|
try:
|
|
if args.cmd == "index":
|
|
cmd_index(args)
|
|
elif args.cmd == "search":
|
|
cmd_search(args)
|
|
elif args.cmd == "serve":
|
|
cmd_serve(args)
|
|
except SearchError as exc:
|
|
parser.error(str(exc))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|