|
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
|
|
FILTER_MODES: tuple[str, ...] = ("off", "label", "review", "block")
|
|
|
|
ALWAYS_BLOCK_CATEGORIES: frozenset[str] = frozenset({"sexual", "exploitative"})
|
|
|
|
MATURE_CATEGORIES: frozenset[str] = frozenset({"sexual", "violence", "self_harm"})
|
|
|
|
DEFAULT_REVIEW_SCORE = 2
|
|
DEFAULT_BLOCK_SCORE = 6
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Rule:
|
|
category: str
|
|
weight: int
|
|
pattern: re.Pattern[str]
|
|
|
|
|
|
def _rule(category: str, weight: int, expression: str) -> Rule:
|
|
return Rule(
|
|
category=category,
|
|
weight=weight,
|
|
pattern=re.compile(expression, re.IGNORECASE),
|
|
)
|
|
|
|
|
|
RULES: tuple[Rule, ...] = (
|
|
_rule(
|
|
"hate",
|
|
6,
|
|
r"\b(?:gas|exterminate|eradicate|purge)\s+(?:the\s+)?"
|
|
r"(?:jews|muslims|blacks|whites|asians|gays|trans(?:\s?people)?|immigrants)\b",
|
|
),
|
|
_rule(
|
|
"hate",
|
|
4,
|
|
r"\b(?:all|every)\s+(?:jews|muslims|blacks|whites|asians|gays|trans(?:\s?people)?|"
|
|
r"immigrants|women|men)\s+(?:are|should\s+be)\s+"
|
|
r"(?:subhuman|vermin|animals|scum|killed|deported|removed)\b",
|
|
),
|
|
_rule("hate", 5, r"\b(?:white|racial)\s+(?:power|supremacy)\b.{0,40}\b(?:rise|fight|war)\b"),
|
|
_rule(
|
|
"violence",
|
|
6,
|
|
r"\b(?:i\s+(?:will|am\s+going\s+to|wanna|want\s+to)|we\s+will)\s+"
|
|
r"(?:kill|murder|shoot|stab|behead|burn)\s+(?:you|him|her|them|u)\b",
|
|
),
|
|
_rule("violence", 5, r"\b(?:i\s+know\s+where\s+you\s+live|watch\s+your\s+back,?\s+(?:you|i))\b"),
|
|
_rule("violence", 4, r"\b(?:death|bomb)\s+threat\s+(?:to|against)\s+\w+"),
|
|
_rule(
|
|
"weapons",
|
|
5,
|
|
r"\b(?:how\s+to\s+)?(?:build|make|assemble|construct)\s+(?:a\s+|an\s+)?"
|
|
r"(?:pipe\s?bomb|nail\s?bomb|ied|pressure\s?cooker\s+bomb|nerve\s+agent|"
|
|
r"sarin|ricin|dirty\s+bomb)\b",
|
|
),
|
|
_rule(
|
|
"weapons",
|
|
4,
|
|
r"\b(?:untraceable|ghost)\s+(?:gun|firearm)\b.{0,40}\b(?:build|print|make|assemble)\b",
|
|
),
|
|
_rule(
|
|
"sexual",
|
|
8,
|
|
r"\b(?:child|minor|underage|preteen|loli|shota)\s?(?:porn|pornography|sex|nudes|cp)\b",
|
|
),
|
|
_rule("sexual", 5, r"\b(?:hardcore|explicit)\s+(?:porn|pornography|xxx)\b"),
|
|
_rule("sexual", 4, r"\b(?:nudes|sexting|camgirl|onlyfans)\b.{0,30}\b(?:dm|send|link|free)\b"),
|
|
_rule(
|
|
"exploitative",
|
|
8,
|
|
r"\b(?:sell|buy|trade|traffic(?:king)?)\s+(?:a\s+)?"
|
|
r"(?:child|children|minor|minors|girl|girls|boy|boys)\b",
|
|
),
|
|
_rule(
|
|
"religious",
|
|
4,
|
|
r"\b(?:all|every)\s+(?:christians|muslims|jews|hindus|buddhists|atheists)\s+"
|
|
r"(?:are|should\s+be)\s+(?:killed|removed|banned|scum|vermin)\b",
|
|
),
|
|
_rule(
|
|
"misinformation",
|
|
3,
|
|
r"\b(?:vaccines?\s+(?:cause|causes)\s+autism|drink(?:ing)?\s+bleach\s+"
|
|
r"(?:cures|to\s+cure)|the\s+election\s+was\s+stolen\s+and)\b",
|
|
),
|
|
_rule(
|
|
"harassment",
|
|
4,
|
|
r"\b(?:kill\s+your\s?self|kys|neck\s+your\s?self)\b",
|
|
),
|
|
_rule(
|
|
"harassment",
|
|
3,
|
|
r"\b(?:you\s+(?:are|'re|r)\s+(?:a\s+)?(?:worthless|pathetic|disgusting)\s+"
|
|
r"(?:piece\s+of\s+\w+|human|waste))\b",
|
|
),
|
|
_rule(
|
|
"harassment",
|
|
4,
|
|
r"\b(?:here\s+is|posting)\s+(?:his|her|their|your)\s+"
|
|
r"(?:home\s+address|real\s+name\s+and\s+address|phone\s+number\s+and\s+address)\b",
|
|
),
|
|
_rule(
|
|
"self_harm",
|
|
5,
|
|
r"\b(?:best|painless|easiest)\s+way\s+to\s+(?:kill\s+myself|end\s+my\s+life|"
|
|
r"commit\s+suicide)\b",
|
|
),
|
|
_rule(
|
|
"self_harm",
|
|
4,
|
|
r"\b(?:you\s+should\s+)?(?:go\s+)?(?:kill\s+yourself|end\s+your\s+life)\b",
|
|
),
|
|
_rule(
|
|
"spam",
|
|
3,
|
|
r"\b(?:buy\s+now|100%\s+free\s+money|work\s+from\s+home\s+\$\d+|"
|
|
r"click\s+here\s+to\s+claim|crypto\s+giveaway)\b",
|
|
),
|
|
_rule(
|
|
"illegal",
|
|
5,
|
|
r"\b(?:selling|buying|for\s+sale)\s+(?:stolen\s+)?"
|
|
r"(?:credit\s?cards?|cc\s+dumps|fullz|ssn\s+list|bank\s+logs)\b",
|
|
),
|
|
_rule(
|
|
"illegal",
|
|
4,
|
|
r"\b(?:hire|hiring)\s+(?:a\s+)?hit\s?man\b",
|
|
),
|
|
_rule(
|
|
"intellectual_property",
|
|
2,
|
|
r"\b(?:full\s+)?(?:cracked|nulled|warez)\s+(?:copy|version|download)\b",
|
|
),
|
|
)
|
|
|
|
|
|
TECHNICAL_CONTEXT = re.compile(
|
|
r"\b(?:cve-\d{4}-\d+|exploit|payload|vulnerabilit(?:y|ies)|proof\s+of\s+concept|"
|
|
r"reverse\s+engineer(?:ing)?|malware\s+analysis|penetration\s+test(?:ing)?|"
|
|
r"red\s+team|sandbox|disassembl(?:y|er)|fuzz(?:ing|er)|stack\s+trace|traceback|"
|
|
r"segmentation\s+fault|kernel\s+panic)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
TECHNICAL_DISCOUNT_CATEGORIES: frozenset[str] = frozenset(
|
|
{"weapons", "violence", "illegal"}
|
|
)
|
|
|
|
TECHNICAL_DISCOUNT = 3
|
|
|
|
|
|
def matches(text: str) -> list[Rule]:
|
|
if not text:
|
|
return []
|
|
return [rule for rule in RULES if rule.pattern.search(text)]
|
|
|
|
|
|
def score_for(text: str, matched: list[Rule]) -> int:
|
|
if not matched:
|
|
return 0
|
|
technical = bool(TECHNICAL_CONTEXT.search(text))
|
|
total = 0
|
|
for rule in matched:
|
|
weight = rule.weight
|
|
if technical and rule.category in TECHNICAL_DISCOUNT_CATEGORIES:
|
|
weight = max(0, weight - TECHNICAL_DISCOUNT)
|
|
total += weight
|
|
return total
|