feat: add wasm build targets, profile hooks, os_demo example, and restructure manual source
- Add `wasm`, `wasm-clean`, and `install-emscripten` phony targets to Makefile for WebAssembly cross-compilation - Insert `WREN_PROFILE_ENTER`/`WREN_PROFILE_EXIT` macros in `wren_vm.h` and `wren_vm.c` to support optional runtime profiling via `WREN_PROFILE_ENABLED` - Create `example/os_demo.wren` demonstrating Platform, Process, and conditional exit usage - Update `example/regex_demo.wren` to import `Match` and exercise Match object properties, groups, and `matchAll` - Remove static HTML manual pages (`base64.html`, `dns.html`, `json.html`) and replace with structured `manual_src/` directory containing Jinja2 templates, YAML metadata, and content pages - Expand `README.md` with build targets table, manual building instructions, source layout, and guide for adding new module documentation
This commit is contained in:
+314
-6
@@ -6,9 +6,225 @@ import re
|
||||
import shutil
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from html.parser import HTMLParser
|
||||
from jinja2 import Environment, FileSystemLoader, ChoiceLoader
|
||||
|
||||
|
||||
class SEOGenerator:
|
||||
STOP_WORDS = {
|
||||
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
||||
'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
|
||||
'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
||||
'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
|
||||
'this', 'that', 'these', 'those', 'it', 'its', 'they', 'them',
|
||||
'we', 'us', 'you', 'your', 'he', 'she', 'him', 'her', 'i', 'my',
|
||||
'if', 'then', 'else', 'when', 'where', 'why', 'how', 'what', 'which',
|
||||
'who', 'whom', 'not', 'no', 'yes', 'all', 'any', 'both', 'each',
|
||||
'more', 'most', 'other', 'some', 'such', 'only', 'same', 'so',
|
||||
'than', 'too', 'very', 'just', 'also', 'now', 'here', 'there'
|
||||
}
|
||||
|
||||
WREN_TERMS = {
|
||||
'wren', 'fiber', 'class', 'method', 'module', 'import', 'foreign',
|
||||
'static', 'construct', 'scheduler', 'async', 'await', 'cli', 'api',
|
||||
'json', 'http', 'websocket', 'sqlite', 'crypto', 'tls', 'regex'
|
||||
}
|
||||
|
||||
def extract_keywords(self, text, title, max_keywords=10):
|
||||
words = re.findall(r'\b[a-zA-Z][a-zA-Z0-9_]{2,}\b', text.lower())
|
||||
|
||||
freq = {}
|
||||
for word in words:
|
||||
if word not in self.STOP_WORDS and len(word) > 2:
|
||||
freq[word] = freq.get(word, 0) + 1
|
||||
|
||||
title_words = set(re.findall(r'\b[a-zA-Z][a-zA-Z0-9_]+\b', title.lower()))
|
||||
for word in freq:
|
||||
if word in title_words:
|
||||
freq[word] *= 3
|
||||
if word in self.WREN_TERMS:
|
||||
freq[word] *= 2
|
||||
|
||||
sorted_words = sorted(freq.items(), key=lambda x: x[1], reverse=True)
|
||||
return [word for word, _ in sorted_words[:max_keywords]]
|
||||
|
||||
def generate_description(self, text, title, max_length=155):
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
text = re.sub(r'^Skip to main content\s*', '', text)
|
||||
text = re.sub(r'^Menu\s+', '', text)
|
||||
text = re.sub(r'Wren-CLI\s+v[\d.]+\s*', '', text)
|
||||
text = re.sub(r'Previous:.*?Next:.*?$', '', text)
|
||||
text = text.strip()
|
||||
|
||||
sentences = re.split(r'(?<=[.!?])\s+', text)
|
||||
sentences = [s for s in sentences if len(s) > 20 and not s.startswith(('import ', 'var ', '//'))]
|
||||
|
||||
if not sentences:
|
||||
return f"{title} - Wren-CLI documentation and reference."
|
||||
|
||||
description = sentences[0]
|
||||
|
||||
if len(description) > max_length:
|
||||
description = description[:max_length-3].rsplit(' ', 1)[0] + '...'
|
||||
elif len(description) < 80 and len(sentences) > 1:
|
||||
for s in sentences[1:]:
|
||||
if len(description) + len(s) + 1 <= max_length:
|
||||
description += ' ' + s
|
||||
else:
|
||||
break
|
||||
|
||||
return description
|
||||
|
||||
def extract_title(self, html):
|
||||
match = re.search(r'<h1[^>]*>([^<]+)</h1>', html)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
match = re.search(r'<title[^>]*>([^<]+)</title>', html)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
if ' - ' in title:
|
||||
return title.split(' - ')[0]
|
||||
return title
|
||||
return 'Wren-CLI Documentation'
|
||||
|
||||
|
||||
class TextExtractor(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.text = []
|
||||
self.skip_tags = {'script', 'style', 'nav', 'head', 'header', 'footer', 'aside'}
|
||||
self.skip_depth = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in self.skip_tags:
|
||||
self.skip_depth += 1
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in self.skip_tags and self.skip_depth > 0:
|
||||
self.skip_depth -= 1
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.skip_depth == 0:
|
||||
text = data.strip()
|
||||
if text:
|
||||
self.text.append(text)
|
||||
|
||||
def get_text(self):
|
||||
return ' '.join(self.text)
|
||||
|
||||
|
||||
class LinkExtractor(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.links = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs_dict = dict(attrs)
|
||||
if tag == 'a' and 'href' in attrs_dict:
|
||||
self.links.append(attrs_dict['href'])
|
||||
elif tag in ('img', 'script') and 'src' in attrs_dict:
|
||||
self.links.append(attrs_dict['src'])
|
||||
elif tag == 'link' and 'href' in attrs_dict:
|
||||
self.links.append(attrs_dict['href'])
|
||||
|
||||
|
||||
def extract_text(html):
|
||||
parser = TextExtractor()
|
||||
parser.feed(html)
|
||||
return parser.get_text()
|
||||
|
||||
|
||||
def escape_json_for_html(json_string):
|
||||
return (json_string
|
||||
.replace('</', '<\\/')
|
||||
.replace('<!--', '<\\!--'))
|
||||
|
||||
|
||||
class TemplateFormatter:
|
||||
INDENT = ' '
|
||||
|
||||
def __init__(self):
|
||||
self.fixes = []
|
||||
|
||||
def format_file(self, path, section=''):
|
||||
content = path.read_text()
|
||||
original = content
|
||||
|
||||
content = self._ensure_author_comment(content)
|
||||
content = self._fix_article_indentation(content)
|
||||
content = self._fix_navigation_urls(content, section)
|
||||
|
||||
if content != original:
|
||||
path.write_text(content)
|
||||
self.fixes.append(str(path))
|
||||
return True
|
||||
return False
|
||||
|
||||
def _ensure_author_comment(self, content):
|
||||
if not content.startswith('{# retoor'):
|
||||
return '{# retoor <retoor@molodetz.nl> #}\n' + content
|
||||
return content
|
||||
|
||||
def _fix_article_indentation(self, content):
|
||||
lines = content.split('\n')
|
||||
result = []
|
||||
in_article = False
|
||||
|
||||
for line in lines:
|
||||
if '{% block article %}' in line:
|
||||
in_article = True
|
||||
result.append(line)
|
||||
continue
|
||||
if '{% endblock %}' in line and in_article:
|
||||
in_article = False
|
||||
result.append(line)
|
||||
continue
|
||||
|
||||
if in_article and line and not line.startswith(' '):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('<') and not stripped.startswith('<!'):
|
||||
line = self.INDENT + stripped
|
||||
result.append(line)
|
||||
|
||||
return '\n'.join(result)
|
||||
|
||||
def _fix_navigation_urls(self, content, section):
|
||||
import re
|
||||
if not section or section == '.':
|
||||
return content
|
||||
|
||||
def fix_url(match):
|
||||
prefix = match.group(1)
|
||||
url = match.group(2)
|
||||
if '/' in url:
|
||||
return match.group(0)
|
||||
return f'{prefix}{section}/{url}'
|
||||
|
||||
def fix_prev_line(match):
|
||||
line = match.group(0)
|
||||
if '"url": "index.html"' in line and '"title": "Home"' in line:
|
||||
return line
|
||||
return re.sub(r'("url":\s*")([a-z0-9_-]+\.html")', lambda m: m.group(1) + section + '/' + m.group(2) if '/' not in m.group(2) else m.group(0), line)
|
||||
|
||||
content = re.sub(
|
||||
r'{% set prev_page = \{[^}]+\} %}',
|
||||
fix_prev_line,
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'(next_page\s*=\s*\{"url":\s*")([^"]+\.html")',
|
||||
fix_url,
|
||||
content
|
||||
)
|
||||
return content
|
||||
|
||||
def report(self):
|
||||
if self.fixes:
|
||||
print(f" Auto-formatted {len(self.fixes)} file(s):")
|
||||
for f in self.fixes:
|
||||
print(f" {f}")
|
||||
|
||||
|
||||
class ManualBuilder:
|
||||
def __init__(self):
|
||||
self.root = Path(__file__).parent.parent
|
||||
@@ -16,6 +232,7 @@ class ManualBuilder:
|
||||
self.output = self.root / 'bin' / 'manual'
|
||||
self.site = self.load_yaml('data/site.yaml')
|
||||
self.nav = self.load_yaml('data/navigation.yaml')
|
||||
self.seo = SEOGenerator()
|
||||
|
||||
templates_loader = FileSystemLoader(str(self.src / 'templates'))
|
||||
pages_loader = FileSystemLoader(str(self.src))
|
||||
@@ -35,16 +252,29 @@ class ManualBuilder:
|
||||
with open(self.src / path) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def format_templates(self):
|
||||
formatter = TemplateFormatter()
|
||||
pages_dir = self.src / 'pages'
|
||||
for html_file in pages_dir.rglob('*.html'):
|
||||
rel_path = html_file.relative_to(pages_dir)
|
||||
section = str(rel_path.parent) if rel_path.parent != Path('.') else ''
|
||||
formatter.format_file(html_file, section)
|
||||
formatter.report()
|
||||
|
||||
def build(self):
|
||||
print("Formatting templates...")
|
||||
self.format_templates()
|
||||
|
||||
if self.output.exists():
|
||||
shutil.rmtree(self.output)
|
||||
self.output.mkdir(parents=True)
|
||||
|
||||
search_index = self.build_search_index()
|
||||
self.env.globals['search_index_json'] = json.dumps(search_index)
|
||||
|
||||
self.build_pages()
|
||||
search_index = self.build_search_index()
|
||||
self.env.globals['search_index_json'] = escape_json_for_html(json.dumps(search_index))
|
||||
self.rebuild_pages_with_index()
|
||||
self.copy_static()
|
||||
self.validate_links()
|
||||
|
||||
print(f"Built manual to {self.output}")
|
||||
|
||||
@@ -54,6 +284,12 @@ class ManualBuilder:
|
||||
rel_path = html_file.relative_to(pages_dir)
|
||||
self.build_page(html_file, rel_path)
|
||||
|
||||
def rebuild_pages_with_index(self):
|
||||
pages_dir = self.src / 'pages'
|
||||
for html_file in pages_dir.rglob('*.html'):
|
||||
rel_path = html_file.relative_to(pages_dir)
|
||||
self.build_page(html_file, rel_path)
|
||||
|
||||
def build_page(self, src_path, rel_path):
|
||||
template_path = f'pages/{rel_path}'
|
||||
template = self.env.get_template(template_path)
|
||||
@@ -64,7 +300,25 @@ class ManualBuilder:
|
||||
html = template.render(
|
||||
current_path=str(rel_path),
|
||||
static_prefix=static_prefix,
|
||||
depth=depth
|
||||
depth=depth,
|
||||
seo={}
|
||||
)
|
||||
|
||||
text = extract_text(html)
|
||||
title = self.seo.extract_title(html)
|
||||
|
||||
seo = {
|
||||
'keywords': self.seo.extract_keywords(text, title),
|
||||
'description': self.seo.generate_description(text, title),
|
||||
'og_title': f"{title} - Wren-CLI",
|
||||
'og_type': 'article' if 'api/' in str(rel_path) or 'tutorials/' in str(rel_path) else 'website',
|
||||
}
|
||||
|
||||
html = template.render(
|
||||
current_path=str(rel_path),
|
||||
static_prefix=static_prefix,
|
||||
depth=depth,
|
||||
seo=seo
|
||||
)
|
||||
|
||||
out_path = self.output / rel_path
|
||||
@@ -81,6 +335,15 @@ class ManualBuilder:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(item, dest)
|
||||
|
||||
wasm_src = self.root / 'bin' / 'wasm'
|
||||
wasm_dest = self.output / 'wasm'
|
||||
if wasm_src.exists():
|
||||
wasm_dest.mkdir(parents=True, exist_ok=True)
|
||||
for item in wasm_src.iterdir():
|
||||
if item.is_file():
|
||||
shutil.copy2(item, wasm_dest / item.name)
|
||||
print(f" Copied WASM files to {wasm_dest}")
|
||||
|
||||
def build_search_index(self):
|
||||
index = {'pages': []}
|
||||
|
||||
@@ -89,14 +352,25 @@ class ManualBuilder:
|
||||
section_dir = section['directory']
|
||||
|
||||
for page in section.get('pages', []):
|
||||
url = f"{section_dir}/{page['file']}.html"
|
||||
if section_dir == ".":
|
||||
url = f"{page['file']}.html"
|
||||
else:
|
||||
url = f"{section_dir}/{page['file']}.html"
|
||||
rendered_path = self.output / url
|
||||
|
||||
content = ''
|
||||
if rendered_path.exists():
|
||||
html = rendered_path.read_text()
|
||||
content = extract_text(html)
|
||||
content = ' '.join(content.split()[:500])
|
||||
|
||||
index['pages'].append({
|
||||
'url': url,
|
||||
'title': page['title'],
|
||||
'section': section_title,
|
||||
'description': page.get('description', ''),
|
||||
'methods': page.get('methods', []),
|
||||
'content': ''
|
||||
'content': content
|
||||
})
|
||||
|
||||
(self.output / 'search-index.json').write_text(
|
||||
@@ -105,6 +379,40 @@ class ManualBuilder:
|
||||
|
||||
return index
|
||||
|
||||
def _is_local_link(self, link):
|
||||
if not link:
|
||||
return False
|
||||
if link.startswith(('http://', 'https://', 'mailto:', 'javascript:', '#', 'data:')):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _resolve_link(self, base_dir, link):
|
||||
link = link.split('#')[0].split('?')[0]
|
||||
if not link:
|
||||
return base_dir / 'index.html'
|
||||
return (base_dir / link).resolve()
|
||||
|
||||
def validate_links(self):
|
||||
broken = []
|
||||
for html_file in self.output.rglob('*.html'):
|
||||
rel_path = html_file.relative_to(self.output)
|
||||
html = html_file.read_text()
|
||||
|
||||
extractor = LinkExtractor()
|
||||
extractor.feed(html)
|
||||
|
||||
for link in extractor.links:
|
||||
if self._is_local_link(link):
|
||||
target = self._resolve_link(html_file.parent, link)
|
||||
if not target.exists():
|
||||
broken.append((rel_path, link))
|
||||
|
||||
if broken:
|
||||
print("Broken links found:")
|
||||
for page, link in broken:
|
||||
print(f" {page}: {link}")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def main():
|
||||
builder = ManualBuilder()
|
||||
|
||||
Vendored
+373
@@ -0,0 +1,373 @@
|
||||
// retoor <retoor@molodetz.nl>
|
||||
|
||||
import "io" for File, Directory
|
||||
import "os" for Process
|
||||
import "pathlib" for Path
|
||||
import "yaml" for Yaml
|
||||
import "json" for Json
|
||||
import "jinja" for Environment, FileSystemLoader, ChoiceLoader
|
||||
import "regex" for Regex
|
||||
import "strutil" for Str
|
||||
|
||||
class TextExtractor {
|
||||
static SKIP_TAGS { ["script", "style", "nav", "head", "header", "footer", "aside"] }
|
||||
|
||||
static extract(html) {
|
||||
var text = html
|
||||
for (tag in TextExtractor.SKIP_TAGS) {
|
||||
var pattern = Regex.new("<" + tag + "[^>]*>([^<]|<[^/]|</[^" + tag[0] + "])*</" + tag + ">", "gi")
|
||||
text = pattern.replaceAll(text, "")
|
||||
}
|
||||
var tagPattern = Regex.new("<[^>]+>", "g")
|
||||
text = tagPattern.replaceAll(text, " ")
|
||||
var whitespace = Regex.new("[ \t\n\r\f]+", "g")
|
||||
text = whitespace.replaceAll(text, " ")
|
||||
return text.trim()
|
||||
}
|
||||
}
|
||||
|
||||
class SEOGenerator {
|
||||
static STOP_WORDS {
|
||||
return [
|
||||
"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for",
|
||||
"of", "with", "by", "from", "as", "is", "was", "are", "were", "been",
|
||||
"be", "have", "has", "had", "do", "does", "did", "will", "would",
|
||||
"could", "should", "may", "might", "must", "shall", "can", "need",
|
||||
"this", "that", "these", "those", "it", "its", "they", "them",
|
||||
"we", "us", "you", "your", "he", "she", "him", "her", "i", "my",
|
||||
"if", "then", "else", "when", "where", "why", "how", "what", "which",
|
||||
"who", "whom", "not", "no", "yes", "all", "any", "both", "each",
|
||||
"more", "most", "other", "some", "such", "only", "same", "so",
|
||||
"than", "too", "very", "just", "also", "now", "here", "there"
|
||||
]
|
||||
}
|
||||
|
||||
static WREN_TERMS {
|
||||
return [
|
||||
"wren", "fiber", "class", "method", "module", "import", "foreign",
|
||||
"static", "construct", "scheduler", "async", "await", "cli", "api",
|
||||
"json", "http", "websocket", "sqlite", "crypto", "tls", "regex"
|
||||
]
|
||||
}
|
||||
|
||||
static extractKeywords(text, title, maxKeywords) {
|
||||
var wordPattern = Regex.new("\\b[a-zA-Z][a-zA-Z0-9_]{2,}\\b", "g")
|
||||
var matches = wordPattern.matchAll(Str.toLower(text))
|
||||
var freq = {}
|
||||
var stopWords = SEOGenerator.STOP_WORDS
|
||||
for (match in matches) {
|
||||
var word = match.text
|
||||
if (word.count > 2 && !stopWords.contains(word)) {
|
||||
if (freq.containsKey(word)) {
|
||||
freq[word] = freq[word] + 1
|
||||
} else {
|
||||
freq[word] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
var titleMatches = wordPattern.matchAll(Str.toLower(title))
|
||||
var titleWords = []
|
||||
for (m in titleMatches) titleWords.add(m.text)
|
||||
var wrenTerms = SEOGenerator.WREN_TERMS
|
||||
for (word in freq.keys) {
|
||||
if (titleWords.contains(word)) freq[word] = freq[word] * 3
|
||||
if (wrenTerms.contains(word)) freq[word] = freq[word] * 2
|
||||
}
|
||||
var sorted = freq.keys.toList
|
||||
sorted.sort {|a, b| freq[b] - freq[a] }
|
||||
var result = []
|
||||
var count = 0
|
||||
for (word in sorted) {
|
||||
if (count >= maxKeywords) break
|
||||
result.add(word)
|
||||
count = count + 1
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
static generateDescription(text, title, maxLength) {
|
||||
var ws = Regex.new("\\s+", "g")
|
||||
text = ws.replaceAll(text, " ").trim()
|
||||
var skipPattern = Regex.new("^Skip to main content\\s*", "")
|
||||
text = skipPattern.replaceAll(text, "")
|
||||
var menuPattern = Regex.new("^Menu\\s+", "")
|
||||
text = menuPattern.replaceAll(text, "")
|
||||
var versionPattern = Regex.new("Wren-CLI\\s+v[\\d.]+\\s*", "g")
|
||||
text = versionPattern.replaceAll(text, "")
|
||||
var navPattern = Regex.new("Previous:.*?Next:.*$", "")
|
||||
text = navPattern.replaceAll(text, "")
|
||||
text = text.trim()
|
||||
var sentencePattern = Regex.new("[.!?][ \t\n\r\f]+", "g")
|
||||
var sentences = sentencePattern.split(text)
|
||||
var filtered = []
|
||||
for (s in sentences) {
|
||||
var sLen = s.bytes.count
|
||||
if (sLen > 20 && !s.startsWith("import ") && !s.startsWith("var ") && !s.startsWith("//")) {
|
||||
filtered.add(s)
|
||||
}
|
||||
}
|
||||
if (filtered.isEmpty) return "%(title) - Wren-CLI documentation and reference."
|
||||
var description = filtered[0]
|
||||
var descLen = description.bytes.count
|
||||
if (descLen > maxLength) {
|
||||
description = description[0...(maxLength - 3)]
|
||||
descLen = description.bytes.count
|
||||
var lastSpace = descLen - 1
|
||||
while (lastSpace > 0 && description[lastSpace] != " ") lastSpace = lastSpace - 1
|
||||
if (lastSpace > 0) description = description[0...lastSpace]
|
||||
description = description + "..."
|
||||
} else if (descLen < 80 && filtered.count > 1) {
|
||||
var i = 1
|
||||
var fCount = filtered.count
|
||||
while (i < fCount) {
|
||||
var addLen = filtered[i].bytes.count
|
||||
if (descLen + addLen + 1 <= maxLength) {
|
||||
description = description + " " + filtered[i]
|
||||
descLen = descLen + addLen + 1
|
||||
} else {
|
||||
break
|
||||
}
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
return description
|
||||
}
|
||||
|
||||
static extractTitle(html) {
|
||||
var h1Pattern = Regex.new("<h1[^>]*>([^<]+)</h1>", "i")
|
||||
var match = h1Pattern.match(html)
|
||||
if (match) return match.group(1).trim()
|
||||
var titlePattern = Regex.new("<title[^>]*>([^<]+)</title>", "i")
|
||||
match = titlePattern.match(html)
|
||||
if (match) {
|
||||
var title = match.group(1).trim()
|
||||
if (title.contains(" - ")) {
|
||||
var parts = title.split(" - ")
|
||||
return parts[0]
|
||||
}
|
||||
return title
|
||||
}
|
||||
return "Wren-CLI Documentation"
|
||||
}
|
||||
}
|
||||
|
||||
class TemplateFormatter {
|
||||
static INDENT { " " }
|
||||
|
||||
construct new() {
|
||||
_fixes = []
|
||||
}
|
||||
|
||||
fixes { _fixes }
|
||||
|
||||
formatFile(path, section) {
|
||||
var content = path.readText()
|
||||
var original = content
|
||||
content = ensureAuthorComment_(content)
|
||||
content = fixArticleIndentation_(content)
|
||||
content = fixNavigationUrls_(content, section)
|
||||
if (content != original) {
|
||||
path.writeText(content)
|
||||
_fixes.add(path.toString)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
ensureAuthorComment_(content) {
|
||||
if (!content.startsWith("{# retoor")) {
|
||||
return "{# retoor <retoor@molodetz.nl> #}" + "\n" + content
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
fixArticleIndentation_(content) {
|
||||
var lines = content.split("\n")
|
||||
var result = []
|
||||
var inArticle = false
|
||||
for (line in lines) {
|
||||
if (line.contains("{\x25 block article \x25}")) {
|
||||
inArticle = true
|
||||
result.add(line)
|
||||
continue
|
||||
}
|
||||
if (line.contains("{\x25 endblock \x25}") && inArticle) {
|
||||
inArticle = false
|
||||
result.add(line)
|
||||
continue
|
||||
}
|
||||
if (inArticle && !line.isEmpty && !line.startsWith(" ")) {
|
||||
var stripped = line.trim()
|
||||
if (stripped.startsWith("<") && !stripped.startsWith("<!")) {
|
||||
line = TemplateFormatter.INDENT + stripped
|
||||
}
|
||||
}
|
||||
result.add(line)
|
||||
}
|
||||
return result.join("\n")
|
||||
}
|
||||
|
||||
fixNavigationUrls_(content, section) {
|
||||
return content
|
||||
}
|
||||
|
||||
report() {
|
||||
if (!_fixes.isEmpty) {
|
||||
System.print(" Auto-formatted %(_fixes.count) file(s):")
|
||||
for (f in _fixes) {
|
||||
System.print(" %(f)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ManualBuilder {
|
||||
construct new() {
|
||||
_root = Path.new(Process.cwd)
|
||||
_src = _root / "manual_src"
|
||||
_output = _root / "bin" / "manual"
|
||||
_site = Yaml.parse((_src / "data/site.yaml").readText())
|
||||
_nav = Yaml.parse((_src / "data/navigation.yaml").readText())
|
||||
var templatesLoader = FileSystemLoader.new((_src / "templates").toString)
|
||||
var pagesLoader = FileSystemLoader.new(_src.toString)
|
||||
_env = Environment.new(ChoiceLoader.new([templatesLoader, pagesLoader]))
|
||||
_searchIndexJson = ""
|
||||
}
|
||||
|
||||
build() {
|
||||
System.print("[DEBUG] Starting build...")
|
||||
System.print("[DEBUG] Checking output exists...")
|
||||
if (_output.exists()) {
|
||||
System.print("[DEBUG] Removing old output...")
|
||||
_output.rmtree()
|
||||
}
|
||||
System.print("[DEBUG] Creating output directory...")
|
||||
_output.mkdir(true)
|
||||
System.print("[DEBUG] Building pages...")
|
||||
buildPages()
|
||||
System.print("[DEBUG] Building search index...")
|
||||
var searchIndex = buildSearchIndex()
|
||||
System.print("[DEBUG] Converting search index to JSON...")
|
||||
_searchIndexJson = escapeJsonForHtml_(Json.stringify(searchIndex))
|
||||
System.print("[DEBUG] Rebuilding pages with index...")
|
||||
rebuildPagesWithIndex()
|
||||
System.print("[DEBUG] Copying static files...")
|
||||
copyStatic()
|
||||
System.print("Built manual to %(_output)")
|
||||
}
|
||||
|
||||
formatTemplates() {
|
||||
System.print("[DEBUG] formatTemplates: skipped (slow in Wren)")
|
||||
}
|
||||
|
||||
buildPages() {
|
||||
var pagesDir = _src / "pages"
|
||||
for (htmlFile in pagesDir.rglob("*.html")) {
|
||||
var relPath = htmlFile.relativeTo(pagesDir)
|
||||
buildPage(htmlFile, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
rebuildPagesWithIndex() {
|
||||
var pagesDir = _src / "pages"
|
||||
for (htmlFile in pagesDir.rglob("*.html")) {
|
||||
var relPath = htmlFile.relativeTo(pagesDir)
|
||||
buildPage(htmlFile, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
buildPage(srcPath, relPath) {
|
||||
System.print("[DEBUG] buildPage: %(relPath)")
|
||||
var templatePath = "pages/" + relPath.toString
|
||||
System.print("[DEBUG] Getting template: %(templatePath)")
|
||||
var template = _env.getTemplate(templatePath)
|
||||
System.print("[DEBUG] Template loaded")
|
||||
var depth = relPath.parts.count - 1
|
||||
var staticPrefix = depth > 0 ? ("../" * depth) : "./"
|
||||
var context = {
|
||||
"current_path": relPath.toString,
|
||||
"static_prefix": staticPrefix,
|
||||
"depth": depth,
|
||||
"seo": {},
|
||||
"site": _site,
|
||||
"nav": _nav,
|
||||
"search_index_json": _searchIndexJson
|
||||
}
|
||||
var html = template.render(context)
|
||||
var text = TextExtractor.extract(html)
|
||||
var title = SEOGenerator.extractTitle(html)
|
||||
var relStr = relPath.toString
|
||||
var ogType = (relStr.contains("api/") || relStr.contains("tutorials/")) ? "article" : "website"
|
||||
var seo = {
|
||||
"keywords": SEOGenerator.extractKeywords(text, title, 10),
|
||||
"description": SEOGenerator.generateDescription(text, title, 155),
|
||||
"og_title": "%(title) - Wren-CLI",
|
||||
"og_type": ogType
|
||||
}
|
||||
context["seo"] = seo
|
||||
html = template.render(context)
|
||||
var outPath = _output / relPath
|
||||
outPath.parent.mkdir(true)
|
||||
outPath.writeText(html)
|
||||
System.print(" %(relPath)")
|
||||
}
|
||||
|
||||
copyStatic() {
|
||||
var staticSrc = _src / "static"
|
||||
for (entry in staticSrc.walk()) {
|
||||
var root = entry[0]
|
||||
var files = entry[2]
|
||||
for (f in files) {
|
||||
var item = root / f
|
||||
var rel = item.relativeTo(staticSrc)
|
||||
var dest = _output / rel
|
||||
dest.parent.mkdir(true)
|
||||
item.copyfile(dest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buildSearchIndex() {
|
||||
var index = {"pages": []}
|
||||
for (section in _nav["sections"]) {
|
||||
var sectionTitle = section["title"]
|
||||
var sectionDir = section["directory"]
|
||||
var pages = section["pages"]
|
||||
if (pages == null) pages = []
|
||||
for (page in pages) {
|
||||
var url = "%(sectionDir)/%(page["file"]).html"
|
||||
var renderedPath = _output / url
|
||||
var content = ""
|
||||
if (renderedPath.exists()) {
|
||||
var html = renderedPath.readText()
|
||||
content = TextExtractor.extract(html)
|
||||
var words = content.split(" ")
|
||||
if (words.count > 500) {
|
||||
content = words[0...500].join(" ")
|
||||
}
|
||||
}
|
||||
var description = page["description"]
|
||||
if (description == null) description = ""
|
||||
var methods = page["methods"]
|
||||
if (methods == null) methods = []
|
||||
index["pages"].add({
|
||||
"url": url,
|
||||
"title": page["title"],
|
||||
"section": sectionTitle,
|
||||
"description": description,
|
||||
"methods": methods,
|
||||
"content": content
|
||||
})
|
||||
}
|
||||
}
|
||||
(_output / "search-index.json").writeText(Json.stringify(index, 2))
|
||||
return index
|
||||
}
|
||||
|
||||
escapeJsonForHtml_(jsonString) {
|
||||
return jsonString.replace("</", "<\\/").replace("<!--", "<\\!--")
|
||||
}
|
||||
}
|
||||
|
||||
var builder = ManualBuilder.new()
|
||||
builder.build()
|
||||
Vendored
+431
@@ -0,0 +1,431 @@
|
||||
// retoor <retoor@molodetz.nl>
|
||||
|
||||
import "os" for Process, Platform
|
||||
import "pathlib" for Path
|
||||
import "regex" for Regex
|
||||
import "argparse" for ArgumentParser
|
||||
import "subprocess" for Popen
|
||||
import "io" for Stdout
|
||||
|
||||
class Colors {
|
||||
static green(text) { Platform.isWindows ? text : "\x1b[32m%(text)\x1b[0m" }
|
||||
static red(text) { Platform.isWindows ? text : "\x1b[31m%(text)\x1b[0m" }
|
||||
static pink(text) { Platform.isWindows ? text : "\x1b[91m%(text)\x1b[0m" }
|
||||
static yellow(text) { Platform.isWindows ? text : "\x1b[33m%(text)\x1b[0m" }
|
||||
}
|
||||
|
||||
class Patterns {
|
||||
static expectPattern { __expectPattern }
|
||||
static expectErrorLinePattern { __expectErrorLinePattern }
|
||||
static expectErrorPattern { __expectErrorPattern }
|
||||
static expectHandledRuntimeErrorPattern { __expectHandledRuntimeErrorPattern }
|
||||
static expectRuntimeErrorPattern { __expectRuntimeErrorPattern }
|
||||
static stdinPattern { __stdinPattern }
|
||||
static skipPattern { __skipPattern }
|
||||
static nontestPattern { __nontestPattern }
|
||||
static splitPattern { __splitPattern }
|
||||
static errorPattern { __errorPattern }
|
||||
static stackTracePattern { __stackTracePattern }
|
||||
|
||||
static init() {
|
||||
__expectPattern = Regex.new("// expect: ?(.*)")
|
||||
__expectErrorLinePattern = Regex.new("// expect error line ([0-9]+)")
|
||||
__expectErrorPattern = Regex.new("// expect error")
|
||||
__expectHandledRuntimeErrorPattern = Regex.new("// expect handled runtime error: (.+)")
|
||||
__expectRuntimeErrorPattern = Regex.new("// expect runtime error: (.+)")
|
||||
__stdinPattern = Regex.new("// stdin: (.*)")
|
||||
__skipPattern = Regex.new("// skip: (.*)")
|
||||
__nontestPattern = Regex.new("// nontest")
|
||||
__splitPattern = Regex.new("\n|\r\n")
|
||||
__errorPattern = Regex.new("\\[.* line ([0-9]+)\\] Error")
|
||||
__stackTracePattern = Regex.new("test/.* line ([0-9]+)\\] in")
|
||||
}
|
||||
}
|
||||
|
||||
class Test {
|
||||
construct new(path) {
|
||||
_path = path
|
||||
_output = []
|
||||
_compileErrors = []
|
||||
_runtimeErrorLine = 0
|
||||
_runtimeErrorMessage = null
|
||||
_exitCode = 0
|
||||
_inputBytes = null
|
||||
_failures = []
|
||||
}
|
||||
|
||||
path { _path }
|
||||
failures { _failures }
|
||||
|
||||
parse() {
|
||||
var inputLines = []
|
||||
var lineNum = 1
|
||||
|
||||
var content = Path.new(_path).readText()
|
||||
var lines = Patterns.splitPattern.split(content)
|
||||
|
||||
for (line in lines) {
|
||||
if (line.count == 0) {
|
||||
lineNum = lineNum + 1
|
||||
continue
|
||||
}
|
||||
|
||||
var match = Patterns.expectPattern.match(line)
|
||||
if (match) {
|
||||
_output.add([match.groups[1], lineNum])
|
||||
TestRunner.expectations = TestRunner.expectations + 1
|
||||
}
|
||||
|
||||
match = Patterns.expectErrorLinePattern.match(line)
|
||||
if (match) {
|
||||
var errLine = Num.fromString(match.groups[1])
|
||||
if (!listContains_(_compileErrors, errLine)) {
|
||||
_compileErrors.add(errLine)
|
||||
}
|
||||
_exitCode = 65
|
||||
TestRunner.expectations = TestRunner.expectations + 1
|
||||
} else {
|
||||
match = Patterns.expectErrorPattern.match(line)
|
||||
if (match) {
|
||||
if (!listContains_(_compileErrors, lineNum)) {
|
||||
_compileErrors.add(lineNum)
|
||||
}
|
||||
_exitCode = 65
|
||||
TestRunner.expectations = TestRunner.expectations + 1
|
||||
}
|
||||
}
|
||||
|
||||
match = Patterns.expectHandledRuntimeErrorPattern.match(line)
|
||||
if (match) {
|
||||
_runtimeErrorLine = lineNum
|
||||
_runtimeErrorMessage = match.groups[1]
|
||||
TestRunner.expectations = TestRunner.expectations + 1
|
||||
} else {
|
||||
match = Patterns.expectRuntimeErrorPattern.match(line)
|
||||
if (match) {
|
||||
_runtimeErrorLine = lineNum
|
||||
_runtimeErrorMessage = match.groups[1]
|
||||
_exitCode = 70
|
||||
TestRunner.expectations = TestRunner.expectations + 1
|
||||
}
|
||||
}
|
||||
|
||||
match = Patterns.stdinPattern.match(line)
|
||||
if (match) {
|
||||
inputLines.add(match.groups[1])
|
||||
}
|
||||
|
||||
match = Patterns.skipPattern.match(line)
|
||||
if (match) {
|
||||
TestRunner.numSkipped = TestRunner.numSkipped + 1
|
||||
var reason = match.groups[1]
|
||||
if (!TestRunner.skipped.containsKey(reason)) {
|
||||
TestRunner.skipped[reason] = 0
|
||||
}
|
||||
TestRunner.skipped[reason] = TestRunner.skipped[reason] + 1
|
||||
return false
|
||||
}
|
||||
|
||||
match = Patterns.nontestPattern.match(line)
|
||||
if (match) {
|
||||
return false
|
||||
}
|
||||
|
||||
lineNum = lineNum + 1
|
||||
}
|
||||
|
||||
if (inputLines.count > 0) {
|
||||
_inputBytes = inputLines.join("\n")
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
listContains_(list, item) {
|
||||
for (i in list) {
|
||||
if (i == item) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
run(app, type) {
|
||||
var proc = Popen.new([app, _path])
|
||||
|
||||
if (_inputBytes) {
|
||||
proc.stdin.write(_inputBytes)
|
||||
}
|
||||
proc.stdin.close()
|
||||
|
||||
var exitCode = proc.wait()
|
||||
var out = proc.stdout.read()
|
||||
var err = proc.stderr.read()
|
||||
|
||||
validate(type == "example", exitCode, out, err)
|
||||
}
|
||||
|
||||
validate(isExample, exitCode, out, err) {
|
||||
if (_compileErrors.count > 0 && _runtimeErrorMessage) {
|
||||
fail("Test error: Cannot expect both compile and runtime errors.")
|
||||
return
|
||||
}
|
||||
|
||||
out = out.replace("\r\n", "\n")
|
||||
err = err.replace("\r\n", "\n")
|
||||
|
||||
var errorLines = Patterns.splitPattern.split(err)
|
||||
|
||||
if (_runtimeErrorMessage) {
|
||||
validateRuntimeError(errorLines)
|
||||
} else {
|
||||
validateCompileErrors(errorLines)
|
||||
}
|
||||
|
||||
validateExitCode(exitCode, errorLines)
|
||||
|
||||
if (isExample) return
|
||||
|
||||
validateOutput(out)
|
||||
}
|
||||
|
||||
validateRuntimeError(errorLines) {
|
||||
if (errorLines.count < 2) {
|
||||
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got none.")
|
||||
return
|
||||
}
|
||||
|
||||
var line = 0
|
||||
while (line < errorLines.count && Patterns.errorPattern.test(errorLines[line])) {
|
||||
line = line + 1
|
||||
}
|
||||
|
||||
if (line >= errorLines.count) {
|
||||
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" but only found compile errors.")
|
||||
return
|
||||
}
|
||||
|
||||
if (errorLines[line] != _runtimeErrorMessage) {
|
||||
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got:")
|
||||
fail(errorLines[line])
|
||||
}
|
||||
|
||||
var match = null
|
||||
var stackLines = []
|
||||
for (i in (line + 1)...errorLines.count) {
|
||||
stackLines.add(errorLines[i])
|
||||
match = Patterns.stackTracePattern.match(errorLines[i])
|
||||
if (match) break
|
||||
}
|
||||
|
||||
if (!match) {
|
||||
fail("Expected stack trace and got:")
|
||||
for (stackLine in stackLines) {
|
||||
fail(stackLine)
|
||||
}
|
||||
} else {
|
||||
var stackLine = Num.fromString(match.groups[1])
|
||||
if (stackLine != _runtimeErrorLine) {
|
||||
fail("Expected runtime error on line %(_runtimeErrorLine) but was on line %(stackLine).")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateCompileErrors(errorLines) {
|
||||
var foundErrors = []
|
||||
|
||||
for (line in errorLines) {
|
||||
var match = Patterns.errorPattern.match(line)
|
||||
if (match) {
|
||||
var errorLine = Num.fromString(match.groups[1])
|
||||
if (listContains_(_compileErrors, errorLine)) {
|
||||
if (!listContains_(foundErrors, errorLine)) {
|
||||
foundErrors.add(errorLine)
|
||||
}
|
||||
} else {
|
||||
fail("Unexpected error:")
|
||||
fail(line)
|
||||
}
|
||||
} else if (line != "") {
|
||||
fail("Unexpected output on stderr:")
|
||||
fail(line)
|
||||
}
|
||||
}
|
||||
|
||||
for (expected in _compileErrors) {
|
||||
if (!listContains_(foundErrors, expected)) {
|
||||
fail("Missing expected error on line %(expected).")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateExitCode(exitCode, errorLines) {
|
||||
if (exitCode == _exitCode) return
|
||||
|
||||
fail("Expected return code %(_exitCode) and got %(exitCode). Stderr:")
|
||||
for (line in errorLines) {
|
||||
_failures.add(line)
|
||||
}
|
||||
}
|
||||
|
||||
validateOutput(out) {
|
||||
var outLines = Patterns.splitPattern.split(out)
|
||||
if (outLines.count > 0 && outLines[-1] == "") {
|
||||
outLines = outLines[0...-1]
|
||||
}
|
||||
|
||||
var index = 0
|
||||
for (line in outLines) {
|
||||
if (index >= _output.count) {
|
||||
fail("Got output \"%(line)\" when none was expected.")
|
||||
} else if (_output[index][0] != line) {
|
||||
fail("Expected output \"%(_output[index][0])\" on line %(_output[index][1]) and got \"%(line)\".")
|
||||
}
|
||||
index = index + 1
|
||||
}
|
||||
|
||||
while (index < _output.count) {
|
||||
fail("Missing expected output \"%(_output[index][0])\" on line %(_output[index][1]).")
|
||||
index = index + 1
|
||||
}
|
||||
}
|
||||
|
||||
fail(message) {
|
||||
_failures.add(message)
|
||||
}
|
||||
}
|
||||
|
||||
class TestRunner {
|
||||
static passed { __passed }
|
||||
static passed=(value) { __passed = value }
|
||||
static failed { __failed }
|
||||
static failed=(value) { __failed = value }
|
||||
static numSkipped { __numSkipped }
|
||||
static numSkipped=(value) { __numSkipped = value }
|
||||
static skipped { __skipped }
|
||||
static skipped=(value) { __skipped = value }
|
||||
static expectations { __expectations }
|
||||
static expectations=(value) { __expectations = value }
|
||||
|
||||
construct new() {
|
||||
TestRunner.passed = 0
|
||||
TestRunner.failed = 0
|
||||
TestRunner.numSkipped = 0
|
||||
TestRunner.skipped = {}
|
||||
TestRunner.expectations = 0
|
||||
|
||||
var parser = ArgumentParser.new()
|
||||
parser.addArgument("--suffix", {"default": ""})
|
||||
parser.addArgument("suite", {"required": false, "default": null})
|
||||
parser.addArgument("--skip-tests", {"action": "storeTrue"})
|
||||
parser.addArgument("--skip-examples", {"action": "storeTrue"})
|
||||
|
||||
_args = parser.parseArgs()
|
||||
_wrenDir = Path.new(Process.cwd)
|
||||
_wrenApp = _wrenDir / "bin" / ("wren_cli" + _args["suffix"])
|
||||
}
|
||||
|
||||
run() {
|
||||
var testDir = _wrenDir / "test"
|
||||
walk(testDir) {|path| runTest(path) }
|
||||
|
||||
printLine()
|
||||
if (TestRunner.failed == 0) {
|
||||
System.print("All " + Colors.green(TestRunner.passed) + " tests passed (" + TestRunner.expectations.toString + " expectations).")
|
||||
} else {
|
||||
System.print(Colors.green(TestRunner.passed) + " tests passed. " + Colors.red(TestRunner.failed) + " tests failed.")
|
||||
}
|
||||
|
||||
var sortedKeys = sortKeys_(TestRunner.skipped)
|
||||
for (key in sortedKeys) {
|
||||
System.print("Skipped " + Colors.yellow(TestRunner.skipped[key]) + " tests: " + key)
|
||||
}
|
||||
|
||||
if (TestRunner.failed != 0) {
|
||||
Process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
sortKeys_(map) {
|
||||
var keys = []
|
||||
for (key in map.keys) {
|
||||
keys.add(key)
|
||||
}
|
||||
keys.sort()
|
||||
return keys
|
||||
}
|
||||
|
||||
walk(dir, callback) {
|
||||
var entries = dir.iterdir()
|
||||
var files = []
|
||||
var dirs = []
|
||||
|
||||
for (entry in entries) {
|
||||
if (entry.isDir()) {
|
||||
dirs.add(entry)
|
||||
} else {
|
||||
files.add(entry)
|
||||
}
|
||||
}
|
||||
|
||||
dirs.sort {|a, b| a.toString < b.toString }
|
||||
files.sort {|a, b| a.toString < b.toString }
|
||||
|
||||
for (file in files) {
|
||||
callback.call(file)
|
||||
}
|
||||
|
||||
for (subdir in dirs) {
|
||||
walk(subdir, callback)
|
||||
}
|
||||
}
|
||||
|
||||
printLine() { printLine(null) }
|
||||
|
||||
printLine(line) {
|
||||
System.write("\x1b[2K")
|
||||
System.write("\r")
|
||||
if (line) {
|
||||
System.write(line)
|
||||
Stdout.flush()
|
||||
}
|
||||
}
|
||||
|
||||
runTest(path) {
|
||||
var pathStr = path.toString
|
||||
|
||||
if (!pathStr.endsWith(".wren")) return
|
||||
|
||||
if (_args["suite"]) {
|
||||
var testPath = path.relativeTo(_wrenDir / "test").toString
|
||||
if (!testPath.startsWith(_args["suite"])) return
|
||||
}
|
||||
|
||||
var appRelPath = _wrenApp.relativeTo(_wrenDir).toString
|
||||
printLine("(" + appRelPath + ") Passed: " + Colors.green(TestRunner.passed) + " Failed: " + Colors.red(TestRunner.failed) + " Skipped: " + Colors.yellow(TestRunner.numSkipped) + " ")
|
||||
|
||||
var normalizedPath = path.relativeTo(Path.cwd).toString.replace("\\", "/")
|
||||
|
||||
var test = Test.new(normalizedPath)
|
||||
|
||||
if (!test.parse()) {
|
||||
return
|
||||
}
|
||||
|
||||
test.run(_wrenApp.toString, "test")
|
||||
|
||||
if (test.failures.count == 0) {
|
||||
TestRunner.passed = TestRunner.passed + 1
|
||||
} else {
|
||||
TestRunner.failed = TestRunner.failed + 1
|
||||
printLine(Colors.red("FAIL") + ": " + normalizedPath)
|
||||
System.print("")
|
||||
for (failure in test.failures) {
|
||||
System.print(" " + Colors.pink(failure))
|
||||
}
|
||||
System.print("")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Patterns.init()
|
||||
var runner = TestRunner.new()
|
||||
runner.run()
|
||||
Reference in New Issue
Block a user