feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+62
@@ -0,0 +1,62 @@
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username VARCHAR(64) NOT NULL UNIQUE,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
password CHAR(64) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
role VARCHAR(32) NOT NULL DEFAULT 'user',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE posts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
author_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
title VARCHAR(200) NOT NULL,
|
||||
slug VARCHAR(200) NOT NULL UNIQUE,
|
||||
body TEXT NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'draft'
|
||||
CHECK (status IN ('draft', 'published', 'archived')),
|
||||
view_count INTEGER NOT NULL DEFAULT 0,
|
||||
published_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
SELECT
|
||||
u.username,
|
||||
u.email,
|
||||
p.title,
|
||||
p.status,
|
||||
p.view_count
|
||||
FROM users u
|
||||
JOIN posts p ON p.author_id = u.id
|
||||
WHERE u.is_active = TRUE
|
||||
AND p.status != 'archived'
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT 20;
|
||||
|
||||
WITH cte AS (
|
||||
SELECT author_id, COUNT(*) AS cnt
|
||||
FROM posts
|
||||
GROUP BY author_id
|
||||
)
|
||||
SELECT u.username, cte.cnt
|
||||
FROM users u
|
||||
JOIN cte ON cte.author_id = u.id;
|
||||
|
||||
UPDATE posts
|
||||
SET status = 'archived',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE published_at < datetime('now', '-1 year')
|
||||
AND status = 'published';
|
||||
|
||||
SELECT
|
||||
NULL AS maybe_value,
|
||||
COALESCE(u.email, 'no-email') AS safe_email,
|
||||
CASE
|
||||
WHEN p.view_count > 1000 THEN 'popular'
|
||||
WHEN p.view_count > 100 THEN 'moderate'
|
||||
ELSE 'quiet'
|
||||
END AS popularity
|
||||
FROM users u
|
||||
LEFT JOIN posts p ON p.author_id = u.id;
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username VARCHAR(64) NOT NULL UNIQUE,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
password CHAR(64) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
role VARCHAR(32) NOT NULL DEFAULT 'user',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
);
|
||||
|
||||
SELECT
|
||||
u.username,
|
||||
u.email,
|
||||
p.title
|
||||
FROM users u
|
||||
JOIN posts p ON p.author_id = u.id
|
||||
WHERE u.is_active = TRUE
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT 20
|
||||
|
||||
SELECT * FROM;
|
||||
|
||||
CREATE INDEX ON posts(status);
|
||||
|
||||
INSERT INTO users (username, email, password)
|
||||
VALUES ('alice', 'alice@example.com');
|
||||
|
||||
UPDATE posts
|
||||
SET status = 'archived',
|
||||
WHERE published_at < datetime('now', '-1 year');
|
||||
Vendored
+91
@@ -0,0 +1,91 @@
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username VARCHAR(64) NOT NULL UNIQUE,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
password CHAR(64) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
role VARCHAR(32) NOT NULL DEFAULT 'user',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE posts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
author_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
title VARCHAR(200) NOT NULL,
|
||||
slug VARCHAR(200) NOT NULL UNIQUE,
|
||||
body TEXT NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'draft'
|
||||
CHECK (status IN ('draft', 'published', 'archived')),
|
||||
view_count INTEGER NOT NULL DEFAULT 0,
|
||||
published_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE tags (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(50) NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE post_tags (
|
||||
post_id INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE,
|
||||
tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (post_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_posts_author ON posts(author_id);
|
||||
CREATE INDEX idx_posts_status ON posts(status);
|
||||
CREATE INDEX idx_posts_slug ON posts(slug);
|
||||
CREATE INDEX idx_users_email ON users(email);
|
||||
|
||||
INSERT INTO users (username, email, password, role)
|
||||
VALUES ('alice', 'alice@example.com', 'hashed_placeholder', 'admin');
|
||||
|
||||
INSERT INTO tags (name) VALUES ('nim'), ('parsing'), ('devtools');
|
||||
|
||||
INSERT INTO posts (author_id, title, slug, body, status)
|
||||
VALUES (1, 'Introducing Nimcheck', 'introducing-nimcheck',
|
||||
'Nimcheck is a universal syntax validator.', 'published');
|
||||
|
||||
INSERT INTO post_tags (post_id, tag_id)
|
||||
SELECT p.id, t.id
|
||||
FROM posts p, tags t
|
||||
WHERE p.slug = 'introducing-nimcheck'
|
||||
AND t.name IN ('nim', 'parsing', 'devtools');
|
||||
|
||||
SELECT
|
||||
u.username,
|
||||
u.email,
|
||||
p.title,
|
||||
p.status,
|
||||
p.view_count,
|
||||
GROUP_CONCAT(t.name, ', ') AS tag_list
|
||||
FROM users u
|
||||
JOIN posts p ON p.author_id = u.id
|
||||
LEFT JOIN post_tags pt ON pt.post_id = p.id
|
||||
LEFT JOIN tags t ON t.id = pt.tag_id
|
||||
WHERE u.is_active = TRUE
|
||||
AND p.status != 'archived'
|
||||
GROUP BY p.id
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT 20;
|
||||
|
||||
UPDATE posts
|
||||
SET status = 'archived',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE published_at < datetime('now', '-1 year')
|
||||
AND status = 'published';
|
||||
|
||||
WITH post_stats AS (
|
||||
SELECT
|
||||
author_id,
|
||||
COUNT(*) AS total_posts,
|
||||
SUM(view_count) AS total_views,
|
||||
AVG(view_count) AS avg_views
|
||||
FROM posts
|
||||
GROUP BY author_id
|
||||
)
|
||||
SELECT u.username, ps.total_posts, ps.total_views, ps.avg_views
|
||||
FROM users u
|
||||
JOIN post_stats ps ON ps.author_id = u.id
|
||||
ORDER BY ps.total_views DESC;
|
||||
Reference in New Issue
Block a user