feat: add extended language validators and fix tokenizer crash bugs
Expand nimcheck with validators and tokenizers for C, C++, C#, Go, Rust, Ruby, CSS, SQL, Markdown, Dockerfile, Makefile, Kotlin, Lua, Swift, TypeScript, and XML. Register all flavors in the validator factory and improve auto-detection scoring for the new languages. Fix infinite tokenizer loops that caused OOM kills: closeBracket now advances position, finishTokenizeStep guards stalled tokenization, and Jinja/JS tokenizers no longer double-advance on brackets. Fix block-balance false positives in Lua (for/do) and Ruby (postfix unless), SQL trailing-comma detection across whitespace, and Makefile tab literals in test fixtures.
This commit is contained in:
Vendored
+181
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxRetries = 3
|
||||
BaseDelay = 100 * time.Millisecond
|
||||
MaxDelay = 5 * time.Second
|
||||
)
|
||||
|
||||
type Executor[T any] struct {
|
||||
workers int
|
||||
tasks chan func() T
|
||||
results chan T
|
||||
}
|
||||
|
||||
func NewExecutor[T any](workers int) *Executor[T] {
|
||||
return &Executor[T]{
|
||||
workers: workers,
|
||||
tasks: make(chan func() T, workers*2),
|
||||
results: make(chan T, workers*2),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Executor[T]) Submit(task func() T) {
|
||||
e.tasks <- task
|
||||
}
|
||||
|
||||
func (e *Executor[T]) Run(ctx context.Context) []T {
|
||||
for i := 0; i < e.workers; i++ {
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case task := <-e.tasks:
|
||||
e.results <- task()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
var results []T
|
||||
for i := 0; i < cap(e.tasks); i++ {
|
||||
select {
|
||||
case r := <-e.results:
|
||||
results = append(results, r)
|
||||
case <-ctx.Done():
|
||||
return results
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func retryWithBackoff(operation func() error) error {
|
||||
var err error
|
||||
delay := BaseDelay
|
||||
|
||||
for i := 0; i < MaxRetries; i++ {
|
||||
if err = operation(); err == nil {
|
||||
return nil
|
||||
}
|
||||
if i < MaxRetries-1 {
|
||||
jitter := time.Duration(rand.Int63n(int64(delay / 2)))
|
||||
time.Sleep(delay + jitter)
|
||||
delay = time.Duration(math.Min(float64(delay*2), float64(MaxDelay)))
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("operation failed after %d retries: %w", MaxRetries, err)
|
||||
}
|
||||
|
||||
func HashContent(r io.Reader) (string, error) {
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, r); err != nil {
|
||||
return "", fmt.Errorf("hashing failed: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func SliceToMap[K comparable, V any](items []V, keyFn func(V) K) map[K]V {
|
||||
result := make(map[K]V, len(items))
|
||||
for _, item := range items {
|
||||
result[keyFn(item)] = item
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func InverseMap[K, V comparable](m map[K]V) map[V]K {
|
||||
result := make(map[V]K, len(m))
|
||||
for k, v := range m {
|
||||
result[v] = k
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
var counter atomic.Int64
|
||||
|
||||
func nextID() int64 {
|
||||
return counter.Add(1)
|
||||
}
|
||||
|
||||
type Enum interface {
|
||||
~int
|
||||
String() string
|
||||
}
|
||||
|
||||
type Color int
|
||||
|
||||
const (
|
||||
Red Color = iota
|
||||
Green
|
||||
Blue
|
||||
)
|
||||
|
||||
func (c Color) String() string {
|
||||
return [...]string{"red", "green", "blue"}[c]
|
||||
}
|
||||
|
||||
func Must[T any](val T, err error) T {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
type TransformFunc[T, U any] func(T) U
|
||||
|
||||
func Chain[T, U, V any](f TransformFunc[T, U], g TransformFunc[U, V]) TransformFunc[T, V] {
|
||||
return func(t T) V {
|
||||
return g(f(t))
|
||||
}
|
||||
}
|
||||
|
||||
func zero[T any]() T {
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
|
||||
func Map[T, U any](items []T, fn func(T) U) []U {
|
||||
result := make([]U, len(items))
|
||||
for i, item := range items {
|
||||
result[i] = fn(item)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func Filter[T any](items []T, fn func(T) bool) []T {
|
||||
var result []T
|
||||
for _, item := range items {
|
||||
if fn(item) {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func Reduce[T, U any](items []T, init U, fn func(U, T) U) U {
|
||||
acc := init
|
||||
for _, item := range items {
|
||||
acc = fn(acc, item)
|
||||
}
|
||||
return acc
|
||||
}
|
||||
|
||||
func init() {
|
||||
fmt.Println("initializing package main")
|
||||
}
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main() {
|
||||
x := 42
|
||||
fmt.Println(x
|
||||
|
||||
func brokenFunc(x int) string {
|
||||
return x
|
||||
}
|
||||
|
||||
type User struct {
|
||||
Name string
|
||||
Age int
|
||||
}
|
||||
|
||||
func (u User) Greet() string {
|
||||
return "Hello, " + .Name
|
||||
|
||||
switch x {
|
||||
case 1:
|
||||
fmt.Println("one")
|
||||
case 2:
|
||||
fmt.Println("two")
|
||||
}
|
||||
Vendored
+149
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
Active bool `json:"active"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type UserService struct {
|
||||
mu sync.RWMutex
|
||||
users map[int]*User
|
||||
}
|
||||
|
||||
func NewUserService() *UserService {
|
||||
return &UserService{
|
||||
users: make(map[int]*User),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *UserService) Add(user *User) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if user.Name == "" {
|
||||
return errors.New("name is required")
|
||||
}
|
||||
if !strings.Contains(user.Email, "@") {
|
||||
return errors.New("invalid email")
|
||||
}
|
||||
if _, exists := s.users[user.ID]; exists {
|
||||
return errors.New("user already exists")
|
||||
}
|
||||
s.users[user.ID] = user
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UserService) FindByID(id int) (*User, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
user, ok := s.users[id]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("user %d not found", id)
|
||||
}
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func (s *UserService) FindByEmail(email string) *User {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
for _, u := range s.users {
|
||||
if u.Email == email {
|
||||
return u
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UserService) All() []*User {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
result := make([]*User, 0, len(s.users))
|
||||
for _, u := range s.users {
|
||||
result = append(result, u)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func loggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
log.Printf("%s %s %s", r.Method, r.URL.Path, r.RemoteAddr)
|
||||
next.ServeHTTP(w, r)
|
||||
log.Printf("%s completed in %v", r.URL.Path, time.Since(start))
|
||||
})
|
||||
}
|
||||
|
||||
func handleUsers(svc *UserService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
users := svc.All()
|
||||
json.NewEncoder(w).Encode(users)
|
||||
|
||||
case http.MethodPost:
|
||||
var user User
|
||||
if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := svc.Add(&user); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
json.NewEncoder(w).Encode(user)
|
||||
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
svc := NewUserService()
|
||||
svc.Add(&User{
|
||||
ID: 1,
|
||||
Name: "Alice",
|
||||
Email: "alice@example.com",
|
||||
Active: true,
|
||||
})
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/users", handleUsers(svc))
|
||||
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: ":" + port,
|
||||
Handler: loggingMiddleware(mux),
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
log.Printf("Server starting on :%s", port)
|
||||
log.Fatal(server.ListenAndServe())
|
||||
}
|
||||
Reference in New Issue
Block a user