Compare commits

..
6 Commits
Author SHA1 Message Date
retoor b16c73fe52 Added runner config
SORM tests / Compile (push) Successful in 5s
2024-11-22 14:42:25 +01:00
retoor 09ed5d9957 Upgraded english quality 2024-11-22 14:42:25 +01:00
retoor 79c2d6ed12 Updated Readme 2024-11-22 14:42:25 +01:00
retoor 96414b6606 Added content 2024-11-22 14:42:25 +01:00
retoor c85802e8ee Updated Readme 2024-11-22 14:42:25 +01:00
Gerben Aaltink 7b6b2a1cbf Initial commit 2024-11-22 13:40:39 +01:00
11 changed files with 494 additions and 8911 deletions
-21
View File
@@ -1,21 +0,0 @@
name: SORM build
run-name: Build 🚀
on: [push]
jobs:
build:
runs-on: ubuntu-latest
steps:
- run: echo "Update apt repository"
- run: apt update
- run: echo "Installing dependencies"
- run: apt install build-essential sqlite3 libsqlite3-dev libreadline-dev libpython3-dev make -y
- name: Check out repository code
uses: actions/checkout@v4
- name: List files in the repository
run: |
ls ${{ gitea.workspace }}
- run: cp rlib.h /usr/include/
- run: echo "Build project (misses rlib.h)"
- run: make build
- run: echo "This job's status is ${{ job.status }}."
+2 -3
View File
@@ -1,10 +1,9 @@
all: build run
build:
gcc main.c -Wextra -Wall -lsqlite3 -lreadline -o sorm
gcc main.c -lsqlite3 -lreadline -o sorm
gcc -shared -o sorm.so -fPIC main.c -lsqlite3 -lreadline
run:
./sorm
+3 -27
View File
@@ -1,9 +1,5 @@
# SORM
## Todo
- auto complete table / view names.
## Description
SORM stands for SQL ORM. I made this because I have a love/hate relationship with both. I combined them together to have the best of both worlds!
@@ -14,34 +10,14 @@ Examples of common used functions are:
- `sormq(int conn, char *sql, ...)` execute query. Variadic arguments. Works like `printf`. Returns result in CSV format in case of `SELECT`.
- `sorm_csvd(char *csv_data)` dumps your CSV result data to a nice fixed content width table in the terminal.
## Thread safety
I wonder if I have configured sqlite3 the right way for thread safety. It maybe requires a manual compilation of the shared object file. Will look into that. SORM is written with thread safety in mind.
## Design choices
I use mainly native types and not custom structs. For example, the db parameter is an int. This is so it can easily communicate with other languages using a shared object file.
Same argument is for the result set of `sormq` (the query function) resulting in a `char *` containing CSV data.
While the performance is nice, it's not written with performance in mind at all.
## Thread safety
I wonder if I have configured sqlite3 the right way for thread safety. It maybe requires a manual compilation of the shared object file. Will look into that. SORM is written with thread safety in mind.
## Building
It has only one build command for the shared object file and the REPL/CLI.
It has not been compiled with optimizations / warnings enabled. Don't recommend for now.
```bash
make build
```
### Requirements
This project has the following dependencies:
- sqlite3-dev (`apt install sqlite3-dev`)
- readline(-dev?) (no idea how to install)
- gcc
- make
The project probably works from C99 to C2X.
### Caution
This project won't compile on your local PC because `rlib.h` is missing. This is my bundle of libraries that I regularely use. I will add this later as 3rd party lib. Maybe I will cherry pick only the functions that SORM uses out of it to keep the source clean, else I will have thousands of lines nothing to do with this project in the source.
## Python support
The Python library is low quality. I made it just for fun and test. This is not a defitive version. But it show very well how to communicate with a shared object file. I'm sure someone will be happy with examples how to use variadic functions trough Python to C. See `sorm.py`
+32 -16
View File
@@ -1,21 +1,19 @@
#ifndef SORM_CLI_H
#define SORM_CLI_H
#include "sorm.h"
#include <fcntl.h>
#include <readline/history.h>
#include <readline/readline.h>
#include <rlib.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <readline/readline.h>
#include <readline/history.h>
#include "sorm.h"
const char * history_filename = ".sorm_history";
int _sorm_readline_accept_line(int count, int key) {
(void)count;
(void)key;
if (strchr(rl_line_buffer, ';')) {
rl_done = 1;
@@ -26,7 +24,16 @@ int _sorm_readline_accept_line(int count, int key) {
}
char* _sorm_autocompletion_generator(const char* text, int state) {
const char *completions[] = {"exit", sorm_last_query, sorm_last_query_expanded, "python", "history", "memory", "truncate", NULL};
const char* completions[] = {
"exit",
sorm_last_query,
sorm_last_query_expanded,
"python",
"history",
"memory",
"truncate",
NULL
};
int list_index;
if (!state) {
@@ -44,13 +51,15 @@ char *_sorm_autocompletion_generator(const char *text, int state) {
}
char** _sorm_autocomplete(const char* text, int start, int end) {
(void)start;
(void)end;
rl_attempted_completion_over = 1;
return rl_completion_matches(text, _sorm_autocompletion_generator);
}
int _hs_read_file(const char *filename, char *buffer, size_t size) {
int
_hs_read_file (const char *filename, char *buffer, size_t size)
{
int fd;
ssize_t bytes_read;
@@ -68,7 +77,10 @@ int _hs_read_file(const char *filename, char *buffer, size_t size) {
return 0;
}
int sorm_cli_history_dump(const char *filename) {
int
sorm_cli_history_dump (const char *filename)
{
register int line_start, line_end;
char *input;
struct stat finfo;
@@ -88,7 +100,7 @@ int sorm_cli_history_dump(const char *filename) {
}
input[file_size] = '\0';
for (line_start = line_end = 0; (size_t)line_end < file_size; line_end++) {
for (line_start = line_end = 0; line_end < file_size; line_end++) {
if (input[line_end] == '\n') {
input[line_end] = '\0';
@@ -98,7 +110,7 @@ int sorm_cli_history_dump(const char *filename) {
}
}
if ((size_t)line_start < file_size)
if (line_start < file_size)
printf ("%s\n",input + line_start);
free (input);
@@ -135,6 +147,7 @@ char *sorm_cli_readline(char *prompt) {
return result;
}
bool sormrepl_handle_command(char * command){
if(!strncmp(command, "history",7)){
sorm_cli_history_dump(history_filename);
@@ -143,6 +156,9 @@ bool sormrepl_handle_command(char *command) {
return false;
}
void sormrepl(int sorm){
sorm_t * db = sormg(sorm);
char sql[4097];
sorm_cli_init(history_filename);
char * query;
while((query = sorm_cli_readline("sql> "))){
@@ -151,6 +167,7 @@ void sormrepl(int sorm) {
sorm_ptr res = sormq(sorm,query);
if(res){
if(sormqt(query) == SORM_SELECT){
int length = sormlv(res);
sormfmtd(res);
free(res);
}else if(sormqt(query) == SORM_DELETE){
@@ -160,8 +177,7 @@ void sormrepl(int sorm) {
}
}
printf("Rows: %lld, Execute %s, Format: %s\n", sorm_row_count, format_time(_sorm_query_duration),
format_time(_sorm_result_format_duration));
printf("Rows: %lld, Execute %s, Format: %s\n",sorm_row_count, format_time(_sorm_query_duration),format_time(_sorm_result_format_duration));
printf("%s\n",rmalloc_stats());
}
}
+22 -7
View File
@@ -1,16 +1,31 @@
#include "cli.h"
#include "sorm.h"
#include "cli.h"
int main() {
int db = sormc("db.sqlite3");
//sormq(db,"DROP TABLE IF EXISTS pony;");
printf("%d\n",db);
sormq(db, "CREATE TABLE IF NOT EXISTS pony (id INTEGER PRIMARY KEY AUTOINCREMENT,name,age);",NULL);
sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);", "Teenii", 19);
sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);", "Amber", 20);
sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);", "Feuerherz", 20);
sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);", "Retoor", 34);
sorm_str csv2 = (sorm_str)sormq(db, "SELECT * FROM pony WHERE id = %d and age = %d ", 1, 33);
sorm_str csv3 = (sorm_str)sormq(db, "SELECT * FROM pony LIMIT 2");
sorm_pk iid = sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);",
"Teenii",
19
);
iid = sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);",
"Amber",
20
);
iid = sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);",
"Feuerherz",
20
);
iid = sormq(db, "INSERT INTO pony (id,name,age) VALUES (NULL,%s,%d);",
"Retoor",
34
);
sorm_str csv = sormq(db, "SELECT * FROM pony WHERE id in (?i,?i,?i)",1,2,3);
sorm_str csv2 = sormq(db, "SELECT * FROM pony WHERE id = %d and age = %d ", 1,33);
sorm_str csv3 = sormq(db, "SELECT * FROM pony LIMIT 2");
//free(csv3);
//free(csv2);
if(csv2)
-8448
View File
File diff suppressed because it is too large Load Diff
BIN
View File
Binary file not shown.
+68 -23
View File
@@ -1,18 +1,19 @@
#ifndef SORM_H
#define SORM_H
#include "str.h"
#include <ctype.h>
#include "rlib.h"
#include <rlib.h>
#include <sqlite3.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <stdarg.h>
#include <ctype.h>
#include "str.h"
sqlite3 *db;
sqlite3_stmt *stmt;
char * sorm_last_query = NULL;
char * sorm_last_query_expanded = NULL;
@@ -51,11 +52,14 @@ void sormd(int db);
char * sormpt(char * sql, int number);
unsigned int sormcq(char * sql, char * out);
unsigned int sormpc(char * sql);
sqlite3_stmt * sormb(sorm_t* db, char * sql, ...);
sorm_ptr sormq(int db, char * sql, ...);
char * sorm_csvc(int db, sqlite3_stmt * stmt);
char * sorm_csvd(int db, sqlite3_stmt * stmt);
char * sorm_csv(int db,sqlite3_stmt * stmt);
typedef enum sorm_query_t {
SORM_UNKNOWN = 0,
SORM_SELECT = 1,
@@ -67,19 +71,26 @@ typedef enum sorm_query_t {
const int sorm_null = -1337;
sorm_t ** sorm_instances = NULL;
int sorm_instance_count = 0;
int sormc(char * path){
// sorm connect
printf("HIERR\n");
sorm_instance_count++;
sorm_instance_count++;
sorm_instances = realloc(sorm_instances,sorm_instance_count * sizeof(sorm_t *) + sorm_instance_count * sizeof(sorm_t));
sorm_t *db = (sorm_t *)&sorm_instances[sorm_instance_count - 1];
printf("HIERR\n");
sorm_t * db = &sorm_instances[sorm_instance_count - 1];
printf("HIERR\n");
db->conn = NULL;
printf("HIERR\n");
db->csv = NULL;
db->current_column = 0;
db->current_row = 0;
@@ -90,13 +101,17 @@ int sormc(char *path) {
db->time_result_format_end = 0;
db->time_result_format_start = 0;
if (sqlite3_open(path, &db->conn)) {
if(sqlite3_open(path,&db->conn))
{
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db->conn));
return 0;
}
printf("DONE!\n");
return sorm_instance_count;
}
sorm_t *sormg(int ptr) { return (sorm_t *)&sorm_instances[ptr - 1]; }
sorm_t * sormg(int ptr){
return &sorm_instances[ptr -1];
}
char * sormgcsv(int ptr){
/* sorm get csv*/
@@ -106,7 +121,8 @@ char *sormgcsv(int ptr) {
void sormd(int sorm){
sorm_t * db = sormg(sorm);
if (sqlite3_close(db->conn)) {
if(sqlite3_close(db->conn))
{
fprintf(stderr, "Error closing database: %s\n", sqlite3_errmsg(db->conn));
}
if(sorm_last_query){
@@ -117,6 +133,7 @@ void sormd(int sorm) {
free(sorm_last_query_expanded);
sorm_last_query_expanded = NULL;
}
}
char * sormpt(char * sql, int number){
@@ -146,6 +163,7 @@ char *sormpt(char *sql, int number) {
}
sqlp++;
index++;
}
if(index == number){
return result;
@@ -160,6 +178,7 @@ char *sormpt(char *sql, int number) {
return NULL;
}
unsigned int sormcq(char * sql, char * out){
// clean query
// converts %s %i parameters to ?
@@ -207,10 +226,9 @@ Execute 3.35s, Format: 36.77s
Memory usage: 537 GB, 96.026 allocated, 96.024 freed, 2 in use.
*/
char * sorm_csvc(int db, sqlite3_stmt * stmt){
(void)db;
sormstr_t * str = sormstrn(512);
unsigned int column_count = sqlite3_column_count(stmt);
for (uint i = 0; i < column_count; i++) {
for(int i = 0; i < column_count; i++){
const char * column_name = sqlite3_column_name(stmt,i);
sormstra(str,column_name);
sormstra(str,"(");
@@ -225,7 +243,7 @@ char *sorm_csvc(int db, sqlite3_stmt *stmt) {
return sormstrc(str);
}
char * sorm_csvd(int sorm, sqlite3_stmt * stmt) {
(void)sorm;
sorm_t * db = sormg(sorm);
int rc = SQLITE_ROW;
int column_count = sqlite3_column_count(stmt);
/*
@@ -252,7 +270,7 @@ char *sorm_csvd(int sorm, sqlite3_stmt *stmt) {
sprintf(temp, "%f",sqlite3_column_double(stmt,field_index));
sormstra(str,temp);
} else if(sqlite3_column_type(stmt,field_index) == SQLITE3_TEXT){
const char *temp = ( char *)sqlite3_column_text(stmt, field_index);
const char * temp = sqlite3_column_text(stmt,field_index);
sormstra(str,temp);
} else {
// exit(1);
@@ -271,6 +289,7 @@ char *sorm_csvd(int sorm, sqlite3_stmt *stmt) {
}
char * sorm_csv(int sorm,sqlite3_stmt * stmt){
sorm_t * db = sormg(sorm);
sorm_row_count = 0;
char * column_names = sorm_csvc(sorm,stmt);
char * data = sorm_csvd(sorm,stmt);
@@ -285,16 +304,31 @@ char *sorm_csv(int sorm, sqlite3_stmt *stmt) {
return result;
}
sqlite3_stmt * sormb(sorm_t* db, char * sql, ...){
// Bind parameters to statement and return amount of parameters
int rc = 0;
sqlite3_stmt * stmt;
va_list args;
va_start(args,sql);
unsigned int number = 0;
char * clean_query = (char *)malloc(strlen(sql) + 1);
unsigned int parameter_count = sormcq(sql,clean_query);
free(clean_query);
return stmt;
}
char * sormm(sorm_t * db){
(void)db;
/* sormmemory */
return rmalloc_stats();
}
sorm_ptr sormq(int sorm, char * sql, ...){
sorm_t * db = sormg(sorm);
if(db->csv){
//free(db->csv);
//db->csv = NULL;
}
_sorm_query_start = nsecs();
db->time_query_start = nsecs();
va_list args;
@@ -302,22 +336,22 @@ sorm_ptr sormq(int sorm, char *sql, ...) {
sqlite3_stmt * stmt;
sorm_ptr result = NULL;
char * clean_query = malloc(strlen(sql) + 1);
uint parameter_count = sormcq(sql, clean_query);
unsigned int parameter_count = sormcq(sql,clean_query);
int rc = sqlite3_prepare_v2(db->conn, clean_query, -1, &stmt, 0);
if (rc != SQLITE_OK) {
fprintf(stderr, "%s\n", sqlite3_errmsg(db->conn));
}
free(clean_query);
int number = 0;
for (uint i = 0; i < parameter_count; i++) {
for(int i = 0; i < parameter_count; i++){
number++;
char * column_type = sormpt(sql,number);
int arg_index = number - 1;
if(!strcmp(column_type, "int") || !strcmp(column_type, "integer") || !strcmp(column_type, "number")) {
rc = sqlite3_bind_int(stmt, number, va_arg(args,int));
}else if(!strcmp(column_type, "int64")){
rc = sqlite3_bind_int64(stmt, number,va_arg(args,__uint64_t));
} else if (!strcmp(column_type, "double") || !strcmp(column_type, "dec") || !strcmp(column_type, "decimal") ||
!strcmp(column_type, "float")) {
}else if(!strcmp(column_type, "double") || !strcmp(column_type, "dec") || !strcmp(column_type, "decimal") || !strcmp(column_type, "float")){
rc = sqlite3_bind_double(stmt, number,va_arg(args,double));
}else if(!strcmp(column_type, "blob")){
size_t size = (size_t)va_arg(args,size_t);
@@ -325,7 +359,7 @@ sorm_ptr sormq(int sorm, char *sql, ...) {
rc = sqlite3_bind_blob(stmt, number, data, size, SQLITE_STATIC);
}else if(!strcmp(column_type,"text") || !strcmp(column_type,"str") || !strcmp(column_type,"string")) {
unsigned char * data = va_arg(args, unsigned char *);
rc = sqlite3_bind_text(stmt, number, (char *)data, -1, SQLITE_STATIC);
rc = sqlite3_bind_text(stmt, number, data, -1, SQLITE_STATIC);
}
if (rc != SQLITE_OK) {
fprintf(stderr, "Failed to bind parameters: %s\n", sqlite3_errmsg(db->conn));
@@ -347,6 +381,7 @@ sorm_ptr sormq(int sorm, char *sql, ...) {
else{
fprintf(stderr, "Execution failed: %s\n", sqlite3_errmsg(db->conn));
}
if(sorm_last_query){
free(sorm_last_query);
@@ -365,6 +400,8 @@ sorm_ptr sormq(int sorm, char *sql, ...) {
return result;
}
char sormlc(char * sql){
// returns last char
char last_char = 0;
@@ -384,7 +421,7 @@ int sormlv(char *csv) {
while(*csv){
char * found = strstr(csv,";");
if(found){
if (found - csv > (long int)longest)
if(found - csv > longest)
longest = found-csv;
csv = csv + (found - csv) + 1;
}else{
@@ -419,7 +456,8 @@ char *sormrq(FILE *f) {
bool in_string = false;
while((c = fgetc(f)) != EOF && strlen(buffer) != sizeof(buffer) -2){
*bufferp = c;
if (c == '"') {
if(c == '"')
{
in_string = !in_string;
}
if(!in_string && c == ';'){
@@ -429,6 +467,7 @@ char *sormrq(FILE *f) {
*bufferp = 0;
}
return strdup(buffer);
}
char * sormcsvn(char * csv){
@@ -475,7 +514,7 @@ char *sormfmt(char *csv) {
sormstr_t * str = sormstrn(longest + 2);
while(*csv && (field = sormcsvn(csv))){
sormstra(str,field);
for (size_t i = 0; i < longest - strlen(field); i++)
for(int i = 0; i < longest - strlen(field); i++)
sormstra(str," ");
csv += strlen(field);
@@ -499,7 +538,8 @@ void apply_colors(char *csv) {
printf("%s\n",csv);
end = strstr(csv,"\n");
char * line;
if (end) {
if(end)
{
line = (char *)malloc(end -csv + 1024);
strncpy(line,csv,end-csv);
}else{
@@ -520,6 +560,7 @@ void apply_colors(char *csv) {
if(*csv && *(csv + 1))
csv++;
}
}
void sormfmtd(char * csv){
@@ -528,4 +569,8 @@ void sormfmtd(char *csv) {
free(formatted);
}
#endif
+29 -28
View File
@@ -5,7 +5,6 @@ import io
import tempfile
import time
class DictReader:
def get_column_types(self):
@@ -26,13 +25,12 @@ class DictReader:
name = column.split("(")[0]
names.append(name)
return names
def __init__(self, data):
self.result = 0
if type(data) == int:
self.data = ""
self.data = ''
else:
self.data = data.decode()
self.data = data.decode();
self.rows = [row.split(";")[:-1] for row in self.data.split("\n")]
self.columns = self.rows[0]
@@ -47,38 +45,43 @@ class DictReader:
row[index] = self.column_types[index](field)
self.column_names = self.get_column_names()
def __iter__(self):
return self.rows.__iter__()
libc = ctypes.CDLL("libc.so.6")
libc = ctypes.CDLL('libc.so.6');
class Sorm:
def __init__(self):
self.lib = ctypes.CDLL("./sorm.so")
self.lib = ctypes.CDLL('./sorm.so')
self.sormq = self.lib.sormq
self.sormq.argtypes = [ctypes.c_int, ctypes.c_char_p]
self.sormq.argtypes = [ctypes.c_int, ctypes.c_char_p];
self.sormq.restype = ctypes.c_char_p
self.sormc = self.lib.sormc
self.sormc.argtypes = [ctypes.c_char_p]
self.sormc.restype = ctypes.c_int
self.sormc.argtypes = [ctypes.c_char_p];
self.sormc.restype = ctypes.c_int;
self.sormd = self.lib.sormd
self.sormd.argtypes = [ctypes.c_int]
self.sormd.argtypes = [ctypes.c_int];
self.sormd.restype = None
self.sormm = self.lib.sormm
self.sormm.argtypes = [ctypes.c_int]
self.sormm.argtypes = [ctypes.c_int];
self.sormm.restype = ctypes.c_char_p
class SormDb(Sorm):
def __init__(self,path):
super().__init__()
super().__init__(
)
self.path = path
self.conn = None
@@ -106,10 +109,7 @@ class SormDb(Sorm):
self.sormq.restype = ctypes.c_int
else:
self.sormq.restype = ctypes.c_char_p
params = tuple(
[self.conn, sql.encode()]
+ list(arg.encode() if type(arg) == str else arg for arg in args)
)
params = tuple([self.conn, sql.encode()] + list(arg.encode() if type(arg) == str else arg for arg in args))
result = DictReader(self.sormq(*params))
self.m = self.sormm(self.conn).decode()
return result
@@ -120,42 +120,43 @@ class SormDb(Sorm):
self.sormd(self.conn)
self.conn = None
# Load the shared library
lib = ctypes.CDLL("./sorm.so")
lib = ctypes.CDLL('./sorm.so')
free = libc.free
free.argtypes = [ctypes.c_void_p]
free.restype = None
rsomm = lib.sormm
rsomm.argtypes = [ctypes.c_int]
rsomm.restype = ctypes.c_char_p
rsomm.argtypes = [ctypes.c_int];
rsomm.restype = ctypes.c_char_p;
disconnect = lib.sormd
disconnect.argtypes = [ctypes.c_int]
disconnect.argtypes = [ctypes.c_int];
start = time.time()
for x in range(1):
with SormDb("db.sqlite3") as db:
for x in range(1):
#db.q("BEGIN TRANSACTION")
#for x in range(100000):
# db.q("INSERT INTO pony (name,age) VALUES (?s,?d);","Python Pony",1337)
#db.q("COMMIT")
result = db.q(
"SELECT * FROM pony WHERE id > ?d AND name like ?s ORDER BY id",
1337,
"%hon Pon%",
)
result = db.q("SELECT * FROM pony WHERE id > ?d AND name like ?s ORDER BY id",1337, "%hon Pon%");
#for row in result:
# print(row)
print(result.column_names)
print(len(result.rows),"records")
print(db.m)
print(db.m);
end = time.time()
duration = end - start
print("Duration: {}s".format(duration))
BIN
View File
Binary file not shown.
+2 -2
View File
@@ -1,9 +1,9 @@
#ifndef SORM_STR_H
#define SORM_STR_H
#include "rlib.h"
#include <rlib.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct sormstr_t {
char * content;