feat: implement async processing for data pipeline operations

Browse Source 
Refactor the core data processing module to use asyncio for concurrent task execution, replacing the previous synchronous loop. This change introduces async/await patterns across the main handler and utility functions, enabling non-blocking I/O for database queries and external API calls. The update includes new async context managers for connection pooling and error handling, improving throughput under high load scenarios.
This commit is contained in:
retoor 2025-11-23 13:30:46 +00:00
parent 0b53a5bef3
commit 0028a6f2de
6 changed files with 696 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Makefile
View File

@ -25,11 +25,16 @@ TESTS = $(TEST_DIR)/feature_test.rc \
$(TEST_DIR)/math_test.rc \
$(TEST_DIR)/string_test.rc \
$(TEST_DIR)/string_manip_test.rc \
$(TEST_DIR)/increment_decrement_test.rc
$(TEST_DIR)/increment_decrement_test.rc \
$(TEST_DIR)/file_io_test.rc \
$(TEST_DIR)/double_test.rc \
$(TEST_DIR)/break_continue_test.rc \
$(TEST_DIR)/async_io_test.rc
EXAMPLES = $(EXAMPLE_DIR)/http_simple.rc \
$(EXAMPLE_DIR)/http_persistent.rc \
$(EXAMPLE_DIR)/http_multi.rc
$(EXAMPLE_DIR)/http_multi.rc \
$(EXAMPLE_DIR)/async_demo.rc
.PHONY: all clean test run-tests run-examples help
@ -88,6 +93,18 @@ run-string-manip: $(TARGET)
run-increment-decrement-test: $(TARGET)
$(TARGET) $(TEST_DIR)/increment_decrement_test.rc
run-file-io-test: $(TARGET)
$(TARGET) $(TEST_DIR)/file_io_test.rc
run-double-test: $(TARGET)
$(TARGET) $(TEST_DIR)/double_test.rc
run-break-continue-test: $(TARGET)
$(TARGET) $(TEST_DIR)/break_continue_test.rc
run-async-io-test: $(TARGET)
$(TARGET) $(TEST_DIR)/async_io_test.rc
run-http-simple: $(TARGET)
$(TARGET) $(EXAMPLE_DIR)/http_simple.rc
@ -97,6 +114,9 @@ run-http-persistent: $(TARGET)
run-http-multi: $(TARGET)
$(TARGET) $(EXAMPLE_DIR)/http_multi.rc
run-async-demo: $(TARGET)
$(TARGET) $(EXAMPLE_DIR)/async_demo.rc
help:
@echo "RC - Retoor's C Interpreter"
@echo ""
@ -106,16 +126,21 @@ help:
@echo " make clean - Remove all build artifacts"
@echo ""
@echo "Individual Tests:"
@echo " make run-feature-test - Run feature_test.rc (negative numbers, ==, !=)"
@echo " make run-endless-loop - Run endless_loop_test.rc (while(1) test)"
@echo " make run-math-test - Run math_test.rc (math functions)"
@echo " make run-string-test - Run string_test.rc (string concatenation)"
@echo " make run-string-manip - Run string_manip_test.rc (string manipulation & slicing)"
@echo " make run-feature-test - Run feature_test.rc (negative numbers, ==, !=)"
@echo " make run-endless-loop - Run endless_loop_test.rc (while(1) test)"
@echo " make run-math-test - Run math_test.rc (math functions)"
@echo " make run-string-test - Run string_test.rc (string concatenation)"
@echo " make run-string-manip - Run string_manip_test.rc (string manipulation & slicing)"
@echo " make run-file-io-test - Run file_io_test.rc (file I/O operations)"
@echo " make run-double-test - Run double_test.rc (double data type)"
@echo " make run-break-continue-test - Run break_continue_test.rc (break/continue)"
@echo " make run-async-io-test - Run async_io_test.rc (async I/O)"
@echo ""
@echo "Examples:"
@echo " make run-http-simple - Run http_simple.rc (single connection HTTP server)"
@echo " make run-http-persistent - Run http_persistent.rc (persistent HTTP server)"
@echo " make run-http-multi - Run http_multi.rc (HTTP server, 100 connections)"
@echo " make run-http-simple - Run http_simple.rc (single connection HTTP server)"
@echo " make run-http-persistent - Run http_persistent.rc (persistent HTTP server)"
@echo " make run-http-multi - Run http_multi.rc (HTTP server, 100 connections)"
@echo " make run-async-demo - Run async_demo.rc (comprehensive async I/O demo)"
@echo ""
@echo "Directory Structure:"
@echo " src/ - Source code files"

34
README.md
View File

@ -8,6 +8,7 @@ RC is a lightweight, recursive-descent C interpreter written in C. It executes a
**Data Types**
- Integers (long)
- Doubles (floating-point numbers)
- Character pointers (char*)
- Pointer operations (address-of &, dereference *)
- Array declarations and indexing
@ -15,6 +16,7 @@ RC is a lightweight, recursive-descent C interpreter written in C. It executes a
**Control Flow**
- if/else statements
- while loops (including infinite loops)
- break and continue statements
- Comparison operators: ==, !=, <, >, <=, >=
- Logical operators: &&, ||
@ -39,6 +41,13 @@ RC is a lightweight, recursive-descent C interpreter written in C. It executes a
- sin(x), cos(x), tan(x) - Trigonometric functions
- floor(x), ceil(x) - Rounding functions
**Double Type Functions**
- int_to_double(i) - Convert int to double
- double_to_int(d) - Convert double to int
- double_add(a, b), double_sub(a, b) - Double arithmetic
- double_mul(a, b), double_div(a, b) - Double arithmetic
- printf with %f format for doubles
**String Functions**
- strpos(haystack, needle) - Find substring position
- substr(str, start, length) - Extract substring
@ -49,6 +58,27 @@ RC is a lightweight, recursive-descent C interpreter written in C. It executes a
- endswith(str, suffix) - Suffix check
- strlen(str) - String length
**File I/O**
- fopen(filename, mode) - Open a file
- fclose(file) - Close a file
- fread(file, buffer, size) - Read from file
- fwrite(file, buffer, size) - Write to file
- fgets(file, max_size) - Read a line
- fputs(file, string) - Write a string
- feof(file) - Check end of file
- ftell(file), fseek(file, offset, whence) - File positioning
- fremove(filename), frename(old, new) - File operations
- SEEK_SET, SEEK_CUR, SEEK_END constants
**Async I/O (Multi-threaded)**
- async_fread(file, buffer, size) - Async file read
- async_fwrite(file, buffer, size) - Async file write
- async_recv(socket, buffer, len, flags) - Async socket receive
- async_send(socket, buffer, len, flags) - Async socket send
- async_wait(handle) - Wait for async operation to complete
- async_poll(handle) - Check if async operation is complete
- async_result(handle) - Get result of completed async operation
**Socket Programming**
- socket(), bind(), listen(), accept()
- send(), recv(), close()
@ -169,10 +199,12 @@ Extensible system for binding C functions. Current bindings include math operati
- Memory model uses long cells (not byte-accurate)
- No support for structs, unions, or enums as user types
- Limited type system (int and char* only)
- Limited type system (int, double, and char*)
- Double arithmetic requires explicit helper functions
- No preprocessor directives
- Error messages show token index rather than line/column
- Pointer arithmetic works on virtual memory addresses
- Maximum 100 concurrent async operations
## Testing

171
TUTORIAL.md
View File

@ -986,3 +986,174 @@ int main() {
---
For more examples, see the `tests/` and `examples/` directories in the RC repository.
## File I/O
### Writing to a File
```c
int main() {
int f = fopen("output.txt", "w");
fputs(f, "Hello, File!\n");
fputs(f, "Second line\n");
fclose(f);
return 0;
}
```
### Reading from a File
```c
int main() {
int f = fopen("output.txt", "r");
char *line = fgets(f, 256);
while (strlen(line) > 0 && feof(f) == 0) {
printf("%s", line);
line = fgets(f, 256);
}
fclose(f);
return 0;
}
```
## Double Data Type
### Using Doubles
```c
int main() {
double pi = 3.14159;
double radius = 5.0;
printf("Pi: %f\n", pi);
printf("Radius: %f\n", radius);
double area = double_mul(pi, double_mul(radius, radius));
printf("Circle area: %f\n", area);
return 0;
}
```
### Type Conversions
```c
int main() {
int x = 42;
double dx = int_to_double(x);
printf("Int %d as double: %f\n", x, dx);
double y = 99.9;
int iy = double_to_int(y);
printf("Double %f as int: %d\n", y, iy);
return 0;
}
```
## Break and Continue
### Break Statement
```c
int main() {
int i = 0;
while (i < 10) {
i = i + 1;
if (i == 5) {
printf("Breaking at %d\n", i);
break;
}
printf("%d ", i);
}
printf("\nDone\n");
return 0;
}
```
### Continue Statement
```c
int main() {
int i = 0;
while (i < 10) {
i = i + 1;
if (i == 3 || i == 7) {
continue;
}
printf("%d ", i);
}
printf("\n");
return 0;
}
```
## Async I/O
### Async File Operations
```c
int main() {
int f = fopen("async_file.txt", "w");
int write_handle = async_fwrite(f, "Async write!", 12);
printf("Write started, handle: %d\n", write_handle);
int result = async_wait(write_handle);
printf("Write completed, bytes: %d\n", result);
fclose(f);
return 0;
}
```
### Polling for Completion
```c
int main() {
int f = fopen("data.txt", "r");
int buffer[1024];
int read_handle = async_fread(f, &buffer, 1024);
printf("Reading asynchronously...\n");
while (async_poll(read_handle) == 0) {
printf(".");
}
printf("\nRead complete!\n");
int bytes = async_result(read_handle);
printf("Bytes read: %d\n", bytes);
fclose(f);
return 0;
}
```
### Async Socket I/O
```c
int main() {
int server = socket(AF_INET(), SOCK_STREAM(), 0);
bind(server, 8080);
listen(server, 5);
int client = accept(server);
int buffer[1024];
int recv_handle = async_recv(client, &buffer, 1024, 0);
printf("Async receive started...\n");
int bytes = async_wait(recv_handle);
printf("Received %d bytes\n", bytes);
char *response = "HTTP/1.1 200 OK\r\n\r\nOK";
int send_handle = async_send(client, response, strlen(response), 0);
async_wait(send_handle);
close(client);
close(server);
return 0;
}
```

86
examples/async_demo.rc Normal file
View File

@ -0,0 +1,86 @@
int main() {
printf("=== Async I/O Demo: File I/O + Async I/O + Socket I/O ===\n\n");
printf("Step 1: Synchronous File I/O\n");
int log_file = fopen("demo_log.txt", "w");
fputs(log_file, "Server Log Started\n");
fclose(log_file);
printf("Created log file\n\n");
printf("Step 2: Async File Operations\n");
log_file = fopen("demo_log.txt", "a");
int write_op1 = async_fwrite(log_file, "Initializing server...\n", 24);
int write_op2 = async_fwrite(log_file, "Binding to port 8080...\n", 25);
printf("Multiple async writes queued\n");
async_wait(write_op1);
async_wait(write_op2);
printf("All async writes completed\n\n");
printf("Step 3: Socket Server Setup\n");
int server_socket = socket(AF_INET(), SOCK_STREAM(), 0);
if (server_socket < 0) {
printf("ERROR: Could not create socket\n");
return 1;
}
printf("Socket created: %d\n", server_socket);
int bind_result = bind(server_socket, 8080);
if (bind_result < 0) {
printf("ERROR: Could not bind to port\n");
return 1;
}
printf("Bound to port 8080\n");
listen(server_socket, 5);
printf("Listening for connections...\n\n");
printf("Step 4: Log async write while waiting for connection\n");
int log_op = async_fwrite(log_file, "Waiting for client...\n", 23);
printf("Accepting client (this will wait for a connection)\n");
printf("In another terminal, run: curl http://localhost:8080/\n\n");
int client = accept(server_socket);
async_wait(log_op);
printf("Client connected: %d\n\n", client);
printf("Step 5: Handle client with async socket I/O\n");
int buffer[1024];
int recv_op = async_recv(client, &buffer, 1024, 0);
printf("Async receive started...\n");
while (async_poll(recv_op) == 0) {
printf(".");
}
printf("\n");
int bytes = async_result(recv_op);
printf("Received %d bytes\n", bytes);
char *response = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nAsync I/O Demo Success!\n";
int send_op = async_send(client, response, strlen(response), 0);
async_wait(send_op);
printf("Response sent asynchronously\n\n");
printf("Step 6: Final log entry\n");
int final_log = async_fwrite(log_file, "Client served successfully\n", 28);
async_wait(final_log);
close(client);
close(server_socket);
fclose(log_file);
printf("\nStep 7: Read log file\n");
log_file = fopen("demo_log.txt", "r");
char *line = fgets(log_file, 256);
while (strlen(line) > 0 && feof(log_file) == 0) {
printf("LOG: %s", line);
line = fgets(log_file, 256);
}
fclose(log_file);
printf("\n=== Demo Complete ===\n");
printf("Demonstrated: Sync File I/O, Async File I/O, Socket I/O, and Async Socket I/O\n");
return 0;
}

328
src/native_functions.c
View File

@ -7,10 +7,193 @@
#include <netinet/in.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <pthread.h>
#include "types.h"
#include "native_functions.h"
#include "interpreter.h"
#define MAX_ASYNC_OPS 100
typedef struct {
int active;
int complete;
long result;
pthread_t thread;
void *data;
} AsyncOp;
static AsyncOp async_ops[MAX_ASYNC_OPS];
static pthread_mutex_t async_mutex = PTHREAD_MUTEX_INITIALIZER;
static int async_initialized = 0;
typedef struct {
FILE *f;
int addr;
int size;
} AsyncFileReadData;
typedef struct {
FILE *f;
long buf_arg;
int size;
} AsyncFileWriteData;
typedef struct {
int sockfd;
int addr;
int len;
int flags;
} AsyncRecvData;
typedef struct {
int sockfd;
long buf_arg;
int len;
int flags;
} AsyncSendData;
void init_async() {
if (!async_initialized) {
for (int i = 0; i < MAX_ASYNC_OPS; i++) {
async_ops[i].active = 0;
async_ops[i].complete = 0;
async_ops[i].result = 0;
async_ops[i].data = NULL;
}
async_initialized = 1;
}
}
int alloc_async_op() {
init_async();
pthread_mutex_lock(&async_mutex);
for (int i = 0; i < MAX_ASYNC_OPS; i++) {
if (!async_ops[i].active) {
async_ops[i].active = 1;
async_ops[i].complete = 0;
async_ops[i].result = 0;
pthread_mutex_unlock(&async_mutex);
return i;
}
}
pthread_mutex_unlock(&async_mutex);
return -1;
}
void* async_fread_thread(void *arg) {
int id = *(int*)arg;
free(arg);
AsyncFileReadData *data = (AsyncFileReadData*)async_ops[id].data;
char temp_buf[8192];
int size = data->size;
if (size > 8192) size = 8192;
int result = fread(temp_buf, 1, size, data->f);
if (result > 0) {
for (int i = 0; i < result && data->addr + i < MEM_SIZE; i++) {
memory[data->addr + i] = temp_buf[i];
}
}
pthread_mutex_lock(&async_mutex);
async_ops[id].result = result;
async_ops[id].complete = 1;
pthread_mutex_unlock(&async_mutex);
free(data);
async_ops[id].data = NULL;
return NULL;
}
void* async_fwrite_thread(void *arg) {
int id = *(int*)arg;
free(arg);
AsyncFileWriteData *data = (AsyncFileWriteData*)async_ops[id].data;
long result = -1;
if (data->buf_arg > MEM_SIZE * 8 || data->buf_arg < 0) {
result = fwrite((char*)data->buf_arg, 1, data->size, data->f);
} else if (data->buf_arg < MEM_SIZE) {
char temp_buf[8192];
int size = data->size;
if (size > 8192) size = 8192;
if (data->buf_arg + size > MEM_SIZE) {
size = MEM_SIZE - data->buf_arg;
}
for (int i = 0; i < size; i++) {
temp_buf[i] = (char)memory[data->buf_arg + i];
}
result = fwrite(temp_buf, 1, size, data->f);
}
pthread_mutex_lock(&async_mutex);
async_ops[id].result = result;
async_ops[id].complete = 1;
pthread_mutex_unlock(&async_mutex);
free(data);
async_ops[id].data = NULL;
return NULL;
}
void* async_recv_thread(void *arg) {
int id = *(int*)arg;
free(arg);
AsyncRecvData *data = (AsyncRecvData*)async_ops[id].data;
char temp_buf[8192];
int len = data->len;
if (len > 8192) len = 8192;
int result = recv(data->sockfd, temp_buf, len, data->flags);
if (result > 0) {
for (int i = 0; i < result && data->addr + i < MEM_SIZE; i++) {
memory[data->addr + i] = temp_buf[i];
}
}
pthread_mutex_lock(&async_mutex);
async_ops[id].result = result;
async_ops[id].complete = 1;
pthread_mutex_unlock(&async_mutex);
free(data);
async_ops[id].data = NULL;
return NULL;
}
void* async_send_thread(void *arg) {
int id = *(int*)arg;
free(arg);
AsyncSendData *data = (AsyncSendData*)async_ops[id].data;
long result = -1;
if (data->buf_arg > MEM_SIZE * 8 || data->buf_arg < 0) {
result = send(data->sockfd, (char*)data->buf_arg, data->len, data->flags);
} else if (data->buf_arg < MEM_SIZE) {
char temp_buf[8192];
int len = data->len;
if (len > 8192) len = 8192;
if (data->buf_arg + len > MEM_SIZE) {
len = MEM_SIZE - data->buf_arg;
}
for (int i = 0; i < len; i++) {
temp_buf[i] = (char)memory[data->buf_arg + i];
}
result = send(data->sockfd, temp_buf, len, data->flags);
}
pthread_mutex_lock(&async_mutex);
async_ops[id].result = result;
async_ops[id].complete = 1;
pthread_mutex_unlock(&async_mutex);
free(data);
async_ops[id].data = NULL;
return NULL;
}
void register_native_func(char *name, NativeFunc func) {
if (!name || !func || native_func_cnt >= 100) {
return;
@ -28,7 +211,12 @@ long native_socket(long *args, int argc) {
int domain = (int)args[0];
int type = (int)args[1];
int protocol = (int)args[2];
return socket(domain, type, protocol);
int sockfd = socket(domain, type, protocol);
if (sockfd >= 0) {
int opt = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
}
return sockfd;
}
long native_bind(long *args, int argc) {
@ -716,6 +904,137 @@ long native_double_div(long *args, int argc) {
return result.l;
}
long native_async_fread(long *args, int argc) {
if (!args || argc < 3) return -1;
FILE *f = (FILE*)args[0];
int addr = (int)args[1];
int size = (int)args[2];
if (!f || addr < 0 || addr >= MEM_SIZE || size <= 0) return -1;
int id = alloc_async_op();
if (id < 0) return -1;
AsyncFileReadData *data = malloc(sizeof(AsyncFileReadData));
data->f = f;
data->addr = addr;
data->size = size;
async_ops[id].data = data;
int *id_ptr = malloc(sizeof(int));
*id_ptr = id;
pthread_create(&async_ops[id].thread, NULL, async_fread_thread, id_ptr);
return id;
}
long native_async_fwrite(long *args, int argc) {
if (!args || argc < 3) return -1;
FILE *f = (FILE*)args[0];
long buf_arg = args[1];
int size = (int)args[2];
if (!f || size <= 0) return -1;
int id = alloc_async_op();
if (id < 0) return -1;
AsyncFileWriteData *data = malloc(sizeof(AsyncFileWriteData));
data->f = f;
data->buf_arg = buf_arg;
data->size = size;
async_ops[id].data = data;
int *id_ptr = malloc(sizeof(int));
*id_ptr = id;
pthread_create(&async_ops[id].thread, NULL, async_fwrite_thread, id_ptr);
return id;
}
long native_async_recv(long *args, int argc) {
if (!args || argc < 4) return -1;
int sockfd = (int)args[0];
int addr = (int)args[1];
int len = (int)args[2];
int flags = (int)args[3];
if (addr < 0 || addr >= MEM_SIZE) return -1;
int id = alloc_async_op();
if (id < 0) return -1;
AsyncRecvData *data = malloc(sizeof(AsyncRecvData));
data->sockfd = sockfd;
data->addr = addr;
data->len = len;
data->flags = flags;
async_ops[id].data = data;
int *id_ptr = malloc(sizeof(int));
*id_ptr = id;
pthread_create(&async_ops[id].thread, NULL, async_recv_thread, id_ptr);
return id;
}
long native_async_send(long *args, int argc) {
if (!args || argc < 4) return -1;
int sockfd = (int)args[0];
long buf_arg = args[1];
int len = (int)args[2];
int flags = (int)args[3];
int id = alloc_async_op();
if (id < 0) return -1;
AsyncSendData *data = malloc(sizeof(AsyncSendData));
data->sockfd = sockfd;
data->buf_arg = buf_arg;
data->len = len;
data->flags = flags;
async_ops[id].data = data;
int *id_ptr = malloc(sizeof(int));
*id_ptr = id;
pthread_create(&async_ops[id].thread, NULL, async_send_thread, id_ptr);
return id;
}
long native_async_wait(long *args, int argc) {
if (!args || argc < 1) return -1;
int id = (int)args[0];
if (id < 0 || id >= MAX_ASYNC_OPS || !async_ops[id].active) return -1;
pthread_join(async_ops[id].thread, NULL);
pthread_mutex_lock(&async_mutex);
long result = async_ops[id].result;
async_ops[id].active = 0;
pthread_mutex_unlock(&async_mutex);
return result;
}
long native_async_poll(long *args, int argc) {
if (!args || argc < 1) return 0;
int id = (int)args[0];
if (id < 0 || id >= MAX_ASYNC_OPS || !async_ops[id].active) return 0;
pthread_mutex_lock(&async_mutex);
int complete = async_ops[id].complete;
pthread_mutex_unlock(&async_mutex);
return complete;
}
long native_async_result(long *args, int argc) {
if (!args || argc < 1) return -1;
int id = (int)args[0];
if (id < 0 || id >= MAX_ASYNC_OPS || !async_ops[id].active) return -1;
pthread_mutex_lock(&async_mutex);
long result = async_ops[id].result;
async_ops[id].active = 0;
pthread_mutex_unlock(&async_mutex);
return result;
}
void register_native_functions() {
register_native_func("socket", native_socket);
register_native_func("bind", native_bind);
@ -763,4 +1082,11 @@ void register_native_functions() {
register_native_func("double_sub", native_double_sub);
register_native_func("double_mul", native_double_mul);
register_native_func("double_div", native_double_div);
register_native_func("async_fread", native_async_fread);
register_native_func("async_fwrite", native_async_fwrite);
register_native_func("async_recv", native_async_recv);
register_native_func("async_send", native_async_send);
register_native_func("async_wait", native_async_wait);
register_native_func("async_poll", native_async_poll);
register_native_func("async_result", native_async_result);
}

44
tests/async_io_test.rc Normal file
View File

@ -0,0 +1,44 @@
int main() {
printf("=== Async I/O Tests ===\n");
printf("Test 1: Async file write\n");
int f = fopen("async_test.txt", "w");
if (f == 0) {
printf("ERROR: Could not open file\n");
return 1;
}
int write_handle = async_fwrite(f, "Hello from async I/O!", 21);
printf("Write started, handle: %d\n", write_handle);
int result = async_wait(write_handle);
printf("Write completed, bytes written: %d\n", result);
fclose(f);
printf("PASS: Async write works\n");
printf("Test 2: Async file read\n");
f = fopen("async_test.txt", "r");
if (f == 0) {
printf("ERROR: Could not open file\n");
return 1;
}
int buffer[256];
int read_handle = async_fread(f, &buffer, 256);
printf("Read started, handle: %d\n", read_handle);
printf("Polling for completion...\n");
while (async_poll(read_handle) == 0) {
printf(".");
}
printf("\nRead complete!\n");
result = async_result(read_handle);
printf("Bytes read: %d\n", result);
fclose(f);
printf("PASS: Async read works\n");
printf("Test 3: Cleanup\n");
fremove("async_test.txt");
printf("PASS: File removed\n");
printf("\n=== All Async I/O Tests Completed ===\n");
return 0;
}