retoor/isspam
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add initial SIMD analysis and io_uring module, drop core_affinity and memmap2 deps

Browse Source 
Introduce a new `analyze_simd` function in main.rs using AVX2 256-bit registers for parallel text scanning, replacing the previous sequential analysis path. Add a stub `uring` module for future async I/O integration. Remove `core_affinity` and `memmap2` crate dependencies from Cargo.toml and Cargo.lock, along with their associated `libc`, `num_cpus`, `hermit-abi`, and `winapi` transitive dependencies. Comment out `mimalloc` and `io-uring` crate references. Uncomment file reading timing instrumentation and add a `bench_rust_only` Makefile target for direct benchmark execution.
This commit is contained in:
12bitfloat
2025-10-06 22:55:06 +00:00
parent 83f3729670
commit e516f5e869
6 changed files with 291 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
12bitfloat_rust/risspam/src/main.rs
+273 -12
View File
@@ -5,6 +5,8 @@
//mod books;
mod uring;
use rayon::prelude::*;
use std::cell::RefCell;
use std::ffi::OsStr;
@@ -15,6 +17,10 @@ use std::thread::available_parallelism;
use std::time::{Duration, Instant};
use std::{array, env, hint, process};
// NOTE: mimalloc is actually slower
//#[global_allocator]
//static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[inline]
fn is_ascii_whitespace(b: u8) -> bool {
matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ')
@@ -74,9 +80,9 @@ extern "rust-cold" fn die() -> ! {
fn work(file_path: &OsStr, stats: &mut Stats) {
WORK_STATE.with_borrow_mut(|state: &mut WorkState| {
// // Load file
// let start_time = Instant::now();
// Load file
let start_time = Instant::now();
// let Ok(text) = fs::read(file_path) else {
// eprintln!("invalid file!");
// process::abort();
@@ -102,19 +108,260 @@ fn work(file_path: &OsStr, stats: &mut Stats) {
read_offset += rb;
}
let text = &state.work_mem[..read_offset];
// file.read_exact(&mut state.work_mem[..file_len]).unwrap();
// let text = include_bytes!("../../../books/Advanced Techniques in Web Intelligence – Part II.txt").as_slice();
let time_reading = start_time.elapsed();
{
let mut guard = TIME_SPENT_READING_FILES.lock().unwrap();
*guard += time_reading;
}
// analyze(&text, stats);
// analyze2(&text, stats);
analyze_simd(&text, stats);
});
}
// let time_reading = start_time.elapsed();
// {
// let mut guard = TIME_SPENT_READING_FILES.lock().unwrap();
// *guard += time_reading;
fn analyze_simd(text: &[u8], stats: &mut Stats) {
use std::arch::x86_64::*;
let mut sentences = 0;
let mut words = 0;
let mut capitalizeds = 0;
let mut numbers = 0;
let mut forbiddens = 0;
const BLOCK_ITEMS: usize = 32; // MUST be 32! 256 bit ymm regs
const REGION_NUM_BLOCKS: usize = 16;
if text.len() < (BLOCK_ITEMS * REGION_NUM_BLOCKS) {
return;
}
let mut offset = 0;
while offset < text.len() {
for block_idx in 0..REGION_NUM_BLOCKS {
// Get block ptr from mem
let block_slice: &[u8; 32] = unsafe {
&*text.as_ptr()
.byte_offset((offset + block_idx * BLOCK_ITEMS) as isize)
.cast()
};
// NOTE: This has *truly* terrible codegen
// for i in 0..32 {
// dots += (chunk[i] == b'.') as u32;
// numbers += (chunk[i] >= b'0' && chunk[i] <= b'9') as u32;
// }
unsafe {
let m_chars = _mm256_loadu_si256(block_slice.as_ptr().cast());
// Count dots
let m_dots_masks = _mm256_cmpeq_epi8(m_chars, _mm256_set1_epi8(b'.' as i8));
sentences += i32::count_ones(_mm256_movemask_epi8(m_dots_masks));
// Count digits
let m_ge_0_masks = _mm256_cmpgt_epi8(m_chars, _mm256_set1_epi8(b'0' as i8 - 1));
let m_le_9_masks = _mm256_cmpgt_epi8(_mm256_set1_epi8(b'9' as i8 + 1), m_chars);
numbers += i32::count_ones(_mm256_movemask_epi8(_mm256_and_si256(m_ge_0_masks, m_le_9_masks)));
}
}
offset += BLOCK_ITEMS * REGION_NUM_BLOCKS;
}
if text.len() == 0 {
return;
}
let mut idx = 0;
'full_loop: loop {
unsafe {
hint::assert_unchecked(idx < text.len());
}
// Skip whitespace
while is_ascii_whitespace(text[idx]) { // Pretty sure this is UB. There have been out of bounds panics
idx += 1;
if hint::unlikely(idx >= text.len()) {
break 'full_loop;
}
}
// Find end of word
let word_start = idx;
let mut has_non_upper = false;
'find_word_end: while let b = text[idx] && !is_ascii_whitespace(b) {
idx += 1;
if hint::unlikely(idx >= text.len()) {
break 'find_word_end;
}
if !is_ascii_upper(b) {
has_non_upper = true;
}
}
unsafe {
hint::assert_unchecked(word_start < idx && idx <= text.len());
}
let word = &text[word_start..idx];
words += 1;
if !has_non_upper {
capitalizeds += 1;
}
// Check forbidden
if unsafe { FW_TAB.lookup(word) } {
// if FW_PHF.contains(word) { // phf is a lot slower than my FwTab
forbiddens += 1;
}
}
stats.sentences = sentences;
stats.words = words;
stats.capitalizeds = capitalizeds;
stats.numbers = numbers;
stats.forbiddens = forbiddens;
}
fn analyze2(text: &[u8], stats: &mut Stats) {
// // NOTE: mmap is quite a bit slower
// // Load file
// let Ok(file) = File::open(file_path) else {
// eprintln!("invalid file!");
// std::process::abort();
// };
// let mmap = unsafe {
// Mmap::map(&file).unwrap()
// };
// mem::forget(file);
// let text = &*mmap;
// // Load file
// let start_time = Instant::now();
// let Ok(text) = fs::read(file_path) else {
// eprintln!("invalid file!");
// process::abort();
// };
// let time_reading = start_time.elapsed();
// {
// let mut guard = TIME_SPENT_READING_FILES.lock().unwrap();
// *guard += time_reading;
// }
let mut sentences = 0;
let mut words = 0;
let mut capitalizeds = 0;
let mut numbers = 0;
let mut forbiddens = 0;
// __A_
// B_BB
if text.len() == 0 {
return;
}
let mut idx = 0;
'full_loop: loop {
unsafe {
hint::assert_unchecked(idx < text.len());
}
// Skip whitespace
while is_ascii_whitespace(text[idx]) { // Pretty sure this is UB. There have been out of bounds panics
idx += 1;
if hint::unlikely(idx >= text.len()) {
break 'full_loop;
}
}
// Find end of word
let word_start = idx;
let mut has_non_upper = false;
'find_word_end: while let b = text[idx] && !is_ascii_whitespace(b) {
idx += 1;
if hint::unlikely(idx >= text.len()) {
break 'find_word_end;
}
if !is_ascii_upper(b) {
has_non_upper = true;
}
if b == b'.' {
sentences += 1;
}
if is_ascii_digit(b) {
numbers += 1;
}
// sentences += (b == b'.') as u32;
// numbers += is_ascii_digit(b) as u32;
}
unsafe {
hint::assert_unchecked(word_start < idx && idx <= text.len());
}
let word = &text[word_start..idx];
// Per-char logic
// for &b in word {
// if !is_ascii_upper(b) {
// has_non_upper = true;
// }
// if b == b'.' {
// sentences += 1;
// }
// if is_ascii_digit(b) {
// numbers += 1;
// }
// }
// for &b in word {
// if !is_ascii_upper(b) {
// has_non_upper = true;
// }
// }
// for &b in word {
// if b == b'.' {
// sentences += 1;
// }
// }
// for &b in word {
// if is_ascii_digit(b) {
// numbers += 1;
// }
// }
analyze(&text, stats);
});
words += 1;
if !has_non_upper {
capitalizeds += 1;
}
// Check forbidden
if unsafe { FW_TAB.lookup(word) } {
// if FW_PHF.contains(word) { // phf is a lot slower than my FwTab
forbiddens += 1;
}
}
stats.sentences = sentences;
stats.words = words;
stats.capitalizeds = capitalizeds;
stats.numbers = numbers;
stats.forbiddens = forbiddens;
}
fn analyze(text: &[u8], stats: &mut Stats) {
@@ -148,6 +395,14 @@ fn analyze(text: &[u8], stats: &mut Stats) {
let mut numbers = 0;
let mut forbiddens = 0;
// __A_
// B_BB
if text.len() == 0 {
return;
}
let mut idx = 0;
'full_loop: loop {
// Skip whitespace
@@ -184,7 +439,7 @@ fn analyze(text: &[u8], stats: &mut Stats) {
let word = &text[word_start..idx];
// let word = unsafe { &text.get_unchecked(word_start..idx) };
// dbg!(str::from_utf8(word).unwrap());
words += 1;
@@ -360,6 +615,12 @@ fn analyze_old(file_path: &OsStr, stats: &mut Stats) {
*/
fn main() {
// // DEBUG:
// uring::test();
// if 1 == 1 {
// return;
// }
// Read in files from args
let mut files = Vec::with_capacity(env::args().len());
// let mut do_parallel = false;
12bitfloat_rust/risspam/src/uring.rs
+8
View File
@@ -0,0 +1,8 @@
pub fn test() {
// let ring = io_uring::Builder::<io_uring::squeue::Entry, io_uring::cqueue::Entry>::default()
// .build(128)
// .unwrap();
//
// ring.
}