feat: replace rayon parallel iteration with tokio async mpsc channels for file processing

This commit is contained in:
JestDotty
2025-03-24 02:29:54 +00:00
parent 144fc9b2ac
commit 32fda51edd
4 changed files with 389 additions and 43 deletions
+34 -13
View File
@@ -1,5 +1,5 @@
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use std::{env, fmt::Display, fs};
use std::{env, fmt::Display, fs, ops::AddAssign};
use tokio::sync::mpsc;
static FORBIDDEN_WORDS: &'static [&'static str] = &[
"recovery",
@@ -93,6 +93,19 @@ impl Stats {
}
}
}
impl AddAssign for Stats {
fn add_assign(&mut self, rhs: Self) {
self.file_count += rhs.file_count;
self.failed_file_count += rhs.failed_file_count;
self.sentence_count += rhs.sentence_count;
self.word_count += rhs.word_count;
self.capitalized_count += rhs.capitalized_count;
self.numeric_count += rhs.numeric_count;
self.forbidden_count += rhs.forbidden_count;
}
}
impl Display for Stats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, "file count: {}", self.file_count)?;
@@ -124,18 +137,26 @@ impl Display for Stats {
}
}
fn main() {
#[tokio::main]
async fn main() {
let files = env::args().skip(1);
// let mut stats = Stats::default();
// for file in files {
// stats.process(&file);
// }
let files = files.collect::<Vec<_>>();
files.par_iter().for_each(|file| {
let mut stats = Stats::default();
stats.process(&file);
println!("{stats}");
});
let mut stats = Stats::default();
let mut rx = {
let (tx, rx) = mpsc::unbounded_channel();
for file in files {
let tx = tx.clone();
tokio::spawn(async move {
let mut stats = Stats::default();
stats.process(&file);
tx.send(stats).unwrap();
});
}
rx
};
while let Some(file_stat) = rx.recv().await {
stats += file_stat;
}
println!("{stats}");
}
#[test]