feat: replace rayon parallel iteration with tokio async mpsc channels for file processing
This commit is contained in:
+34
-13
@@ -1,5 +1,5 @@
|
||||
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
|
||||
use std::{env, fmt::Display, fs};
|
||||
use std::{env, fmt::Display, fs, ops::AddAssign};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
static FORBIDDEN_WORDS: &'static [&'static str] = &[
|
||||
"recovery",
|
||||
@@ -93,6 +93,19 @@ impl Stats {
|
||||
}
|
||||
}
|
||||
}
|
||||
impl AddAssign for Stats {
|
||||
fn add_assign(&mut self, rhs: Self) {
|
||||
self.file_count += rhs.file_count;
|
||||
self.failed_file_count += rhs.failed_file_count;
|
||||
|
||||
self.sentence_count += rhs.sentence_count;
|
||||
self.word_count += rhs.word_count;
|
||||
|
||||
self.capitalized_count += rhs.capitalized_count;
|
||||
self.numeric_count += rhs.numeric_count;
|
||||
self.forbidden_count += rhs.forbidden_count;
|
||||
}
|
||||
}
|
||||
impl Display for Stats {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
writeln!(f, "file count: {}", self.file_count)?;
|
||||
@@ -124,18 +137,26 @@ impl Display for Stats {
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let files = env::args().skip(1);
|
||||
// let mut stats = Stats::default();
|
||||
// for file in files {
|
||||
// stats.process(&file);
|
||||
// }
|
||||
let files = files.collect::<Vec<_>>();
|
||||
files.par_iter().for_each(|file| {
|
||||
let mut stats = Stats::default();
|
||||
stats.process(&file);
|
||||
println!("{stats}");
|
||||
});
|
||||
let mut stats = Stats::default();
|
||||
let mut rx = {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
for file in files {
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut stats = Stats::default();
|
||||
stats.process(&file);
|
||||
tx.send(stats).unwrap();
|
||||
});
|
||||
}
|
||||
rx
|
||||
};
|
||||
while let Some(file_stat) = rx.recv().await {
|
||||
stats += file_stat;
|
||||
}
|
||||
println!("{stats}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user