chore: delete all isspam_v1 through v4 Rust spam analysis files
This commit is contained in:
@@ -1,138 +1 @@
|
||||
#![feature(let_chains)]
|
||||
|
||||
use rayon::prelude::*;
|
||||
//use rayon::prelude::*;
|
||||
use std::{env, fs};
|
||||
|
||||
fn clean_content(content: &str) -> String {
|
||||
let alloed_ichars = "01234567891abcdefghijklmnopqrstuvwxyz \n.,!?";
|
||||
|
||||
let clean_content = content.chars()
|
||||
.filter(|&c| alloed_ichars.contains(c))
|
||||
.collect::<String>();
|
||||
|
||||
clean_content
|
||||
}
|
||||
|
||||
fn get_sentences(content: &str) -> Vec<&str> {
|
||||
let mut sentences = content.split('.')
|
||||
.map(|s| s.trim_start()) // Remove leading whitespace
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Remove last "sentence" if didn't end with a dot
|
||||
if let Some(last) = sentences.last() && !last.ends_with('.') {
|
||||
sentences.pop();
|
||||
}
|
||||
|
||||
sentences
|
||||
}
|
||||
|
||||
fn get_words(sentences: &str) -> impl Iterator<Item = &str> + Clone {
|
||||
sentences.split_whitespace()
|
||||
}
|
||||
|
||||
fn is_fully_capitalized_word(word: &str) -> bool {
|
||||
word.chars()
|
||||
.all(|c| !c.is_ascii_alphanumeric() || c.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn get_capitalized_words(content: &str) -> Vec<&str> {
|
||||
let sentences = get_sentences(content);
|
||||
let mut cap_words = vec![];
|
||||
|
||||
for sentence in sentences {
|
||||
// Always skip the first word since sentences start with
|
||||
for word in get_words(sentence).skip(1) {
|
||||
if is_fully_capitalized_word(word) {
|
||||
cap_words.push(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cap_words
|
||||
}
|
||||
|
||||
fn get_numbers(clean_content: &str) -> Vec<&str> {
|
||||
clean_content.split(|c: char| c.is_ascii_digit())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn get_forbidden_words(content: &str) -> Vec<&str> {
|
||||
fn check_forbidden(w: &str) -> bool {
|
||||
FORBIDDEN_WORDS.iter()
|
||||
.find(|fw| str::eq_ignore_ascii_case(w, fw))
|
||||
.is_some()
|
||||
}
|
||||
|
||||
get_words(content)
|
||||
.filter(|w| check_forbidden(w))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn analyze(data: &str) {
|
||||
let clean_data = clean_content(data);
|
||||
// drop(clean_data); // You aren't actually using clean_data :O
|
||||
|
||||
// All capitalized words
|
||||
let cap_words = get_capitalized_words(data);
|
||||
println!("All capitalized words: {}", cap_words.len());
|
||||
|
||||
// All sentences
|
||||
let sentences = get_sentences(data);
|
||||
println!("Sentences: {}", sentences.len());
|
||||
|
||||
// All words
|
||||
let words = get_words(data);
|
||||
println!("Words: {}", words.clone().count());
|
||||
|
||||
// Numbers
|
||||
let numbers = get_numbers(&clean_data);
|
||||
println!("Numbers: {}", numbers.len());
|
||||
|
||||
// Forbidden words
|
||||
let fw = get_forbidden_words(data);
|
||||
println!("Forbidden words: {}", fw.len());
|
||||
|
||||
if sentences.len() > 0 {
|
||||
let word_count_per_sentence = words.count() / sentences.len();
|
||||
println!("Word count per sentence: {}", word_count_per_sentence);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// Read in files from args
|
||||
let mut files = Vec::with_capacity(env::args().len());
|
||||
let mut do_parallel = false;
|
||||
|
||||
for arg in env::args().skip(1) { // skip program arg
|
||||
if arg == "-p" {
|
||||
do_parallel = true;
|
||||
} else {
|
||||
files.push(arg);
|
||||
}
|
||||
}
|
||||
|
||||
// Do the work
|
||||
let work = |file| {
|
||||
let Ok(text) = fs::read_to_string(&file) else {
|
||||
eprintln!("{file} isn't a valid file or couldn't be read");
|
||||
return;
|
||||
};
|
||||
|
||||
analyze(&text);
|
||||
};
|
||||
|
||||
if !do_parallel {
|
||||
files.iter().for_each(work);
|
||||
} else {
|
||||
files.par_iter().for_each(work)
|
||||
}
|
||||
}
|
||||
|
||||
static FORBIDDEN_WORDS: &'static [&'static str] = &[
|
||||
"recovery", "techie", "http", "https", "digital", "hack", "::", "//", "com",
|
||||
"@", "crypto", "bitcoin", "wallet", "hacker", "welcome", "whatsapp", "email", "cryptocurrency",
|
||||
"stolen", "freeze", "quick", "crucial", "tracing", "scammers", "expers", "hire", "century",
|
||||
"transaction", "essential", "managing", "contact", "contacting", "understanding", "assets", "funds"
|
||||
];
|
||||
|
||||
#![feature(let_chains)]\\n\\nuse rayon::prelude::*;\\n//use rayon::prelude::*;\\nuse std::{env, fs};\\n\\nfn clean_content(content: &str) -> String {\\n\\tlet alloed_ichars = \\\"01234567891abcdefghijklmnopqrstuvwxyz \\\\n.,!?\\\";\\n\\t\\n\\tlet clean_content = content.chars()\\n\\t\\t.filter(|&c| alloed_ichars.contains(c))\\n\\t\\t.collect::<String>();\\n\\t\\n\\tclean_content\\n}\\n\\nfn get_sentences(content: &str) -> usize {\\n\\tlet sentences = content.split('.')\\n\\t\\t.map(|s| s.trim_start()) // Remove leading whitespace\\n\\t\\t.count();\\n\\t\\n//\\t// Remove last \\\"sentence\\\" if didn't end with a dot\\n//\\tif let Some(last) = sentences.last() && !last.ends_with('.') {\\n//\\t\\tsentences.pop();\\n//\\t}\\n\\t\\n\\tsentences\\n}\\n\\nfn get_words(content: &str, words: &mut usize, caps: &mut usize, fw: &mut usize) {\\n\\tfn check_forbidden(w: &str) -> bool {\\n\\t\\tFORBIDDEN_WORDS.iter()\\n\\t\\t\\t.find(|fw| str::eq_ignore_ascii_case(w, fw))\\n\\t\\t\\t.is_some()\\n\\t}\\n\\t\\n\\tfor word in content.split_whitespace() {\\n\\t\\t*words += 1;\\n\\t\\t\\n\\t\\tif is_fully_capitalized_word(word) {\\n\\t\\t\\t*caps += 1;\\n\\t\\t}\\n\\t\\tif check_forbidden(word) {\\n\\t\\t\\t*fw += 1;\\n\\t\\t}\\n\\t}\\n}\\n\\nfn is_fully_capitalized_word(word: &str) -> bool {\\n\\tword.chars()\\n\\t\\t.all(|c| !c.is_ascii_alphanumeric() || c.is_ascii_uppercase())\\n}\\n\\nfn get_numbers(clean_content: &str) -> usize {\\n\\tclean_content.split(|c: char| !c.is_ascii_digit())\\n\\t\\t.count()\\n}\\n\\nfn analyze(data: &str) {\\n\\tlet clean_data = clean_content(data);\\n//\\tdrop(clean_data); // You aren't actually using clean_data :O\\n\\t\\n\\t// All capitalized words\\n\\tlet mut words = 0;\\n\\tlet mut fw = 0;\\n\\tlet mut cap_words = 0;\\n\\tget_words(&clean_data, &mut words, &mut fw, &mut cap_words);\\n\\t\\n\\tprintln!(\\\"All capitalized words: {}\\\", cap_words);\\n\\t\\n\\t// All sentences\\n\\tlet sentences = get_sentences(data);\\n\\tprintln!(\\\"Sentences: {}\\\", sentences);\\n\\t\\n\\t// All words\\n\\tprintln!(\\\"Words: {}\\\", words);\\n\\t\\n\\t// Numbers\\n\\tlet numbers = get_numbers(&clean_data);\\n\\tprintln!(\\\"Numbers: {}\\\", numbers);\\n\\t\\n\\t// Forbidden words\\n\\tprintln!(\\\"Forbidden words: {}\\\", fw);\\n\\t\\n\\tif sentences > 0 {\\n\\t\\tlet word_count_per_sentence = words / sentences;\\n\\t\\tprintln!(\\\"Word count per sentence: {}\\\", word_count_per_sentence);\\n\\t}\\n}\\n\\nfn main() {\\n // Read in files from args\\n\\tlet mut files = Vec::with_capacity(env::args().len());\\n\\tlet mut do_parallel = false;\\n\\t\\n\\tfor arg in env::args().skip(1) { // skip program arg\\n\\t\\tif arg == \\\"-p\\\" {\\n\\t\\t\\tdo_parallel = true;\\n\\t\\t} else {\\n\\t\\t\\tfiles.push(arg);\\n\\t\\t}\\n\\t}\\n\\t\\n\\t// Do the work\\n\\tlet work = |file| {\\n\\t let Ok(text) = fs::read_to_string(&file) else {\\n\\t\\t\\teprintln!(\\\"{file} isn't a valid file or couldn't be read\\\");\\n\\t\\t\\treturn;\\n\\t };\\n\\t \\tanalyze(&text);\\n\\t};\\n\\t\\n\\tif !do_parallel {\\n\\t\\tfiles.iter().for_each(work);\\n\\t} else {\\n\\t\\tfiles.par_iter().for_each(work)\\n\\t}\\n}\\n\\nstatic FORBIDDEN_WORDS: &'static [&'static str] = &[\\n \\\"recovery\\\", \\\"techie\\\", \\\"http\\\", \\\"https\\\", \\\"digital\\\", \\\"hack\\\", \\\"::\\\", \\\"//\\\", \\\"com\\\",\\n \\\"@\\\", \\\"crypto\\\", \\\"bitcoin\\\", \\\"wallet\\\", \\\"hacker\\\", \\\"welcome\\\", \\\"whatsapp\\\", \\\"email\\\", \\\"cryptocurrency\\\",\\n \\\"stolen\\\", \\\"freeze\\\", \\\"quick\\\", \\\"crucial\\\", \\\"tracing\\\", \\\"scammers\\\", \\\"expers\\\", \\\"hire\\\", \\\"century\\\",\\n \\\"transaction\\\", \\\"essential\\\", \\\"managing\\\", \\\"contact\\\", \\\"contacting\\\", \\\"understanding\\\", \\\"assets\\\", \\\"funds\\\"\\n];\\n
|
||||
Reference in New Issue
Block a user