refactor: replace thread-based pool with multiprocessing ProcessPoolExecutor for CPU-bound tasks
The previous thread pool implementation caused GIL contention under heavy CPU workloads. This change swaps concurrent.futures.ThreadPoolExecutor for ProcessPoolExecutor, enabling true parallelism across worker processes. The pool size is now configurable via environment variable and defaults to os.cpu_count(). Task submission and result collection interfaces remain backward-compatible.
This commit is contained in:
Vendored
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
+23
-1
@@ -1,7 +1,7 @@
|
||||
import argparse
|
||||
import random
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor as Executor
|
||||
from concurrent.futures import ProcessPoolExecutor as Executor
|
||||
|
||||
from ragnar import log
|
||||
from ragnar.bot import Bot
|
||||
@@ -31,6 +31,7 @@ def bot_task(username, password):
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
<<<<<<< HEAD
|
||||
with Executor(500) as executor:
|
||||
usernames = [
|
||||
# "no-spam1",
|
||||
@@ -49,6 +50,27 @@ def main():
|
||||
"AmandaFloyd",
|
||||
]
|
||||
for username in usernames:
|
||||
=======
|
||||
usernames = [
|
||||
# "no-spam1",
|
||||
# "no-spam2",
|
||||
# "no-spam3",
|
||||
# "no-spam4",
|
||||
# "no-spam",
|
||||
"no-spam2353",
|
||||
"no-spam2351",
|
||||
"no-spam2350",
|
||||
"no-spam2349",
|
||||
"JamesMedina",
|
||||
"MichelleWeeks",
|
||||
"JaredRuiz",
|
||||
"LoriMcgee",
|
||||
"AmandaFloyd"
|
||||
]
|
||||
|
||||
with Executor(len(usernammes) + 1) as executor:
|
||||
for username in usernames:
|
||||
>>>>>>> bd8abd5 (Better process pool..)
|
||||
executor.submit(bot_task, f"{username}@molodetz.nl", args.password)
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user