AI wanted to know why my CPU Usage is low despite using multiprocessing
A1)
ACode
A
JIm not experienced in this, before taking my guess have you tried asking chatgpt?
Jand or claude
ANope but I am using multi-processing
JCan you run this block
Jimport os
Use container-aware CPU detection
num_cores = len(os.sched_getaffinity(0))Jreplace that variable
Jmaybe print the value too
Jimport time
import psutil
def monitor_resources():
while True:
cpu_percent = psutil.cpu_percent(interval=1)
memory_percent = psutil.virtual_memory().percent
print(f"CPU: {cpu_percent}%, Memory: {memory_percent}%")
time.sleep(5)
monitor_thread = threading.Thread(target=monitor_resources, daemon=True)
monitor_thread.start()
you can try this too
import psutil
def monitor_resources():
while True:
cpu_percent = psutil.cpu_percent(interval=1)
memory_percent = psutil.virtual_memory().percent
print(f"CPU: {cpu_percent}%, Memory: {memory_percent}%")
time.sleep(5)
Run monitoring in background
import threadingmonitor_thread = threading.Thread(target=monitor_resources, daemon=True)
monitor_thread.start()
you can try this too
A24
Aum value is actually ver less
A
JTry smth like this
from concurrent.futures import ThreadPoolExecutor
import threading
def extract_file_threaded(args):
full_file_path, output_path = args
thread_id = threading.current_thread().ident
print(f"[Thread {thread_id}] Extracting {full_file_path}")
try:
with tarfile.open(full_file_path, "r:gz") as tar:
tar.extractall(path=output_path)
return f"Success: {full_file_path}"
except Exception as e:
return f"Error: {full_file_path} - {e}"
results = list(executor.map(extract_file_threaded,
[(f, output_path) for f in files_to_extract]))
from concurrent.futures import ThreadPoolExecutor
import threading
def extract_file_threaded(args):
full_file_path, output_path = args
thread_id = threading.current_thread().ident
print(f"[Thread {thread_id}] Extracting {full_file_path}")
try:
with tarfile.open(full_file_path, "r:gz") as tar:
tar.extractall(path=output_path)
return f"Success: {full_file_path}"
except Exception as e:
return f"Error: {full_file_path} - {e}"
Use ThreadPoolExecutor instead of multiprocessing
with ThreadPoolExecutor(max_workers=min(len(files_to_extract), num_cores * 2)) as executor:results = list(executor.map(extract_file_threaded,
[(f, output_path) for f in files_to_extract]))
Anear about same
Aig sharing puts a upper bound
Jsharing what?
Jig its not correct usage on your python code
Acpus are shared
Jyes but 48 vcpu's should be for your own container / pod
Jprobably its just your code, try another library that properly supports multithreading
Jor multiprocessing
Ai think maybe support is low for tasks like decompression
Ai will try the gzip cmd line
