I’m trying to create a way to pass a dataclass object to several different child processes using the Multiprocessing module. Right now I’ve tried:
import queue
from multiprocessing import Process, Queue, Manager
from multiprocessing.managers import BaseManager, NamespaceProxy
class TestManager(BaseManager):
pass
class Test:
def __init__(self):
self.a = []
self.b = []
self.c = []
class TestProxy(NamespaceProxy):
# We need to expose the same __dunder__ methods as NamespaceProxy,
# in addition to the b method.
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
def popManager(queue, manager, i):
manager.a.append(i)
manager.b.append(i+1)
manager.c.append(i**2)
queue.put(True)
def managerParent():
processes = []
queues = []
TestManager.register("Test", Test, TestProxy)
with TestManager() as manager:
t = manager.Test()
for i in range(10):
queues.append(Queue())
processes.append(
Process(target=popManager, args=(queues[i], t, i))
)
[p.start() for p in processes]
print("started!")
while True:
try:
[q.get(timeout=0.2) for i, q in enumerate(queues)]
break
except queue.Empty:
print(f"Processes still running!")
print(t.a, t.a, t.c)
if __name__ == "__main__":
managerParent()
which runs just fine, but the list instances are all empty. It outputs
[] [] []
where I’d like it to output something like
[0, 1, 2, ... 9], [1, 2, 3, ..., 10], [0, 1, 4, ..., 81]