Multiprocessing doesn't work on Windows with 3.0.x
λ python --version
Python 2.7.12
λ pip --version
pip 8.1.2
λ flake8 --version
3.0.1 (pyflakes: 1.2.3, pycodestyle: 2.0.0, mccabe: 0.5.0) CPython 2.7.12 on Windows
λ flake8 krc_debugger.py
Traceback (most recent call last):
File "c:\python27\Lib\runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "c:\python27\Lib\runpy.py", line 72, in _run_code
exec code in run_globals
File "C:\Users\schlaich\Desktop\temp\Scripts\flake8.exe\__main__.py", line 9, in <module>
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\main\cli.py", line 16, in main
app.run(argv)
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\main\application.py", line 316, in run
self._run(argv)
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\main\application.py", line 300, in _run
self.run_checks()
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\main\application.py", line 238, in run_checks
self.file_checker_manager.run()
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\checker.py", line 344, in run
self.run_parallel()
File "c:\users\schlaich\desktop\temp\lib\site-packages\flake8\checker.py", line 315, in run_parallel
proc.start()
File "c:\python27\Lib\multiprocessing\process.py", line 130, in start
self._popen = Popen(self)
File "c:\python27\Lib\multiprocessing\forking.py", line 277, in __init__
dump(process_obj, to_child, HIGHEST_PROTOCOL)
File "c:\python27\Lib\multiprocessing\forking.py", line 199, in dump
ForkingPickler(file, protocol).dump(obj)
File "c:\python27\Lib\pickle.py", line 224, in dump
self.save(obj)
File "c:\python27\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "c:\python27\Lib\pickle.py", line 425, in save_reduce
save(state)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "c:\python27\Lib\pickle.py", line 687, in _batch_setitems
save(v)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\multiprocessing\forking.py", line 67, in dispatcher
self.save_reduce(obj=obj, *rv)
File "c:\python27\Lib\pickle.py", line 401, in save_reduce
save(args)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 554, in save_tuple
save(element)
File "c:\python27\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "c:\python27\Lib\pickle.py", line 425, in save_reduce
save(state)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "c:\python27\Lib\pickle.py", line 687, in _batch_setitems
save(v)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 606, in save_list
self._batch_appends(iter(obj))
File "c:\python27\Lib\pickle.py", line 642, in _batch_appends
save(tmp[0])
File "c:\python27\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "c:\python27\Lib\pickle.py", line 425, in save_reduce
save(state)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "c:\python27\Lib\pickle.py", line 687, in _batch_setitems
save(v)
File "c:\python27\Lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "c:\python27\Lib\pickle.py", line 425, in save_reduce
save(state)
File "c:\python27\Lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "c:\python27\Lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "c:\python27\Lib\pickle.py", line 687, in _batch_setitems
save(v)
File "c:\python27\Lib\pickle.py", line 313, in save
(t.__name__, obj))
pickle.PicklingError: Can't pickle '_subprocess_handle' object: <_subprocess_handle object at 0x025C1BC0>
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "c:\python27\Lib\multiprocessing\forking.py", line 381, in main
self = load(from_parent)
File "c:\python27\Lib\pickle.py", line 1384, in load
return Unpickler(file).load()
File "c:\python27\Lib\pickle.py", line 864, in load
dispatch[key](self)
File "c:\python27\Lib\pickle.py", line 886, in load_eof
raise EOFError
EOFError
I assume the issue is that you store the process objects on the manager (Manager.processes
) and you have to pickle the manager object because you are using a method as Process target (self._run_checks_from_queue
). But processes can't be pickled on Windows apparently.
My proposal: use a standalone function as process target and pass the queues via args
.