parallelize progress bar causes zombie processes and deadlocks #1123
Description
Report
I noticed this when the CI jobs hang up even though they finish.
I am still not sure if this is the case but I want to document this hang-up. Very hard to pinpoint as there is no direct messages that are coming from the root of the problem
This appears for example in https://github.com/scverse/squidpy/actions/runs/22302343298/job/64513191945?pr=1121
~~~~~~~~~~~~~~~~~ Stack of QueueFeederThread (139840620328640) ~~~~~~~~~~~~~~~~~
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1015, in _bootstrap
self._bootstrap_inner()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1044, in _bootstrap_inner
self.run()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 995, in run
self._target(*self._args, **self._kwargs)
File "/home/runner/.local/share/hatch/env/virtual/squidpy/gW4OMqsM/hatch-test.py3.13-stable/lib/python3.13/site-packages/joblib/externals/loky/backend/queues.py", line 147, in _feed
nwait()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 359, in wait
waiter.acquire()
~~~~~~~~~~~~~~~ Stack of ExecutorManagerThread (139841106863808) ~~~~~~~~~~~~~~~
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1015, in _bootstrap
self._bootstrap_inner()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1044, in _bootstrap_inner
self.run()
File "/home/runner/.local/share/hatch/env/virtual/squidpy/gW4OMqsM/hatch-test.py3.13-stable/lib/python3.13/site-packages/joblib/externals/loky/process_executor.py", line 623, in run
result_item, is_broken, bpe = self.wait_result_broken_or_wakeup()
File "/home/runner/.local/share/hatch/env/virtual/squidpy/gW4OMqsM/hatch-test.py3.13-stable/lib/python3.13/site-packages/joblib/externals/loky/process_executor.py", line 681, in wait_result_broken_or_wakeup
ready = wait(readers + worker_sentinels)
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/multiprocessing/connection.py", line 1148, in wait
ready = selector.select(timeout)
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/selectors.py", line 398, in select
fd_event_list = self._selector.poll(timeout)
~~~~~~~~~~~~~~ Stack of ParallelizeUpdateThread (139841090082496) ~~~~~~~~~~~~~~
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1015, in _bootstrap
self._bootstrap_inner()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1044, in _bootstrap_inner
self.run()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 995, in run
self._target(*self._args, **self._kwargs)
File "/home/runner/work/squidpy/squidpy/src/squidpy/_utils.py", line 158, in update
res = queue.get()
File "<string>", line 2, in get
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/multiprocessing/managers.py", line 831, in _callmethod
kind, result = conn.recv()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/multiprocessing/connection.py", line 250, in recv
buf = self._recv_bytes()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/multiprocessing/connection.py", line 430, in _recv_bytes
buf = self._recv(4)
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/multiprocessing/connection.py", line 395, in _recv
chunk = read(handle, remaining)
~~~~~~~~~~~~~~~~~~~~~ Stack of Thread-3 (139839020115648) ~~~~~~~~~~~~~~~~~~~~~~
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1015, in _bootstrap
self._bootstrap_inner()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1044, in _bootstrap_inner
self.run()
File "/home/runner/.local/share/hatch/env/virtual/squidpy/gW4OMqsM/hatch-test.py3.13-stable/lib/python3.13/site-packages/tqdm/_monitor.py", line 60, in run
self.was_killed.wait(self.sleep_interval)
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 660, in wait
signaled = self._cond.wait(timeout)
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 363, in wait
gotit = waiter.acquire(True, timeout)
~~~~~~~~~~~~~~~ Stack of Thread-2 (run_server) (139840353339072) ~~~~~~~~~~~~~~~
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1015, in _bootstrap
self._bootstrap_inner()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 1044, in _bootstrap_inner
self.run()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/threading.py", line 995, in run
self._target(*self._args, **self._kwargs)
File "/home/runner/.local/share/hatch/env/virtual/squidpy/gW4OMqsM/hatch-test.py3.13-stable/lib/python3.13/site-packages/pytest_rerunfailures.py", line 433, in run_server
conn, _ = self.sock.accept()
File "/home/runner/work/_temp/uv-python-dir/cpython-3.13.12-linux-x86_64-gnu/lib/python3.13/socket.py", line 295, in accept
fd, addr = self._accept()
+++++++++++++++++++++++++++++++++++ Timeout ++++++++++++++++++++++++++++++++++++
also fails locally
parallelize.py
================ test session starts =================
platform darwin -- Python 3.12.9, pytest-9.0.2, pluggy-1.6.0
rootdir: /Users/selman/projects/squidpy
configfile: pyproject.toml
plugins: mock-3.15.1, fast-array-utils-1.3.1, xdist-3.8.0, timeout-2.4.0, typeguard-4.4.4, zarr-3.1.5, cov-7.0.0
8 workers [12 items]
..Fatal Python error: Aborted
Thread 0x0000000172237000 (most recent call first):
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 579 in _handle_results
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1012 in run
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1032 in _bootstrap
Thread 0x000000017122b000 (most recent call first):
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 531 in _handle_tasks
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1012 in run
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1032 in _bootstrap
Thread 0x000000017021f000 (most recent call first):
File "/Users/selman/miniforge3/lib/python3.12/selectors.py", line 415 in select
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/connection.py", line 1136 in wait
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 502 in _wait_for_updates
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 522 in _handle_workers
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1012 in run
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1032 in _bootstrap
Current thread 0x000000016f213000 (most recent call first):
File "/Users/selman/projects/squidpy/tests/utils/test_parallelize.py", line 20 in wrap_numba_check
File "/Users/selman/projects/squidpy/tests/utils/test_parallelize.py", line 43 in mock_runner
File "/Users/selman/projects/squidpy/src/squidpy/_utils.py", line 56 in _callback_wrapper
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/joblib/parallel.py", line 607 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/joblib/_utils.py", line 109 in __call__
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 125 in worker
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1012 in run
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1032 in _bootstrap
Thread 0x000000016e207000 (most recent call first):
File "/Users/selman/miniforge3/lib/python3.12/multiprocessing/pool.py", line 114 in worker
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1012 in run
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
File "/Users/selman/miniforge3/lib/python3.12/threading.py", line 1032 in _bootstrap
Thread 0x000000016c8ab000 (most recent call first):
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 534 in read
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 567 in from_io
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 1160 in _thread_receiver
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 341 in run
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 411 in _perform_spawn
Thread 0x00000001fd92ec40 (most recent call first):
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/joblib/parallel.py", line 1800 in _retrieve
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/joblib/parallel.py", line 1682 in _get_outputs
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/joblib/parallel.py", line 2072 in __call__
File "/Users/selman/projects/squidpy/src/squidpy/_utils.py", line 159 in wrapper
File "/Users/selman/projects/squidpy/tests/utils/test_parallelize.py", line 92 in test_parallelize
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/python.py", line 166 in pytest_pyfunc_call
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_callers.py", line 121 in _multicall
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_hooks.py", line 512 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/python.py", line 1720 in runtest
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 179 in pytest_runtest_call
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_callers.py", line 121 in _multicall
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_hooks.py", line 512 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 245 in <lambda>
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 353 in from_call
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 244 in call_and_report
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 137 in runtestprotocol
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/runner.py", line 118 in pytest_runtest_protocol
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_callers.py", line 121 in _multicall
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_hooks.py", line 512 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/xdist/remote.py", line 227 in run_one_test
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/xdist/remote.py", line 206 in pytest_runtestloop
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_callers.py", line 121 in _multicall
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_hooks.py", line 512 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/main.py", line 372 in _main
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/main.py", line 318 in wrap_session
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/_pytest/main.py", line 365 in pytest_cmdline_main
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_callers.py", line 121 in _multicall
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/pluggy/_hooks.py", line 512 in __call__
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/xdist/remote.py", line 427 in <module>
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 1291 in executetask
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 341 in run
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 411 in _perform_spawn
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 389 in integrate_as_primary_thread
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 1273 in serve
File "/Users/selman/projects/squidpy/squidpy/lib/python3.12/site-packages/execnet/gateway_base.py", line 1806 in serve
File "<string>", line 8 in <module>
File "<string>", line 1 in <module>
Extension modules: numpy._core._multiarray_umath, numpy.linalg._umath_linalg, scipy._lib._ccallback_c, charset_normalizer.md, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._pcg64, numpy.random._mt19937, numpy.random._generator, numpy.random._philox, numpy.random._sfc64, numpy.random.mtrand, scipy.sparse._sparsetools, _csparsetools, _cyutility, scipy._cyutility, scipy.sparse._csparsetools, yaml._yaml, pyarrow.lib, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pyarrow._compute, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, scipy.io.matlab._mio_utils, scipy.io.matlab._streams, scipy.io.matlab._mio5_utils, scipy._lib._uarray._uarray, scipy.special._ufuncs_cxx, scipy.special._ellip_harm_2, scipy.special._special_ufuncs, scipy.special._gufuncs, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array_utils, scipy.linalg._solve_toeplitz, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_schur_sqrtm, scipy.linalg._matfuncs_expm, scipy.linalg._linalg_pythran, scipy.linalg.cython_blas, scipy.linalg._decomp_update, scipy.fftpack.convolve, h5py._errors, h5py.defs, h5py._objects, h5py.h5, h5py.utils, h5py.h5t, h5py.h5s, h5py.h5ac, h5py.h5p, h5py.h5r, h5py._npystrings, h5py._proxy, h5py._conv, h5py.h5z, h5py.h5a, h5py.h5d, h5py.h5ds, h5py.h5g, h5py.h5i, h5py.h5o, h5py.h5f, h5py.h5fd, h5py.h5pl, h5py.h5l, h5py._selector, numcodecs.compat_ext, numcodecs.blosc, numcodecs.zstd, numcodecs.lz4, numcodecs._shuffle, numcodecs.jenkins, google_crc32c._crc32c, numcodecs.vlen, numcodecs.fletcher32, numba.core.typeconv._typeconv, numba._helperlib, numba._dynfunc, numba._dispatcher, numba.core.runtime._nrt_python, numba.np.ufunc._internal, numba.experimental.jitclass._box, shapely.lib, shapely._geos, shapely._geometry_helpers, pyproj._compat, pyproj._context, pyproj._network, pyproj._version, pyproj._geod, pyproj.list, pyproj._crs, pyproj.database, pyproj._transformer, pyproj._sync, PIL._imaging, kiwisolver._cext, sklearn.__check_build._check_build, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpack, scipy.sparse.linalg._propack._spropack, scipy.sparse.linalg._propack._dpropack, scipy.sparse.linalg._propack._cpropack, scipy.sparse.linalg._propack._zpropack, scipy.spatial._ckdtree, scipy._lib.messagestream, scipy.spatial._qhull, scipy.spatial._voronoi, scipy.spatial._hausdorff, scipy.spatial._distance_wrap, scipy.spatial.transform._rotation, scipy.spatial.transform._rigid_transform, scipy.optimize._group_columns, scipy.optimize._trlib._trlib, scipy.optimize._lbfgsb, _moduleTNC, scipy.optimize._moduleTNC, scipy.optimize._slsqplib, scipy.optimize._minpack, scipy.optimize._lsq.givens_elimination, scipy.optimize._zeros, scipy.linalg._decomp_interpolative, scipy.optimize._bglu_dense, scipy.optimize._lsap, scipy.optimize._direct, scipy.integrate._odepack, scipy.integrate._quadpack, scipy.integrate._vode, scipy.integrate._dop, scipy.integrate._lsoda, scipy.interpolate._fitpack, scipy.interpolate._dfitpack, scipy.interpolate._dierckx, scipy.interpolate._ppoly, scipy.interpolate._interpnd, scipy.interpolate._rbfinterp_pythran, scipy.interpolate._rgi_cython, scipy.special.cython_special, scipy.stats._stats, scipy.stats._biasedurn, scipy.stats._stats_pythran, scipy.stats._levy_stable.levyst, scipy.stats._ansari_swilk_statistics, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, scipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, scipy.stats._sobol, scipy.stats._qmc_cy, scipy.stats._rcont.rcont, scipy.stats._qmvnt_cy, scipy.ndimage._nd_image, scipy.ndimage._rank_filter_1d, _ni_label, scipy.ndimage._ni_label, sklearn._cyutility, sklearn.utils._isfinite, sklearn.utils.sparsefuncs_fast, sklearn.utils.murmurhash, sklearn.utils._openmp_helpers, sklearn.utils._random, pyarrow._fs, pyarrow._azurefs, pyarrow._hdfs, pyarrow._gcsfs, pyarrow._s3fs, pyarrow._acero, pyarrow._csv, pyarrow._json, pyarrow._substrait, pyarrow._dataset, pyarrow._dataset_orc, pyarrow._parquet, pyarrow._parquet_encryption, pyarrow._dataset_parquet_encryption, pyarrow._dataset_parquet, scipy.cluster._vq, scipy.cluster._hierarchy, scipy.cluster._optimal_leaf_ordering, skimage.segmentation._slic, numba.mviewbuf, numba.types.itertools, requests.packages.charset_normalizer.md, requests.packages.chardet.md, skimage.measure._moments_cy, skimage.measure._find_contours_cy, skimage.measure._marching_cubes_lewiner_cy, skimage.transform._warps_cy, skimage.filters._multiotsu, skimage.morphology._misc_cy, skimage.measure._ccomp, _skeletonize_lee_cy, skimage.morphology._skeletonize_lee_cy, skimage.morphology._skeletonize_various_cy, skimage._shared.geometry, skimage.measure._pnpoly, skimage.morphology._convex_hull, skimage.morphology._grayreconstruct, skimage.morphology._extrema_cy, skimage.morphology._flood_fill_cy, skimage.morphology._max_tree, skimage.segmentation._felzenszwalb_cy, sklearn.metrics.cluster._expected_mutual_info_fast, sklearn.metrics._dist_metrics, sklearn.metrics._pairwise_distances_reduction._datasets_pair, sklearn.utils._cython_blas, sklearn.metrics._pairwise_distances_reduction._base, sklearn.metrics._pairwise_distances_reduction._middle_term_computer, sklearn.utils._heap, sklearn.utils._sorting, sklearn.metrics._pairwise_distances_reduction._argkmin, sklearn.metrics._pairwise_distances_reduction._argkmin_classmode, sklearn.utils._vector_sentinel, sklearn.metrics._pairwise_distances_reduction._radius_neighbors, sklearn.metrics._pairwise_distances_reduction._radius_neighbors_classmode, sklearn.metrics._pairwise_fast, sklearn.preprocessing._csr_polynomial_expansion, sklearn.preprocessing._target_encoder_fast, sklearn.neighbors._partition_nodes, sklearn.neighbors._ball_tree, sklearn.neighbors._kd_tree, sklearn.utils.arrayfuncs, sklearn.utils._seq_dataset, sklearn.linear_model._cd_fast, _loss, sklearn._loss._loss, sklearn.linear_model._sag_fast, sklearn.svm._liblinear, sklearn.svm._libsvm, sklearn.svm._libsvm_sparse, sklearn.utils._weight_vector, sklearn.linear_model._sgd_fast, sklearn.decomposition._online_lda_fast, sklearn.decomposition._cdnmf_fast, sklearn.tree._utils, sklearn.neighbors._quad_tree, sklearn.tree._tree, sklearn.tree._partitioner, sklearn.tree._splitter, sklearn.tree._criterion, sklearn.ensemble._gradient_boosting, sklearn.ensemble._hist_gradient_boosting.common, sklearn.ensemble._hist_gradient_boosting._gradient_boosting, sklearn.ensemble._hist_gradient_boosting._binning, sklearn.ensemble._hist_gradient_boosting._bitset, sklearn.ensemble._hist_gradient_boosting.histogram, sklearn.ensemble._hist_gradient_boosting._predictor, sklearn.ensemble._hist_gradient_boosting.splitting, sklearn.utils._fast_dict, sklearn.cluster._hierarchical_fast, sklearn.cluster._k_means_common, sklearn.cluster._k_means_elkan, sklearn.cluster._k_means_lloyd, sklearn.cluster._k_means_minibatch, sklearn.cluster._dbscan_inner, sklearn.cluster._hdbscan._tree, sklearn.cluster._hdbscan._linkage, sklearn.cluster._hdbscan._reachability, sklearn._isotonic, sklearn.manifold._barnes_hut_tsne, sklearn.manifold._utils, skimage._shared.transform, skimage.feature._texture, skimage.segmentation._watershed_cy (total: 310)
[gw0] node down: Not properly terminated
F
replacing crashed worker gw0
Metadata
Metadata
Assignees
Labels
No labels