Skip to content

Commit 95f9f20

Browse files
lblasctwassman
lblasc
authored and
twassman
committed
python3Packages.gensim: fix build with python 3.14
Apply piskvorky/gensim#3649 for the forkserver multiprocessing default.
1 parent 667aa53 commit 95f9f20

2 files changed

Lines changed: 164 additions & 4 deletions

File tree

‎pkgs/development/python-modules/gensim/default.nix‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,28 @@
1919
testfixtures,
2020
}:
2121

22-
buildPythonPackage rec {
22+
buildPythonPackage (finalAttrs: {
2323
pname = "gensim";
2424
version = "4.4.0";
2525
pyproject = true;
2626

27+
__structuredAttrs = true;
28+
2729
src = fetchFromGitHub {
2830
owner = "piskvorky";
2931
repo = "gensim";
30-
tag = version;
32+
tag = finalAttrs.version;
3133
hash = "sha256-TXutcU43ReBj9ss9+zBJFUxb5JqVHpl+B0c7hqcJAJY=";
3234
};
3335

36+
patches = [
37+
# Fall back to serial chunkize when the multiprocessing start method is not
38+
# "fork" (the default changed to "forkserver" on Linux with Python 3.14).
39+
# Vendored (filtered to gensim/utils.py) from the not yet merged
40+
# https://github.com/piskvorky/gensim/pull/3649
41+
./python314-chunkize-forkserver.patch
42+
];
43+
3444
build-system = [
3545
cython
3646
setuptools
@@ -70,7 +80,8 @@ buildPythonPackage rec {
7080
description = "Topic-modelling library";
7181
homepage = "https://radimrehurek.com/gensim/";
7282
downloadPage = "https://github.com/piskvorky/gensim";
73-
changelog = "https://github.com/RaRe-Technologies/gensim/blob/${version}/CHANGELOG.md";
83+
changelog = "https://github.com/RaRe-Technologies/gensim/blob/${finalAttrs.version}/CHANGELOG.md";
7484
license = lib.licenses.lgpl21Only;
85+
maintainers = [ ];
7586
};
76-
}
87+
})

‎pkgs/development/python-modules/gensim/python314-chunkize-forkserver.patch‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
diff --git a/gensim/utils.py b/gensim/utils.py
2+
index 2abd5179e09671de1ea9ef48a9dc3fc4bce9b465..20bad4857bf4e159bc0e244522aa21bd41af8712 100644
3+
--- a/gensim/utils.py
4+
+++ b/gensim/utils.py
5+
@@ -1302,90 +1302,68 @@
6+
self.q.put(wrapped_chunk.pop(), block=True)
7+
8+
9+
-# Multiprocessing on Windows (and on OSX with python3.8+) uses "spawn" mode, which
10+
-# causes issues with pickling.
11+
-# So for these two platforms, use simpler serial processing in `chunkize`.
12+
-# See https://github.com/RaRe-Technologies/gensim/pull/2800#discussion_r410890171
13+
-if os.name == 'nt' or (sys.platform == "darwin" and sys.version_info >= (3, 8)):
14+
- def chunkize(corpus, chunksize, maxsize=0, as_numpy=False):
15+
- """Split `corpus` into fixed-sized chunks, using :func:`~gensim.utils.chunkize_serial`.
16+
+def chunkize(corpus, chunksize, maxsize=0, as_numpy=False):
17+
+ """Split `corpus` into fixed-sized chunks, using :func:`~gensim.utils.chunkize_serial`.
18+
19+
- Parameters
20+
- ----------
21+
- corpus : iterable of object
22+
- An iterable.
23+
- chunksize : int
24+
- Split `corpus` into chunks of this size.
25+
- maxsize : int, optional
26+
- Ignored. For interface compatibility only.
27+
- as_numpy : bool, optional
28+
- Yield chunks as `np.ndarray` s instead of lists?
29+
-
30+
- Yields
31+
- ------
32+
- list OR np.ndarray
33+
- "chunksize"-ed chunks of elements from `corpus`.
34+
-
35+
- """
36+
- if maxsize > 0:
37+
- entity = "Windows" if os.name == 'nt' else "OSX with python3.8+"
38+
- warnings.warn("detected %s; aliasing chunkize to chunkize_serial" % entity)
39+
- for chunk in chunkize_serial(corpus, chunksize, as_numpy=as_numpy):
40+
- yield chunk
41+
-else:
42+
- def chunkize(corpus, chunksize, maxsize=0, as_numpy=False):
43+
- """Split `corpus` into fixed-sized chunks, using :func:`~gensim.utils.chunkize_serial`.
44+
-
45+
- Parameters
46+
- ----------
47+
- corpus : iterable of object
48+
- An iterable.
49+
- chunksize : int
50+
- Split `corpus` into chunks of this size.
51+
- maxsize : int, optional
52+
- If > 0, prepare chunks in a background process, filling a chunk queue of size at most `maxsize`.
53+
- as_numpy : bool, optional
54+
- Yield chunks as `np.ndarray` instead of lists?
55+
-
56+
- Yields
57+
- ------
58+
- list OR np.ndarray
59+
- "chunksize"-ed chunks of elements from `corpus`.
60+
+ Parameters
61+
+ ----------
62+
+ corpus : iterable of object
63+
+ An iterable.
64+
+ chunksize : int
65+
+ Split `corpus` into chunks of this size.
66+
+ maxsize : int, optional
67+
+ If > 0, prepare chunks in a background process, filling a chunk queue of size at most `maxsize`.
68+
+ as_numpy : bool, optional
69+
+ Yield chunks as `np.ndarray` instead of lists?
70+
71+
- Notes
72+
- -----
73+
- Each chunk is of length `chunksize`, except the last one which may be smaller.
74+
- A once-only input stream (`corpus` from a generator) is ok, chunking is done efficiently via itertools.
75+
+ Yields
76+
+ ------
77+
+ list OR np.ndarray
78+
+ "chunksize"-ed chunks of elements from `corpus`.
79+
80+
- If `maxsize > 0`, don't wait idly in between successive chunk `yields`, but rather keep filling a short queue
81+
- (of size at most `maxsize`) with forthcoming chunks in advance. This is realized by starting a separate process,
82+
- and is meant to reduce I/O delays, which can be significant when `corpus` comes from a slow medium
83+
- like HDD, database or network.
84+
+ Notes
85+
+ -----
86+
+ Each chunk is of length `chunksize`, except the last one which may be smaller.
87+
+ A once-only input stream (`corpus` from a generator) is ok, chunking is done efficiently via itertools.
88+
89+
- If `maxsize == 0`, don't fool around with parallelism and simply yield the chunksize
90+
- via :func:`~gensim.utils.chunkize_serial` (no I/O optimizations).
91+
+ If `maxsize > 0`, don't wait idly in between successive chunk `yields`, but rather keep filling a short queue
92+
+ (of size at most `maxsize`) with forthcoming chunks in advance. This is realized by starting a separate process,
93+
+ and is meant to reduce I/O delays, which can be significant when `corpus` comes from a slow medium
94+
+ like HDD, database or network.
95+
96+
- Yields
97+
- ------
98+
- list of object OR np.ndarray
99+
- Groups based on `iterable`
100+
+ If `maxsize == 0`, don't fool around with parallelism and simply yield the chunksize
101+
+ via :func:`~gensim.utils.chunkize_serial` (no I/O optimizations).
102+
103+
- """
104+
- assert chunksize > 0
105+
+ Yields
106+
+ ------
107+
+ list of object OR np.ndarray
108+
+ Groups based on `iterable`
109+
110+
+ """
111+
+ assert chunksize > 0
112+
+
113+
+ # 1. Evaluate the start method dynamically at runtime
114+
+ # 2. Only attempt multiprocessing if maxsize > 0 AND the method is 'fork'
115+
+ # See https://github.com/RaRe-Technologies/gensim/pull/2800#discussion_r410890171
116+
+ if maxsize > 0 and multiprocessing.get_start_method() == "fork":
117+
+ q = multiprocessing.Queue(maxsize=maxsize)
118+
+ worker = InputQueue(q, corpus, chunksize, maxsize=maxsize, as_numpy=as_numpy)
119+
+ worker.daemon = True
120+
+ worker.start()
121+
+ while True:
122+
+ chunk = [q.get(block=True)]
123+
+ if chunk[0] is None:
124+
+ break
125+
+ yield chunk.pop()
126+
+ else:
127+
+ # Fallback to serial processing
128+
if maxsize > 0:
129+
- q = multiprocessing.Queue(maxsize=maxsize)
130+
- worker = InputQueue(q, corpus, chunksize, maxsize=maxsize, as_numpy=as_numpy)
131+
- worker.daemon = True
132+
- worker.start()
133+
- while True:
134+
- chunk = [q.get(block=True)]
135+
- if chunk[0] is None:
136+
- break
137+
- yield chunk.pop()
138+
- else:
139+
- for chunk in chunkize_serial(corpus, chunksize, as_numpy=as_numpy):
140+
- yield chunk
141+
+ # We only warn if they asked for multiprocessing but we have to deny it
142+
+ current_method = multiprocessing.get_start_method()
143+
+ warnings.warn(f"start method is '{current_method}', not 'fork'; aliasing chunkize to chunkize_serial")
144+
+
145+
+ for chunk in chunkize_serial(corpus, chunksize, as_numpy=as_numpy):
146+
+ yield chunk
147+
148+
149+
def smart_extension(fname, ext):

0 commit comments

Comments
 (0)