From c2d4d2a452ff7ade9bd607c51b5fc2617164cebd Mon Sep 17 00:00:00 2001 From: Dmitry Teryaev Date: Sat, 13 Jun 2026 17:02:41 +0300 Subject: [PATCH] fix: raise RLIMIT_NOFILE and use real cocoindex inflight env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #293 fix (#300) set COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS — an env var CocoIndex never reads. The real semaphore var is COCOINDEX_MAX_INFLIGHT_COMPONENTS (default 1024, see cocoindex/_internal/app.py), so the throttle was a no-op and the EMFILE "Too many open files (os error 24)" recurred (#306). Layer A (correctness): centralize the throttle in cocoindex_subprocess_env_defaults() using the real env var; both cocoindex subprocess sites (pipeline.run_cocoindex_update + server._cocoindex_subprocess_env) apply it via setdefault so an operator override still wins. Layer B (deterministic): raise_fd_limit() raises the process soft RLIMIT_NOFILE toward its hard limit (capped 65536, never infinity) at cli.main / server.main startup. rlimits are inherited across fork+exec, so cocoindex children get headroom regardless of launch context — macOS GUI/launchd/IDE-launched processes inherit a 256 FD ceiling, not the shell's raised limit, which is why the error recurred even on hosts whose terminal shows a high ulimit. No ontology/schema/re-index impact. Fixes #306. Co-Authored-By: Claude --- java_codebase_rag/_fdlimit.py | 48 +++++++++++++++++++++ java_codebase_rag/cli.py | 2 + java_codebase_rag/config.py | 21 ++++++++++ java_codebase_rag/pipeline.py | 8 ++-- server.py | 10 +++-- tests/test_config.py | 16 +++++++ tests/test_fd_limit.py | 78 +++++++++++++++++++++++++++++++++++ tests/test_mcp_tools.py | 17 ++++++++ 8 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 java_codebase_rag/_fdlimit.py create mode 100644 tests/test_fd_limit.py diff --git a/java_codebase_rag/_fdlimit.py b/java_codebase_rag/_fdlimit.py new file mode 100644 index 00000000..b5ee655f --- /dev/null +++ b/java_codebase_rag/_fdlimit.py @@ -0,0 +1,48 @@ +"""Raise the process soft file-descriptor limit to avoid LanceDB EMFILE. + +LanceDB's merge-insert path opens many file handles concurrently; under the +default OS soft ``RLIMIT_NOFILE`` (256 on macOS processes launched by GUI / +launchd / IDE hosts, *not* the shell's raised limit) this exhausts file +descriptors and surfaces as:: + + RuntimeError: lance error: LanceError(IO): ... Too many open files (os error 24) + lance-io-4.0.0/src/local.rs:133:24 + +``raise_fd_limit`` raises the process's *own* soft limit toward its hard limit. +``RLIMIT_NOFILE`` is inherited across ``fork``+``exec``, so every CocoIndex / +``cocoindex-code`` child spawned afterwards inherits the headroom. This fixes the +failure regardless of launch context (shell vs IDE vs MCP host) and regardless of +Lance's internal IO concurrency. + +Never raise to ``RLIM_INFINITY`` — that breaks ``select()``/kqueue and Python +selectors on macOS; ``cap`` bounds the target to a safe value. + +See https://github.com/HumanBean17/java-codebase-rag/issues/306 +""" + +from __future__ import annotations + +import resource + +# Safe ceiling well above LanceDB's appetite, comfortably below macOS libc +# quirks. The hard limit caps it further if lower (locked-down servers). +_DEFAULT_CAP = 65536 + + +def raise_fd_limit(cap: int = _DEFAULT_CAP) -> None: + """Raise this process's soft ``RLIMIT_NOFILE`` toward its hard limit. + + Best-effort and silent: never raises. No-op where ``RLIMIT_NOFILE`` is + unsupported (Windows) or where the soft limit already meets ``min(hard, cap)``. + """ + if not hasattr(resource, "RLIMIT_NOFILE"): + return + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + target = min(hard, cap) + if soft >= target: + return + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard)) + except (ValueError, OSError): + # Best-effort: a locked-down environment shouldn't fail the run. + pass diff --git a/java_codebase_rag/cli.py b/java_codebase_rag/cli.py index b288655e..2a15aa07 100644 --- a/java_codebase_rag/cli.py +++ b/java_codebase_rag/cli.py @@ -21,6 +21,7 @@ index_dir_has_existing_artifacts, resolve_operator_config, ) +from java_codebase_rag._fdlimit import raise_fd_limit from java_codebase_rag.pipeline import clip, run_build_ast_graph, run_cocoindex_drop, run_cocoindex_update, run_incremental_graph from java_ontology import VALID_UNRESOLVED_CALL_REASONS @@ -902,6 +903,7 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: + raise_fd_limit() raw = list(argv if argv is not None else sys.argv[1:]) if raw and raw[0] == "refresh": print(_REFRESH_DEPRECATION, file=sys.stderr) diff --git a/java_codebase_rag/config.py b/java_codebase_rag/config.py index bb05d9a7..6603dbc3 100644 --- a/java_codebase_rag/config.py +++ b/java_codebase_rag/config.py @@ -25,6 +25,27 @@ ENV_DEBUG_CONTEXT = "JAVA_CODEBASE_RAG_DEBUG_CONTEXT" ENV_RUN_HEAVY = "JAVA_CODEBASE_RAG_RUN_HEAVY" +# CocoIndex inflight-component throttle. CocoIndex's default is 1024 inflight +# components (cocoindex/_internal/app.py: ``_ENV_MAX_INFLIGHT_COMPONENTS``), +# which spawns enough concurrent LanceDB merge-inserts to exhaust OS file +# descriptors under default ulimits -> "Too many open files (os error 24)". +# NOTE: this is the REAL env var. An earlier fix (#293) set the non-existent +# ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS`` — CocoIndex never reads it, so it was a +# no-op and the EMFILE error recurred (#306). +COCOINDEX_MAX_INFLIGHT_COMPONENTS_ENV = "COCOINDEX_MAX_INFLIGHT_COMPONENTS" +COCOINDEX_DEFAULT_MAX_INFLIGHT_COMPONENTS = "256" + + +def cocoindex_subprocess_env_defaults() -> dict[str, str]: + """Env defaults applied to every CocoIndex subprocess to bound concurrency. + + Apply with ``env.setdefault(...)`` so a caller-provided (operator) value + always wins. See :issue:`306`. + """ + return { + COCOINDEX_MAX_INFLIGHT_COMPONENTS_ENV: COCOINDEX_DEFAULT_MAX_INFLIGHT_COMPONENTS + } + _DEFAULT_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2" # Matches either $VAR or ${VAR} (POSIX shell variable syntax). diff --git a/java_codebase_rag/pipeline.py b/java_codebase_rag/pipeline.py index f37bfe81..d75e04d5 100644 --- a/java_codebase_rag/pipeline.py +++ b/java_codebase_rag/pipeline.py @@ -11,6 +11,7 @@ from java_codebase_rag.cli_format import Spinner, is_noise_line, stderr_is_tty from java_codebase_rag.cli_progress import emit_vectors_finish, emit_vectors_start +from java_codebase_rag.config import cocoindex_subprocess_env_defaults COCOINDEX_TARGET = "java_index_flow_lancedb.py:JavaCodeIndexLance" @@ -128,10 +129,11 @@ def run_cocoindex_update( stdout="", stderr=f"java_index_flow_lancedb.py not found under {bd}", ) - # Set CocoIndex concurrency limits to prevent "too many open files" error - # See: https://github.com/HumanBean17/java-codebase-rag/issues/293 + # Cap CocoIndex concurrency to avoid EMFILE ("too many open files") under + # default OS fd limits. See: https://github.com/HumanBean17/java-codebase-rag/issues/306 env = env.copy() - env.setdefault("COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS", "256") + for _k, _v in cocoindex_subprocess_env_defaults().items(): + env.setdefault(_k, _v) cmd: list[str] = [str(exe), "update", COCOINDEX_TARGET] if full_reprocess: cmd.extend(["--full-reprocess", "-f"]) diff --git a/server.py b/server.py index 5ac83dd1..bbfbea14 100644 --- a/server.py +++ b/server.py @@ -16,7 +16,9 @@ emit_vectors_finish, emit_vectors_start, ) +from java_codebase_rag._fdlimit import raise_fd_limit from java_codebase_rag.config import ( + cocoindex_subprocess_env_defaults, discover_project_root, emit_legacy_env_hints_if_present, resolved_sbert_model_for_process_env, @@ -162,9 +164,10 @@ def _cocoindex_subprocess_env(project_root: Path) -> dict[str, str]: idx = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip() if idx: sub_env["JAVA_CODEBASE_RAG_INDEX_DIR"] = str(Path(idx).expanduser().resolve()) - # Set CocoIndex concurrency limits to prevent "too many open files" error - # See: https://github.com/HumanBean17/java-codebase-rag/issues/293 - sub_env.setdefault("COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS", "256") + # Cap CocoIndex concurrency to avoid EMFILE ("too many open files") under + # default OS fd limits. See: https://github.com/HumanBean17/java-codebase-rag/issues/306 + for _k, _v in cocoindex_subprocess_env_defaults().items(): + sub_env.setdefault(_k, _v) return sub_env @@ -622,6 +625,7 @@ async def resolve( def main() -> None: + raise_fd_limit() emit_legacy_env_hints_if_present() # Load YAML config and apply embedding settings to environment diff --git a/tests/test_config.py b/tests/test_config.py index b9403fb4..4b2aefc8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -239,3 +239,19 @@ def test_existing_behavior_unchanged(self, tmp_path, monkeypatch): # Also test that index_dir derives from source_root assert result.index_dir == tmp_path / ".java-codebase-rag" + + +def test_cocoindex_subprocess_env_defaults_uses_real_inflight_env_var() -> None: + """The throttle must use CocoIndex's REAL env var name. + + The earlier #293 "fix" set ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS``, an env + var CocoIndex never reads (it reads ``COCOINDEX_MAX_INFLIGHT_COMPONENTS``, + default 1024), so it was a no-op and the EMFILE error recurred (#306). + """ + from java_codebase_rag.config import cocoindex_subprocess_env_defaults + + defaults = cocoindex_subprocess_env_defaults() + + assert defaults["COCOINDEX_MAX_INFLIGHT_COMPONENTS"] == "256" + # The bogus name from the broken #293 fix must NOT leak back in. + assert "COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS" not in defaults diff --git a/tests/test_fd_limit.py b/tests/test_fd_limit.py new file mode 100644 index 00000000..b3259598 --- /dev/null +++ b/tests/test_fd_limit.py @@ -0,0 +1,78 @@ +"""Tests for ``raise_fd_limit`` — EMFILE / "too many open files" mitigation. + +LanceDB's merge-insert path opens many file handles concurrently; under the +default OS soft ``RLIMIT_NOFILE`` (256 on macOS GUI/launchd-launched processes) +this exhausts file descriptors -> ``Too many open files (os error 24)`` in +``lance-io/local.rs``. ``raise_fd_limit`` raises the process's own soft limit +toward its hard limit so cocoindex children (which inherit rlimits) get headroom. + +See https://github.com/HumanBean17/java-codebase-rag/issues/306 +""" + +from __future__ import annotations + +from java_codebase_rag import _fdlimit + + +def test_raises_soft_limit_up_to_cap(monkeypatch): + """When soft < min(hard, cap), raise soft to the target and keep hard.""" + monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 65536)) + calls: list[tuple] = [] + monkeypatch.setattr( + _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits)) + ) + + _fdlimit.raise_fd_limit(cap=4096) + + assert calls == [(_fdlimit.resource.RLIMIT_NOFILE, (4096, 65536))] + + +def test_caps_target_at_hard_limit(monkeypatch): + """Never exceed the hard limit even when cap > hard.""" + monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 1024)) + calls: list[tuple] = [] + monkeypatch.setattr( + _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits)) + ) + + _fdlimit.raise_fd_limit(cap=65536) # target = min(1024, 65536) = 1024 + + assert calls == [(_fdlimit.resource.RLIMIT_NOFILE, (1024, 1024))] + + +def test_noop_when_soft_already_at_or_above_target(monkeypatch): + """No setrlimit call when the soft limit is already high enough.""" + monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (1048576, 1048576)) + calls: list[tuple] = [] + monkeypatch.setattr( + _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits)) + ) + + _fdlimit.raise_fd_limit(cap=65536) + + assert calls == [] + + +def test_noop_when_rlimit_nofile_unsupported(monkeypatch): + """Windows-like host with no RLIMIT_NOFILE: no error, no setrlimit.""" + monkeypatch.delattr(_fdlimit.resource, "RLIMIT_NOFILE") + calls: list[tuple] = [] + monkeypatch.setattr( + _fdlimit.resource, "setrlimit", lambda *a, **k: calls.append((a, k)) + ) + + _fdlimit.raise_fd_limit() # must not raise + + assert calls == [] + + +def test_swallows_setrlimit_errors(monkeypatch): + """Best-effort: a failing setrlimit must never propagate.""" + monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 65536)) + + def boom(rlim, limits): + raise OSError("permission denied") + + monkeypatch.setattr(_fdlimit.resource, "setrlimit", boom) + + _fdlimit.raise_fd_limit(cap=4096) # must not raise diff --git a/tests/test_mcp_tools.py b/tests/test_mcp_tools.py index 2c1de8c0..fabbac5d 100644 --- a/tests/test_mcp_tools.py +++ b/tests/test_mcp_tools.py @@ -94,3 +94,20 @@ def test_cocoindex_subprocess_env_sets_project_root(monkeypatch, tmp_path) -> No env = server._cocoindex_subprocess_env(resolved) assert env["JAVA_CODEBASE_RAG_SOURCE_ROOT"] == str(resolved) assert env["PRESERVE_ME_FOR_SUBPROCESS"] == "ok" + + +def test_cocoindex_subprocess_env_applies_inflight_default(monkeypatch, tmp_path) -> None: + """The MCP-triggered cocoindex subprocess must carry the real inflight throttle. + + Regression guard for #306: the throttle env var must be CocoIndex's real + ``COCOINDEX_MAX_INFLIGHT_COMPONENTS`` (default 1024 -> capped to 256), not the + non-existent ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS`` from the broken #293 fix. + """ + import server + + proj = tmp_path / "external-java-repo" + proj.mkdir() + env = server._cocoindex_subprocess_env(proj.resolve()) + + assert env["COCOINDEX_MAX_INFLIGHT_COMPONENTS"] == "256" + assert "COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS" not in env