fix: raise RLIMIT_NOFILE and use real cocoindex inflight env var (#306)

java_codebase_rag/_fdlimit.py

@@ -0,0 +1,48 @@
+"""Raise the process soft file-descriptor limit to avoid LanceDB EMFILE.
+
+LanceDB's merge-insert path opens many file handles concurrently; under the
+default OS soft ``RLIMIT_NOFILE`` (256 on macOS processes launched by GUI /
+launchd / IDE hosts, *not* the shell's raised limit) this exhausts file
+descriptors and surfaces as::
+
+    RuntimeError: lance error: LanceError(IO): ... Too many open files (os error 24)
+        lance-io-4.0.0/src/local.rs:133:24
+
+``raise_fd_limit`` raises the process's *own* soft limit toward its hard limit.
+``RLIMIT_NOFILE`` is inherited across ``fork``+``exec``, so every CocoIndex /
+``cocoindex-code`` child spawned afterwards inherits the headroom. This fixes the
+failure regardless of launch context (shell vs IDE vs MCP host) and regardless of
+Lance's internal IO concurrency.
+
+Never raise to ``RLIM_INFINITY`` — that breaks ``select()``/kqueue and Python
+selectors on macOS; ``cap`` bounds the target to a safe value.
+
+See https://github.com/HumanBean17/java-codebase-rag/issues/306
+"""
+
+from __future__ import annotations
+
+import resource
+
+# Safe ceiling well above LanceDB's appetite, comfortably below macOS libc
+# quirks. The hard limit caps it further if lower (locked-down servers).
+_DEFAULT_CAP = 65536
+
+
+def raise_fd_limit(cap: int = _DEFAULT_CAP) -> None:
+    """Raise this process's soft ``RLIMIT_NOFILE`` toward its hard limit.
+
+    Best-effort and silent: never raises. No-op where ``RLIMIT_NOFILE`` is
+    unsupported (Windows) or where the soft limit already meets ``min(hard, cap)``.
+    """
+    if not hasattr(resource, "RLIMIT_NOFILE"):
+        return
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    target = min(hard, cap)
+    if soft >= target:
+        return
+    try:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
+    except (ValueError, OSError):
+        # Best-effort: a locked-down environment shouldn't fail the run.
+        pass

java_codebase_rag/cli.py

@@ -21,6 +21,7 @@
     index_dir_has_existing_artifacts,
     resolve_operator_config,
 )
+from java_codebase_rag._fdlimit import raise_fd_limit
 from java_codebase_rag.pipeline import clip, run_build_ast_graph, run_cocoindex_drop, run_cocoindex_update, run_incremental_graph
 from java_ontology import VALID_UNRESOLVED_CALL_REASONS
 
@@ -902,6 +903,7 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def main(argv: list[str] | None = None) -> int:
+    raise_fd_limit()
     raw = list(argv if argv is not None else sys.argv[1:])
     if raw and raw[0] == "refresh":
         print(_REFRESH_DEPRECATION, file=sys.stderr)

java_codebase_rag/config.py

@@ -25,6 +25,27 @@
 ENV_DEBUG_CONTEXT = "JAVA_CODEBASE_RAG_DEBUG_CONTEXT"
 ENV_RUN_HEAVY = "JAVA_CODEBASE_RAG_RUN_HEAVY"
 
+# CocoIndex inflight-component throttle. CocoIndex's default is 1024 inflight
+# components (cocoindex/_internal/app.py: ``_ENV_MAX_INFLIGHT_COMPONENTS``),
+# which spawns enough concurrent LanceDB merge-inserts to exhaust OS file
+# descriptors under default ulimits -> "Too many open files (os error 24)".
+# NOTE: this is the REAL env var. An earlier fix (#293) set the non-existent
+# ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS`` — CocoIndex never reads it, so it was a
+# no-op and the EMFILE error recurred (#306).
+COCOINDEX_MAX_INFLIGHT_COMPONENTS_ENV = "COCOINDEX_MAX_INFLIGHT_COMPONENTS"
+COCOINDEX_DEFAULT_MAX_INFLIGHT_COMPONENTS = "256"
+
+
+def cocoindex_subprocess_env_defaults() -> dict[str, str]:
+    """Env defaults applied to every CocoIndex subprocess to bound concurrency.
+
+    Apply with ``env.setdefault(...)`` so a caller-provided (operator) value
+    always wins. See :issue:`306`.
+    """
+    return {
+        COCOINDEX_MAX_INFLIGHT_COMPONENTS_ENV: COCOINDEX_DEFAULT_MAX_INFLIGHT_COMPONENTS
+    }
+
 _DEFAULT_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 
 # Matches either $VAR or ${VAR} (POSIX shell variable syntax).

java_codebase_rag/pipeline.py

@@ -11,6 +11,7 @@
 
 from java_codebase_rag.cli_format import Spinner, is_noise_line, stderr_is_tty
 from java_codebase_rag.cli_progress import emit_vectors_finish, emit_vectors_start
+from java_codebase_rag.config import cocoindex_subprocess_env_defaults
 
 COCOINDEX_TARGET = "java_index_flow_lancedb.py:JavaCodeIndexLance"
 
@@ -128,10 +129,11 @@ def run_cocoindex_update(
             stdout="",
             stderr=f"java_index_flow_lancedb.py not found under {bd}",
         )
-    # Set CocoIndex concurrency limits to prevent "too many open files" error
-    # See: https://github.com/HumanBean17/java-codebase-rag/issues/293
+    # Cap CocoIndex concurrency to avoid EMFILE ("too many open files") under
+    # default OS fd limits. See: https://github.com/HumanBean17/java-codebase-rag/issues/306
     env = env.copy()
-    env.setdefault("COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS", "256")
+    for _k, _v in cocoindex_subprocess_env_defaults().items():
+        env.setdefault(_k, _v)
     cmd: list[str] = [str(exe), "update", COCOINDEX_TARGET]
     if full_reprocess:
         cmd.extend(["--full-reprocess", "-f"])

server.py

@@ -16,7 +16,9 @@
     emit_vectors_finish,
     emit_vectors_start,
 )
+from java_codebase_rag._fdlimit import raise_fd_limit
 from java_codebase_rag.config import (
+    cocoindex_subprocess_env_defaults,
     discover_project_root,
     emit_legacy_env_hints_if_present,
     resolved_sbert_model_for_process_env,
@@ -162,9 +164,10 @@ def _cocoindex_subprocess_env(project_root: Path) -> dict[str, str]:
     idx = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
     if idx:
         sub_env["JAVA_CODEBASE_RAG_INDEX_DIR"] = str(Path(idx).expanduser().resolve())
-    # Set CocoIndex concurrency limits to prevent "too many open files" error
-    # See: https://github.com/HumanBean17/java-codebase-rag/issues/293
-    sub_env.setdefault("COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS", "256")
+    # Cap CocoIndex concurrency to avoid EMFILE ("too many open files") under
+    # default OS fd limits. See: https://github.com/HumanBean17/java-codebase-rag/issues/306
+    for _k, _v in cocoindex_subprocess_env_defaults().items():
+        sub_env.setdefault(_k, _v)
     return sub_env
 
 
@@ -622,6 +625,7 @@ async def resolve(
 
 
 def main() -> None:
+    raise_fd_limit()
     emit_legacy_env_hints_if_present()
 
     # Load YAML config and apply embedding settings to environment

tests/test_config.py

@@ -239,3 +239,19 @@ def test_existing_behavior_unchanged(self, tmp_path, monkeypatch):
 
         # Also test that index_dir derives from source_root
         assert result.index_dir == tmp_path / ".java-codebase-rag"
+
+
+def test_cocoindex_subprocess_env_defaults_uses_real_inflight_env_var() -> None:
+    """The throttle must use CocoIndex's REAL env var name.
+
+    The earlier #293 "fix" set ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS``, an env
+    var CocoIndex never reads (it reads ``COCOINDEX_MAX_INFLIGHT_COMPONENTS``,
+    default 1024), so it was a no-op and the EMFILE error recurred (#306).
+    """
+    from java_codebase_rag.config import cocoindex_subprocess_env_defaults
+
+    defaults = cocoindex_subprocess_env_defaults()
+
+    assert defaults["COCOINDEX_MAX_INFLIGHT_COMPONENTS"] == "256"
+    # The bogus name from the broken #293 fix must NOT leak back in.
+    assert "COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS" not in defaults

tests/test_fd_limit.py

@@ -0,0 +1,78 @@
+"""Tests for ``raise_fd_limit`` — EMFILE / "too many open files" mitigation.
+
+LanceDB's merge-insert path opens many file handles concurrently; under the
+default OS soft ``RLIMIT_NOFILE`` (256 on macOS GUI/launchd-launched processes)
+this exhausts file descriptors -> ``Too many open files (os error 24)`` in
+``lance-io/local.rs``. ``raise_fd_limit`` raises the process's own soft limit
+toward its hard limit so cocoindex children (which inherit rlimits) get headroom.
+
+See https://github.com/HumanBean17/java-codebase-rag/issues/306
+"""
+
+from __future__ import annotations
+
+from java_codebase_rag import _fdlimit
+
+
+def test_raises_soft_limit_up_to_cap(monkeypatch):
+    """When soft < min(hard, cap), raise soft to the target and keep hard."""
+    monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 65536))
+    calls: list[tuple] = []
+    monkeypatch.setattr(
+        _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits))
+    )
+
+    _fdlimit.raise_fd_limit(cap=4096)
+
+    assert calls == [(_fdlimit.resource.RLIMIT_NOFILE, (4096, 65536))]
+
+
+def test_caps_target_at_hard_limit(monkeypatch):
+    """Never exceed the hard limit even when cap > hard."""
+    monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 1024))
+    calls: list[tuple] = []
+    monkeypatch.setattr(
+        _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits))
+    )
+
+    _fdlimit.raise_fd_limit(cap=65536)  # target = min(1024, 65536) = 1024
+
+    assert calls == [(_fdlimit.resource.RLIMIT_NOFILE, (1024, 1024))]
+
+
+def test_noop_when_soft_already_at_or_above_target(monkeypatch):
+    """No setrlimit call when the soft limit is already high enough."""
+    monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (1048576, 1048576))
+    calls: list[tuple] = []
+    monkeypatch.setattr(
+        _fdlimit.resource, "setrlimit", lambda rlim, limits: calls.append((rlim, limits))
+    )
+
+    _fdlimit.raise_fd_limit(cap=65536)
+
+    assert calls == []
+
+
+def test_noop_when_rlimit_nofile_unsupported(monkeypatch):
+    """Windows-like host with no RLIMIT_NOFILE: no error, no setrlimit."""
+    monkeypatch.delattr(_fdlimit.resource, "RLIMIT_NOFILE")
+    calls: list[tuple] = []
+    monkeypatch.setattr(
+        _fdlimit.resource, "setrlimit", lambda *a, **k: calls.append((a, k))
+    )
+
+    _fdlimit.raise_fd_limit()  # must not raise
+
+    assert calls == []
+
+
+def test_swallows_setrlimit_errors(monkeypatch):
+    """Best-effort: a failing setrlimit must never propagate."""
+    monkeypatch.setattr(_fdlimit.resource, "getrlimit", lambda _rlim: (256, 65536))
+
+    def boom(rlim, limits):
+        raise OSError("permission denied")
+
+    monkeypatch.setattr(_fdlimit.resource, "setrlimit", boom)
+
+    _fdlimit.raise_fd_limit(cap=4096)  # must not raise

tests/test_mcp_tools.py

@@ -94,3 +94,20 @@ def test_cocoindex_subprocess_env_sets_project_root(monkeypatch, tmp_path) -> No
     env = server._cocoindex_subprocess_env(resolved)
     assert env["JAVA_CODEBASE_RAG_SOURCE_ROOT"] == str(resolved)
     assert env["PRESERVE_ME_FOR_SUBPROCESS"] == "ok"
+
+
+def test_cocoindex_subprocess_env_applies_inflight_default(monkeypatch, tmp_path) -> None:
+    """The MCP-triggered cocoindex subprocess must carry the real inflight throttle.
+
+    Regression guard for #306: the throttle env var must be CocoIndex's real
+    ``COCOINDEX_MAX_INFLIGHT_COMPONENTS`` (default 1024 -> capped to 256), not the
+    non-existent ``COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS`` from the broken #293 fix.
+    """
+    import server
+
+    proj = tmp_path / "external-java-repo"
+    proj.mkdir()
+    env = server._cocoindex_subprocess_env(proj.resolve())
+
+    assert env["COCOINDEX_MAX_INFLIGHT_COMPONENTS"] == "256"
+    assert "COCOINDEX_SOURCE_MAX_INFLIGHT_ROWS" not in env