macos bug2 bisect/repro (no-compile, nightly wheel or run-artifact) #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: macos bug2 bisect/repro (no-compile, nightly wheel or run-artifact) | |
| # Reproduce + bisect the intermittent macOS Sparsity-cache heap corruption with NO | |
| # compilation: install a prebuilt wheel (nightly-release-3.8.0, or the fixed build's | |
| # artifact from a casadi/casadi run), load Julia/libMad (madnlp warmup), then run a | |
| # CONFIGURABLE subset of test modules x N parallel reps. Bisect by re-dispatching with | |
| # different `modules`. Crash = rc 139 (SIGSEGV) / 134 (SIGABRT). | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| modules: | |
| description: "comma-separated test modules to run (empty = full default set)" | |
| default: "" | |
| reps: | |
| description: "parallel reps" | |
| default: "6" | |
| fixed_run: | |
| description: "casadi/casadi run id to pull the FIXED casadi-osx-arm64-py311 artifact from (empty = use nightly wheel)" | |
| default: "" | |
| no_julia: | |
| description: "set to 'yes' to remove the madnlp plugin so libMad/Julia never loads (necessity test)" | |
| default: "" | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| reps: ${{ steps.g.outputs.reps }} | |
| steps: | |
| - id: g | |
| run: | | |
| N="${{ github.event.inputs.reps }}"; N="${N:-6}" | |
| echo "reps=[$(seq -s, 1 "$N")]" >> $GITHUB_OUTPUT | |
| repro: | |
| needs: setup-matrix | |
| runs-on: macos-14 | |
| env: | |
| HAVE_COMM: ${{ secrets.JGILLIS_RESTRICTED != '' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| rep: ${{ fromJson(needs.setup-matrix.outputs.reps) }} | |
| steps: | |
| - name: Checkout casadi release-3.8.0 (for test files + the subset runner) | |
| run: | | |
| set -eux | |
| curl -fsSL -o /tmp/cs.zip "https://github.com/casadi/casadi/releases/download/nightly-release-3.8.0/casadi-source-v3.8.0.zip" | |
| mkdir -p src && unzip -q /tmp/cs.zip -d src | |
| ls src/test/python/alltests.py | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| python-version: "3.11" | |
| activate-environment: test | |
| auto-update-conda: true | |
| - name: Python deps | |
| shell: bash -el {0} | |
| run: pip install -q --upgrade pip numpy scipy pandas looseversion pyright | |
| - name: Install casadi -- FIXED artifact if fixed_run set, else nightly wheel | |
| shell: bash -el {0} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| FIXED_RUN: ${{ github.event.inputs.fixed_run }} | |
| run: | | |
| set -eux | |
| if [ -n "$FIXED_RUN" ]; then | |
| # FIXED build: download the package-zip artifact from the casadi/casadi run | |
| gh run download "$FIXED_RUN" --repo casadi/casadi -n casadi-osx-arm64-py311 -D /tmp/fix | |
| unzip -q /tmp/fix/casadi-osx-arm64-py311.zip -d /tmp/fixpkg | |
| echo "CASADI_PKG=$(python -c "import os;print(os.path.abspath('/tmp/fixpkg'))")" >> $GITHUB_ENV | |
| else | |
| WHL=casadi-3.8.0-cp311-abi3-macosx_11_0_arm64.whl | |
| curl -fsSL -o "/tmp/$WHL" "https://github.com/casadi/casadi/releases/download/nightly-release-3.8.0/$WHL" | |
| pip install -q "/tmp/$WHL" | |
| echo "CASADI_PKG=" >> $GITHUB_ENV | |
| fi | |
| - name: Report whether the commercial_solvers token is available | |
| run: echo "HAVE_COMM=$HAVE_COMM" | |
| - name: commercial_solvers (real ORT/clarabel/madnlp -- the test-python-long env) | |
| uses: casadi/commercial_solvers@master | |
| with: | |
| token: ${{ secrets.JGILLIS_RESTRICTED }} | |
| env: | |
| JGILLIS_RESTRICTED: ${{ secrets.JGILLIS_RESTRICTED != '' }} | |
| if: ${{ env.HAVE_COMM == 'true' }} | |
| - name: POLYGLOT libMad (the one commercial_solvers actually uses) -> /usr/local/lib | |
| if: ${{ env.HAVE_COMM != 'true' }} | |
| run: | | |
| set -eux | |
| curl -fsSL -o /tmp/lm.zip "https://github.com/casadi/polyglot/releases/download/evergreen/libMad-osx-arm64.zip" | |
| mkdir -p /tmp/lm && unzip -q /tmp/lm.zip -d /tmp/lm | |
| sudo cp -R /tmp/lm/lib/ /usr/local/lib/ | |
| [ -d /tmp/lm/share ] && sudo cp -R /tmp/lm/share/ /usr/local/share/ || true | |
| - name: libomp (OpenMP) for the wheel | |
| run: brew install libomp || true | |
| - name: Download REAL ONNX Runtime (only when NOT using commercial_solvers) | |
| if: ${{ env.HAVE_COMM != 'true' }} | |
| run: | | |
| set -eux | |
| curl -fL -o "$RUNNER_TEMP/ort.tgz" https://github.com/microsoft/onnxruntime/releases/download/v1.22.0/onnxruntime-osx-arm64-1.22.0.tgz | |
| tar -xzf "$RUNNER_TEMP/ort.tgz" -C "$RUNNER_TEMP" | |
| ls -l "$RUNNER_TEMP/onnxruntime-osx-arm64-1.22.0/lib"/libonnxruntime* || true | |
| echo "ORT_LIB=$RUNNER_TEMP/onnxruntime-osx-arm64-1.22.0/lib" >> $GITHUB_ENV | |
| - name: pip onnx (enables the real-ORT runtime gate) | |
| shell: bash -el {0} | |
| run: pip install -q onnx | |
| - name: Fetch test data | |
| shell: bash -el {0} | |
| run: | | |
| set -eux | |
| cd src/test/python | |
| curl -fL -O https://github.com/casadi/testbot/releases/download/perpetual/serialize_3.5.5.zip && unzip -o serialize_3.5.5.zip || true | |
| cd ../data | |
| for f in VanDerPol2.fmu VanDerPol3.fmu vdp.fmu cstr.fmu car_t_fmu2.fmu; do curl -fL -O "https://github.com/casadi/testbot/releases/download/perpetual/$f" || true; done | |
| - name: Write the subset runner | |
| run: | | |
| cat > src/test/python/bisect_run.py <<'PY' | |
| import os, sys | |
| sys.argv = ['bisect', '--run_slow'] # helpers.py argparse reads this at import | |
| import unittest, importlib | |
| DEFAULT = ["mx","sx","typemaps","integration","ocp","nlp","implicitfunction","ad", | |
| "sparsity","linearsolver","matrix","conic","misc","function","tools", | |
| "simulator","vectortools","optistack","feasiblesqpmethod","serialize", | |
| "threads","daebuilder","graph_builder","pyright_stubs"] | |
| mods = [m for m in os.environ.get("CASADI_BISECT_MODULES","").split(",") if m] or DEFAULT | |
| import casadi as ca | |
| try: | |
| x = ca.MX.sym('x', 2) | |
| nlp = {'x': x, 'f': ca.sumsqr(x - 1)} | |
| ca.nlpsol('s', 'madnlp', nlp)(x0=[0.2, 0.3]) | |
| print("JULIA_WARMUP_OK", flush=True) | |
| except Exception as e: | |
| print("JULIA_WARMUP_FAIL", repr(e), flush=True) | |
| print("BISECT_MODULES", mods, flush=True) | |
| loader = unittest.TestLoader(); suite = unittest.TestSuite() | |
| for m in mods: | |
| suite.addTests(loader.loadTestsFromModule(importlib.import_module(m))) | |
| res = unittest.TextTestRunner(verbosity=1).run(suite) | |
| sys.exit(0 if res.wasSuccessful() else 1) | |
| PY | |
| - name: Run subset (rep ${{ matrix.rep }}) | |
| shell: bash -el {0} | |
| env: | |
| CASADI_BISECT_MODULES: ${{ github.event.inputs.modules }} | |
| NO_JULIA: ${{ github.event.inputs.no_julia }} | |
| PYTHONFAULTHANDLER: "1" | |
| run: | | |
| set +e | |
| if [ -n "$NO_JULIA" ]; then | |
| # Necessity test: remove the madnlp plugin so libMad/Julia is never loaded, | |
| # while keeping the rest of the commercial env (ORT/clarabel/TBB). | |
| CASDIR=$(python -c "import casadi,os;print(os.path.dirname(casadi.__file__))") | |
| ls "$CASDIR"/libcasadi_nlpsol_madnlp.* 2>/dev/null && rm -f "$CASDIR"/libcasadi_nlpsol_madnlp.* && echo "REMOVED madnlp plugin (no-Julia run)" | |
| fi | |
| if [ "$HAVE_COMM" = "true" ]; then | |
| # commercial_solvers already wired DYLD/env via $GITHUB_ENV (real libMad + | |
| # ORT + clarabel etc.) -- preserve it, just add libomp for the wheel. | |
| export DYLD_LIBRARY_PATH="$(brew --prefix libomp)/lib:${DYLD_LIBRARY_PATH:-}" | |
| else | |
| # fallback polyglot libMad: rely on /usr/local/lib default; don't override rpath | |
| unset DYLD_LIBRARY_PATH | |
| fi | |
| if [ -n "$CASADI_PKG" ]; then export PYTHONPATH="$CASADI_PKG"; fi | |
| # Enable core dumps so we can post-mortem the SIGABRT that Julia's handler masks. | |
| sudo mkdir -p /cores && sudo chmod 1777 /cores || true | |
| ulimit -c unlimited; ulimit -c | |
| cd src/test/python | |
| # Mirror test-python-long: load all real plugins first (commercial_solvers env) | |
| python -c "from casadi.tools import *; loadAllCompiledPlugins()" || echo "loadAllCompiledPlugins rc=$?" | |
| PYEXE=$(python -c 'import sys;print(sys.executable)') | |
| if [ -z "$CASADI_BISECT_MODULES" ]; then | |
| python -u alltests.py --run_slow | |
| else | |
| python -u bisect_run.py | |
| fi | |
| rc=$? | |
| echo "BISECT_RC=$rc (rep ${{ matrix.rep }})" | |
| if [ "$rc" = "139" ] || [ "$rc" = "134" ]; then echo "CRASH_DETECTED rc=$rc"; fi | |
| # Post-mortem the core (if any) to UNMASK Julia: classify malloc-corruption | |
| # (heap UAF/double-free) vs libc++abi-terminate (exception/unwind flavour) and | |
| # show the real casadi frames at the abort. | |
| CORE=$(ls -t /cores/core.* 2>/dev/null | head -1) | |
| if [ -n "$CORE" ]; then | |
| echo "=== CORE POST-MORTEM ($CORE) ===" | |
| lldb -b -o "bt all" -o "quit" "$PYEXE" -c "$CORE" 2>&1 \ | |
| | grep -iE "stop reason|frame #|casadi|malloc|free|terminat|__cxa|abort|Sparsity|SXNode|GenericShared|_Unwind|EXC_" | head -80 | |
| else | |
| echo "NO CORE produced" | |
| fi | |
| exit 0 |