Skip to content

Commit 3681ddd

Browse files
committed
build: move Windows toolchain to .dx/windows/
Store the downloaded toolchain in .dx/windows/ instead of papermario-dx-windows/ at the repo root, keeping ephemeral data together in one gitignored directory. Also fix clangd index portability: build in YAML format and replace absolute CI paths with a $$ROOT$$ placeholder, which configure.py substitutes with the local project root at download time.
1 parent 98a568f commit 3681ddd

8 files changed

Lines changed: 197 additions & 120 deletions

File tree

.github/workflows/release-dx.yaml

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -37,30 +37,12 @@ jobs:
3737
with:
3838
string-case: paramCase
3939
- name: Build clangd index
40-
run: |
41-
nix develop --command bash -c '
42-
./configure
43-
ninja -t compdb > compile_commands.json
44-
'
45-
46-
TAG=$(curl -s https://api.github.com/repos/clangd/clangd/releases/latest | jq -r .tag_name)
47-
curl -L -o clangd_indexing_tools.zip \
48-
"https://github.com/clangd/clangd/releases/download/$TAG/clangd_indexing_tools-linux-$TAG.zip"
49-
unzip clangd_indexing_tools.zip
50-
INDEXER=$(find . -name 'clangd-indexer' -type f)
51-
chmod +x "$INDEXER"
52-
53-
# Build index in YAML format so we can make paths portable
54-
"$INDEXER" --format=yaml --executor=all-TUs compile_commands.json > papermario-dx.yaml
55-
56-
# Replace absolute CI paths with $$ROOT$$ placeholder
57-
sed -i "s|$(pwd)/|\\$\\$ROOT\\$\\$/|g" papermario-dx.yaml
58-
mv papermario-dx.yaml papermario-dx.idx
40+
run: nix build .#clangd-index --out-link result-clangd-index
5941
- name: Copy patch file
6042
run: cp result/papermario.bps ${{ steps.repo-name.outputs.repository-name }}.bps
6143
- uses: softprops/action-gh-release@v2
6244
with:
6345
files: |
6446
${{ steps.repo-name.outputs.repository-name }}.bps
6547
result-toolchain/papermario-dx-windows.zip
66-
papermario-dx.idx
48+
result-clangd-index/papermario-dx.idx.gz

.github/workflows/release-nightly.yaml

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -34,32 +34,13 @@ jobs:
3434
- name: Build Windows toolchain
3535
run: nix build .#windows-toolchain --out-link result-toolchain
3636
- name: Build clangd index
37-
run: |
38-
nix develop --command bash -c '
39-
./configure
40-
ninja -t compdb > compile_commands.json
41-
'
42-
43-
# Fetch clangd-indexer
44-
TAG=$(curl -s https://api.github.com/repos/clangd/clangd/releases/latest | jq -r .tag_name)
45-
curl -L -o clangd_indexing_tools.zip \
46-
"https://github.com/clangd/clangd/releases/download/$TAG/clangd_indexing_tools-linux-$TAG.zip"
47-
unzip clangd_indexing_tools.zip
48-
INDEXER=$(find . -name 'clangd-indexer' -type f)
49-
chmod +x "$INDEXER"
50-
51-
# Build index in YAML format so we can make paths portable
52-
"$INDEXER" --format=yaml --executor=all-TUs compile_commands.json > papermario-dx.yaml
53-
54-
# Replace absolute CI paths with $$ROOT$$ placeholder
55-
sed -i "s|$(pwd)/|\\$\\$ROOT\\$\\$/|g" papermario-dx.yaml
56-
mv papermario-dx.yaml papermario-dx.idx
37+
run: nix build .#clangd-index --out-link result-clangd-index
5738
- uses: softprops/action-gh-release@v2
5839
with:
5940
tag_name: dx-nightly
6041
files: |
6142
result-toolchain/papermario-dx-windows.zip
62-
papermario-dx.idx
43+
result-clangd-index/papermario-dx.idx.gz
6344
- name: Notify docs repo
6445
run: |
6546
gh api repos/star-haven/docs/dispatches \

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ ctx.c.m2c
1818
/data2c
1919
.nix-profile*
2020
/result
21-
/papermario-dx-windows/
2221

2322
# Build artifacts
2423
build.ninja

build.bat

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,12 @@
22
setlocal enabledelayedexpansion
33

44
set "REPO=bates64/papermario-dx"
5-
set "TOOLCHAIN_DIR=%~dp0papermario-dx-windows"
6-
set "TOOLCHAIN_ZIP=%~dp0papermario-dx-windows.zip"
5+
set "DX_DIR=%~dp0.dx"
6+
set "TOOLCHAIN_DIR=%DX_DIR%\windows"
7+
set "TOOLCHAIN_ZIP=%DX_DIR%\papermario-dx-windows.zip"
8+
9+
:: Ensure .dx directory exists
10+
if not exist "%DX_DIR%" mkdir "%DX_DIR%"
711

812
:: Check for git
913
where git >nul 2>nul
@@ -30,8 +34,8 @@ del "%TEMP%\dx-tag-hash.txt" 2>nul
3034
:: Check if toolchain needs downloading
3135
set "NEED_DOWNLOAD=0"
3236
if not exist "%TOOLCHAIN_DIR%\bin\ninja.exe" set "NEED_DOWNLOAD=1"
33-
if exist "%TOOLCHAIN_DIR%\.tag" (
34-
set /p CURRENT_TAG=<"%TOOLCHAIN_DIR%\.tag"
37+
if exist "%DX_DIR%\windows-tag" (
38+
set /p CURRENT_TAG=<"%DX_DIR%\windows-tag"
3539
if not "!CURRENT_TAG!"=="%TAG_HASH%" set "NEED_DOWNLOAD=1"
3640
) else (
3741
if exist "%TOOLCHAIN_DIR%" set "NEED_DOWNLOAD=1"
@@ -52,17 +56,18 @@ if "%NEED_DOWNLOAD%"=="1" (
5256
exit /b 1
5357
)
5458

55-
:: Extract
59+
:: Extract (zip contains papermario-dx-windows/ dir, rename to .dx/windows/)
5660
echo Extracting toolchain...
57-
tar -xf "%TOOLCHAIN_ZIP%" -C "%~dp0."
61+
tar -xf "%TOOLCHAIN_ZIP%" -C "%DX_DIR%"
5862
if errorlevel 1 (
5963
echo Error: failed to extract toolchain.
6064
exit /b 1
6165
)
66+
ren "%DX_DIR%\papermario-dx-windows" windows
6267
del "%TOOLCHAIN_ZIP%"
6368

6469
:: Record the tag commit hash so we can detect updates (including force-moved tags)
65-
echo %TAG_HASH%> "%TOOLCHAIN_DIR%\.tag"
70+
echo %TAG_HASH%> "%DX_DIR%\windows-tag"
6671
)
6772

6873
:: Set up PATH

flake.nix

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,59 @@
115115
${pkgs.lib.optionalString pkgs.stdenv.isLinux
116116
"flips --create --bps ${baseRom} ver/us/build/papermario.z64 $out/papermario.bps"}
117117
'';
118+
119+
clangdIndexingTools = pkgs.callPackage ./tools/clangd-indexing-tools.nix {};
120+
clangdIndex = pkgs.runCommand "papermario-dx-clangd-index" {
121+
nativeBuildInputs = [
122+
pkgsCross.stdenv.cc
123+
binutils2_39
124+
pkgs.python3
125+
pkgs.python3Packages.pip
126+
pkgs.python3Packages.virtualenv
127+
pkgs.ninja
128+
pkgs.gcc
129+
pkgs.git
130+
pkgs.libyaml
131+
pkgs.iconv
132+
(pkgs.callPackage ./tools/pigment64.nix {})
133+
(pkgs.callPackage ./tools/crunch64.nix {})
134+
clangdIndexingTools
135+
];
136+
NIX_HARDENING_ENABLE = "";
137+
} ''
138+
cp -r --no-preserve=mode ${self} build
139+
chmod -R u+w build
140+
find build -name '*.py' -exec chmod +x {} +
141+
cd build
142+
git init --quiet
143+
cp ${baseRom} ver/us/baserom.z64
144+
patchShebangs --build tools/
145+
146+
virtualenv venv --quiet
147+
source venv/bin/activate
148+
pip install --no-index --find-links=${pythonDeps} -r requirements.txt --quiet
149+
150+
export PAPERMARIO_LD="${binutils2_39}/bin/mips-linux-gnu-ld"
151+
python3 tools/build/configure.py --no-ccache
152+
ninja
153+
154+
# Build YAML index (clangd reads YAML natively) and replace the
155+
# build directory with a $$ROOT$$ placeholder for portability.
156+
clangd-indexer --format=yaml --executor=all-TUs compile_commands.json \
157+
| sed "s|$(pwd)/|\$\$ROOT\$\$/|g" \
158+
| gzip > papermario-dx.idx.gz
159+
160+
mkdir -p $out
161+
mv papermario-dx.idx.gz $out/papermario-dx.idx.gz
162+
'';
163+
118164
in {
119165
packages = {
120166
default = linuxRom;
121167
} // pkgs.lib.optionalAttrs (system == "x86_64-linux") {
122168
windows-toolchain = windowsToolchain;
123169
windows-rom = windowsToolchain.passthru.wineRom;
170+
clangd-index = clangdIndex;
124171
};
125172

126173
checks = pkgs.lib.optionalAttrs (system == "x86_64-linux") {

tools/build/clangd_index.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""Download and configure a pre-built clangd index from GitHub releases."""
2+
3+
import gzip
4+
import subprocess
5+
import urllib.request
6+
import urllib.error
7+
from pathlib import Path
8+
9+
10+
def exec_shell(command):
11+
ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
12+
return ret.stdout
13+
14+
15+
def fetch_clangd_index(root: Path):
16+
"""Fetch the clangd index for the nearest dx-* tag and configure .clangd."""
17+
tag = exec_shell(["git", "describe", "--tags", "--abbrev=0", "--match", "dx-*"]).strip()
18+
if not tag:
19+
return
20+
21+
dx_dir = root / ".dx"
22+
dx_dir.mkdir(exist_ok=True)
23+
idx_path = dx_dir / "papermario-dx.idx"
24+
tag_hash = exec_shell(["git", "rev-parse", f"{tag}^{{}}"]).strip()
25+
tag_file = dx_dir / "configure-tag"
26+
27+
# Check if we already have the index for this tag
28+
need_download = True
29+
if idx_path.exists() and tag_file.exists():
30+
current_tag = tag_file.read_text().strip()
31+
if current_tag == tag_hash:
32+
need_download = False
33+
34+
if need_download:
35+
repo = "bates64/papermario-dx"
36+
url = f"https://github.com/{repo}/releases/download/{tag}/papermario-dx.idx"
37+
print(f"configure: downloading clangd index for {tag}...")
38+
try:
39+
gz_path = idx_path.with_suffix(".idx.gz")
40+
url_gz = url.replace(".idx", ".idx.gz")
41+
urllib.request.urlretrieve(url_gz, str(gz_path))
42+
# Decompress and replace $$ROOT$$ with local project root
43+
abs_root = str(root.resolve())
44+
with gzip.open(gz_path, "rt") as f:
45+
content = f.read()
46+
content = content.replace("$$ROOT$$", abs_root)
47+
idx_path.write_text(content)
48+
gz_path.unlink()
49+
tag_file.write_text(tag_hash + "\n")
50+
print("configure: clangd index downloaded")
51+
except urllib.error.HTTPError as e:
52+
print(f"configure: clangd index not available for {tag} ({e.code}), skipping")
53+
except Exception as e:
54+
print(f"configure: failed to download clangd index: {e}")
55+
56+
# Update .clangd config with index path
57+
if idx_path.exists():
58+
_update_clangd_config(root, idx_path)
59+
60+
61+
def _update_clangd_config(root: Path, idx_path: Path):
62+
"""Add or update the Index.External section in .clangd."""
63+
clangd_path = root / ".clangd"
64+
abs_idx = str(idx_path.resolve())
65+
abs_root = str(root.resolve()) + "/"
66+
67+
# Read existing config, preserving other sections
68+
existing_lines = []
69+
if clangd_path.exists():
70+
with open(clangd_path) as f:
71+
existing_lines = f.readlines()
72+
73+
# Remove any existing Index section
74+
filtered = []
75+
in_index = False
76+
for line in existing_lines:
77+
if line.rstrip() == "Index:":
78+
in_index = True
79+
continue
80+
if in_index and (line.startswith(" ") or line.startswith("\t")):
81+
continue
82+
in_index = False
83+
filtered.append(line)
84+
85+
# Append Index.External section
86+
if filtered and not filtered[-1].endswith("\n"):
87+
filtered.append("\n")
88+
filtered.append("Index:\n")
89+
filtered.append(" External:\n")
90+
filtered.append(f" File: {abs_idx}\n")
91+
filtered.append(f" MountPoint: {abs_root}\n")
92+
93+
with open(clangd_path, "w") as f:
94+
f.writelines(filtered)

tools/build/configure.py

Lines changed: 2 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1454,78 +1454,9 @@ def make_current(self, ninja: ninja_syntax.Writer):
14541454
ninja.default("all")
14551455

14561456
# Fetch pre-built clangd index from the matching dx-* GitHub release.
1457-
# This gives clangd project-wide symbols without needing a full background index.
14581457
try:
1459-
import urllib.request
1460-
import urllib.error
1461-
1462-
tag = exec_shell(["git", "describe", "--tags", "--abbrev=0", "--match", "dx-*"]).strip()
1463-
if tag:
1464-
dx_dir = ROOT / ".dx"
1465-
dx_dir.mkdir(exist_ok=True)
1466-
idx_path = dx_dir / "papermario-dx.idx"
1467-
tag_hash = exec_shell(["git", "rev-parse", f"{tag}^{{}}"]).strip()
1468-
tag_file = dx_dir / "configure-tag"
1469-
1470-
# Check if we already have the index for this tag
1471-
need_download = True
1472-
if idx_path.exists() and tag_file.exists():
1473-
current_tag = tag_file.read_text().strip()
1474-
if current_tag == tag_hash:
1475-
need_download = False
1476-
1477-
if need_download:
1478-
repo = "bates64/papermario-dx"
1479-
url = f"https://github.com/{repo}/releases/download/{tag}/papermario-dx.idx"
1480-
print(f"configure: downloading clangd index for {tag}...")
1481-
try:
1482-
urllib.request.urlretrieve(url, str(idx_path))
1483-
# Replace $$ROOT$$ placeholder with local project root
1484-
abs_root = str(ROOT.resolve())
1485-
idx_content = idx_path.read_text()
1486-
idx_content = idx_content.replace("$$ROOT$$", abs_root)
1487-
idx_path.write_text(idx_content)
1488-
tag_file.write_text(tag_hash + "\n")
1489-
print("configure: clangd index downloaded")
1490-
except urllib.error.HTTPError as e:
1491-
print(f"configure: clangd index not available for {tag} ({e.code}), skipping")
1492-
except Exception as e:
1493-
print(f"configure: failed to download clangd index: {e}")
1494-
1495-
# Update .clangd config with index path
1496-
if idx_path.exists():
1497-
clangd_path = ROOT / ".clangd"
1498-
abs_idx = str(idx_path.resolve())
1499-
abs_root = str(ROOT.resolve()) + "/"
1500-
1501-
# Read existing config, preserving other sections
1502-
existing_lines = []
1503-
if clangd_path.exists():
1504-
with open(clangd_path) as f:
1505-
existing_lines = f.readlines()
1506-
1507-
# Remove any existing Index section
1508-
filtered = []
1509-
in_index = False
1510-
for line in existing_lines:
1511-
if line.rstrip() == "Index:":
1512-
in_index = True
1513-
continue
1514-
if in_index and (line.startswith(" ") or line.startswith("\t")):
1515-
continue
1516-
in_index = False
1517-
filtered.append(line)
1518-
1519-
# Append Index.External section
1520-
if filtered and not filtered[-1].endswith("\n"):
1521-
filtered.append("\n")
1522-
filtered.append("Index:\n")
1523-
filtered.append(" External:\n")
1524-
filtered.append(f" File: {abs_idx}\n")
1525-
filtered.append(f" MountPoint: {abs_root}\n")
1526-
1527-
with open(clangd_path, "w") as f:
1528-
f.writelines(filtered)
1458+
from clangd_index import fetch_clangd_index
1459+
fetch_clangd_index(ROOT)
15291460
except Exception:
15301461
pass
15311462

tools/clangd-indexing-tools.nix

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{ lib, stdenv, fetchurl, autoPatchelfHook, libgcc, unzip }:
2+
3+
let
4+
version = "21.1.8";
5+
systems = {
6+
x86_64-linux = {
7+
url = "https://github.com/clangd/clangd/releases/download/${version}/clangd_indexing_tools-linux-${version}.zip";
8+
hash = "sha256-fFoSY1zp4/B5B4kxGgvNPTIFRRz6jJjIsj0FDIIBkyA=";
9+
};
10+
};
11+
platform = systems.${stdenv.hostPlatform.system}
12+
or (throw "clangd-indexing-tools: unsupported system ${stdenv.hostPlatform.system}");
13+
in
14+
stdenv.mkDerivation {
15+
pname = "clangd-indexing-tools";
16+
inherit version;
17+
18+
src = fetchurl {
19+
inherit (platform) url hash;
20+
};
21+
22+
sourceRoot = "clangd_${version}";
23+
24+
nativeBuildInputs = [ autoPatchelfHook unzip ];
25+
buildInputs = [ stdenv.cc.cc.lib libgcc ];
26+
27+
installPhase = ''
28+
mkdir -p $out/bin
29+
cp bin/* $out/bin/
30+
'';
31+
32+
meta = {
33+
description = "clangd indexing tools (clangd-indexer, clangd-index-server)";
34+
homepage = "https://github.com/clangd/clangd";
35+
license = lib.licenses.asl20;
36+
platforms = builtins.attrNames systems;
37+
};
38+
}

0 commit comments

Comments
 (0)