Skip to content

Fix test runner not working for tests using relative paths in workspace projects #2236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dan-niles merged 6 commits into from
May 19, 2026
+73 −120
Merged

Fix test runner not working for tests using relative paths in workspace projects #2236

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2153d87
Fix test runner not working in workspaces
dan-niles May 13, 2026
2734c20
Clean up unused imports and validate projects before test runs
dan-niles May 13, 2026
287600a
Refactor test runner to use a single evaluation check for AI evaluations
dan-niles May 13, 2026
477314d
Address copilot suggestions
dan-niles May 14, 2026
587c4e5
Fix project path extraction for Windows in test item IDs
dan-niles May 16, 2026
6f1264f
Add error handling in runHandler by including error messages in test …
dan-niles May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 73 additions & 120 deletions workspaces/ballerina/ballerina-extension/src/features/test-explorer/runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,18 @@ import path from 'path';
import { EvaluationReportWebview } from '../../views/evaluation-report/webview';
import { captureGitState, createSnapshot, pinSnapshot, ensureEvalReportsGitignored } from '../../utils/git-utils';
import { quoteShellPath } from '../../utils/config';
import { cleanAndValidateProject } from '../config-generator/configGenerator';

/**
* Extract project path from a test item
* Test IDs have the format: test:${projectPath}:${fileName}:${functionName}
*/
function getProjectPathFromTestItem(test: TestItem): string | undefined {
if (isTestFunctionItem(test)) {
// Extract from test ID: test:${projectPath}:${fileName}:${functionName}
// ID: test:${projectPath}:${fileName}:${functionName} — projectPath may contain ':' on Windows
const parts = test.id.split(':');
if (parts.length >= 2 && parts[0] === 'test') {
return parts[1];
if (parts.length >= 4 && parts[0] === 'test') {
return parts.slice(1, -2).join(':');
}
} else if (isProjectGroupItem(test)) {
// For project groups, we need to get a child test to extract the path
Expand All @@ -63,26 +64,6 @@ function getProjectPathFromTestItem(test: TestItem): string | undefined {
return StateMachine.context().projectPath;
}

/**
* Check if we're in a workspace context (multiple projects)
* Returns the project name if in workspace, undefined otherwise
*/
function getProjectNameIfWorkspace(projectPath: string): string | undefined {
const projectInfo = StateMachine.context().projectInfo;

// Check if this is a workspace with multiple child projects
if (projectInfo?.children?.length > 0) {
// Find the matching child project
for (const child of projectInfo.children) {
if (child.projectPath === projectPath) {
return path.basename(projectPath);
}
}
}

return undefined;
}

function isAiEvaluations(test: TestItem): boolean {
// Check if the test item itself is the evaluations group
if (isTestGroupItem(test) && test.label === EVALUATION_GROUP) {
Expand Down Expand Up @@ -113,20 +94,16 @@ function isAiEvaluations(test: TestItem): boolean {
return false;
}

function buildTestCommand(test: TestItem, executor: string, projectName: string | undefined, testCaseNames?: string[]): string {
// Always invoked with the package dir as CWD — workaround for workspace `bal test` CWD bug.
function buildTestCommand(test: TestItem, executor: string, testCaseNames?: string[]): string {
if (isAiEvaluations(test)) {
// Evaluations tests use group-based execution with test report
const projectPath = getProjectPathFromTestItem(test);
if (projectPath) { ensureEvalReportsGitignored(projectPath); }
const testsPart = testCaseNames && testCaseNames.length > 0 ? ` --tests ${testCaseNames.join(',')}` : '';
const projectPart = projectName ? ` ${quoteShellPath(projectName)}` : '';
const reportDir = projectName ? `${projectName}/tests/evaluation-reports` : 'tests/evaluation-reports';
return `${quoteShellPath(executor)} test --groups ${EVALUATION_GROUP} --test-report --test-report-dir=${quoteShellPath(reportDir)}${testsPart}${projectPart}`;
return `${quoteShellPath(executor)} test --groups ${EVALUATION_GROUP} --test-report --test-report-dir=${quoteShellPath('tests/evaluation-reports')}${testsPart}`;
} else {
// Standard tests use code coverage and optional test filtering
const testsPart = testCaseNames && testCaseNames.length > 0 ? ` --tests ${testCaseNames.join(',')}` : '';
const projectPart = projectName ? ` ${quoteShellPath(projectName)}` : '';
return `${quoteShellPath(executor)} test --code-coverage${testsPart}${projectPart}`;
return `${quoteShellPath(executor)} test --code-coverage${testsPart}`;
}
}

Expand Down Expand Up @@ -250,6 +227,26 @@ export async function runHandler(request: TestRunRequest, token: CancellationTok
return;
}

// Match the run/debug flow: clean up unused imports before invoking `bal test`.
const projectPaths = new Set<string>();
include.forEach((test) => {
const projectPath = getProjectPathFromTestItem(test);
if (projectPath) {
projectPaths.add(projectPath);
}
});
const langClient = extension.ballerinaExtInstance.langClient;
for (const projectPath of projectPaths) {
if (token.isCancellationRequested) {
break;
}
try {
await cleanAndValidateProject(langClient, projectPath);
} catch (err) {
console.error(`Failed to clean project before test run: ${projectPath}`, err);
}
}
Comment thread
dan-niles marked this conversation as resolved.

// Handle Test Run
include.forEach((test) => {
if (token.isCancellationRequested) {
Expand All @@ -267,9 +264,6 @@ export async function runHandler(request: TestRunRequest, token: CancellationTok
return;
}

// Check if we're in a workspace with multiple projects
const projectName = getProjectNameIfWorkspace(projectPath);

let command: string;
const executor = extension.ballerinaExtInstance.getBallerinaCmd();

Expand Down Expand Up @@ -298,45 +292,31 @@ export async function runHandler(request: TestRunRequest, token: CancellationTok
}
});

command = buildTestCommand(test, executor, projectName, testCaseNames.length > 0 ? testCaseNames : undefined);
const isEval = isAiEvaluations(test);
command = buildTestCommand(test, executor, testCaseNames.length > 0 ? testCaseNames : undefined);

const startTime = Date.now();
// For workspace, run from workspace root; for single project, run from project path
const workingDirectory = projectName ? StateMachine.context().workspacePath || projectPath : projectPath;
runCommand(command, workingDirectory, run).then(async () => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);

if (isAiEvaluations(test)) {
handleEvalReport(run, testItems, timeElapsed, projectPath).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch(async () => {
runCommand(command, projectPath, run).then(async ({ stdout }) => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);
const reportPathOverride = extractTestReportPath(stdout);

if (isAiEvaluations(test)) {
if (isEval) {
handleEvalReport(run, testItems, timeElapsed, projectPath).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath).then(() => {
reportTestResults(run, testItems, timeElapsed, projectPath, false, reportPathOverride).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch((err) => {
testItems.forEach((item) => run.failed(item, new TestMessage(`Failed to run bal test: ${err.message}`)));
endGroup(test, false, run);
});
} else if (isTestGroupItem(test)) {
let testCaseNames: string[] = [];
Expand All @@ -348,48 +328,35 @@ export async function runHandler(request: TestRunRequest, token: CancellationTok
run.started(child);
});

command = buildTestCommand(test, executor, projectName, testCaseNames);
const isEval = isAiEvaluations(test);
command = buildTestCommand(test, executor, testCaseNames);

const startTime = Date.now();
// For workspace, run from workspace root; for single project, run from project path
const workingDirectory = projectName ? StateMachine.context().workspacePath || projectPath : projectPath;
runCommand(command, workingDirectory, run).then(async () => {
runCommand(command, projectPath, run).then(async ({ stdout }) => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);
const reportPathOverride = extractTestReportPath(stdout);

if (isAiEvaluations(test)) {
if (isEval) {
handleEvalReport(run, testItems, timeElapsed, projectPath).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch(async () => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);

if (isAiEvaluations(test)) {
handleEvalReport(run, testItems, timeElapsed, projectPath).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath).then(() => {
reportTestResults(run, testItems, timeElapsed, projectPath, false, reportPathOverride).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch((err) => {
testItems.forEach((item) => run.failed(item, new TestMessage(`Failed to run bal test: ${err.message}`)));
endGroup(test, false, run);
});
} else if (isTestFunctionItem(test)) {
command = buildTestCommand(test, executor, projectName, [test.label]);
command = buildTestCommand(test, executor, [test.label]);
const isEval = isAiEvaluations(test);

const parentGroup = test.parent;
let testItems: TestItem[] = [];
Expand All @@ -402,42 +369,27 @@ export async function runHandler(request: TestRunRequest, token: CancellationTok
}

const startTime = Date.now();
// For workspace, run from workspace root; for single project, run from project path
const workingDirectory = projectName ? StateMachine.context().workspacePath || projectPath : projectPath;
runCommand(command, workingDirectory, run).then(async () => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);

if (isAiEvaluations(test)) {
handleEvalReport(run, testItems, timeElapsed, projectPath, true).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath, true).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch(async () => {
runCommand(command, projectPath, run).then(async ({ stdout }) => {
const endTime = Date.now();
const timeElapsed = calculateTimeElapsed(startTime, endTime, testItems);
const reportPathOverride = extractTestReportPath(stdout);

if (isAiEvaluations(test)) {
if (isEval) {
handleEvalReport(run, testItems, timeElapsed, projectPath, true).then((allPassed) => {
endGroup(test, allPassed, run);
}).catch(() => {
endGroup(test, false, run);
});
} else {
reportTestResults(run, testItems, timeElapsed, projectPath, true).then(() => {
reportTestResults(run, testItems, timeElapsed, projectPath, true, reportPathOverride).then(() => {
endGroup(test, true, run);
}).catch(() => {
endGroup(test, false, run);
});
}
}).catch((err) => {
run.failed(test, new TestMessage(`Failed to run bal test: ${err.message}`));
endGroup(test, false, run);
});
}
});
Expand All @@ -461,19 +413,16 @@ enum TEST_STATUS {
SKIPPED = 'SKIPPED',
}

async function reportTestResults(run: TestRun, testItems: TestItem[], timeElapsed: number, projectPath: string, individualTest: boolean = false) {
// reading test results
// For workspace projects, results are in workspace/target, not project/target
const projectInfo = StateMachine.context().projectInfo;
const workspacePath = StateMachine.context().workspacePath;

async function reportTestResults(run: TestRun, testItems: TestItem[], timeElapsed: number, projectPath: string, individualTest: boolean = false, reportPathOverride?: string) {
let testResultsPath: string;
if (projectInfo?.children?.length > 0 && workspacePath) {
// Workspace with multiple projects - results are at workspace root
testResultsPath = path.join(workspacePath, TEST_RESULTS_PATH);
if (reportPathOverride) {
testResultsPath = reportPathOverride;
} else {
// Single project - results are in project directory
testResultsPath = path.join(projectPath, TEST_RESULTS_PATH);
const projectInfo = StateMachine.context().projectInfo;
const workspacePath = StateMachine.context().workspacePath;
testResultsPath = projectInfo?.children?.length > 0 && workspacePath
? path.join(workspacePath, TEST_RESULTS_PATH)
: path.join(projectPath, TEST_RESULTS_PATH);
}

let testsJson: JSON | undefined = undefined;
Expand Down Expand Up @@ -610,7 +559,7 @@ function endGroup(test: TestItem, allPassed: boolean, run: TestRun) {
run.end();
}

async function runCommand(command: string, projectPath: string, run?: TestRun): Promise<{ stdout: string; stderr: string }> {
async function runCommand(command: string, projectPath: string, run?: TestRun): Promise<{ stdout: string; stderr: string; exitCode: number | null }> {
return new Promise((resolve, reject) => {
let stdout = '';
let stderr = '';
Expand All @@ -636,16 +585,20 @@ async function runCommand(command: string, projectPath: string, run?: TestRun):
reject(err);
});

// Always resolve — `bal test` exits non-zero on test failures but still emits a report.
// We need stdout in both cases so the caller can locate the report file.
proc.on('close', (code) => {
if (code !== 0) {
reject(new Error(stderr || 'Test failed!'));
} else {
resolve({ stdout, stderr });
}
resolve({ stdout, stderr, exitCode: code });
});
});
}

// Parses `bal test`'s "Generating Test Report\n\t<path>" line. Returns undefined if absent.
function extractTestReportPath(stdout: string): string | undefined {
const match = stdout.match(/Generating Test Report\s+(\S.*?test_results\.json)/);
return match ? match[1].trim() : undefined;
}

/**
* Start debugging
*/
Expand Down
Loading