Skip to content

Commit b0ec328

Browse files
authored
Merge branch 'cloudwego:main' into main
2 parents 7ffb19a + 105eb91 commit b0ec328

File tree

8 files changed

+885
-76
lines changed

8 files changed

+885
-76
lines changed

go.mod

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.23.4
44

55
require (
66
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
7-
github.com/bytedance/sonic v1.13.3
7+
github.com/bytedance/sonic v1.14.1
88
github.com/cloudwego/eino v0.3.52
99
github.com/cloudwego/eino-ext/components/model/ark v0.1.16
1010
github.com/cloudwego/eino-ext/components/model/claude v0.1.1
@@ -40,9 +40,10 @@ require (
4040
github.com/aws/smithy-go v1.22.1 // indirect
4141
github.com/bahlo/generic-list-go v0.2.0 // indirect
4242
github.com/buger/jsonparser v1.1.1 // indirect
43-
github.com/bytedance/sonic/loader v0.2.4 // indirect
43+
github.com/bytedance/gopkg v0.1.3 // indirect
44+
github.com/bytedance/sonic/loader v0.3.0 // indirect
4445
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
45-
github.com/cloudwego/base64x v0.1.5 // indirect
46+
github.com/cloudwego/base64x v0.1.6 // indirect
4647
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250626133421-3c142631c961 // indirect
4748
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
4849
github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -107,13 +107,14 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
107107
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
108108
github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
109109
github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
110+
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
111+
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
110112
github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8=
111113
github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY=
112-
github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0=
113-
github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
114-
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
115-
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
116-
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
114+
github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
115+
github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
116+
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
117+
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
117118
github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
118119
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
119120
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
@@ -131,8 +132,8 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D
131132
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
132133
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
133134
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
134-
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
135-
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
135+
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
136+
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
136137
github.com/cloudwego/eino v0.3.52 h1:jYs1qsEobXS5RC0VamtSVKyM4d26O/cCaYW/s5+Da1o=
137138
github.com/cloudwego/eino v0.3.52/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY=
138139
github.com/cloudwego/eino-ext/components/model/ark v0.1.16 h1:D1kDC0kmpVV3XYEY8jFytjv8qbAvq5yn316Bu9pXEyc=
@@ -147,7 +148,6 @@ github.com/cloudwego/eino-ext/components/tool/mcp v0.0.3 h1:kT8yynQh8q92BPofkHxx
147148
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.3/go.mod h1:D1Yircehgx7em8EH6vw8If+ATjtXl60IpBzVaqFBx3c=
148149
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250626133421-3c142631c961 h1:fGE3RFHaAsrLjA+2fkE0YMsPrkFI6pEKKZmbhD42L7E=
149150
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250626133421-3c142631c961/go.mod h1:iB0W8l+OqKNL5LtJQ9JaGYXekhsxVxrDMfnfD9L+5gc=
150-
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
151151
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
152152
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
153153
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -372,10 +372,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
372372
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
373373
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
374374
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
375-
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
376375
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
377376
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
378-
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
379377
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
380378
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
381379
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -1018,7 +1016,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
10181016
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
10191017
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
10201018
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
1021-
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
10221019
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
10231020
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
10241021
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

ts-parser/README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,15 @@ See `./index.ts` for more information.
3737

3838
4. If the repository you're analyzing is too large, you may need to adjust Node.js's maximum memory allocation.
3939

40+
## Terminology
41+
42+
**Package vs Module**: In JavaScript/TypeScript terminology, a "Package" typically refers to an npm package (defined by `package.json`). However, in our UniAST output, what JavaScript/TypeScript calls a "Package" corresponds to a "Module" in the UniAST structure. This means:
43+
44+
- **TypeScript/JavaScript Package** (npm package with `package.json`) → **UniAST Module**
45+
- **TypeScript/JavaScript Module** (individual `.ts`/`.js` files) → **UniAST Package**
46+
47+
This terminology mapping is used consistently throughout the parser to align with the UniAST specification, but it may initially seem counterintuitive to developers familiar with JavaScript/TypeScript conventions.
48+
4049

4150
## Some known issues
4251

ts-parser/src/parser/RepositoryParser.ts

Lines changed: 96 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,78 @@ import * as fs from 'fs';
44
import { Repository, Node, Relation, Identity, Function } from '../types/uniast';
55
import { ModuleParser } from './ModuleParser';
66
import { TsConfigCache } from '../utils/tsconfig-cache';
7+
import { MonorepoUtils } from '../utils/monorepo';
78

89
export class RepositoryParser {
9-
private project: Project;
10-
private moduleParser: ModuleParser;
10+
private project?: Project;
11+
private moduleParser?: ModuleParser;
1112
private tsConfigCache: TsConfigCache;
13+
private projectRoot: string;
14+
private tsConfigPath?: string;
1215

1316
constructor(projectRoot: string, tsConfigPath?: string) {
1417
this.tsConfigCache = TsConfigCache.getInstance();
18+
this.projectRoot = projectRoot;
19+
this.tsConfigPath = tsConfigPath;
20+
}
1521

22+
async parseRepository(repoPath: string, options: { loadExternalSymbols?: boolean, noDist?: boolean, srcPatterns?: string[] } = {}): Promise<Repository> {
23+
const absolutePath = path.resolve(repoPath);
1624

17-
let configPath = path.join(projectRoot, 'tsconfig.json');
25+
const repository: Repository = {
26+
ASTVersion: "v0.1.3",
27+
id: path.basename(absolutePath),
28+
Modules: {},
29+
Graph: {}
30+
};
31+
32+
const isMonorepo = MonorepoUtils.isMonorepo(absolutePath);
33+
34+
if (isMonorepo) {
35+
const packages = MonorepoUtils.getMonorepoPackages(absolutePath);
36+
console.log(`Monorepo detected. Found ${packages.length} packages.`);
37+
38+
for (const pkg of packages) {
39+
const packageTsConfigPath = path.join(pkg.absolutePath, 'tsconfig.json');
40+
try {
41+
let project: Project;
42+
if (fs.existsSync(packageTsConfigPath)) {
43+
console.log(`Parsing package ${pkg.name || pkg.path} with tsconfig ${packageTsConfigPath}`);
44+
project = new Project({
45+
tsConfigFilePath: packageTsConfigPath,
46+
compilerOptions: {
47+
allowJs: true,
48+
skipLibCheck: true,
49+
forceConsistentCasingInFileNames: true
50+
}
51+
});
52+
} else {
53+
console.log(`No tsconfig.json found for package ${pkg.name || pkg.path}, using default configuration.`);
54+
project = this.createProjectWithDefaultConfig();
55+
}
56+
57+
const moduleParser = new ModuleParser(project, this.projectRoot);
58+
const module = await moduleParser.parseModule(pkg.absolutePath, pkg.path, options);
59+
repository.Modules[module.Name] = module;
60+
} catch (error) {
61+
console.warn(`Failed to parse package ${pkg.name || pkg.path}:`, error);
62+
}
63+
}
64+
} else {
65+
console.log('Single project detected.');
66+
this.project = this.createProjectForSingleRepo(this.projectRoot, this.tsConfigPath);
67+
this.moduleParser = new ModuleParser(this.project, this.projectRoot);
68+
const module = await this.moduleParser.parseModule(absolutePath, '.', options);
69+
repository.Modules[module.Name] = module;
70+
}
71+
72+
this.buildGlobalGraph(repository);
73+
return repository;
74+
}
75+
76+
private createProjectForSingleRepo(projectRoot: string, tsConfigPath?: string): Project {
77+
let configPath = path.join(projectRoot, 'tsconfig.json');
1878

19-
// If a custom tsconfig path is provided, use it
2079
if (tsConfigPath) {
2180
let absoluteTsConfigPath = tsConfigPath;
2281
if (!path.isAbsolute(absoluteTsConfigPath)) {
@@ -27,8 +86,7 @@ export class RepositoryParser {
2786
}
2887

2988
if (fs.existsSync(configPath)) {
30-
// if tsconfig.json exists, use it to configure the project
31-
this.project = new Project({
89+
const project = new Project({
3290
tsConfigFilePath: configPath,
3391
compilerOptions: {
3492
allowJs: true,
@@ -62,8 +120,7 @@ export class RepositoryParser {
62120
console.warn("parse tsconfig warning:", err.messageText)
63121
});
64122
}
65-
this.project.addSourceFilesAtPaths(parsedConfig.fileNames);
66-
// Get references
123+
project.addSourceFilesAtPaths(parsedConfig.fileNames);
67124
const references = parsedConfig.projectReferences;
68125
if (!references) {
69126
continue;
@@ -78,63 +135,43 @@ export class RepositoryParser {
78135
}
79136
}
80137
}
138+
return project;
81139
} else {
82-
// if tsconfig.json does not exist, use default configuration
83-
this.project = new Project({
84-
compilerOptions: {
85-
target: 99,
86-
module: 1,
87-
allowJs: true,
88-
checkJs: false,
89-
skipLibCheck: true,
90-
skipDefaultLibCheck: true,
91-
strict: false,
92-
noImplicitAny: false,
93-
strictNullChecks: false,
94-
strictFunctionTypes: false,
95-
strictBindCallApply: false,
96-
strictPropertyInitialization: false,
97-
noImplicitReturns: false,
98-
noFallthroughCasesInSwitch: false,
99-
noUncheckedIndexedAccess: false,
100-
noImplicitOverride: false,
101-
noPropertyAccessFromIndexSignature: false,
102-
allowUnusedLabels: false,
103-
allowUnreachableCode: false,
104-
exactOptionalPropertyTypes: false,
105-
noImplicitThis: false,
106-
alwaysStrict: false,
107-
noImplicitUseStrict: false,
108-
forceConsistentCasingInFileNames: true
109-
}
110-
});
140+
return this.createProjectWithDefaultConfig();
111141
}
112-
113-
this.moduleParser = new ModuleParser(this.project, projectRoot);
114142
}
115143

116-
117-
async parseRepository(repoPath: string, options: { loadExternalSymbols?: boolean, noDist?: boolean, srcPatterns?: string[] } = {}): Promise<Repository> {
118-
const absolutePath = path.resolve(repoPath);
119-
120-
const repository: Repository = {
121-
ASTVersion: "v0.1.3",
122-
id: path.basename(absolutePath),
123-
Modules: {},
124-
Graph: {}
125-
};
126-
127-
// Parse main module only
128-
const mainModule = await this.moduleParser.parseModule(absolutePath, '.', options);
129-
repository.Modules[mainModule.Name] = mainModule;
130-
131-
// Build global symbol graph
132-
this.buildGlobalGraph(repository);
133-
134-
return repository;
144+
private createProjectWithDefaultConfig(): Project {
145+
return new Project({
146+
compilerOptions: {
147+
target: 99,
148+
module: 1,
149+
allowJs: true,
150+
checkJs: false,
151+
skipLibCheck: true,
152+
skipDefaultLibCheck: true,
153+
strict: false,
154+
noImplicitAny: false,
155+
strictNullChecks: false,
156+
strictFunctionTypes: false,
157+
strictBindCallApply: false,
158+
strictPropertyInitialization: false,
159+
noImplicitReturns: false,
160+
noFallthroughCasesInSwitch: false,
161+
noUncheckedIndexedAccess: false,
162+
noImplicitOverride: false,
163+
noPropertyAccessFromIndexSignature: false,
164+
allowUnusedLabels: false,
165+
allowUnreachableCode: false,
166+
exactOptionalPropertyTypes: false,
167+
noImplicitThis: false,
168+
alwaysStrict: false,
169+
noImplicitUseStrict: false,
170+
forceConsistentCasingInFileNames: true
171+
}
172+
});
135173
}
136174

137-
138175
private buildGlobalGraph(repository: Repository): void {
139176
// First pass: Create all nodes from functions, types, and variables
140177
for (const [, module] of Object.entries(repository.Modules)) {

0 commit comments

Comments
 (0)