Skip to content

Commit e1944f6

Browse files
hefanlihefanli
authored
add apireader collection plug-in (#301)
* feature: add apireader collection plug-in * feature: add apireader collection plug-in * feature: add apireader collection plug-in
1 parent 87b8b6d commit e1944f6

File tree

10 files changed

+598
-12
lines changed

10 files changed

+598
-12
lines changed

‎frontend/src/pages/DataCollection/Create/CreateTask.tsx‎

Lines changed: 160 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type TemplateFieldDef = {
3434
required?: boolean;
3535
options?: Array<{ label: string; value: string | number } | string | number>;
3636
defaultValue?: any;
37+
index?: number;
38+
properties?: Record<string, TemplateFieldDef>;
3739
};
3840

3941
export default function CollectionTaskCreate() {
@@ -78,10 +80,110 @@ export default function CollectionTaskCreate() {
7880
run()
7981
}, []);
8082

83+
const parseJsonObjectInput = (value: any) => {
84+
if (value === undefined || value === null) return value;
85+
if (typeof value === "string") {
86+
const trimmed = value.trim();
87+
if (!trimmed) return undefined;
88+
const parsed = JSON.parse(trimmed);
89+
if (parsed === null || Array.isArray(parsed) || typeof parsed !== "object") {
90+
throw new Error("必须是JSON对象");
91+
}
92+
return parsed;
93+
}
94+
if (typeof value === "object") {
95+
if (Array.isArray(value) || value === null) {
96+
throw new Error("必须是JSON对象");
97+
}
98+
return value;
99+
}
100+
throw new Error("必须是JSON对象");
101+
};
102+
103+
const tryFormatJsonValue = (value: any) => {
104+
const parsed = parseJsonObjectInput(value);
105+
if (parsed === undefined) return undefined;
106+
return JSON.stringify(parsed, null, 2);
107+
};
108+
109+
const handleFormatJsonField = (name: (string | number)[]) => {
110+
const currentValue = form.getFieldValue(name);
111+
try {
112+
const formatted = tryFormatJsonValue(currentValue);
113+
if (formatted !== undefined) {
114+
form.setFieldValue(name, formatted);
115+
}
116+
} catch (error: any) {
117+
message.error(error?.message || "JSON格式错误");
118+
}
119+
};
120+
121+
const normalizeConfigSection = (
122+
sectionValue: any,
123+
defs?: Record<string, TemplateFieldDef>
124+
) => {
125+
if (!defs || typeof defs !== "object") return sectionValue;
126+
const normalized =
127+
Array.isArray(sectionValue) ? [...sectionValue] : { ...(sectionValue || {}) };
128+
129+
Object.entries(defs).forEach(([key, def]) => {
130+
const fieldType = (def?.type || "input").toLowerCase();
131+
const required = def?.required !== false;
132+
const value = sectionValue?.[key];
133+
134+
if (fieldType === "jsonobject") {
135+
const parsed = parseJsonObjectInput(value);
136+
if (parsed === undefined && !required) {
137+
if (normalized && !Array.isArray(normalized)) {
138+
delete normalized[key];
139+
}
140+
} else if (normalized && !Array.isArray(normalized)) {
141+
normalized[key] = parsed;
142+
}
143+
return;
144+
}
145+
146+
if (fieldType === "multiple") {
147+
if (value && typeof value === "object") {
148+
normalized[key] = normalizeConfigSection(value, def?.properties);
149+
}
150+
return;
151+
}
152+
153+
if (fieldType === "multiplelist") {
154+
if (Array.isArray(value)) {
155+
normalized[key] = value.map((item) =>
156+
normalizeConfigSection(item, def?.properties)
157+
);
158+
}
159+
}
160+
});
161+
162+
return normalized;
163+
};
164+
81165
const handleSubmit = async () => {
82166
try {
83-
await form.validateFields();
84-
await createTaskUsingPost(newTask);
167+
const values = await form.validateFields();
168+
const payload = { ...newTask, ...values };
169+
if (selectedTemplate?.templateContent) {
170+
payload.config = {
171+
...(payload.config || {}),
172+
parameter: normalizeConfigSection(
173+
payload.config?.parameter,
174+
selectedTemplate.templateContent.parameter
175+
),
176+
reader: normalizeConfigSection(
177+
payload.config?.reader,
178+
selectedTemplate.templateContent.reader
179+
),
180+
writer: normalizeConfigSection(
181+
payload.config?.writer,
182+
selectedTemplate.templateContent.writer
183+
),
184+
};
185+
}
186+
await createTaskUsingPost(payload);
85187
message.success("任务创建成功");
86188
navigate("/data/collection");
87189
} catch (error) {
@@ -107,9 +209,33 @@ export default function CollectionTaskCreate() {
107209
const description = def?.description;
108210
const fieldType = (def?.type || "input").toLowerCase();
109211
const required = def?.required !== false;
110-
const rules = required
111-
? [{ required: true, message: `请输入${label}` }]
112-
: undefined;
212+
const rules: any[] = [];
213+
if (required) {
214+
rules.push({ required: true, message: `请输入${label}` });
215+
}
216+
if (fieldType === "jsonobject") {
217+
rules.push({
218+
validator: (_: any, value: any) => {
219+
if (
220+
value === undefined ||
221+
value === null ||
222+
(typeof value === "string" && value.trim() === "")
223+
) {
224+
return Promise.resolve();
225+
}
226+
try {
227+
parseJsonObjectInput(value);
228+
return Promise.resolve();
229+
} catch (e) {
230+
return Promise.reject(
231+
new Error(
232+
`JSON格式错误：${(e as Error)?.message || "请输入合法的JSON对象"}`
233+
)
234+
);
235+
}
236+
},
237+
});
238+
}
113239
const name = section.concat(key)
114240

115241
switch (fieldType) {
@@ -126,18 +252,43 @@ export default function CollectionTaskCreate() {
126252
</Form.Item>
127253
));
128254
break;
255+
case "jsonobject":
256+
items_.push((
257+
<Form.Item
258+
key={`${section}.${key}`}
259+
name={name}
260+
label={label}
261+
tooltip={description}
262+
rules={rules.length ? rules : undefined}
263+
extra={(
264+
<div className="flex justify-end">
265+
<Button size="small" onClick={() => handleFormatJsonField(name)}>
266+
格式化JSON
267+
</Button>
268+
</div>
269+
)}
270+
>
271+
<TextArea
272+
placeholder={description || `请输入${label}`}
273+
autoSize={{ minRows: 4, maxRows: 12 }}
274+
className="font-mono"
275+
/>
276+
</Form.Item>
277+
));
278+
break;
129279
case "selecttag":
130280
items_.push((
131281
<Form.Item
132282
name={name}
133283
label={label}
134-
rules={rules}
284+
rules={rules.length ? rules : undefined}
135285
>
136286
<Select placeholder={description || `请输入${label}`} mode="tags" />
137287
</Form.Item>
138288
));
139289
break;
140290
case "select":
291+
case "option":
141292
const options = (def?.options || []).map((opt: any) => {
142293
if (typeof opt === "string" || typeof opt === "number") {
143294
return { label: String(opt), value: opt };
@@ -150,7 +301,7 @@ export default function CollectionTaskCreate() {
150301
name={name}
151302
label={label}
152303
tooltip={description}
153-
rules={rules}
304+
rules={rules.length ? rules : undefined}
154305
>
155306
<Select placeholder={description || `请选择${label}`} options={options} />
156307
</Form.Item>
@@ -172,7 +323,7 @@ export default function CollectionTaskCreate() {
172323
name={name.concat(0)}
173324
label={label}
174325
tooltip={description}
175-
rules={rules}
326+
rules={rules.length ? rules : undefined}
176327
>
177328
<Input placeholder={description || `请输入${label}`} />
178329
</Form.Item>
@@ -185,7 +336,7 @@ export default function CollectionTaskCreate() {
185336
name={name}
186337
label={label}
187338
tooltip={description}
188-
rules={rules}
339+
rules={rules.length ? rules : undefined}
189340
>
190341
<Input placeholder={description || `请输入${label}`} />
191342
</Form.Item>

‎runtime/datamate-python/app/module/collection/client/datax_client.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ def generate_datx_config(task_config: CollectionConfig, template: CollectionTemp
6060
dest_parameter = {
6161
"path": target_path,
6262
"fileName": "collection_result",
63-
"writeMode": "truncate"
63+
"writeMode": "truncate",
64+
"fileFormat": "csv"
6465
}
6566
elif dest_path_target.__contains__(template.target_type):
6667
dest_parameter = {

‎runtime/datax/apireader/pom.xml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>com.alibaba.datax</groupId>
8+
<artifactId>datax-all</artifactId>
9+
<version>0.0.1-SNAPSHOT</version>
10+
</parent>
11+
12+
<artifactId>apireader</artifactId>
13+
14+
<dependencies>
15+
<dependency>
16+
<groupId>com.alibaba.datax</groupId>
17+
<artifactId>datax-core</artifactId>
18+
<version>${datax-project-version}</version>
19+
</dependency>
20+
<dependency>
21+
<groupId>com.alibaba.datax</groupId>
22+
<artifactId>datax-common</artifactId>
23+
<version>${datax-project-version}</version>
24+
</dependency>
25+
<dependency>
26+
<groupId>org.slf4j</groupId>
27+
<artifactId>slf4j-api</artifactId>
28+
</dependency>
29+
<dependency>
30+
<groupId>com.alibaba</groupId>
31+
<artifactId>fastjson</artifactId>
32+
<version>1.2.83_noneautotype</version>
33+
<scope>compile</scope>
34+
</dependency>
35+
<dependency>
36+
<groupId>org.apache.httpcomponents.client5</groupId>
37+
<artifactId>httpclient5</artifactId>
38+
<version>5.3.1</version>
39+
</dependency>
40+
</dependencies>
41+
42+
<build>
43+
<resources>
44+
<resource>
45+
<directory>src/main/java</directory>
46+
<includes>
47+
<include>**/*.properties</include>
48+
</includes>
49+
</resource>
50+
</resources>
51+
<plugins>
52+
<plugin>
53+
<artifactId>maven-compiler-plugin</artifactId>
54+
<configuration>
55+
<source>${jdk-version}</source>
56+
<target>${jdk-version}</target>
57+
<encoding>${project-sourceEncoding}</encoding>
58+
</configuration>
59+
</plugin>
60+
<plugin>
61+
<artifactId>maven-assembly-plugin</artifactId>
62+
<configuration>
63+
<descriptors>
64+
<descriptor>src/main/assembly/package.xml</descriptor>
65+
</descriptors>
66+
<finalName>datax</finalName>
67+
</configuration>
68+
<executions>
69+
<execution>
70+
<id>dwzip</id>
71+
<phase>package</phase>
72+
<goals>
73+
<goal>single</goal>
74+
</goals>
75+
</execution>
76+
</executions>
77+
</plugin>
78+
</plugins>
79+
</build>
80+
81+
</project>

‎runtime/datax/apireader/src/main/assembly/package.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<assembly
2+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
5+
<id></id>
6+
<formats>
7+
<format>dir</format>
8+
</formats>
9+
<includeBaseDirectory>false</includeBaseDirectory>
10+
<fileSets>
11+
<fileSet>
12+
<directory>src/main/resources</directory>
13+
<includes>
14+
<include>plugin.json</include>
15+
<include>plugin_job_template.json</include>
16+
</includes>
17+
<outputDirectory>plugin/reader/apireader</outputDirectory>
18+
</fileSet>
19+
<fileSet>
20+
<directory>target/</directory>
21+
<includes>
22+
<include>apireader-0.0.1-SNAPSHOT.jar</include>
23+
</includes>
24+
<outputDirectory>plugin/reader/apireader</outputDirectory>
25+
</fileSet>
26+
</fileSets>
27+
28+
<dependencySets>
29+
<dependencySet>
30+
<useProjectArtifact>false</useProjectArtifact>
31+
<outputDirectory>plugin/reader/apireader/libs</outputDirectory>
32+
<scope>runtime</scope>
33+
</dependencySet>
34+
</dependencySets>
35+
</assembly>

0 commit comments

Comments
 (0)