Skip to content

Commit 64e9094

Browse files
hefanlihefanli
authored
feature: added universal relational database collection template (#302)
1 parent e1944f6 commit 64e9094

File tree

5 files changed

+196
-44
lines changed

5 files changed

+196
-44
lines changed

‎runtime/datax/package.xml‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -67,27 +67,27 @@
6767
<!-- </includes>-->
6868
<!-- <outputDirectory>datax</outputDirectory>-->
6969
<!-- </fileSet>-->
70-
<fileSet>
71-
<directory>postgresqlreader/target/datax/</directory>
72-
<includes>
73-
<include>**/*.*</include>
74-
</includes>
75-
<outputDirectory>datax</outputDirectory>
76-
</fileSet>
7770
<!-- <fileSet>-->
78-
<!-- <directory>kingbaseesreader/target/datax/</directory>-->
71+
<!-- <directory>postgresqlreader/target/datax/</directory>-->
7972
<!-- <includes>-->
8073
<!-- <include>**/*.*</include>-->
8174
<!-- </includes>-->
8275
<!-- <outputDirectory>datax</outputDirectory>-->
8376
<!-- </fileSet>-->
8477
<!-- <fileSet>-->
85-
<!-- <directory>rdbmsreader/target/datax/</directory>-->
78+
<!-- <directory>kingbaseesreader/target/datax/</directory>-->
8679
<!-- <includes>-->
8780
<!-- <include>**/*.*</include>-->
8881
<!-- </includes>-->
8982
<!-- <outputDirectory>datax</outputDirectory>-->
9083
<!-- </fileSet>-->
84+
<fileSet>
85+
<directory>rdbmsreader/target/datax/</directory>
86+
<includes>
87+
<include>**/*.*</include>
88+
</includes>
89+
<outputDirectory>datax</outputDirectory>
90+
</fileSet>
9191

9292
<!-- <fileSet>-->
9393
<!-- <directory>odpsreader/target/datax/</directory>-->
@@ -257,13 +257,13 @@
257257
<!-- </includes>-->
258258
<!-- <outputDirectory>datax</outputDirectory>-->
259259
<!-- </fileSet>-->
260-
<fileSet>
261-
<directory>gaussdbreader/target/datax/</directory>
262-
<includes>
263-
<include>**/*.*</include>
264-
</includes>
265-
<outputDirectory>datax</outputDirectory>
266-
</fileSet>
260+
<!-- <fileSet>-->
261+
<!-- <directory>gaussdbreader/target/datax/</directory>-->
262+
<!-- <includes>-->
263+
<!-- <include>**/*.*</include>-->
264+
<!-- </includes>-->
265+
<!-- <outputDirectory>datax</outputDirectory>-->
266+
<!-- </fileSet>-->
267267
<fileSet>
268268
<directory>nfsreader/target/datax/</directory>
269269
<includes>
@@ -294,13 +294,13 @@
294294
</fileSet>
295295

296296
<!-- writer -->
297-
<fileSet>
298-
<directory>mysqlwriter/target/datax/</directory>
299-
<includes>
300-
<include>**/*.*</include>
301-
</includes>
302-
<outputDirectory>datax</outputDirectory>
303-
</fileSet>
297+
<!-- <fileSet>-->
298+
<!-- <directory>mysqlwriter/target/datax/</directory>-->
299+
<!-- <includes>-->
300+
<!-- <include>**/*.*</include>-->
301+
<!-- </includes>-->
302+
<!-- <outputDirectory>datax</outputDirectory>-->
303+
<!-- </fileSet>-->
304304
<!-- <fileSet>-->
305305
<!-- <directory>tdenginewriter/target/datax/</directory>-->
306306
<!-- <includes>-->
@@ -399,13 +399,13 @@
399399
<!-- </includes>-->
400400
<!-- <outputDirectory>datax</outputDirectory>-->
401401
<!-- </fileSet>-->
402-
<fileSet>
403-
<directory>postgresqlwriter/target/datax/</directory>
404-
<includes>
405-
<include>**/*.*</include>
406-
</includes>
407-
<outputDirectory>datax</outputDirectory>
408-
</fileSet>
402+
<!-- <fileSet>-->
403+
<!-- <directory>postgresqlwriter/target/datax/</directory>-->
404+
<!-- <includes>-->
405+
<!-- <include>**/*.*</include>-->
406+
<!-- </includes>-->
407+
<!-- <outputDirectory>datax</outputDirectory>-->
408+
<!-- </fileSet>-->
409409
<!-- <fileSet>-->
410410
<!-- <directory>kingbaseeswriter/target/datax/</directory>-->
411411
<!-- <includes>-->
@@ -581,13 +581,13 @@
581581
<!-- </includes>-->
582582
<!-- <outputDirectory>datax</outputDirectory>-->
583583
<!-- </fileSet>-->
584-
<fileSet>
585-
<directory>gaussdbwriter/target/datax/</directory>
586-
<includes>
587-
<include>**/*.*</include>
588-
</includes>
589-
<outputDirectory>datax</outputDirectory>
590-
</fileSet>
584+
<!-- <fileSet>-->
585+
<!-- <directory>gaussdbwriter/target/datax/</directory>-->
586+
<!-- <includes>-->
587+
<!-- <include>**/*.*</include>-->
588+
<!-- </includes>-->
589+
<!-- <outputDirectory>datax</outputDirectory>-->
590+
<!-- </fileSet>-->
591591
<!-- <fileSet>-->
592592
<!-- <directory>milvuswriter/target/datax/</directory>-->
593593
<!-- <includes>-->

‎runtime/datax/pom.xml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,13 @@
5151
<module>mysqlreader</module>
5252
<!-- <module>drdsreader</module>-->
5353
<!-- <module>sqlserverreader</module>-->
54-
<module>postgresqlreader</module>
54+
<!-- <module>postgresqlreader</module>-->
5555
<!-- <module>kingbaseesreader</module>-->
5656
<!-- <module>oraclereader</module>-->
5757
<!-- <module>cassandrareader</module>-->
5858
<!-- <module>oceanbasev10reader</module>-->
5959
<!-- <module>obhbasereader</module>-->
60-
<!-- <module>rdbmsreader</module>-->
60+
<module>rdbmsreader</module>
6161

6262
<!-- <module>odpsreader</module>-->
6363
<!-- <module>otsreader</module>-->
@@ -88,13 +88,13 @@
8888
<module>glusterfsreader</module>
8989
<module>s3reader</module>
9090
<!-- writer -->
91-
<module>mysqlwriter</module>
91+
<!-- <module>mysqlwriter</module>-->
9292
<module>starrockswriter</module>
9393
<!-- <module>drdswriter</module>-->
9494
<!-- <module>databendwriter</module>-->
9595
<!-- <module>oraclewriter</module>-->
9696
<!-- <module>sqlserverwriter</module>-->
97-
<module>postgresqlwriter</module>
97+
<!-- <module>postgresqlwriter</module>-->
9898
<!-- <module>kingbaseeswriter</module>-->
9999
<!-- <module>adswriter</module>-->
100100
<!-- <module>oceanbasev10writer</module>-->
@@ -137,8 +137,8 @@
137137
<module>nfswriter</module>
138138
<module>glusterfswriter</module>
139139
<module>s3writer</module>
140-
<module>gaussdbreader</module>
141-
<module>gaussdbwriter</module>
140+
<!-- <module>gaussdbreader</module>-->
141+
<!-- <module>gaussdbwriter</module>-->
142142
<module>apireader</module>
143143
<!-- common support module -->
144144
<module>plugin-rdbms-util</module>

‎runtime/datax/rdbmsreader/pom.xml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<parent>
5+
<artifactId>datax-all</artifactId>
6+
<groupId>com.alibaba.datax</groupId>
7+
<version>0.0.1-SNAPSHOT</version>
8+
</parent>
9+
<modelVersion>4.0.0</modelVersion>
10+
11+
<artifactId>rdbmsreader</artifactId>
12+
<name>rdbmsreader</name>
13+
<packaging>jar</packaging>
14+
15+
<dependencies>
16+
<dependency>
17+
<groupId>com.alibaba.datax</groupId>
18+
<artifactId>datax-common</artifactId>
19+
<version>${datax-project-version}</version>
20+
<exclusions>
21+
<exclusion>
22+
<artifactId>slf4j-log4j12</artifactId>
23+
<groupId>org.slf4j</groupId>
24+
</exclusion>
25+
</exclusions>
26+
</dependency>
27+
28+
<!-- dm7 dm8 driver -->
29+
<dependency>
30+
<groupId>com.dameng</groupId>
31+
<artifactId>Dm7JdbcDriver17</artifactId>
32+
<version>7.6.0.142</version>
33+
</dependency>
34+
<!-- sybase driver -->
35+
<dependency>
36+
<groupId>com.sybase</groupId>
37+
<artifactId>jconn3</artifactId>
38+
<version>1.0.0-SNAPSHOT</version>
39+
<scope>system</scope>
40+
<systemPath>${basedir}/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar</systemPath>
41+
</dependency>
42+
<!-- ppas driver -->
43+
<dependency>
44+
<groupId>ppas</groupId>
45+
<artifactId>ppas</artifactId>
46+
<version>16</version>
47+
<scope>system</scope>
48+
<systemPath>${basedir}/src/main/libs/edb-jdbc16.jar</systemPath>
49+
</dependency>
50+
<!-- db2 driver -->
51+
<dependency>
52+
<groupId>com.ibm.db2.jcc</groupId>
53+
<artifactId>db2jcc</artifactId>
54+
<version>db2jcc4</version>
55+
</dependency>
56+
<!-- sqlserver driver -->
57+
<dependency>
58+
<groupId>com.microsoft.sqlserver</groupId>
59+
<artifactId>sqljdbc4</artifactId>
60+
<version>4.0</version>
61+
</dependency>
62+
<!-- postgresql driver -->
63+
<dependency>
64+
<groupId>org.postgresql</groupId>
65+
<artifactId>postgresql</artifactId>
66+
<version>42.3.3</version>
67+
</dependency>
68+
<!-- mysql driver -->
69+
<dependency>
70+
<groupId>mysql</groupId>
71+
<artifactId>mysql-connector-java</artifactId>
72+
<version>${mysql.driver.version}</version>
73+
</dependency>
74+
<!-- opengauss driver -->
75+
<dependency>
76+
<groupId>org.opengauss</groupId>
77+
<artifactId>opengauss-jdbc</artifactId>
78+
<version>3.0.0</version>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>org.slf4j</groupId>
83+
<artifactId>slf4j-api</artifactId>
84+
</dependency>
85+
86+
<dependency>
87+
<groupId>ch.qos.logback</groupId>
88+
<artifactId>logback-classic</artifactId>
89+
</dependency>
90+
91+
<dependency>
92+
<groupId>com.alibaba.datax</groupId>
93+
<artifactId>plugin-rdbms-util</artifactId>
94+
<version>${datax-project-version}</version>
95+
</dependency>
96+
97+
</dependencies>
98+
99+
<build>
100+
<plugins>
101+
<!-- compiler plugin -->
102+
<plugin>
103+
<artifactId>maven-compiler-plugin</artifactId>
104+
<configuration>
105+
<source>${jdk-version}</source>
106+
<target>${jdk-version}</target>
107+
<encoding>${project-sourceEncoding}</encoding>
108+
</configuration>
109+
</plugin>
110+
<!-- assembly plugin -->
111+
<plugin>
112+
<artifactId>maven-assembly-plugin</artifactId>
113+
<configuration>
114+
<descriptors>
115+
<descriptor>src/main/assembly/package.xml</descriptor>
116+
</descriptors>
117+
<finalName>datax</finalName>
118+
</configuration>
119+
<executions>
120+
<execution>
121+
<id>dwzip</id>
122+
<phase>package</phase>
123+
<goals>
124+
<goal>single</goal>
125+
</goals>
126+
</execution>
127+
</executions>
128+
</plugin>
129+
</plugins>
130+
</build>
131+
132+
</project>

‎runtime/datax/rdbmsreader/src/main/resources/plugin.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "rdbmsreader",
3+
"class": "com.alibaba.datax.plugin.reader.rdbmsreader.RdbmsReader",
4+
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5+
"developer": "alibaba",
6+
"drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver", "com.ibm.db2.jcc.DB2Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "org.postgresql.Driver", "org.opengauss.Driver", "com.mysql.cj.jdbc.Driver"]
7+
}

‎scripts/db/data-collection-init.sql‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,20 @@ INSERT INTO t_dc_collection_templates (
233233
'{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式（默认为GET）。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数，主要针对POST请求。示例：\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例：\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例：\n {\"dataPath\": \"返回体内定位到具体数据的路径，形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}',
234234
TRUE,
235235
'system',
236-
'system')
236+
'system'),
237+
(
238+
'7',
239+
'通用关系型数据库归集模板',
240+
'将关系型数据库中的数据以csv文件的形式归集到DataMate平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。',
241+
'rdbmsreader',
242+
'rdbmsreader',
243+
'txtfilewriter',
244+
'txtfilewriter',
245+
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
246+
TRUE,
247+
'system',
248+
'system'
249+
)
237250
ON CONFLICT (id) DO UPDATE SET
238251
name = EXCLUDED.name,
239252
description = EXCLUDED.description,

0 commit comments

Comments
 (0)