getMessage(String key);
22 | }
23 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.plugin;
2 |
3 |
4 | /**
5 | * 这里只是一个标示类
6 | * */
7 | public interface PluginCollector {
8 |
9 | }
10 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.plugin;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 |
5 | public interface Pluginable {
6 | String getDeveloper();
7 |
8 | String getDescription();
9 |
10 | void setPluginConf(Configuration pluginConf);
11 |
12 | void init();
13 |
14 | void destroy();
15 |
16 | String getPluginName();
17 |
18 | Configuration getPluginJobConf();
19 |
20 | Configuration getPeerPluginJobConf();
21 |
22 | public String getPeerPluginName();
23 |
24 | void setPluginJobConf(Configuration jobConf);
25 |
26 | void setPeerPluginJobConf(Configuration peerPluginJobConf);
27 |
28 | public void setPeerPluginName(String peerPluginName);
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java:
--------------------------------------------------------------------------------
1 | /**
2 | * (C) 2010-2013 Alibaba Group Holding Limited.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.alibaba.datax.common.plugin;
18 |
19 | import com.alibaba.datax.common.element.Record;
20 |
21 | public interface RecordReceiver {
22 |
23 | public Record getFromReader();
24 |
25 | public void shutdown();
26 | }
27 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java:
--------------------------------------------------------------------------------
1 | /**
2 | * (C) 2010-2013 Alibaba Group Holding Limited.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.alibaba.datax.common.plugin;
18 |
19 | import com.alibaba.datax.common.element.Record;
20 |
21 | public interface RecordSender {
22 |
23 | public Record createRecord();
24 |
25 | public void sendToWriter(Record record);
26 |
27 | public void flush();
28 |
29 | public void terminate();
30 |
31 | public void shutdown();
32 | }
33 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/spi/ErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.spi;
2 |
3 | /**
4 | * 尤其注意：最好提供toString()实现。例如：
5 | *
6 | *
7 | *
8 | * @Override
9 | * public String toString() {
10 | * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
11 | * }
12 | *
13 | *
14 | */
15 | public interface ErrorCode {
16 | // 错误码编号
17 | String getCode();
18 |
19 | // 错误码描述
20 | String getDescription();
21 |
22 | /** 必须提供toString的实现
23 | *
24 | *
25 | * @Override
26 | * public String toString() {
27 | * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
28 | * }
29 | *
30 | *
31 | */
32 | String toString();
33 | }
34 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/spi/Hook.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.spi;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 |
5 | import java.util.Map;
6 |
7 | /**
8 | * Created by xiafei.qiuxf on 14/12/17.
9 | */
10 | public interface Hook {
11 |
12 | /**
13 | * 返回名字
14 | *
15 | * @return
16 | */
17 | public String getName();
18 |
19 | /**
20 | * TODO 文档
21 | *
22 | * @param jobConf
23 | * @param msg
24 | */
25 | public void invoke(Configuration jobConf, Map msg);
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/common/src/main/java/com/alibaba/datax/common/spi/Writer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.spi;
2 |
3 | import com.alibaba.datax.common.base.BaseObject;
4 | import com.alibaba.datax.common.plugin.AbstractJobPlugin;
5 | import com.alibaba.datax.common.plugin.AbstractTaskPlugin;
6 | import com.alibaba.datax.common.util.Configuration;
7 | import com.alibaba.datax.common.plugin.RecordReceiver;
8 |
9 | import java.util.List;
10 |
11 | /**
12 | * 每个Writer插件需要实现Writer类，并在其内部实现Job、Task两个内部类。
13 | *
14 | *
15 | * */
16 | public abstract class Writer extends BaseObject {
17 | /**
18 | * 每个Writer插件必须实现Job内部类
19 | */
20 | public abstract static class Job extends AbstractJobPlugin {
21 | /**
22 | * 切分任务。
23 | *
24 | * @param mandatoryNumber
25 | * 为了做到Reader、Writer任务数对等，这里要求Writer插件必须按照源端的切分数进行切分。否则框架报错！
26 | *
27 | * */
28 | public abstract List split(int mandatoryNumber);
29 | }
30 |
31 | /**
32 | * 每个Writer插件必须实现Task内部类
33 | */
34 | public abstract static class Task extends AbstractTaskPlugin {
35 |
36 | public abstract void startWrite(RecordReceiver lineReceiver);
37 |
38 | public boolean supportFailOver(){return false;}
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/common/src/test/java/com/alibaba/datax/common/base/BaseTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.base;
2 |
3 | import java.io.File;
4 | import java.io.IOException;
5 |
6 | import org.apache.commons.io.FileUtils;
7 | import org.apache.commons.lang3.StringUtils;
8 | import org.junit.BeforeClass;
9 |
10 | import com.alibaba.datax.common.element.ColumnCast;
11 | import com.alibaba.datax.common.element.ColumnCastTest;
12 | import com.alibaba.datax.common.util.Configuration;
13 | import org.junit.Test;
14 |
15 | public class BaseTest {
16 |
17 | @BeforeClass
18 | public static void beforeClass() throws IOException {
19 | String path = ColumnCastTest.class.getClassLoader().getResource(".")
20 | .getFile();
21 | ColumnCast.bind(Configuration.from(FileUtils.readFileToString(new File(
22 | StringUtils.join(new String[] { path, "all.json" },
23 | File.separator)))));
24 | }
25 |
26 | @Test
27 | public void emptyTest() {
28 |
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/common/src/test/java/com/alibaba/datax/common/element/ScientificTester.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.element;
2 |
3 | import org.apache.commons.lang3.math.NumberUtils;
4 | import org.junit.Test;
5 |
6 | public class ScientificTester {
7 | @Test
8 | public void test() {
9 | System.out.println(NumberUtils.createBigDecimal("10E+6").toBigInteger().toString());
10 | System.err.println((String) null);
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/common/src/test/java/com/alibaba/datax/common/exception/DataXExceptionTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.exception;
2 |
3 | import org.junit.Assert;
4 | import org.junit.Test;
5 |
6 | import com.alibaba.datax.common.spi.ErrorCode;
7 |
8 | public class DataXExceptionTest {
9 |
10 | private DataXException dataXException;
11 |
12 | @Test
13 | public void basicTest() {
14 | ErrorCode errorCode = FakeErrorCode.FAKE_ERROR_CODE_ONLY_FOR_TEST_00;
15 | String errorMsg = "basicTest";
16 | dataXException = DataXException.asDataXException(errorCode, errorMsg);
17 | Assert.assertEquals(errorCode.toString() + " - " + errorMsg,
18 | dataXException.getMessage());
19 | }
20 |
21 | @Test
22 | public void basicTest_中文() {
23 | ErrorCode errorCode = FakeErrorCode.FAKE_ERROR_CODE_ONLY_FOR_TEST_01;
24 | String errorMsg = "basicTest中文";
25 | dataXException = DataXException.asDataXException(errorCode, errorMsg);
26 | Assert.assertEquals(errorCode.toString() + " - " + errorMsg,
27 | dataXException.getMessage());
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/common/src/test/java/com/alibaba/datax/common/exception/FakeErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.common.exception;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum FakeErrorCode implements ErrorCode {
6 |
7 | FAKE_ERROR_CODE_ONLY_FOR_TEST_00("FakeErrorCode-00",
8 | "only a test, FakeErrorCode."), FAKE_ERROR_CODE_ONLY_FOR_TEST_01(
9 | "FakeErrorCode-01",
10 | "only a test, FakeErrorCode，测试中文."),
11 |
12 | ;
13 |
14 | private final String code;
15 | private final String description;
16 |
17 | private FakeErrorCode(String code, String description) {
18 | this.code = code;
19 | this.description = description;
20 | }
21 |
22 | @Override
23 | public String getCode() {
24 | return this.code;
25 | }
26 |
27 | @Override
28 | public String getDescription() {
29 | return this.description;
30 | }
31 |
32 | @Override
33 | public String toString() {
34 | return String.format("Code:[%s], Describe:[%s]", this.code,
35 | this.description);
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/common/src/test/resources/logback-test.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | UTF-8
9 |
10 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{0} - %msg%n
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/common/src/test/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | UTF-8
9 |
10 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{0} - %msg%n
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/core/src/main/conf/.secret.properties:
--------------------------------------------------------------------------------
1 | #ds basicAuth config
2 | auth.user=
3 | auth.pass=
4 | current.keyVersion=
5 | current.publicKey=
6 | current.privateKey=
7 | current.service.username=
8 | current.service.password=
9 |
10 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/AbstractContainer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator;
5 | import org.apache.commons.lang.Validate;
6 |
7 | /**
8 | * 执行容器的抽象类，持有该容器全局的配置 configuration
9 | */
10 | public abstract class AbstractContainer {
11 | protected Configuration configuration;
12 |
13 | protected AbstractContainerCommunicator containerCommunicator;
14 |
15 | public AbstractContainer(Configuration configuration) {
16 | Validate.notNull(configuration, "Configuration can not be null.");
17 |
18 | this.configuration = configuration;
19 | }
20 |
21 | public Configuration getConfiguration() {
22 | return configuration;
23 | }
24 |
25 | public AbstractContainerCommunicator getContainerCommunicator() {
26 | return containerCommunicator;
27 | }
28 |
29 | public void setContainerCommunicator(AbstractContainerCommunicator containerCommunicator) {
30 | this.containerCommunicator = containerCommunicator;
31 | }
32 |
33 | public abstract void start();
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/job/meta/ExecuteMode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.job.meta;
2 |
3 | /**
4 | * Created by liupeng on 15/12/21.
5 | */
6 | public enum ExecuteMode {
7 | STANDALONE("standalone"), ;
8 |
9 | String value;
10 |
11 | private ExecuteMode(String value) {
12 | this.value = value;
13 | }
14 |
15 | public String value() {
16 | return this.value;
17 | }
18 |
19 | public String getValue() {
20 | return this.value;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/job/meta/State.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.job.meta;
2 |
3 | /**
4 | * Created by liupeng on 15/12/21.
5 | */
6 | public enum State {
7 | SUBMITTING(10),
8 | WAITING(20),
9 | RUNNING(30),
10 | KILLING(40),
11 | KILLED(50),
12 | FAILED(60),
13 | SUCCEEDED(70), ;
14 |
15 | int value;
16 |
17 | private State(int value) {
18 | this.value = value;
19 | }
20 |
21 | public int value() {
22 | return this.value;
23 | }
24 |
25 | public boolean isFinished() {
26 | return this == KILLED || this == FAILED || this == SUCCEEDED;
27 | }
28 |
29 | public boolean isRunning() {
30 | return !this.isFinished();
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/StandAloneScheduler.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.job.scheduler.processinner;
2 |
3 | import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator;
4 |
5 | /**
6 | * Created by hongjiao.hj on 2014/12/22.
7 | */
8 | public class StandAloneScheduler extends ProcessInnerScheduler{
9 |
10 | public StandAloneScheduler(AbstractContainerCommunicator containerCommunicator) {
11 | super(containerCommunicator);
12 | }
13 |
14 | @Override
15 | protected boolean isJobKilling(Long jobId) {
16 | return false;
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.container.collector;
2 |
3 | import com.alibaba.datax.core.statistics.communication.Communication;
4 | import com.alibaba.datax.core.statistics.communication.LocalTGCommunicationManager;
5 |
6 | public class ProcessInnerCollector extends AbstractCollector {
7 |
8 | public ProcessInnerCollector(Long jobId) {
9 | super.setJobId(jobId);
10 | }
11 |
12 | @Override
13 | public Communication collectFromTaskGroup() {
14 | return LocalTGCommunicationManager.getJobCommunication();
15 | }
16 |
17 | }
18 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.container.communicator.taskgroup;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.core.statistics.container.report.ProcessInnerReporter;
5 | import com.alibaba.datax.core.statistics.communication.Communication;
6 |
7 | public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommunicator {
8 |
9 | public StandaloneTGContainerCommunicator(Configuration configuration) {
10 | super(configuration);
11 | super.setReporter(new ProcessInnerReporter());
12 | }
13 |
14 | @Override
15 | public void report(Communication communication) {
16 | super.getReporter().reportTGCommunication(super.taskGroupId, communication);
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.container.report;
2 |
3 | import com.alibaba.datax.core.statistics.communication.Communication;
4 |
5 | public abstract class AbstractReporter {
6 |
7 | public abstract void reportJobCommunication(Long jobId, Communication communication);
8 |
9 | public abstract void reportTGCommunication(Integer taskGroupId, Communication communication);
10 |
11 | }
12 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.container.report;
2 |
3 | import com.alibaba.datax.core.statistics.communication.Communication;
4 | import com.alibaba.datax.core.statistics.communication.LocalTGCommunicationManager;
5 |
6 | public class ProcessInnerReporter extends AbstractReporter {
7 |
8 | @Override
9 | public void reportJobCommunication(Long jobId, Communication communication) {
10 | // do nothing
11 | }
12 |
13 | @Override
14 | public void reportTGCommunication(Integer taskGroupId, Communication communication) {
15 | LocalTGCommunicationManager.updateTaskGroupCommunication(taskGroupId, communication);
16 | }
17 | }
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/plugin/DefaultJobPluginCollector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.plugin;
2 |
3 | import com.alibaba.datax.common.plugin.JobPluginCollector;
4 | import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator;
5 | import com.alibaba.datax.core.statistics.communication.Communication;
6 |
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | /**
11 | * Created by jingxing on 14-9-9.
12 | */
13 | public final class DefaultJobPluginCollector implements JobPluginCollector {
14 | private AbstractContainerCommunicator jobCollector;
15 |
16 | public DefaultJobPluginCollector(AbstractContainerCommunicator containerCollector) {
17 | this.jobCollector = containerCollector;
18 | }
19 |
20 | @Override
21 | public Map> getMessage() {
22 | Communication totalCommunication = this.jobCollector.collect();
23 | return totalCommunication.getMessage();
24 | }
25 |
26 | @Override
27 | public List getMessage(String key) {
28 | Communication totalCommunication = this.jobCollector.collect();
29 | return totalCommunication.getMessage(key);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/HttpPluginCollector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.statistics.plugin.task;
2 |
3 | import com.alibaba.datax.common.constant.PluginType;
4 | import com.alibaba.datax.common.element.Record;
5 | import com.alibaba.datax.common.util.Configuration;
6 | import com.alibaba.datax.core.statistics.communication.Communication;
7 |
8 | /**
9 | * Created by jingxing on 14-9-9.
10 | */
11 | public class HttpPluginCollector extends AbstractTaskPluginCollector {
12 | public HttpPluginCollector(Configuration configuration, Communication Communication,
13 | PluginType type) {
14 | super(configuration, Communication, type);
15 | }
16 |
17 | @Override
18 | public void collectDirtyRecord(Record dirtyRecord, Throwable t,
19 | String errorMessage) {
20 | super.collectDirtyRecord(dirtyRecord, t, errorMessage);
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.transport.record;
2 |
3 | import com.alibaba.datax.common.element.Column;
4 | import com.alibaba.datax.common.element.Record;
5 |
6 | /**
7 | * 作为标示 生产者已经完成生产的标志
8 | *
9 | */
10 | public class TerminateRecord implements Record {
11 | private final static TerminateRecord SINGLE = new TerminateRecord();
12 |
13 | private TerminateRecord() {
14 | }
15 |
16 | public static TerminateRecord get() {
17 | return SINGLE;
18 | }
19 |
20 | @Override
21 | public void addColumn(Column column) {
22 | }
23 |
24 | @Override
25 | public Column getColumn(int i) {
26 | return null;
27 | }
28 |
29 | @Override
30 | public int getColumnNumber() {
31 | return 0;
32 | }
33 |
34 | @Override
35 | public int getByteSize() {
36 | return 0;
37 | }
38 |
39 | @Override
40 | public int getMemorySize() {
41 | return 0;
42 | }
43 |
44 | @Override
45 | public void setColumn(int i, Column column) {
46 | return;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/transport/transformer/ComplexTransformerProxy.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.transport.transformer;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 | import com.alibaba.datax.transformer.ComplexTransformer;
5 | import com.alibaba.datax.transformer.Transformer;
6 |
7 | import java.util.Map;
8 |
9 | /**
10 | * no comments.
11 | * Created by liqiang on 16/3/8.
12 | */
13 | public class ComplexTransformerProxy extends ComplexTransformer {
14 | private Transformer realTransformer;
15 |
16 | public ComplexTransformerProxy(Transformer transformer) {
17 | setTransformerName(transformer.getTransformerName());
18 | this.realTransformer = transformer;
19 | }
20 |
21 | @Override
22 | public Record evaluate(Record record, Map tContext, Object... paras) {
23 | return this.realTransformer.evaluate(record, paras);
24 | }
25 |
26 | public Transformer getRealTransformer() {
27 | return realTransformer;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.transport.transformer;
2 |
3 | /**
4 | * GroovyTransformer的帮助类，供groovy代码使用，必须全是static的方法
5 | * Created by liqiang on 16/3/4.
6 | */
7 | public class GroovyTransformerStaticUtil {
8 |
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerInfo.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.transport.transformer;
2 |
3 | import com.alibaba.datax.transformer.ComplexTransformer;
4 |
5 | /**
6 | * 单实例.
7 | * Created by liqiang on 16/3/9.
8 | */
9 | public class TransformerInfo {
10 |
11 | /**
12 | * function基本信息
13 | */
14 | private ComplexTransformer transformer;
15 | private ClassLoader classLoader;
16 | private boolean isNative;
17 |
18 |
19 | public ComplexTransformer getTransformer() {
20 | return transformer;
21 | }
22 |
23 | public ClassLoader getClassLoader() {
24 | return classLoader;
25 | }
26 |
27 | public boolean isNative() {
28 | return isNative;
29 | }
30 |
31 | public void setTransformer(ComplexTransformer transformer) {
32 | this.transformer = transformer;
33 | }
34 |
35 | public void setClassLoader(ClassLoader classLoader) {
36 | this.classLoader = classLoader;
37 | }
38 |
39 | public void setIsNative(boolean isNative) {
40 | this.isNative = isNative;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/util/ClassSize.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.util;
2 |
3 | /**
4 | * Created by liqiang on 15/12/12.
5 | */
6 | public class ClassSize {
7 |
8 | public static final int DefaultRecordHead;
9 | public static final int ColumnHead;
10 |
11 | //objectHead的大小
12 | public static final int REFERENCE;
13 | public static final int OBJECT;
14 | public static final int ARRAY;
15 | public static final int ARRAYLIST;
16 | static {
17 | //only 64位
18 | REFERENCE = 8;
19 |
20 | OBJECT = 2 * REFERENCE;
21 |
22 | ARRAY = align(3 * REFERENCE);
23 |
24 | // 16+8+24+16
25 | ARRAYLIST = align(OBJECT + align(REFERENCE) + align(ARRAY) +
26 | (2 * Long.SIZE / Byte.SIZE));
27 | // 8+64+8
28 | DefaultRecordHead = align(align(REFERENCE) + ClassSize.ARRAYLIST + 2 * Integer.SIZE / Byte.SIZE);
29 | //16+4
30 | ColumnHead = align(2 * REFERENCE + Integer.SIZE / Byte.SIZE);
31 | }
32 |
33 | public static int align(int num) {
34 | return (int)(align((long)num));
35 | }
36 |
37 | public static long align(long num) {
38 | //The 7 comes from that the alignSize is 8 which is the number of bytes
39 | //stored and sent together
40 | return ((num + 7) >> 3) << 3;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/util/ClassUtil.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.util;
2 |
3 | import java.lang.reflect.Constructor;
4 |
5 | public final class ClassUtil {
6 |
7 | /**
8 | * 通过反射构造类对象
9 | *
10 | * @param className
11 | * 反射的类名称
12 | * @param t
13 | * 反射类的类型Class对象
14 | * @param args
15 | * 构造参数
16 | *
17 | * */
18 | @SuppressWarnings({ "rawtypes", "unchecked" })
19 | public static T instantiate(String className, Class t,
20 | Object... args) {
21 | try {
22 | Constructor constructor = (Constructor) Class.forName(className)
23 | .getConstructor(ClassUtil.toClassType(args));
24 | return (T) constructor.newInstance(args);
25 | } catch (Exception e) {
26 | throw new IllegalArgumentException(e);
27 | }
28 | }
29 |
30 | private static Class[] toClassType(Object[] args) {
31 | Class[] clazzs = new Class[args.length];
32 |
33 | for (int i = 0, length = args.length; i < length; i++) {
34 | clazzs[i] = args[i].getClass();
35 | }
36 |
37 | return clazzs;
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/util/ConfigurationValidate.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.util;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import org.apache.commons.lang.Validate;
5 |
6 | /**
7 | * Created by jingxing on 14-9-16.
8 | *
9 | * 对配置文件做整体检查
10 | */
11 | public class ConfigurationValidate {
12 | public static void doValidate(Configuration allConfig) {
13 | Validate.isTrue(allConfig!=null, "");
14 |
15 | coreValidate(allConfig);
16 |
17 | pluginValidate(allConfig);
18 |
19 | jobValidate(allConfig);
20 | }
21 |
22 | private static void coreValidate(Configuration allconfig) {
23 | return;
24 | }
25 |
26 | private static void pluginValidate(Configuration allConfig) {
27 | return;
28 | }
29 |
30 | private static void jobValidate(Configuration allConfig) {
31 | return;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/core/util/ExceptionTracker.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.util;
2 |
3 | import java.io.PrintWriter;
4 | import java.io.StringWriter;
5 |
6 | public class ExceptionTracker {
7 | public static final int STRING_BUFFER = 4096;
8 |
9 | public static String trace(Throwable ex) {
10 | StringWriter sw = new StringWriter(STRING_BUFFER);
11 | PrintWriter pw = new PrintWriter(sw);
12 | ex.printStackTrace(pw);
13 | return sw.toString();
14 | }
15 | }
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumStrVal.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.dataxservice.face.domain.enums;
2 |
3 | public interface EnumStrVal {
4 | public String value();
5 | }
6 |
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumVal.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.dataxservice.face.domain.enums;
2 |
3 | public interface EnumVal {
4 | public int value();
5 | }
--------------------------------------------------------------------------------
/core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/State.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.dataxservice.face.domain.enums;
2 |
3 | public enum State implements EnumVal {
4 |
5 | SUBMITTING(10),
6 | WAITING(20),
7 | RUNNING(30),
8 | KILLING(40),
9 | KILLED(50),
10 | FAILED(60),
11 | SUCCEEDED(70);
12 |
13 |
14 | /* 一定会被初始化的 */
15 | int value;
16 |
17 | State(int value) {
18 | this.value = value;
19 | }
20 |
21 | @Override
22 | public int value() {
23 | return value;
24 | }
25 |
26 |
27 | public boolean isFinished() {
28 | return this == KILLED || this == FAILED || this == SUCCEEDED;
29 | }
30 |
31 | public boolean isRunning() {
32 | return !isFinished();
33 | }
34 |
35 | }
--------------------------------------------------------------------------------
/core/src/main/log/datax.log:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/main/log/datax.log
--------------------------------------------------------------------------------
/core/src/main/script/Readme.md:
--------------------------------------------------------------------------------
1 | some script here.
--------------------------------------------------------------------------------
/core/src/main/tmp/readme.txt:
--------------------------------------------------------------------------------
1 | If you are developing DataX Plugin, In your Plugin you can use this directory to store temporary resources .
2 |
3 | NOTE:
4 | Each time install DataX, this directory will be cleaned up !
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/constant/CoreConstantTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.constant;
2 |
3 | import com.alibaba.datax.core.util.container.CoreConstant;
4 | import org.junit.Test;
5 |
6 | public class CoreConstantTest {
7 | @Test
8 | public void test() {
9 | System.out.println(System.getProperties());
10 | System.out.println(CoreConstant.DATAX_PLUGIN_HOME);
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/container/ClassLoaderSwapperTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.container;
2 |
3 | import com.alibaba.datax.core.util.container.ClassLoaderSwapper;
4 | import org.junit.Assert;
5 | import org.junit.Test;
6 |
7 | import java.net.URL;
8 | import java.net.URLClassLoader;
9 |
10 | /**
11 | * Created by jingxing on 14-9-4.
12 | */
13 | public class ClassLoaderSwapperTest {
14 | @Test
15 | public void test() {
16 | ClassLoaderSwapper classLoaderSwapper =
17 | ClassLoaderSwapper.newCurrentThreadClassLoaderSwapper();
18 | ClassLoader newClassLoader = new URLClassLoader(new URL[]{});
19 | classLoaderSwapper.setCurrentThreadClassLoader(newClassLoader);
20 | Assert.assertTrue("", newClassLoader ==
21 | classLoaderSwapper.restoreCurrentThreadClassLoader());
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/faker/FakeGroovyTransformer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.faker;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 | import com.alibaba.datax.transformer.Transformer;
5 |
6 | /**
7 | * no comments.
8 | * Created by liqiang on 16/3/4.
9 | */
10 | public class FakeGroovyTransformer extends Transformer {
11 | public FakeGroovyTransformer() {
12 | setTransformerName("dx_fackGroovy");
13 | }
14 |
15 | @Override
16 | public Record evaluate(Record record, Object... paras) {
17 | return null;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/faker/FakeJobContainer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.faker;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.core.AbstractContainer;
5 |
6 | /**
7 | * Created by jingxing on 14-9-25.
8 | */
9 | public class FakeJobContainer extends AbstractContainer {
10 | public FakeJobContainer(Configuration configuration) {
11 | super(configuration);
12 | }
13 |
14 | @Override
15 | public void start() {
16 | System.out.println("Fake Job start ..");
17 | }
18 | }
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/faker/FakeReplaceTransformer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.faker;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 | import com.alibaba.datax.transformer.Transformer;
5 |
6 | /**
7 | * no comments.
8 | * Created by liqiang on 16/3/4.
9 | */
10 | public class FakeReplaceTransformer extends Transformer {
11 | public FakeReplaceTransformer() {
12 | setTransformerName("dx_fakeReplace");
13 | }
14 |
15 | @Override
16 | public Record evaluate(Record record, Object... paras) {
17 | return null;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/faker/FakeSubstrTransformer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.faker;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 | import com.alibaba.datax.transformer.Transformer;
5 |
6 | /**
7 | * no comments.
8 | * Created by liqiang on 16/3/4.
9 | */
10 | public class FakeSubstrTransformer extends Transformer {
11 | public FakeSubstrTransformer() {
12 | setTransformerName("dx_fakeSubstr");
13 | }
14 |
15 | @Override
16 | public Record evaluate(Record record, Object... paras) {
17 | return null;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/scaffold/ColumnProducer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.scaffold;
2 |
3 | import com.alibaba.datax.common.element.BoolColumn;
4 | import com.alibaba.datax.common.element.BytesColumn;
5 | import com.alibaba.datax.common.element.Column;
6 | import com.alibaba.datax.common.element.DateColumn;
7 | import com.alibaba.datax.common.element.LongColumn;
8 | import com.alibaba.datax.common.element.StringColumn;
9 |
10 | public class ColumnProducer {
11 | public static Column produceLongColumn(int i) {
12 | return new LongColumn(i);
13 | }
14 |
15 | public static Column produceStringColumn(String s) {
16 | return new StringColumn(s);
17 | }
18 |
19 | public static Column produceDateColumn(long time) {
20 | return new DateColumn(time);
21 | }
22 |
23 | public static Column produceBytesColumn(byte[] bytes) {
24 | return new BytesColumn(bytes);
25 | }
26 |
27 | public static Column produceBoolColumn(boolean bool) {
28 | return new BoolColumn(bool);
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/scaffold/ConfigurationProducer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.scaffold;
2 |
3 | import java.io.File;
4 |
5 | import com.alibaba.datax.common.util.Configuration;
6 | import com.alibaba.datax.core.util.ConfigParser;
7 |
8 | public final class ConfigurationProducer {
9 |
10 | public static Configuration produce() {
11 | String path = ConfigurationProducer.class.getClassLoader()
12 | .getResource(".").getFile();
13 | return ConfigParser.parse(path + File.separator + "all.json");
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/scheduler/standalone/StandAloneTestTaskGroupContainer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.scheduler.standalone;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.core.taskgroup.TaskGroupContainer;
5 |
6 | /**
7 | * Created by jingxing on 14-9-4.
8 | */
9 | public class StandAloneTestTaskGroupContainer extends TaskGroupContainer {
10 | public StandAloneTestTaskGroupContainer(Configuration configuration) {
11 | super(configuration);
12 | }
13 |
14 | @Override
15 | public void start() {
16 | try {
17 | Thread.sleep(200);
18 | } catch (InterruptedException e) {
19 | e.printStackTrace();
20 | }
21 | System.out.println("start standAlone test task container");
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/core/src/test/java/com/alibaba/datax/core/transport/record/RecordTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.core.transport.record;
2 |
3 | import org.junit.Assert;
4 | import org.junit.Test;
5 |
6 | import com.alibaba.datax.common.element.Record;
7 | import com.alibaba.datax.common.util.Configuration;
8 | import com.alibaba.datax.core.scaffold.RecordProducer;
9 |
10 | public class RecordTest {
11 | @Test
12 | public void test() {
13 | Record record = RecordProducer.produceRecord();
14 | System.out.println(record.toString());
15 |
16 | Configuration configuration = Configuration.from(record.toString());
17 | Assert.assertTrue(configuration.getInt("size") == 5);
18 | Assert.assertTrue(configuration.getList("data").size() == 5);
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/core/src/test/resources/conf/.secret.properties:
--------------------------------------------------------------------------------
1 | #ds basicAuth config
2 | auth.user=datax
3 | auth.pass=datax
4 | last.keyVersion=
5 | last.publicKey=
6 | last.privateKey=
7 | current.keyVersion=201412091312
8 | current.publicKey=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDE7hkiZukRMdrb3pCsjG149BObagDjn7y4B+qde1pET9RxbzXP9bhmS63C4e8WzKmYbMd6o8paCtYn+S7R60vYFjoEwgb4p3aGPC8sp5AkpBeQXAADFZFJ7zxN1se3LMa2/UKxIjO9OA3aSieAczKd3ChmGo4Vi3hFIrecTG7oXQIDAQAB
9 | current.privateKey=MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAMTuGSJm6REx2tvekKyMbXj0E5tqAOOfvLgH6p17WkRP1HFvNc/1uGZLrcLh7xbMqZhsx3qjyloK1if5LtHrS9gWOgTCBvindoY8LyynkCSkF5BcAAMVkUnvPE3Wx7csxrb9QrEiM704DdpKJ4BzMp3cKGYajhWLeEUit5xMbuhdAgMBAAECgYEAn+CFe059LT6CZjpczhj7z1SojmYS7rmCZw3WRaAdepQs7yLQV1MwL6yFF1CB4MqrbVny4PgUkeF2V+GPR1F1sj2emYdBDBqmTj8gn5aF8xXvfHUl1uH3VnIyDQHnMG0FTSH0f1QgEv7L4WkpuLM1lcBpegkJCcuTcMt+fm3GoEECQQDvDxPyoB2K1PMLzta9aO4ZaaBLS/ggdVaPFcqlDdY0hDSuquHeWqTfvPwI3yZkc6nyhUvQNiKTKqasucSXnTGlAkEA0uK75OWQkGmuLyPwOSGTbFhW6Z0DzvtdhO1TtIBZ3y8DGr8G1a/kzNYWcEQjOK50Ula7jLymSA/lO79Dx9kuWQJBAN6bmMSu6rOT9rsBIaABLO6HGFflZym6eh8FeM1X5CbFEVWxFGD84Vji34LXYSXbOt711xIMxwdpiQmAdxuDqm0CQHKxQ5VS0RPplgUnW5AG1cH4LZSyg46/oPYZiQvDPp2mWN7kA9iV6C8LRHrcY/eA0dyyNSBuvVS16GtdM4TudkkCQQCyo2hFVn/zSbZYV02LPR47IkN2dEkNTr8j4dXcPqAy3rkx18Me86RwraHBJ0TLs6mMbRBRo5AimpH47Z9iVDdb
10 |
11 |
12 |
--------------------------------------------------------------------------------
/core/src/test/resources/job/job.json:
--------------------------------------------------------------------------------
1 | {
2 | "job": {
3 | "setting": {},
4 | "content": [
5 | {
6 | "reader": {
7 | "name": "fakereader",
8 | "parameter": {}
9 | },
10 | "writer": {
11 | "name": "fakewriter",
12 | "parameter": {}
13 | }
14 | },
15 | {
16 | "reader": {
17 | "name": "fakereader",
18 | "parameter": {}
19 | },
20 | "writer": {
21 | "name": "fakewriter",
22 | "parameter": {}
23 | }
24 | }
25 | ]
26 | }
27 | }
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/nullJarTransformer/transformer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "userUDF1",
3 | "class": "com.alibaba.datax.plugin.tranformer.userudf",
4 | "version": 1,
5 | "md5": 1234567,
6 | "developer": "user@user.com"
7 | }
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/nullJsonTransformer/test:
--------------------------------------------------------------------------------
1 | dfdfdf
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/nullclassTransformer/transformer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "userUDF1",
3 | "version": 1,
4 | "md5": 1234567,
5 | "developer": "user@user.com"
6 | }
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/userComplexTransformerTest/libs/guava-16.0.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/test/resources/local_storage/transformer/userComplexTransformerTest/libs/guava-16.0.jar
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/userComplexTransformerTest/oracle_oss_transformer-1.0-SNAPSHOT.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/test/resources/local_storage/transformer/userComplexTransformerTest/oracle_oss_transformer-1.0-SNAPSHOT.jar
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/userComplexTransformerTest/transformer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "userComplexTransformerTest",
3 | "class": "com.alibaba.transformer.gongan.ComplexTranformerTest",
4 | "version":1,
5 | "md5":1234567,
6 | "developer": "user@user.com"
7 | }
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/userTransformerTest/oracle_oss_transformer-1.0-SNAPSHOT.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/test/resources/local_storage/transformer/userTransformerTest/oracle_oss_transformer-1.0-SNAPSHOT.jar
--------------------------------------------------------------------------------
/core/src/test/resources/local_storage/transformer/userTransformerTest/transformer.json:
--------------------------------------------------------------------------------
1 | {
2 | "class": "com.alibaba.transformer.gongan.TransformerTest",
3 | "version":1,
4 | "md5":1234567,
5 | "developer": "user@user.com"
6 | }
--------------------------------------------------------------------------------
/core/src/test/resources/plugin/reader/fakereader/FakePluginer.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/test/resources/plugin/reader/fakereader/FakePluginer.jar
--------------------------------------------------------------------------------
/core/src/test/resources/plugin/reader/fakereader/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "fakereader",
3 | "class": "com.alibaba.datax.core.faker.FakeReader",
4 | "description": {
5 | "useScene": "only for performance test.",
6 | "mechanism": "Produce Record from memory.",
7 | "warn": "Never use it in your real job."
8 | },
9 | "developer": "someBody,bug reported to : someBody@someSite"
10 | }
--------------------------------------------------------------------------------
/core/src/test/resources/plugin/writer/fakewriter/FakePluginer.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/core/src/test/resources/plugin/writer/fakewriter/FakePluginer.jar
--------------------------------------------------------------------------------
/core/src/test/resources/plugin/writer/fakewriter/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "fakewriter",
3 | "class": "com.alibaba.datax.core.faker.FakeWriter",
4 | "description": {
5 | "useScene": "only for performance test.",
6 | "mechanism": "Produce Record from memory.",
7 | "warn": "Never use it in your real job."
8 | },
9 | "developer": "someBody,bug reported to : someBody@someSite"
10 | }
--------------------------------------------------------------------------------
/datax-opensource-dingding.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/datax-opensource-dingding.png
--------------------------------------------------------------------------------
/drdsreader/src/main/java/com/alibaba/datax/plugin/reader/drdsreader/DrdsReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.drdsreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum DrdsReaderErrorCode implements ErrorCode {
6 | GET_TOPOLOGY_FAILED("DrdsReader-01", "获取 drds 表的拓扑结构失败."),;
7 |
8 | private final String code;
9 | private final String description;
10 |
11 | private DrdsReaderErrorCode(String code, String description) {
12 | this.code = code;
13 | this.description = description;
14 | }
15 |
16 | @Override
17 | public String getCode() {
18 | return this.code;
19 | }
20 |
21 | @Override
22 | public String getDescription() {
23 | return this.description;
24 | }
25 |
26 | @Override
27 | public String toString() {
28 | return String.format("Code:[%s], Description:[%s]. ", this.code,
29 | this.description);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/drdsreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "drdsreader",
3 | "class": "com.alibaba.datax.plugin.reader.drdsreader.DrdsReader",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/drdsreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "drdsreader",
3 | "parameter": {
4 | "jdbcUrl": "",
5 | "username": "",
6 | "password": "",
7 | "table": "",
8 | "column": [],
9 | "where": ""
10 | }
11 | }
--------------------------------------------------------------------------------
/drdswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "drdswriter",
3 | "class": "com.alibaba.datax.plugin.writer.drdswriter.DrdsWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/drdswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "drdswriter",
3 | "parameter": {
4 | "jdbcUrl": "",
5 | "username": "",
6 | "password": "",
7 | "table": "",
8 | "column": [],
9 | "writeMode": "",
10 | "preSql": [],
11 | "postSql": []
12 | }
13 | }
--------------------------------------------------------------------------------
/elasticsearchwriter/README.md:
--------------------------------------------------------------------------------
1 | 本插件仅在Elasticsearch 5.x上测试
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/elasticsearchwriter/build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | SCRIPT_HOME=$(cd $(dirname $0); pwd)
4 | cd $SCRIPT_HOME/..
5 | mvn clean package -DskipTests assembly:assembly
6 |
7 | cd $SCRIPT_HOME/target/datax/plugin/writer/
8 |
9 | if [ -d "eswriter" ]; then
10 | tar -zcvf eswriter.tgz eswriter
11 | cp eswriter.tgz $SCRIPT_HOME
12 | cd $SCRIPT_HOME
13 | ansible-playbook -i hosts main.yml -u vagrant -k
14 | fi
15 |
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESFieldType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.elasticsearchwriter;
2 |
3 | /**
4 | * Created by xiongfeng.bxf on 17/3/1.
5 | */
6 | public enum ESFieldType {
7 | ID,
8 | STRING,
9 | TEXT,
10 | KEYWORD,
11 | LONG,
12 | INTEGER,
13 | SHORT,
14 | BYTE,
15 | DOUBLE,
16 | FLOAT,
17 | DATE,
18 | BOOLEAN,
19 | BINARY,
20 | INTEGER_RANGE,
21 | FLOAT_RANGE,
22 | LONG_RANGE,
23 | DOUBLE_RANGE,
24 | DATE_RANGE,
25 | GEO_POINT,
26 | GEO_SHAPE,
27 |
28 | IP,
29 | COMPLETION,
30 | TOKEN_COUNT,
31 |
32 | ARRAY,
33 | OBJECT,
34 | NESTED;
35 |
36 | public static ESFieldType getESFieldType(String type) {
37 | if (type == null) {
38 | return null;
39 | }
40 | for (ESFieldType f : ESFieldType.values()) {
41 | if (f.name().compareTo(type.toUpperCase()) == 0) {
42 | return f;
43 | }
44 | }
45 | return null;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.elasticsearchwriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum ESWriterErrorCode implements ErrorCode {
6 | BAD_CONFIG_VALUE("ESWriter-00", "您配置的值不合法."),
7 | ES_INDEX_DELETE("ESWriter-01", "删除index错误."),
8 | ES_INDEX_CREATE("ESWriter-02", "创建index错误."),
9 | ES_MAPPINGS("ESWriter-03", "mappings错误."),
10 | ES_INDEX_INSERT("ESWriter-04", "插入数据错误."),
11 | ES_ALIAS_MODIFY("ESWriter-05", "别名修改错误."),
12 | ;
13 |
14 | private final String code;
15 | private final String description;
16 |
17 | ESWriterErrorCode(String code, String description) {
18 | this.code = code;
19 | this.description = description;
20 | }
21 |
22 | @Override
23 | public String getCode() {
24 | return this.code;
25 | }
26 |
27 | @Override
28 | public String getDescription() {
29 | return this.description;
30 | }
31 |
32 | @Override
33 | public String toString() {
34 | return String.format("Code:[%s], Description:[%s]. ", this.code,
35 | this.description);
36 | }
37 | }
--------------------------------------------------------------------------------
/elasticsearchwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "elasticsearchwriter",
3 | "class": "com.alibaba.datax.plugin.writer.elasticsearchwriter.ESWriter",
4 | "description": "适用于: 生产环境. 原理: TODO",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/ftpreader/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/reader/ftpreader
18 |
19 |
20 | target/
21 |
22 | ftpreader-${version}.jar
23 |
24 | plugin/reader/ftpreader
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/reader/ftpreader/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.ftpreader;
2 |
3 |
4 | public class Constant {
5 | public static final String SOURCE_FILES = "sourceFiles";
6 |
7 | public static final int DEFAULT_FTP_PORT = 21;
8 | public static final int DEFAULT_SFTP_PORT = 22;
9 | public static final int DEFAULT_TIMEOUT = 60000;
10 | public static final int DEFAULT_MAX_TRAVERSAL_LEVEL = 100;
11 | public static final String DEFAULT_FTP_CONNECT_PATTERN = "PASV";
12 |
13 |
14 | }
15 |
--------------------------------------------------------------------------------
/ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.ftpreader;
2 |
3 | public class Key {
4 | public static final String PROTOCOL = "protocol";
5 | public static final String HOST = "host";
6 | public static final String USERNAME = "username";
7 | public static final String PASSWORD = "password";
8 | public static final String PORT = "port";
9 | public static final String TIMEOUT = "timeout";
10 | public static final String CONNECTPATTERN = "connectPattern";
11 | public static final String PATH = "path";
12 | public static final String MAXTRAVERSALLEVEL = "maxTraversalLevel";
13 | }
14 |
--------------------------------------------------------------------------------
/ftpreader/src/main/resources/plugin-template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ftpreader",
3 | "parameter": {
4 | "host": "",
5 | "port": "",
6 | "username": "",
7 | "password": "",
8 | "protocol": "",
9 | "path": [
10 | ""
11 | ],
12 | "encoding": "UTF-8",
13 | "column": [
14 | {
15 | "index": 0,
16 | "type": "long"
17 | },
18 | {
19 | "index": 1,
20 | "type": "boolean"
21 | },
22 | {
23 | "index": 2,
24 | "type": "double"
25 | },
26 | {
27 | "index": 3,
28 | "type": "string"
29 | },
30 | {
31 | "index": 4,
32 | "type": "date",
33 | "format": "yyyy.MM.dd"
34 | }
35 | ],
36 | "fieldDelimiter": ","
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/ftpreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ftpreader",
3 | "class": "com.alibaba.datax.plugin.reader.ftpreader.FtpReader",
4 | "description": "useScene: test. mechanism: use datax framework to transport data from txt file. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/ftpreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ftpreader",
3 | "parameter": {
4 | "host": "",
5 | "protocol": "sftp",
6 | "port":"",
7 | "username": "",
8 | "password": "",
9 | "path": [],
10 | "column": [
11 | {
12 | "index": 0,
13 | "type": ""
14 | }
15 | ],
16 | "fieldDelimiter": ",",
17 | "encoding": "UTF-8"
18 | }
19 | }
--------------------------------------------------------------------------------
/ftpwriter/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/ftpwriter/doc/.gitkeep
--------------------------------------------------------------------------------
/ftpwriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/ftpwriter
18 |
19 |
20 | target/
21 |
22 | ftpwriter-${version}.jar
23 |
24 | plugin/writer/ftpwriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/ftpwriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ftpwriter;
2 |
3 | public class Key {
4 | public static final String PROTOCOL = "protocol";
5 |
6 | public static final String HOST = "host";
7 |
8 | public static final String USERNAME = "username";
9 |
10 | public static final String PASSWORD = "password";
11 |
12 | public static final String PORT = "port";
13 |
14 | public static final String TIMEOUT = "timeout";
15 |
16 | public static final String CONNECTPATTERN = "connectPattern";
17 |
18 | public static final String PATH = "path";
19 | }
20 |
--------------------------------------------------------------------------------
/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ftpwriter.util;
2 |
3 |
4 | public class Constant {
5 | public static final int DEFAULT_FTP_PORT = 21;
6 |
7 | public static final int DEFAULT_SFTP_PORT = 22;
8 |
9 | public static final int DEFAULT_TIMEOUT = 60000;
10 |
11 | public static final int DEFAULT_MAX_TRAVERSAL_LEVEL = 100;
12 |
13 | public static final String DEFAULT_FTP_CONNECT_PATTERN = "PASV";
14 |
15 | public static final String CONTROL_ENCODING = "utf8";
16 | }
17 |
--------------------------------------------------------------------------------
/ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/IFtpHelper.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ftpwriter.util;
2 |
3 | import java.io.OutputStream;
4 | import java.util.Set;
5 |
6 | public interface IFtpHelper {
7 |
8 | //使用被动方式
9 | public void loginFtpServer(String host, String username, String password, int port, int timeout);
10 |
11 | public void logoutFtpServer();
12 |
13 | /**
14 | * warn: 不支持递归创建, 比如 mkdir -p
15 | * */
16 | public void mkdir(String directoryPath);
17 |
18 | /**
19 | * 支持目录递归创建
20 | */
21 | public void mkDirRecursive(String directoryPath);
22 |
23 | public OutputStream getOutputStream(String filePath);
24 |
25 | public String getRemoteFileContent(String filePath);
26 |
27 | public Set getAllFilesInDir(String dir, String prefixFileName);
28 |
29 | /**
30 | * warn: 不支持文件夹删除, 比如 rm -rf
31 | * */
32 | public void deleteFiles(Set filesToDelete);
33 |
34 | public void completePendingCommand();
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/ftpwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ftpwriter",
3 | "class": "com.alibaba.datax.plugin.writer.ftpwriter.FtpWriter",
4 | "description": "useScene: test. mechanism: use datax framework to transport data from ftp txt file. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/ftpwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ftpwriter",
3 | "parameter": {
4 | "name": "ftpwriter",
5 | "parameter": {
6 | "protocol": "",
7 | "host": "",
8 | "port": "",
9 | "username": "",
10 | "password": "",
11 | "timeout": "",
12 | "connectPattern": "",
13 | "path": "",
14 | "fileName": "",
15 | "writeMode": "",
16 | "fieldDelimiter": "",
17 | "encoding": "",
18 | "nullFormat": "",
19 | "dateFormat": "",
20 | "fileFormat": "",
21 | "header": []
22 | }
23 | }
24 | }
--------------------------------------------------------------------------------
/gpdbjsonwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "gpdbjsonwriter",
3 | "class": "cn.hashdata.datax.plugin.writer.gpdbjsonwriter.GpdbJsonWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute copy sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "hashdata"
6 | }
--------------------------------------------------------------------------------
/gpdbjsonwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "gpdbjsonwriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "segment_reject_limit": 0,
7 | "column": [],
8 | "preSql": [],
9 | "connection": [
10 | {
11 | "jdbcUrl": "",
12 | "table": []
13 | }
14 | ],
15 | "preSql": [],
16 | "postSql": []
17 | }
18 | }
19 |
20 |
--------------------------------------------------------------------------------
/gpdbwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "gpdbwriter",
3 | "class": "cn.hashdata.datax.plugin.writer.gpdbwriter.GpdbWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute copy sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "hashdata"
6 | }
--------------------------------------------------------------------------------
/gpdbwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "gpdbwriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "segment_reject_limit": 0,
7 | "column": [],
8 | "preSql": [],
9 | "connection": [
10 | {
11 | "jdbcUrl": "",
12 | "table": []
13 | }
14 | ],
15 | "preSql": [],
16 | "postSql": []
17 | }
18 | }
19 |
20 |
--------------------------------------------------------------------------------
/hbase094xreader/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/hbase094xreader/doc/.gitkeep
--------------------------------------------------------------------------------
/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase094xreader;
2 |
3 | public final class Constant {
4 | public static final String RANGE = "range";
5 |
6 | public static final String ROWKEY_FLAG = "rowkey";
7 |
8 | public static final String DEFAULT_ENCODING = "UTF-8";
9 |
10 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
11 |
12 | public static final int DEFAULT_SCAN_CACHE_SIZE = 256;
13 |
14 | public static final int DEFAULT_SCAN_BATCH_SIZE = 100;
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/ModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase094xreader;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum ModeType {
8 | Normal("normal"),
9 | MultiVersionFixedColumn("multiVersionFixedColumn")
10 | ;
11 |
12 | private String mode;
13 |
14 | ModeType(String mode) {
15 | this.mode = mode.toLowerCase();
16 | }
17 |
18 | public static ModeType getByTypeName(String modeName) {
19 | for (ModeType modeType : values()) {
20 | if (modeType.mode.equalsIgnoreCase(modeName)) {
21 | return modeType;
22 | }
23 | }
24 |
25 | throw DataXException.asDataXException(Hbase094xReaderErrorCode.ILLEGAL_VALUE,
26 | String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values())));
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/MultiVersionFixedColumnTask.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase094xreader;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import org.apache.hadoop.hbase.client.Scan;
5 | import org.apache.hadoop.hbase.util.Bytes;
6 |
7 | import java.util.Map;
8 |
9 | public class MultiVersionFixedColumnTask extends MultiVersionTask {
10 |
11 | public MultiVersionFixedColumnTask(Configuration configuration) {
12 | super(configuration);
13 | }
14 |
15 | @Override
16 | public void initScan(Scan scan) {
17 | for (Map aColumn : column) {
18 | String columnName = aColumn.get(Key.NAME);
19 | if(!Hbase094xHelper.isRowkeyColumn(columnName)){
20 | String[] cfAndQualifier = columnName.split(":");
21 | scan.addColumn(Bytes.toBytes(cfAndQualifier[0].trim()), Bytes.toBytes(cfAndQualifier[1].trim()));
22 | }
23 | }
24 | super.setMaxVersions(scan);
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/hbase094xreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase094xreader",
3 | "class": "com.alibaba.datax.plugin.reader.hbase094xreader.Hbase094xReader",
4 | "description": "useScene: prod. mechanism: Scan to read data.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/hbase094xreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase094xreader",
3 | "parameter": {
4 | "hbaseConfig": {},
5 | "table": "",
6 | "encoding": "",
7 | "mode": "",
8 | "column": [],
9 | "range": {
10 | "startRowkey": "",
11 | "endRowkey": "",
12 | "isBinaryRowkey": true
13 | }
14 | }
15 | }
--------------------------------------------------------------------------------
/hbase094xwriter/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/hbase094xwriter/doc/.gitkeep
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/hbase094xwriter
18 |
19 |
20 | target/
21 |
22 | hbase094xwriter-${version}.jar
23 |
24 | plugin/writer/hbase094xwriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/hbase094xwriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase094xwriter;
2 |
3 | public final class Constant {
4 | public static final String DEFAULT_ENCODING = "UTF-8";
5 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
6 | public static final String DEFAULT_NULL_MODE = "skip";
7 | public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024;
8 | }
9 |
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/ModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase094xwriter;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum ModeType {
8 | Normal("normal"),
9 | MultiVersion("multiVersion")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | ModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static ModeType getByTypeName(String modeName) {
24 | for (ModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE,
30 | String.format("Hbasewriter 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values())));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/NullModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase094xwriter;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum NullModeType {
8 | Skip("skip"),
9 | Empty("empty")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | NullModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static NullModeType getByTypeName(String modeName) {
24 | for (NullModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE,
30 | String.format("Hbasewriter 不支持该 nullMode 类型:%s, 目前支持的 nullMode 类型是:%s", modeName, Arrays.asList(values())));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase094xwriter",
3 | "class": "com.alibaba.datax.plugin.writer.hbase094xwriter.Hbase094xWriter",
4 | "description": "use put: prod. mechanism: use hbase java api put data.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/hbase094xwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase094xwriter",
3 | "parameter": {
4 | "hbaseConfig": {},
5 | "table": "",
6 | "mode": "",
7 | "rowkeyColumn": [
8 | ],
9 | "column": [
10 | ],
11 | "versionColumn":{
12 | "index": "",
13 | "value":""
14 | },
15 | "encoding": ""
16 | }
17 | }
--------------------------------------------------------------------------------
/hbase11xreader/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/hbase11xreader/doc/.gitkeep
--------------------------------------------------------------------------------
/hbase11xreader/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/reader/hbase11xreader
18 |
19 |
20 | target/
21 |
22 | hbase11xreader-${version}.jar
23 |
24 | plugin/reader/hbase11xreader
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/reader/hbase11xreader/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase11xreader;
2 |
3 | public final class Constant {
4 | public static final String RANGE = "range";
5 |
6 | public static final String ROWKEY_FLAG = "rowkey";
7 |
8 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
9 |
10 | public static final String DEFAULT_ENCODING = "UTF-8";
11 |
12 | public static final int DEFAULT_SCAN_CACHE_SIZE = 256;
13 |
14 | public static final int DEFAULT_SCAN_BATCH_SIZE = 100;
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/ModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase11xreader;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum ModeType {
8 | Normal("normal"),
9 | MultiVersionFixedColumn("multiVersionFixedColumn")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | ModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static ModeType getByTypeName(String modeName) {
24 | for (ModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(Hbase11xReaderErrorCode.ILLEGAL_VALUE,
30 | String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values())));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/MultiVersionDynamicColumnTask.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase11xreader;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import org.apache.hadoop.hbase.client.Scan;
5 | import org.apache.hadoop.hbase.util.Bytes;
6 |
7 | import java.util.List;
8 |
9 | public class MultiVersionDynamicColumnTask extends MultiVersionTask {
10 | private List columnFamilies = null;
11 |
12 | public MultiVersionDynamicColumnTask(Configuration configuration){
13 | super(configuration);
14 |
15 | this.columnFamilies = configuration.getList(Key.COLUMN_FAMILY, String.class);
16 | }
17 |
18 | @Override
19 | public void initScan(Scan scan) {
20 | for (String columnFamily : columnFamilies) {
21 | scan.addFamily(Bytes.toBytes(columnFamily.trim()));
22 | }
23 |
24 | super.setMaxVersions(scan);
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/MultiVersionFixedColumnTask.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbase11xreader;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import org.apache.hadoop.hbase.client.Scan;
5 | import org.apache.hadoop.hbase.util.Bytes;
6 |
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | public class MultiVersionFixedColumnTask extends MultiVersionTask {
11 |
12 | public MultiVersionFixedColumnTask(Configuration configuration) {
13 | super(configuration);
14 | }
15 |
16 | @Override
17 | public void initScan(Scan scan) {
18 | for (Map aColumn : column) {
19 | String columnName = aColumn.get(Key.NAME);
20 | if(!Hbase11xHelper.isRowkeyColumn(columnName)){
21 | String[] cfAndQualifier = columnName.split(":");
22 | scan.addColumn(Bytes.toBytes(cfAndQualifier[0].trim()), Bytes.toBytes(cfAndQualifier[1].trim()));
23 | }
24 | }
25 | super.setMaxVersions(scan);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase11xreader",
3 | "class": "com.alibaba.datax.plugin.reader.hbase11xreader.Hbase11xReader",
4 | "description": "useScene: prod. mechanism: Scan to read data.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/hbase11xreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase11xreader",
3 | "parameter": {
4 | "hbaseConfig": {},
5 | "table": "",
6 | "encoding": "",
7 | "mode": "",
8 | "column": [],
9 | "range": {
10 | "startRowkey": "",
11 | "endRowkey": "",
12 | "isBinaryRowkey": true
13 | }
14 | }
15 | }
--------------------------------------------------------------------------------
/hbase11xsqlwriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 |
16 | plugin/writer/hbase11xsqlwriter
17 |
18 |
19 | target/
20 |
21 | hbase11xsqlwriter-${version}.jar
22 |
23 | plugin/writer/hbase11xsqlwriter
24 |
25 |
26 |
27 |
28 |
29 | false
30 | plugin/writer/hbase11xsqlwriter/libs
31 | runtime
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xsqlwriter;
2 |
3 | public final class Constant {
4 | public static final String DEFAULT_ENCODING = "UTF-8";
5 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
6 | public static final String DEFAULT_NULL_MODE = "skip";
7 | public static final String DEFAULT_ZNODE = "/hbase";
8 | public static final boolean DEFAULT_LAST_COLUMN_IS_VERSION = false; // 默认最后一列不是version列
9 | public static final int DEFAULT_BATCH_ROW_COUNT = 256; // 默认一次写256行
10 | public static final boolean DEFAULT_TRUNCATE = false; // 默认开始的时候不清空表
11 |
12 | public static final int TYPE_UNSIGNED_TINYINT = 11;
13 | public static final int TYPE_UNSIGNED_SMALLINT = 13;
14 | public static final int TYPE_UNSIGNED_INTEGER = 9;
15 | public static final int TYPE_UNSIGNED_LONG = 10;
16 | public static final int TYPE_UNSIGNED_FLOAT = 14;
17 | public static final int TYPE_UNSIGNED_DOUBLE = 15;
18 | public static final int TYPE_UNSIGNED_DATE = 19;
19 | public static final int TYPE_UNSIGNED_TIME = 18;
20 | public static final int TYPE_UNSIGNED_TIMESTAMP = 20;
21 | }
22 |
--------------------------------------------------------------------------------
/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xsqlwriter;
2 |
3 | import org.apache.hadoop.hbase.HConstants;
4 |
5 | public final class Key {
6 |
7 | /**
8 | * 【必选】hbase集群配置，连接一个hbase集群需要的最小配置只有两个：zk和znode
9 | */
10 | public final static String HBASE_CONFIG = "hbaseConfig";
11 | public final static String HBASE_ZK_QUORUM = HConstants.ZOOKEEPER_QUORUM;
12 | public final static String HBASE_ZNODE_PARENT = HConstants.ZOOKEEPER_ZNODE_PARENT;
13 |
14 | /**
15 | * 【必选】writer要写入的表的表名
16 | */
17 | public final static String TABLE = "table";
18 |
19 | /**
20 | * 【必选】列配置
21 | */
22 | public final static String COLUMN = "column";
23 | public static final String NAME = "name";
24 |
25 | /**
26 | * 【可选】遇到空值默认跳过
27 | */
28 | public static final String NULL_MODE = "nullMode";
29 |
30 | /**
31 | * 【可选】
32 | * 在writer初始化的时候，是否清空目的表
33 | * 如果全局启动多个writer，则必须确保所有的writer都prepare之后，再开始导数据。
34 | */
35 | public static final String TRUNCATE = "truncate";
36 |
37 | /**
38 | * 【可选】批量写入的最大行数，默认100行
39 | */
40 | public static final String BATCH_SIZE = "batchSize";
41 |
42 |
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/NullModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xsqlwriter;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum NullModeType {
8 | Skip("skip"),
9 | Empty("empty")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | NullModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static NullModeType getByTypeName(String modeName) {
24 | for (NullModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE,
30 | "Hbasewriter 不支持该 nullMode 类型:" + modeName + ", 目前支持的 nullMode 类型是:" + Arrays.asList(values()));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase11xsqlwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase11xsqlwriter",
3 | "class": "com.alibaba.datax.plugin.writer.hbase11xsqlwriter.HbaseSQLWriter",
4 | "description": "useScene: prod. mechanism: use hbase sql UPSERT to put data, index tables will be updated too.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/hbase11xwriter/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/hbase11xwriter/doc/.gitkeep
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/hbase11xwriter
18 |
19 |
20 | target/
21 |
22 | hbase11xwriter-${version}.jar
23 |
24 | plugin/writer/hbase11xwriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/hbase11xwriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xwriter;
2 |
3 | public final class Constant {
4 | public static final String DEFAULT_ENCODING = "UTF-8";
5 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss";
6 | public static final String DEFAULT_NULL_MODE = "skip";
7 | public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024;
8 | }
9 |
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/ModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xwriter;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum ModeType {
8 | Normal("normal"),
9 | MultiVersion("multiVersion")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | ModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static ModeType getByTypeName(String modeName) {
24 | for (ModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE,
30 | String.format("Hbasewriter 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values())));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/NullModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbase11xwriter;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum NullModeType {
8 | Skip("skip"),
9 | Empty("empty")
10 | ;
11 |
12 | private String mode;
13 |
14 |
15 | NullModeType(String mode) {
16 | this.mode = mode.toLowerCase();
17 | }
18 |
19 | public String getMode() {
20 | return mode;
21 | }
22 |
23 | public static NullModeType getByTypeName(String modeName) {
24 | for (NullModeType modeType : values()) {
25 | if (modeType.mode.equalsIgnoreCase(modeName)) {
26 | return modeType;
27 | }
28 | }
29 | throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE,
30 | String.format("Hbasewriter 不支持该 nullMode 类型:%s, 目前支持的 nullMode 类型是:%s", modeName, Arrays.asList(values())));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase11xwriter",
3 | "class": "com.alibaba.datax.plugin.writer.hbase11xwriter.Hbase11xWriter",
4 | "description": "use put: prod. mechanism: use hbase java api put data.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/hbase11xwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbase11xwriter",
3 | "parameter": {
4 | "hbaseConfig": {
5 | "hbase.rootdir": "",
6 | "hbase.cluster.distributed": "",
7 | "hbase.zookeeper.quorum": ""
8 | },
9 | "table": "",
10 | "mode": "",
11 | "rowkeyColumn": [
12 | ],
13 | "column": [
14 | ],
15 | "versionColumn":{
16 | "index": "",
17 | "value":""
18 | },
19 | "encoding": ""
20 | }
21 | }
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/ColumnType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | /**
8 | * 只对 normal 模式读取时有用，多版本读取时，不存在列类型的
9 | */
10 | public enum ColumnType {
11 | STRING("string"),
12 | BINARY_STRING("binarystring"),
13 | BYTES("bytes"),
14 | BOOLEAN("boolean"),
15 | SHORT("short"),
16 | INT("int"),
17 | LONG("long"),
18 | FLOAT("float"),
19 | DOUBLE("double"),
20 | DATE("date"),;
21 |
22 | private String typeName;
23 |
24 | ColumnType(String typeName) {
25 | this.typeName = typeName;
26 | }
27 |
28 | public static ColumnType getByTypeName(String typeName) {
29 | for (ColumnType columnType : values()) {
30 | if (columnType.typeName.equalsIgnoreCase(typeName)) {
31 | return columnType;
32 | }
33 | }
34 |
35 | throw DataXException.asDataXException(HbaseReaderErrorCode.ILLEGAL_VALUE,
36 | String.format("Hbasereader 不支持该类型:%s, 目前支持的类型是:%s", typeName, Arrays.asList(values())));
37 | }
38 |
39 | @Override
40 | public String toString() {
41 | return this.typeName;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | public final class Constant {
4 | public static final String RANGE = "range";
5 |
6 | public static final String ROWKEY_FLAG = "rowkey";
7 |
8 | public static final int DEFAULT_SCAN_CACHE = 256;
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/HTableManager.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | import org.apache.hadoop.conf.Configuration;
4 | import org.apache.hadoop.hbase.client.HBaseAdmin;
5 | import org.apache.hadoop.hbase.client.HTable;
6 |
7 | import java.io.IOException;
8 |
9 | public final class HTableManager {
10 |
11 | public static HTable createHTable(Configuration config, String tableName)
12 | throws IOException {
13 |
14 | return new HTable(config, tableName);
15 | }
16 |
17 | public static HBaseAdmin createHBaseAdmin(Configuration config)
18 | throws IOException {
19 | return new HBaseAdmin(config);
20 | }
21 |
22 | public static void closeHTable(HTable hTable) throws IOException {
23 | if (hTable != null) {
24 | hTable.close();
25 | }
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/HbaseColumnConfig.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | import java.util.Arrays;
4 |
5 | public class HbaseColumnConfig {
6 | public String[] columnTypes = null;
7 | public String[] columnFamilyAndQualifiers = null;
8 |
9 | public HbaseColumnConfig() {
10 | }
11 |
12 | @Override
13 | public String toString() {
14 | if (null != columnTypes && null != columnFamilyAndQualifiers) {
15 | return "columnTypes:" + Arrays.asList(columnTypes) + "\n"
16 | + "columnNames:" + Arrays.toString(columnFamilyAndQualifiers);
17 | } else {
18 | return null;
19 | }
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/HbaseReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum HbaseReaderErrorCode implements ErrorCode {
6 | REQUIRED_VALUE("HbaseReader-00", "您缺失了必须填写的参数值."),
7 | ILLEGAL_VALUE("HbaseReader-01", "您配置的值不合法."),
8 | PREPAR_READ_ERROR("HbaseReader-02", "准备读取 Hbase 时出错."),
9 | SPLIT_ERROR("HbaseReader-03", "切分 Hbase 表时出错."),
10 | INIT_TABLE_ERROR("HbaseReader-04", "初始化 Hbase 抽取表时出错."),
11 |
12 | ;
13 |
14 | private final String code;
15 | private final String description;
16 |
17 | private HbaseReaderErrorCode(String code, String description) {
18 | this.code = code;
19 | this.description = description;
20 | }
21 |
22 | @Override
23 | public String getCode() {
24 | return this.code;
25 | }
26 |
27 | @Override
28 | public String getDescription() {
29 | return this.description;
30 | }
31 |
32 | @Override
33 | public String toString() {
34 | return String.format("Code:[%s], Description:[%s]. ", this.code,
35 | this.description);
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/ModeType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader;
2 |
3 | import com.alibaba.datax.common.exception.DataXException;
4 |
5 | import java.util.Arrays;
6 |
7 | public enum ModeType {
8 | Normal("normal"),
9 | MultiVersionFixedColumn("multiVersionFixedColumn"),
10 | MultiVersionDynamicColumn("multiVersionDynamicColumn"),;
11 |
12 | private String mode;
13 |
14 | ModeType(String mode) {
15 | this.mode = mode.toLowerCase();
16 | }
17 |
18 | public static ModeType getByTypeName(String modeName) {
19 | for (ModeType modeType : values()) {
20 | if (modeType.mode.equalsIgnoreCase(modeName)) {
21 | return modeType;
22 | }
23 | }
24 |
25 | throw DataXException.asDataXException(HbaseReaderErrorCode.ILLEGAL_VALUE,
26 | String.format("Hbasereader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values())));
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/util/MultiVersionDynamicColumnTask.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader.util;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.plugin.reader.hbasereader.Key;
5 | import org.apache.hadoop.hbase.client.Scan;
6 | import org.apache.hadoop.hbase.util.Bytes;
7 |
8 | import java.util.List;
9 |
10 | public class MultiVersionDynamicColumnTask extends MultiVersionTask {
11 | private List columnFamilies = null;
12 |
13 | public MultiVersionDynamicColumnTask(Configuration configuration){
14 | super(configuration);
15 |
16 | this.columnFamilies = configuration.getList(Key.COLUMN_FAMILY, String.class);
17 | }
18 |
19 | @Override
20 | public void initScan(Scan scan) {
21 | for (String columnFamily : columnFamilies) {
22 | scan.addFamily(Bytes.toBytes(columnFamily.trim()));
23 | }
24 |
25 | super.setMaxVersions(scan);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/hbasereader/src/main/java/com/alibaba/datax/plugin/reader/hbasereader/util/MultiVersionFixedColumnTask.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hbasereader.util;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 | import com.alibaba.datax.plugin.reader.hbasereader.Key;
5 | import org.apache.hadoop.hbase.client.Scan;
6 | import org.apache.hadoop.hbase.util.Bytes;
7 |
8 | import java.util.List;
9 |
10 | public class MultiVersionFixedColumnTask extends MultiVersionTask {
11 | private List column = null;
12 |
13 | public MultiVersionFixedColumnTask(Configuration configuration) {
14 | super(configuration);
15 |
16 | this.column = configuration.getList(Key.COLUMN, String.class);
17 | }
18 |
19 | @Override
20 | public void initScan(Scan scan) {
21 | for (String aColumn : this.column) {
22 | String[] cfAndQualifier = aColumn.split(":");
23 | scan.addColumn(Bytes.toBytes(cfAndQualifier[0].trim()), Bytes.toBytes(cfAndQualifier[1].trim()));
24 | }
25 | super.setMaxVersions(scan);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/hbasereader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbasereader",
3 | "class": "com.alibaba.datax.plugin.reader.hbasereader.HbaseReader",
4 | "description": "useScene: prod. mechanism: Scan to read data.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/hbasereader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbasereader",
3 | "parameter": {
4 | "hbaseConfig": {},
5 | "table": "",
6 | "encoding": "",
7 | "mode": "",
8 | "column": [],
9 | "range": {
10 | "startRowkey": "",
11 | "endRowkey": ""
12 | },
13 | "isBinaryRowkey": true
14 | }
15 | }
--------------------------------------------------------------------------------
/hbasewriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/hbasewriter
18 |
19 |
20 | target/
21 |
22 | hbasewriter-${version}.jar
23 |
24 | plugin/writer/hbasewriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/hbasewriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/hbasewriter/src/main/java/com/alibaba/datax/plugin/writer/hbasewriter/Const.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbasewriter;
2 |
3 | /**
4 | * Created by qifeng.sxm on 2015/12/11.
5 | */
6 | public class Const {
7 | public static final String DEFAULT_NULLMODE = "EMPTY_BYTES";
8 | public static final String DEFAULT_ENCODING = "utf-8";
9 | public static final int DEFAULT_BATCHROWS = 100;
10 | public static final int MAX_BATCHROWS = 1000;
11 | public static final String DEFAULT_COLUMN_FAMILY = "cf";
12 | public static final String DEFAULT_QUALIFIER = "unknown";
13 | }
14 |
--------------------------------------------------------------------------------
/hbasewriter/src/main/java/com/alibaba/datax/plugin/writer/hbasewriter/HBaseCell.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbasewriter;
2 |
3 | public class HBaseCell {
4 | private String rowKey;
5 | private String colf;
6 | private String col;
7 | private String value;
8 |
9 | public String getRowKey() {
10 | return rowKey;
11 | }
12 | public void setRowKey(String rowKey) {
13 | this.rowKey = rowKey;
14 | }
15 | public String getColf() {
16 | return colf;
17 | }
18 | public void setColf(String colf) {
19 | this.colf = colf;
20 | }
21 | public String getCol() {
22 | return col;
23 | }
24 | public void setCol(String col) {
25 | this.col = col;
26 | }
27 | public String getValue() {
28 | return value;
29 | }
30 | public void setValue(String value) {
31 | this.value = value;
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/hbasewriter/src/main/java/com/alibaba/datax/plugin/writer/hbasewriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hbasewriter;
2 |
3 | public class Key {
4 | public static final String KEY_HBASE_TABLE = "hbaseTable";
5 | public static final String KEY_HBASE_ROWKEY = "hbaseRowkey";
6 | public static final String KEY_HBASE_COLUMN = "hbaseColumn";
7 | public static final String KEY_HBASE_NULLMODE = "nullMode";
8 | public static final String KEY_HBASE_ENCODE = "encoding";
9 | public static final String KEY_HBASE_BATCHROWS = "batchRows";
10 | public static final String KEY_HBASE_CONF = "configuration";
11 |
12 | public static final String KEY_ZK_PARENT = "zookeeper.znode.parent";
13 | public static final String KEY_ZK_QUORUM = "hbase.zookeeper.quorum";
14 | public static final String KEY_ZK_CLIENTPORT = "hbase.zookeeper.clientPort";
15 | }
16 |
--------------------------------------------------------------------------------
/hbasewriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbasewriter",
3 | "class": "com.alibaba.datax.plugin.writer.hbasewriter.HBaseWriter",
4 | "description": {
5 | "useScene": "prod.",
6 | "mechanism": "TODO",
7 | "warn": "TODO"
8 | },
9 | "developer": "alibaba"
10 | }
--------------------------------------------------------------------------------
/hbasewriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hbasewriter",
3 | "parameter": {
4 | "hbaseConfig": {
5 | "hbase.rootdir": "",
6 | "hbase.cluster.distributed": "",
7 | "hbase.zookeeper.quorum": ""
8 | },
9 | "table": "",
10 | "mode": "",
11 | "rowkeyColumn": [
12 | ],
13 | "column": [
14 | ],
15 | "versionColumn":{
16 | "index": "",
17 | "value":""
18 | },
19 | "encoding": ""
20 | }
21 | }
--------------------------------------------------------------------------------
/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hdfsreader;
2 |
3 | /**
4 | * Created by mingya.wmy on 2015/8/14.
5 | */
6 | public class Constant {
7 | public static final String SOURCE_FILES = "sourceFiles";
8 | public static final String TEXT = "TEXT";
9 | public static final String ORC = "ORC";
10 | public static final String CSV = "CSV";
11 | public static final String SEQ = "SEQ";
12 | public static final String RC = "RC";
13 | }
14 |
--------------------------------------------------------------------------------
/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsFileType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hdfsreader;
2 |
3 | /**
4 | * Created by mingya.wmy on 2015/8/22.
5 | *
6 | */
7 | public enum HdfsFileType {
8 | ORC, SEQ, RC, CSV, TEXT,
9 | }
10 |
--------------------------------------------------------------------------------
/hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.hdfsreader;
2 |
3 | public final class Key {
4 |
5 | /**
6 | * 此处声明插件用到的需要插件使用者提供的配置项
7 | */
8 | public final static String PATH = "path";
9 | public final static String DEFAULT_FS = "defaultFS";
10 | public static final String FILETYPE = "fileType";
11 | public static final String HADOOP_CONFIG = "hadoopConfig";
12 | public static final String HAVE_KERBEROS = "haveKerberos";
13 | public static final String KERBEROS_KEYTAB_FILE_PATH = "kerberosKeytabFilePath";
14 | public static final String KERBEROS_PRINCIPAL = "kerberosPrincipal";
15 | }
16 |
--------------------------------------------------------------------------------
/hdfsreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hdfsreader",
3 | "class": "com.alibaba.datax.plugin.reader.hdfsreader.HdfsReader",
4 | "description": "useScene: test. mechanism: use datax framework to transport data from hdfs. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/hdfsreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hdfsreader",
3 | "parameter": {
4 | "path": "",
5 | "defaultFS": "",
6 | "column": [],
7 | "fileType": "orc",
8 | "encoding": "UTF-8",
9 | "fieldDelimiter": ","
10 | }
11 | }
--------------------------------------------------------------------------------
/hdfswriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/hdfswriter
18 |
19 |
20 | target/
21 |
22 | hdfswriter-${version}.jar
23 |
24 | plugin/writer/hdfswriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/hdfswriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hdfswriter;
2 |
3 | public class Constant {
4 |
5 | public static final String DEFAULT_ENCODING = "UTF-8";
6 | public static final String DEFAULT_NULL_FORMAT = "\\N";
7 | }
8 |
--------------------------------------------------------------------------------
/hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/SupportHiveDataType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.hdfswriter;
2 |
3 | public enum SupportHiveDataType {
4 | TINYINT,
5 | SMALLINT,
6 | INT,
7 | BIGINT,
8 | FLOAT,
9 | DOUBLE,
10 |
11 | TIMESTAMP,
12 | DATE,
13 |
14 | STRING,
15 | VARCHAR,
16 | CHAR,
17 |
18 | BOOLEAN
19 | }
20 |
--------------------------------------------------------------------------------
/hdfswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hdfswriter",
3 | "class": "com.alibaba.datax.plugin.writer.hdfswriter.HdfsWriter",
4 | "description": "useScene: prod. mechanism: via FileSystem connect HDFS write data concurrent.",
5 | "developer": "alibaba"
6 | }
7 |
--------------------------------------------------------------------------------
/hdfswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hdfswriter",
3 | "parameter": {
4 | "defaultFS": "",
5 | "fileType": "",
6 | "path": "",
7 | "fileName": "",
8 | "column": [],
9 | "writeMode": "",
10 | "fieldDelimiter": "",
11 | "compress":""
12 | }
13 | }
--------------------------------------------------------------------------------
/images/DataX-logo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/images/DataX-logo.jpg
--------------------------------------------------------------------------------
/images/datax-enterprise-users.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/images/datax-enterprise-users.jpg
--------------------------------------------------------------------------------
/images/datax-opensource-dingding.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/images/datax-opensource-dingding.png
--------------------------------------------------------------------------------
/license.txt:
--------------------------------------------------------------------------------
1 | Copyright 1999-2017 Alibaba Group Holding Ltd.
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 |
7 | http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 |
--------------------------------------------------------------------------------
/mongodbjsonreader/src/main/java/com/alibaba/datax/plugin/reader/mongodbjsonreader/MongoDBJsonReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.mongodbjsonreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | /**
6 | * Created by jianying.wcj on 2015/3/19 0019.
7 | */
8 | public enum MongoDBJsonReaderErrorCode implements ErrorCode {
9 |
10 | ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE","参数不合法"),
11 | ILLEGAL_ADDRESS("ILLEGAL_ADDRESS","不合法的Mongo地址"),
12 | UNEXCEPT_EXCEPTION("UNEXCEPT_EXCEPTION","未知异常");
13 |
14 | private final String code;
15 |
16 | private final String description;
17 |
18 | private MongoDBJsonReaderErrorCode(String code,String description) {
19 | this.code = code;
20 | this.description = description;
21 | }
22 |
23 | @Override
24 | public String getCode() {
25 | return code;
26 | }
27 |
28 | @Override
29 | public String getDescription() {
30 | return description;
31 | }
32 | }
33 |
34 |
--------------------------------------------------------------------------------
/mongodbjsonreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbjsonreader",
3 | "class": "com.alibaba.datax.plugin.reader.mongodbjsonreader.MongoDBJsonReader",
4 | "description": "useScene: prod. mechanism: via mongoclient connect mongodb reader data concurrent.",
5 | "developer": "alibaba"
6 | }
7 |
--------------------------------------------------------------------------------
/mongodbjsonreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbjsonreader",
3 | "parameter": {
4 | "address": [],
5 | "userName": "",
6 | "userPassword": "",
7 | "dbName": "",
8 | "collectionName": "",
9 | "column": []
10 | }
11 | }
--------------------------------------------------------------------------------
/mongodbreader/src/main/java/com/alibaba/datax/plugin/reader/mongodbreader/MongoDBReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.mongodbreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | /**
6 | * Created by jianying.wcj on 2015/3/19 0019.
7 | */
8 | public enum MongoDBReaderErrorCode implements ErrorCode {
9 |
10 | ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE","参数不合法"),
11 | ILLEGAL_ADDRESS("ILLEGAL_ADDRESS","不合法的Mongo地址"),
12 | UNEXCEPT_EXCEPTION("UNEXCEPT_EXCEPTION","未知异常");
13 |
14 | private final String code;
15 |
16 | private final String description;
17 |
18 | private MongoDBReaderErrorCode(String code,String description) {
19 | this.code = code;
20 | this.description = description;
21 | }
22 |
23 | @Override
24 | public String getCode() {
25 | return code;
26 | }
27 |
28 | @Override
29 | public String getDescription() {
30 | return description;
31 | }
32 | }
33 |
34 |
--------------------------------------------------------------------------------
/mongodbreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbreader",
3 | "class": "com.alibaba.datax.plugin.reader.mongodbreader.MongoDBReader",
4 | "description": "useScene: prod. mechanism: via mongoclient connect mongodb reader data concurrent.",
5 | "developer": "alibaba"
6 | }
7 |
--------------------------------------------------------------------------------
/mongodbreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbreader",
3 | "parameter": {
4 | "address": [],
5 | "userName": "",
6 | "userPassword": "",
7 | "dbName": "",
8 | "collectionName": "",
9 | "column": []
10 | }
11 | }
--------------------------------------------------------------------------------
/mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/MongoDBWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.mongodbwriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum MongoDBWriterErrorCode implements ErrorCode {
6 |
7 | ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE","参数不合法"),
8 | ILLEGAL_ADDRESS("ILLEGAL_ADDRESS","不合法的Mongo地址"),
9 | JSONCAST_EXCEPTION("JSONCAST_EXCEPTION","json类型转换异常"),
10 | UNEXCEPT_EXCEPTION("UNEXCEPT_EXCEPTION","未知异常");
11 |
12 | private final String code;
13 |
14 | private final String description;
15 |
16 | private MongoDBWriterErrorCode(String code,String description) {
17 | this.code = code;
18 | this.description = description;
19 | }
20 |
21 | @Override
22 | public String getCode() {
23 | return code;
24 | }
25 |
26 | @Override
27 | public String getDescription() {
28 | return description;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/mongodbwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbwriter",
3 | "class": "com.alibaba.datax.plugin.writer.mongodbwriter.MongoDBWriter",
4 | "description": "useScene: prod. mechanism: via mongoclient connect mongodb write data concurrent.",
5 | "developer": "alibaba"
6 | }
7 |
--------------------------------------------------------------------------------
/mongodbwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongodbwriter",
3 | "parameter": {
4 | "address": [],
5 | "userName": "",
6 | "userPassword": "",
7 | "dbName": "",
8 | "collectionName": "",
9 | "column": [],
10 | "upsertInfo": {
11 | "isUpsert": "",
12 | "upsertKey": ""
13 | }
14 | }
15 | }
--------------------------------------------------------------------------------
/mysqlreader/src/main/java/com/alibaba/datax/plugin/reader/mysqlreader/MysqlReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.mysqlreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum MysqlReaderErrorCode implements ErrorCode {
6 | ;
7 |
8 | private final String code;
9 | private final String description;
10 |
11 | private MysqlReaderErrorCode(String code, String description) {
12 | this.code = code;
13 | this.description = description;
14 | }
15 |
16 | @Override
17 | public String getCode() {
18 | return this.code;
19 | }
20 |
21 | @Override
22 | public String getDescription() {
23 | return this.description;
24 | }
25 |
26 | @Override
27 | public String toString() {
28 | return String.format("Code:[%s], Description:[%s]. ", this.code,
29 | this.description);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/mysqlreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mysqlreader",
3 | "class": "com.alibaba.datax.plugin.reader.mysqlreader.MysqlReader",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/mysqlreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mysqlreader",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "connection": [
8 | {
9 | "jdbcUrl": [],
10 | "table": []
11 | }
12 | ],
13 | "where": ""
14 | }
15 | }
--------------------------------------------------------------------------------
/mysqlwriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/mysqlwriter
18 |
19 |
20 | target/
21 |
22 | mysqlwriter-${version}.jar
23 |
24 | plugin/writer/mysqlwriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/mysqlwriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/mysqlwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mysqlwriter",
3 | "class": "com.alibaba.datax.plugin.writer.mysqlwriter.MysqlWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/mysqlwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mysqlwriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "writeMode": "",
7 | "column": [],
8 | "session": [],
9 | "preSql": [],
10 | "connection": [
11 | {
12 | "jdbcUrl": "",
13 | "table": []
14 | }
15 | ]
16 | }
17 | }
--------------------------------------------------------------------------------
/ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ocswriter;
2 |
3 | public final class Key {
4 | public final static String USER = "username";
5 |
6 | public final static String PASSWORD = "password";
7 |
8 | public final static String PROXY = "proxy";
9 |
10 | public final static String PORT = "port";
11 |
12 | public final static String WRITE_MODE = "writeMode";
13 |
14 | public final static String WRITE_FORMAT = "writeFormat";
15 |
16 | public final static String FIELD_DELIMITER = "fieldDelimiter";
17 |
18 | public final static String EXPIRE_TIME = "expireTime";
19 |
20 | public final static String BATCH_SIZE = "batchSize";
21 |
22 | public final static String INDEXES = "indexes";
23 | }
24 |
--------------------------------------------------------------------------------
/ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/utils/CommonUtils.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ocswriter.utils;
2 |
3 | public class CommonUtils {
4 |
5 | public static void sleepInMs(long time) {
6 | try{
7 | Thread.sleep(time);
8 | } catch (InterruptedException e) {
9 | //
10 | }
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/utils/OcsWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.ocswriter.utils;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum OcsWriterErrorCode implements ErrorCode {
6 | REQUIRED_VALUE("OcsWriterErrorCode-000", "参数不能为空"),
7 | ILLEGAL_PARAM_VALUE("OcsWriterErrorCode-001", "参数不合法"),
8 | HOST_UNREACHABLE("OcsWriterErrorCode-002", "服务不可用"),
9 | OCS_INIT_ERROR("OcsWriterErrorCode-003", "初始化ocs client失败"),
10 | DIRTY_RECORD("OcsWriterErrorCode-004", "脏数据"),
11 | SHUTDOWN_FAILED("OcsWriterErrorCode-005", "关闭ocs client失败"),
12 | COMMIT_FAILED("OcsWriterErrorCode-006", "提交数据到ocs失败");
13 |
14 | private final String code;
15 | private final String description;
16 |
17 | private OcsWriterErrorCode(String code, String description) {
18 | this.code = code;
19 | this.description = description;
20 | }
21 |
22 | @Override
23 | public String getCode() {
24 | return null;
25 | }
26 |
27 | @Override
28 | public String getDescription() {
29 | return null;
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/ocswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ocswriter",
3 | "class": "com.alibaba.datax.plugin.writer.ocswriter.OcsWriter",
4 | "description": "set|add|replace|append|prepend record into ocs.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/ocswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ocswriter",
3 | "parameter": {
4 | "proxy": "",
5 | "port": "",
6 | "userName": "",
7 | "password": "",
8 | "writeMode": "",
9 | "writeFormat": "",
10 | "fieldDelimiter": "",
11 | "expireTime": "",
12 | "indexes": "",
13 | "batchSize": ""
14 | }
15 | }
--------------------------------------------------------------------------------
/odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/ColumnType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.odpsreader;
2 |
3 | public enum ColumnType {
4 | PARTITION, NORMAL, CONSTANT, UNKNOWN, ;
5 |
6 | @Override
7 | public String toString() {
8 | switch (this) {
9 | case PARTITION:
10 | return "partition";
11 | case NORMAL:
12 | return "normal";
13 | case CONSTANT:
14 | return "constant";
15 | default:
16 | return "unknown";
17 | }
18 | }
19 |
20 | public static ColumnType asColumnType(String columnTypeString) {
21 | if ("partition".equals(columnTypeString)) {
22 | return PARTITION;
23 | } else if ("normal".equals(columnTypeString)) {
24 | return NORMAL;
25 | } else if ("constant".equals(columnTypeString)) {
26 | return CONSTANT;
27 | } else {
28 | return UNKNOWN;
29 | }
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.odpsreader;
2 |
3 | public class Constant {
4 |
5 | public final static String START_INDEX = "startIndex";
6 |
7 | public final static String STEP_COUNT = "stepCount";
8 |
9 | public final static String SESSION_ID = "sessionId";
10 |
11 | public final static String IS_PARTITIONED_TABLE = "isPartitionedTable";
12 |
13 | public static final String DEFAULT_SPLIT_MODE = "record";
14 |
15 | public static final String PARTITION_SPLIT_MODE = "partition";
16 |
17 | public static final String DEFAULT_ACCOUNT_TYPE = "aliyun";
18 |
19 | public static final String TAOBAO_ACCOUNT_TYPE = "taobao";
20 |
21 | // 常量字段用COLUMN_CONSTANT_FLAG 首尾包住即可
22 | public final static String COLUMN_CONSTANT_FLAG = "'";
23 |
24 | /**
25 | * 以下是获取accesskey id 需要用到的常量值
26 | */
27 | public static final String SKYNET_ACCESSID = "SKYNET_ACCESSID";
28 |
29 | public static final String SKYNET_ACCESSKEY = "SKYNET_ACCESSKEY";
30 |
31 | public static final String PARTITION_COLUMNS = "partitionColumns";
32 |
33 | public static final String PARSED_COLUMNS = "parsedColumns";
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.odpsreader;
2 |
3 | public class Key {
4 |
5 | public final static String ACCESS_ID = "accessId";
6 |
7 | public final static String ACCESS_KEY = "accessKey";
8 |
9 | public static final String PROJECT = "project";
10 |
11 | public final static String TABLE = "table";
12 |
13 | public final static String PARTITION = "partition";
14 |
15 | public final static String ODPS_SERVER = "odpsServer";
16 |
17 | // 线上环境不需要填写，线下环境必填
18 | public final static String TUNNEL_SERVER = "tunnelServer";
19 |
20 | public final static String COLUMN = "column";
21 |
22 | // 当值为：partition 则只切分到分区；当值为：record，则当按照分区切分后达不到adviceNum时，继续按照record切分
23 | public final static String SPLIT_MODE = "splitMode";
24 |
25 | // 账号类型，默认为aliyun，也可能为taobao等其他类型
26 | public final static String ACCOUNT_TYPE = "accountType";
27 |
28 | public final static String PACKAGE_AUTHORIZED_PROJECT = "packageAuthorizedProject";
29 |
30 | public final static String IS_COMPRESS = "isCompress";
31 |
32 | public final static String MAX_RETRY_TIME = "maxRetryTime";
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/util/OdpsExceptionMsg.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.odpsreader.util;
2 |
3 | /**
4 | * Created by hongjiao.hj on 2015/6/9.
5 | */
6 | public class OdpsExceptionMsg {
7 |
8 | public static final String ODPS_PROJECT_NOT_FOUNT = "ODPS-0420111: Project not found";
9 |
10 | public static final String ODPS_TABLE_NOT_FOUNT = "ODPS-0130131:Table not found";
11 |
12 | public static final String ODPS_ACCESS_KEY_ID_NOT_FOUND = "ODPS-0410051:Invalid credentials - accessKeyId not found";
13 |
14 | public static final String ODPS_ACCESS_KEY_INVALID = "ODPS-0410042:Invalid signature value - User signature dose not match";
15 |
16 | public static final String ODPS_ACCESS_DENY = "ODPS-0420095: Access Denied - Authorization Failed [4002], You doesn't exist in project";
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/odpsreader/src/main/libs/bcprov-jdk15on-1.52.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/odpsreader/src/main/libs/bcprov-jdk15on-1.52.jar
--------------------------------------------------------------------------------
/odpsreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "odpsreader",
3 | "class": "com.alibaba.datax.plugin.reader.odpsreader.OdpsReader",
4 | "description": {
5 | "useScene": "prod.",
6 | "mechanism": "TODO",
7 | "warn": "TODO"
8 | },
9 | "developer": "alibaba"
10 | }
--------------------------------------------------------------------------------
/odpsreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "odpsreader",
3 | "parameter": {
4 | "accessId": "",
5 | "accessKey": "",
6 | "project": "",
7 | "table": "",
8 | "partition": [],
9 | "column": [],
10 | "packageAuthorizedProject": "",
11 | "splitMode": "",
12 | "odpsServer": ""
13 | }
14 | }
--------------------------------------------------------------------------------
/odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.odpswriter;
2 |
3 |
4 | public class Constant {
5 | public static final String SKYNET_ACCESSID = "SKYNET_ACCESSID";
6 |
7 | public static final String SKYNET_ACCESSKEY = "SKYNET_ACCESSKEY";
8 |
9 | public static final String DEFAULT_ACCOUNT_TYPE = "aliyun";
10 |
11 | public static final String TAOBAO_ACCOUNT_TYPE = "taobao";
12 |
13 | public static final String COLUMN_POSITION = "columnPosition";
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.odpswriter;
2 |
3 |
4 | public final class Key {
5 |
6 | public final static String ODPS_SERVER = "odpsServer";
7 |
8 | public final static String TUNNEL_SERVER = "tunnelServer";
9 |
10 | public final static String ACCESS_ID = "accessId";
11 |
12 | public final static String ACCESS_KEY = "accessKey";
13 |
14 | public final static String PROJECT = "project";
15 |
16 | public final static String TABLE = "table";
17 |
18 | public final static String PARTITION = "partition";
19 |
20 | public final static String COLUMN = "column";
21 |
22 | public final static String TRUNCATE = "truncate";
23 |
24 | public final static String MAX_RETRY_TIME = "maxRetryTime";
25 |
26 | public final static String BLOCK_SIZE_IN_MB = "blockSizeInMB";
27 |
28 | //boolean 类型，default:false
29 | public final static String EMPTY_AS_NULL = "emptyAsNull";
30 |
31 | public final static String ACCOUNT_TYPE = "accountType";
32 |
33 | public final static String IS_COMPRESS = "isCompress";
34 | }
35 |
--------------------------------------------------------------------------------
/odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/util/OdpsExceptionMsg.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.odpswriter.util;
2 |
3 | /**
4 | * Created by hongjiao.hj on 2015/6/9.
5 | */
6 | public class OdpsExceptionMsg {
7 |
8 | public static final String ODPS_PROJECT_NOT_FOUNT = "ODPS-0420111: Project not found";
9 |
10 | public static final String ODPS_TABLE_NOT_FOUNT = "ODPS-0130131:Table not found";
11 |
12 | public static final String ODPS_ACCESS_KEY_ID_NOT_FOUND = "ODPS-0410051:Invalid credentials - accessKeyId not found";
13 |
14 | public static final String ODPS_ACCESS_KEY_INVALID = "ODPS-0410042:Invalid signature value - User signature dose not match";
15 |
16 | public static final String ODPS_ACCESS_DENY = "ODPS-0420095: Access Denied - Authorization Failed [4002], You doesn't exist in project";
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/odpswriter/src/main/libs/bcprov-jdk15on-1.52.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/odpswriter/src/main/libs/bcprov-jdk15on-1.52.jar
--------------------------------------------------------------------------------
/odpswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "odpswriter",
3 | "class": "com.alibaba.datax.plugin.writer.odpswriter.OdpsWriter",
4 | "description": {
5 | "useScene": "prod.",
6 | "mechanism": "TODO",
7 | "warn": "TODO"
8 | },
9 | "developer": "alibaba"
10 | }
--------------------------------------------------------------------------------
/odpswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "odpswriter",
3 | "parameter": {
4 | "project": "",
5 | "table": "",
6 | "partition":"",
7 | "column": [],
8 | "accessId": "",
9 | "accessKey": "",
10 | "truncate": true,
11 | "odpsServer": "",
12 | "tunnelServer": ""
13 | }
14 | }
--------------------------------------------------------------------------------
/oraclereader/src/main/java/com/alibaba/datax/plugin/reader/oraclereader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.oraclereader;
2 |
3 | public class Constant {
4 |
5 | public static final int DEFAULT_FETCH_SIZE = 1024;
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/oraclereader/src/main/java/com/alibaba/datax/plugin/reader/oraclereader/OracleReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.oraclereader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum OracleReaderErrorCode implements ErrorCode {
6 | HINT_ERROR("Oraclereader-00", "您的 Hint 配置出错."),
7 |
8 | ;
9 |
10 | private final String code;
11 | private final String description;
12 |
13 | private OracleReaderErrorCode(String code, String description) {
14 | this.code = code;
15 | this.description = description;
16 | }
17 |
18 | @Override
19 | public String getCode() {
20 | return this.code;
21 | }
22 |
23 | @Override
24 | public String getDescription() {
25 | return this.description;
26 | }
27 |
28 | @Override
29 | public String toString() {
30 | return String.format("Code:[%s], Description:[%s]. ", this.code,
31 | this.description);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/oraclereader/src/main/lib/ojdbc6-11.2.0.3.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/oraclereader/src/main/lib/ojdbc6-11.2.0.3.jar
--------------------------------------------------------------------------------
/oraclereader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "oraclereader",
3 | "class": "com.alibaba.datax.plugin.reader.oraclereader.OracleReader",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/oraclereader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "oraclereader",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "connection": [
8 | {
9 | "table": [],
10 | "jdbcUrl": []
11 | }
12 | ]
13 | }
14 | }
--------------------------------------------------------------------------------
/oraclewriter/src/main/java/com/alibaba/datax/plugin/writer/oraclewriter/OracleWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.oraclewriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum OracleWriterErrorCode implements ErrorCode {
6 | ;
7 |
8 | private final String code;
9 | private final String describe;
10 |
11 | private OracleWriterErrorCode(String code, String describe) {
12 | this.code = code;
13 | this.describe = describe;
14 | }
15 |
16 | @Override
17 | public String getCode() {
18 | return this.code;
19 | }
20 |
21 | @Override
22 | public String getDescription() {
23 | return this.describe;
24 | }
25 |
26 | @Override
27 | public String toString() {
28 | return String.format("Code:[%s], Describe:[%s]. ", this.code,
29 | this.describe);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/oraclewriter/src/main/lib/ojdbc6-11.2.0.3.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/oraclewriter/src/main/lib/ojdbc6-11.2.0.3.jar
--------------------------------------------------------------------------------
/oraclewriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "oraclewriter",
3 | "class": "com.alibaba.datax.plugin.writer.oraclewriter.OracleWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/oraclewriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "oraclewriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "preSql": [],
8 | "connection": [
9 | {
10 | "jdbcUrl": "",
11 | "table": []
12 | }
13 | ]
14 | }
15 | }
--------------------------------------------------------------------------------
/ossreader/src/main/java/com/alibaba/datax/plugin/reader/ossreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.ossreader;
2 |
3 | /**
4 | * Created by mengxin.liumx on 2014/12/7.
5 | */
6 | public class Constant {
7 |
8 | public static final String OBJECT = "object";
9 | public static final int SOCKETTIMEOUT = 5000000;
10 | }
11 |
--------------------------------------------------------------------------------
/ossreader/src/main/java/com/alibaba/datax/plugin/reader/ossreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.ossreader;
2 |
3 | /**
4 | * Created by mengxin.liumx on 2014/12/7.
5 | */
6 | public class Key {
7 | public static final String ENDPOINT = "endpoint";
8 |
9 | public static final String ACCESSID = "accessId";
10 |
11 | public static final String ACCESSKEY = "accessKey";
12 |
13 | public static final String ENCODING = "encoding";
14 |
15 | public static final String BUCKET = "bucket";
16 |
17 | public static final String OBJECT = "object";
18 |
19 | public static final String CNAME = "cname";
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/ossreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ossreader",
3 | "class": "com.alibaba.datax.plugin.reader.ossreader.OssReader",
4 | "description": "",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/ossreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ossreader",
3 | "parameter": {
4 | "endpoint": "",
5 | "accessId": "",
6 | "accessKey": "",
7 | "bucket": "",
8 | "object": [],
9 | "column": [],
10 | "encoding": "",
11 | "fieldDelimiter": "",
12 | "compress": ""
13 | }
14 | }
--------------------------------------------------------------------------------
/osswriter/src/main/java/com/alibaba/datax/plugin/writer/osswriter/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.osswriter;
2 |
3 | /**
4 | * Created by haiwei.luo on 15-02-09.
5 | */
6 | public class Constant {
7 | public static final String OBJECT = "object";
8 | public static final int SOCKETTIMEOUT = 5000000;
9 | }
10 |
--------------------------------------------------------------------------------
/osswriter/src/main/java/com/alibaba/datax/plugin/writer/osswriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.osswriter;
2 |
3 | /**
4 | * Created by haiwei.luo on 15-02-09.
5 | */
6 | public class Key {
7 | public static final String ENDPOINT = "endpoint";
8 |
9 | public static final String ACCESSID = "accessId";
10 |
11 | public static final String ACCESSKEY = "accessKey";
12 |
13 | public static final String BUCKET = "bucket";
14 |
15 | public static final String OBJECT = "object";
16 |
17 | public static final String CNAME = "cname";
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/osswriter/src/main/java/com/alibaba/datax/plugin/writer/osswriter/OssWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.osswriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | /**
6 | * Created by haiwei.luo on 14-9-17.
7 | */
8 | public enum OssWriterErrorCode implements ErrorCode {
9 |
10 | CONFIG_INVALID_EXCEPTION("OssWriter-00", "您的参数配置错误."),
11 | REQUIRED_VALUE("OssWriter-01", "您缺失了必须填写的参数值."),
12 | ILLEGAL_VALUE("OssWriter-02", "您填写的参数值不合法."),
13 | Write_OBJECT_ERROR("OssWriter-03", "您配置的目标Object在写入时异常."),
14 | OSS_COMM_ERROR("OssWriter-05", "执行相应的OSS操作异常."),
15 | ;
16 |
17 | private final String code;
18 | private final String description;
19 |
20 | private OssWriterErrorCode(String code, String description) {
21 | this.code = code;
22 | this.description = description;
23 | }
24 |
25 | @Override
26 | public String getCode() {
27 | return this.code;
28 | }
29 |
30 | @Override
31 | public String getDescription() {
32 | return this.description;
33 | }
34 |
35 | @Override
36 | public String toString() {
37 | return String.format("Code:[%s], Description:[%s].", this.code,
38 | this.description);
39 | }
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/osswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "osswriter",
3 | "class": "com.alibaba.datax.plugin.writer.osswriter.OssWriter",
4 | "description": "",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/osswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "osswriter",
3 | "parameter": {
4 | "endpoint": "",
5 | "accessId": "",
6 | "accessKey": "",
7 | "bucket": "",
8 | "object": "",
9 | "encoding": "",
10 | "fieldDelimiter": "",
11 | "writeMode": ""
12 | }
13 | }
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/OtsReaderError.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public class OtsReaderError implements ErrorCode {
6 |
7 | private String code;
8 |
9 | private String description;
10 |
11 | // TODO
12 | // 这一块需要DATAX来统一定义分类， OTS基于这些分类在细化
13 | // 所以暂定两个基础的Error Code，其他错误统一使用OTS的错误码和错误消息
14 |
15 | public final static OtsReaderError ERROR = new OtsReaderError(
16 | "OtsReaderError",
17 | "该错误表示插件的内部错误，表示系统没有处理到的异常");
18 | public final static OtsReaderError INVALID_PARAM = new OtsReaderError(
19 | "OtsReaderInvalidParameter",
20 | "该错误表示参数错误，表示用户输入了错误的参数格式等");
21 |
22 | public OtsReaderError (String code) {
23 | this.code = code;
24 | this.description = code;
25 | }
26 |
27 | public OtsReaderError (String code, String description) {
28 | this.code = code;
29 | this.description = description;
30 | }
31 |
32 | @Override
33 | public String getCode() {
34 | return this.code;
35 | }
36 |
37 | @Override
38 | public String getDescription() {
39 | return this.description;
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetRangeCallable.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.callable;
2 |
3 | import java.util.concurrent.Callable;
4 |
5 | import com.aliyun.openservices.ots.OTSClientAsync;
6 | import com.aliyun.openservices.ots.model.GetRangeRequest;
7 | import com.aliyun.openservices.ots.model.GetRangeResult;
8 | import com.aliyun.openservices.ots.model.OTSFuture;
9 | import com.aliyun.openservices.ots.model.RangeRowQueryCriteria;
10 |
11 | public class GetRangeCallable implements Callable {
12 |
13 | private OTSClientAsync ots;
14 | private RangeRowQueryCriteria criteria;
15 | private OTSFuture future;
16 |
17 | public GetRangeCallable(OTSClientAsync ots, RangeRowQueryCriteria criteria, OTSFuture future) {
18 | this.ots = ots;
19 | this.criteria = criteria;
20 | this.future = future;
21 | }
22 |
23 | @Override
24 | public GetRangeResult call() throws Exception {
25 | try {
26 | return future.get();
27 | } catch (Exception e) {
28 | GetRangeRequest request = new GetRangeRequest();
29 | request.setRangeRowQueryCriteria(criteria);
30 | future = ots.getRange(request);
31 | throw e;
32 | }
33 | }
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.callable;
2 |
3 | import java.util.concurrent.Callable;
4 |
5 | import com.aliyun.openservices.ots.OTSClient;
6 | import com.aliyun.openservices.ots.model.DescribeTableRequest;
7 | import com.aliyun.openservices.ots.model.DescribeTableResult;
8 | import com.aliyun.openservices.ots.model.TableMeta;
9 |
10 | public class GetTableMetaCallable implements Callable{
11 |
12 | private OTSClient ots = null;
13 | private String tableName = null;
14 |
15 | public GetTableMetaCallable(OTSClient ots, String tableName) {
16 | this.ots = ots;
17 | this.tableName = tableName;
18 | }
19 |
20 | @Override
21 | public TableMeta call() throws Exception {
22 | DescribeTableRequest describeTableRequest = new DescribeTableRequest();
23 | describeTableRequest.setTableName(tableName);
24 | DescribeTableResult result = ots.describeTable(describeTableRequest);
25 | TableMeta tableMeta = result.getTableMeta();
26 | return tableMeta;
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConst.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.model;
2 |
3 | public class OTSConst {
4 | // Reader support type
5 | public final static String TYPE_STRING = "STRING";
6 | public final static String TYPE_INTEGER = "INT";
7 | public final static String TYPE_DOUBLE = "DOUBLE";
8 | public final static String TYPE_BOOLEAN = "BOOL";
9 | public final static String TYPE_BINARY = "BINARY";
10 | public final static String TYPE_INF_MIN = "INF_MIN";
11 | public final static String TYPE_INF_MAX = "INF_MAX";
12 |
13 | // Column
14 | public final static String NAME = "name";
15 | public final static String TYPE = "type";
16 | public final static String VALUE = "value";
17 |
18 | public final static String OTS_CONF = "OTS_CONF";
19 | public final static String OTS_RANGE = "OTS_RANGE";
20 | public final static String OTS_DIRECTION = "OTS_DIRECTION";
21 |
22 | // options
23 | public final static String RETRY = "maxRetryTime";
24 | public final static String SLEEP_IN_MILLI_SECOND = "retrySleepInMillionSecond";
25 | }
26 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.model;
2 |
3 | import com.aliyun.openservices.ots.model.PrimaryKeyType;
4 |
5 | public class OTSPrimaryKeyColumn {
6 | private String name;
7 | private PrimaryKeyType type;
8 |
9 | public String getName() {
10 | return name;
11 | }
12 | public void setName(String name) {
13 | this.name = name;
14 | }
15 | public PrimaryKeyType getType() {
16 | return type;
17 | }
18 | public void setType(PrimaryKeyType type) {
19 | this.type = type;
20 | }
21 |
22 | }
23 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.model;
2 |
3 | import com.aliyun.openservices.ots.model.RowPrimaryKey;
4 |
5 | public class OTSRange {
6 |
7 | private RowPrimaryKey begin = null;
8 | private RowPrimaryKey end = null;
9 |
10 | public OTSRange() {}
11 |
12 | public OTSRange(RowPrimaryKey begin, RowPrimaryKey end) {
13 | this.begin = begin;
14 | this.end = end;
15 | }
16 |
17 | public RowPrimaryKey getBegin() {
18 | return begin;
19 | }
20 | public void setBegin(RowPrimaryKey begin) {
21 | this.begin = begin;
22 | }
23 | public RowPrimaryKey getEnd() {
24 | return end;
25 | }
26 | public void setEnd(RowPrimaryKey end) {
27 | this.end = end;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/DefaultNoRetry.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsreader.utils;
2 |
3 | import com.aliyun.openservices.ots.internal.OTSDefaultRetryStrategy;
4 |
5 | public class DefaultNoRetry extends OTSDefaultRetryStrategy {
6 |
7 | @Override
8 | public boolean shouldRetry(String action, Exception ex, int retries) {
9 | return false;
10 | }
11 |
12 | }
--------------------------------------------------------------------------------
/otsreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "otsreader",
3 | "class": "com.alibaba.datax.plugin.reader.otsreader.OtsReader",
4 | "description": "",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/otsreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "otsreader",
3 | "parameter": {
4 | "endpoint":"",
5 | "accessId":"",
6 | "accessKey":"",
7 | "instanceName":"",
8 | "column":[],
9 | "range":{
10 | "begin":[],
11 | "end":[]
12 | }
13 | }
14 | }
--------------------------------------------------------------------------------
/otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/OTSReaderError.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public class OTSReaderError implements ErrorCode {
6 |
7 | private String code;
8 |
9 | private String description;
10 |
11 | public final static OTSReaderError ERROR = new OTSReaderError("OTSStreamReaderError", "OTS Stream Reader Error");
12 |
13 | public final static OTSReaderError INVALID_PARAM = new OTSReaderError(
14 | "OTSStreamReaderInvalidParameter", "OTS Stream Reader Invalid Parameter");
15 |
16 | public OTSReaderError(String code, String description) {
17 | this.code = code;
18 | this.description = description;
19 | }
20 |
21 | public String getCode() {
22 | return this.code;
23 | }
24 |
25 | public String getDescription() {
26 | return this.description;
27 | }
28 |
29 | public String toString() {
30 | return "[ code:" + this.code + ", message" + this.description + "]";
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/OTSStreamReaderException.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal;
2 |
3 | public class OTSStreamReaderException extends RuntimeException {
4 |
5 | public OTSStreamReaderException(String message) {
6 | super(message);
7 | }
8 |
9 | public OTSStreamReaderException(String message, Exception cause) {
10 | super(message, cause);
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/config/Mode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.config;
2 |
3 | public enum Mode {
4 |
5 | MULTI_VERSION,
6 |
7 | SINGLE_VERSION_AND_UPDATE_ONLY
8 | }
9 |
--------------------------------------------------------------------------------
/otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/core/IStreamRecordSender.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.core;
2 |
3 | import com.alicloud.openservices.tablestore.model.StreamRecord;
4 |
5 | public interface IStreamRecordSender {
6 |
7 | void sendToDatax(StreamRecord streamRecord);
8 |
9 | }
10 |
--------------------------------------------------------------------------------
/otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/utils/ParamChecker.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.utils;
2 |
3 | import com.alibaba.datax.common.util.Configuration;
4 |
5 | public class ParamChecker {
6 |
7 | private static void throwNotExistException() {
8 | throw new IllegalArgumentException("missing the key.");
9 | }
10 |
11 | private static void throwStringLengthZeroException() {
12 | throw new IllegalArgumentException("input the key is empty string.");
13 | }
14 |
15 | public static String checkStringAndGet(Configuration param, String key, boolean isTrim) {
16 | try {
17 | String value = param.getString(key);
18 | if (isTrim) {
19 | value = value != null ? value.trim() : null;
20 | }
21 | if (null == value) {
22 | throwNotExistException();
23 | } else if (value.length() == 0) {
24 | throwStringLengthZeroException();
25 | }
26 | return value;
27 | } catch(RuntimeException e) {
28 | throw e;
29 | }
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/otsstreamreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "otsstreamreader",
3 | "class": "com.alibaba.datax.plugin.reader.otsstreamreader.internal.OTSStreamReader",
4 | "description": "",
5 | "developer": "zhaofeng.zhou@alibaba-inc.com"
6 | }
7 |
--------------------------------------------------------------------------------
/otsstreamreader/tools/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "endpoint" : "",
3 | "accessId" : "",
4 | "accessKey" : "",
5 | "instanceName" : "",
6 | "statusTable" : ""
7 | }
8 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/OtsWriterError.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public class OtsWriterError implements ErrorCode {
6 |
7 | private String code;
8 |
9 | private String description;
10 |
11 | // TODO
12 | // 这一块需要DATAX来统一定义分类， OTS基于这些分类在细化
13 | // 所以暂定两个基础的Error Code，其他错误统一使用OTS的错误码和错误消息
14 |
15 | public final static OtsWriterError ERROR = new OtsWriterError(
16 | "OtsWriterError",
17 | "该错误表示插件的内部错误，表示系统没有处理到的异常");
18 | public final static OtsWriterError INVALID_PARAM = new OtsWriterError(
19 | "OtsWriterInvalidParameter",
20 | "该错误表示参数错误，表示用户输入了错误的参数格式等");
21 |
22 | public OtsWriterError (String code) {
23 | this.code = code;
24 | this.description = code;
25 | }
26 |
27 | public OtsWriterError (String code, String description) {
28 | this.code = code;
29 | this.description = description;
30 | }
31 |
32 | @Override
33 | public String getCode() {
34 | return this.code;
35 | }
36 |
37 | @Override
38 | public String getDescription() {
39 | return this.description;
40 | }
41 |
42 | @Override
43 | public String toString() {
44 | return this.code;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/WriterRetryPolicy.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter;
2 |
3 | import com.alibaba.datax.plugin.writer.otswriter.model.OTSConf;
4 | import com.aliyun.openservices.ots.internal.OTSRetryStrategy;
5 |
6 | public class WriterRetryPolicy implements OTSRetryStrategy {
7 | OTSConf conf;
8 |
9 | public WriterRetryPolicy(OTSConf conf) {
10 | this.conf = conf;
11 | }
12 |
13 | @Override
14 | public boolean shouldRetry(String action, Exception ex, int retries) {
15 | return retries <= conf.getRetry();
16 | }
17 |
18 | @Override
19 | public long getPauseDelay(String action, Exception ex, int retries) {
20 | if (retries <= 0) {
21 | return 0;
22 | }
23 |
24 | int sleepTime = conf.getSleepInMillisecond() * retries;
25 | return sleepTime > 30000 ? 30000 : sleepTime;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/callable/GetTableMetaCallable.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.callable;
2 |
3 | import java.util.concurrent.Callable;
4 |
5 | import com.aliyun.openservices.ots.OTSClient;
6 | import com.aliyun.openservices.ots.model.DescribeTableRequest;
7 | import com.aliyun.openservices.ots.model.DescribeTableResult;
8 | import com.aliyun.openservices.ots.model.TableMeta;
9 |
10 | public class GetTableMetaCallable implements Callable{
11 |
12 | private OTSClient ots = null;
13 | private String tableName = null;
14 |
15 | public GetTableMetaCallable(OTSClient ots, String tableName) {
16 | this.ots = ots;
17 | this.tableName = tableName;
18 | }
19 |
20 | @Override
21 | public TableMeta call() throws Exception {
22 | DescribeTableRequest describeTableRequest = new DescribeTableRequest();
23 | describeTableRequest.setTableName(tableName);
24 | DescribeTableResult result = ots.describeTable(describeTableRequest);
25 | TableMeta tableMeta = result.getTableMeta();
26 | return tableMeta;
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSAttrColumn.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.aliyun.openservices.ots.model.ColumnType;
4 |
5 | public class OTSAttrColumn {
6 | private String name;
7 | private ColumnType type;
8 |
9 | public OTSAttrColumn(String name, ColumnType type) {
10 | this.name = name;
11 | this.type = type;
12 | }
13 |
14 | public String getName() {
15 | return name;
16 | }
17 |
18 | public ColumnType getType() {
19 | return type;
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSOpType.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | public enum OTSOpType {
4 | PUT_ROW,
5 | UPDATE_ROW,
6 | DELETE_ROW
7 | }
8 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSPKColumn.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.aliyun.openservices.ots.model.PrimaryKeyType;
4 |
5 | public class OTSPKColumn {
6 | private String name;
7 | private PrimaryKeyType type;
8 |
9 | public OTSPKColumn(String name, PrimaryKeyType type) {
10 | this.name = name;
11 | this.type = type;
12 | }
13 |
14 | public PrimaryKeyType getType() {
15 | return type;
16 | }
17 |
18 | public String getName() {
19 | return name;
20 | }
21 |
22 | }
23 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowDeleteChangeWithRecord.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 | public class RowDeleteChangeWithRecord extends com.aliyun.openservices.ots.model.RowDeleteChange implements WithRecord {
6 |
7 | private Record record;
8 |
9 | public RowDeleteChangeWithRecord(String tableName) {
10 | super(tableName);
11 | }
12 |
13 | @Override
14 | public Record getRecord() {
15 | return record;
16 | }
17 |
18 | @Override
19 | public void setRecord(Record record) {
20 | this.record = record;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowPutChangeWithRecord.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 | public class RowPutChangeWithRecord extends com.aliyun.openservices.ots.model.RowPutChange implements WithRecord {
6 |
7 | private Record record;
8 |
9 | public RowPutChangeWithRecord(String tableName) {
10 | super(tableName);
11 | }
12 |
13 | @Override
14 | public Record getRecord() {
15 | return record;
16 | }
17 |
18 | @Override
19 | public void setRecord(Record record) {
20 | this.record = record;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowUpdateChangeWithRecord.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 | public class RowUpdateChangeWithRecord extends com.aliyun.openservices.ots.model.RowUpdateChange implements WithRecord {
6 |
7 | private Record record;
8 |
9 | public RowUpdateChangeWithRecord(String tableName) {
10 | super(tableName);
11 | }
12 |
13 | @Override
14 | public Record getRecord() {
15 | return record;
16 | }
17 |
18 | @Override
19 | public void setRecord(Record record) {
20 | this.record = record;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/WithRecord.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.otswriter.model;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 | public interface WithRecord {
6 | Record getRecord();
7 |
8 | void setRecord(Record record);
9 | }
10 |
--------------------------------------------------------------------------------
/otswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "otswriter",
3 | "class": "com.alibaba.datax.plugin.writer.otswriter.OtsWriter",
4 | "description": "",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/otswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "otswriter",
3 | "parameter": {
4 | "endpoint":"",
5 | "accessId":"",
6 | "accessKey":"",
7 | "instanceName":"",
8 | "table":"",
9 | "primaryKey" : [],
10 | "column" : [],
11 | "writeMode" : ""
12 | }
13 | }
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.reader;
2 |
3 | public final class Constant {
4 | public static final String PK_TYPE = "pkType";
5 |
6 | public static final Object PK_TYPE_STRING = "pkTypeString";
7 |
8 | public static final Object PK_TYPE_LONG = "pkTypeLong";
9 |
10 | public static final Object PK_TYPE_MONTECARLO = "pkTypeMonteCarlo";
11 |
12 | public static final String SPLIT_MODE_RANDOMSAMPLE = "randomSampling";
13 |
14 | public static String CONN_MARK = "connection";
15 |
16 | public static String TABLE_NUMBER_MARK = "tableNumber";
17 |
18 | public static String IS_TABLE_MODE = "isTableMode";
19 |
20 | public final static String FETCH_SIZE = "fetchSize";
21 |
22 | public static String QUERY_SQL_TEMPLATE_WITHOUT_WHERE = "select %s from %s ";
23 |
24 | public static String QUERY_SQL_TEMPLATE = "select %s from %s where (%s)";
25 |
26 | public static String TABLE_NAME_PLACEHOLDER = "@table";
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/ConnectionFactory.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.util;
2 |
3 | import java.sql.Connection;
4 |
5 | /**
6 | * Date: 15/3/16 下午2:17
7 | */
8 | public interface ConnectionFactory {
9 |
10 | public Connection getConnecttion();
11 |
12 | public Connection getConnecttionWithoutRetry();
13 |
14 | public String getConnectionInfo();
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/JdbcConnectionFactory.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.util;
2 |
3 | import java.sql.Connection;
4 |
5 | /**
6 | * Date: 15/3/16 下午3:12
7 | */
8 | public class JdbcConnectionFactory implements ConnectionFactory {
9 |
10 | private DataBaseType dataBaseType;
11 |
12 | private String jdbcUrl;
13 |
14 | private String userName;
15 |
16 | private String password;
17 |
18 | public JdbcConnectionFactory(DataBaseType dataBaseType, String jdbcUrl, String userName, String password) {
19 | this.dataBaseType = dataBaseType;
20 | this.jdbcUrl = jdbcUrl;
21 | this.userName = userName;
22 | this.password = password;
23 | }
24 |
25 | @Override
26 | public Connection getConnecttion() {
27 | return DBUtil.getConnection(dataBaseType, jdbcUrl, userName, password);
28 | }
29 |
30 | @Override
31 | public Connection getConnecttionWithoutRetry() {
32 | return DBUtil.getConnectionWithoutRetry(dataBaseType, jdbcUrl, userName, password);
33 | }
34 |
35 | @Override
36 | public String getConnectionInfo() {
37 | return "jdbcUrl:" + jdbcUrl;
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.writer;
2 |
3 | /**
4 | * 用于插件解析用户配置时，需要进行标识（MARK）的常量的声明.
5 | */
6 | public final class Constant {
7 | public static final int DEFAULT_BATCH_SIZE = 2048;
8 |
9 | public static final int DEFAULT_BATCH_BYTE_SIZE = 32 * 1024 * 1024;
10 |
11 | public static String TABLE_NAME_PLACEHOLDER = "@table";
12 |
13 | public static String CONN_MARK = "connection";
14 |
15 | public static String TABLE_NUMBER_MARK = "tableNumber";
16 |
17 | public static String INSERT_OR_REPLACE_TEMPLATE_MARK = "insertOrReplaceTemplate";
18 |
19 | public static final String OB10_SPLIT_STRING = "||_dsc_ob10_dsc_||";
20 | public static final String OB10_SPLIT_STRING_PATTERN = "\\|\\|_dsc_ob10_dsc_\\|\\|";
21 |
22 | }
23 |
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.writer;
2 |
3 | public final class Key {
4 | public final static String JDBC_URL = "jdbcUrl";
5 |
6 | public final static String USERNAME = "username";
7 |
8 | public final static String PASSWORD = "password";
9 |
10 | public final static String TABLE = "table";
11 |
12 | public final static String COLUMN = "column";
13 |
14 | //可选值为：insert,replace，默认为 insert （mysql 支持，oracle 没用 replace 机制，只能 insert,oracle 可以不暴露这个参数）
15 | public final static String WRITE_MODE = "writeMode";
16 |
17 | public final static String PRE_SQL = "preSql";
18 |
19 | public final static String POST_SQL = "postSql";
20 |
21 | public final static String TDDL_APP_NAME = "appName";
22 |
23 | //默认值：256
24 | public final static String BATCH_SIZE = "batchSize";
25 |
26 | //默认值：32m
27 | public final static String BATCH_BYTE_SIZE = "batchByteSize";
28 |
29 | public final static String EMPTY_AS_NULL = "emptyAsNull";
30 |
31 | public final static String DB_NAME_PATTERN = "dbNamePattern";
32 |
33 | public final static String DB_RULE = "dbRule";
34 |
35 | public final static String TABLE_NAME_PATTERN = "tableNamePattern";
36 |
37 | public final static String TABLE_RULE = "tableRule";
38 |
39 | public final static String DRYRUN = "dryRun";
40 | }
--------------------------------------------------------------------------------
/plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/MysqlWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.rdbms.writer;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | //TODO 后续考虑与 util 包种的 DBUTilErrorCode 做合并.（区分读和写的错误码）
6 | public enum MysqlWriterErrorCode implements ErrorCode {
7 | ;
8 |
9 | private final String code;
10 | private final String describe;
11 |
12 | private MysqlWriterErrorCode(String code, String describe) {
13 | this.code = code;
14 | this.describe = describe;
15 | }
16 |
17 | @Override
18 | public String getCode() {
19 | return this.code;
20 | }
21 |
22 | @Override
23 | public String getDescription() {
24 | return this.describe;
25 | }
26 |
27 | @Override
28 | public String toString() {
29 | return String.format("Code:[%s], Describe:[%s]. ", this.code,
30 | this.describe);
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/reader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.unstructuredstorage.reader;
2 |
3 | public class Constant {
4 | public static final String DEFAULT_ENCODING = "UTF-8";
5 |
6 | public static final char DEFAULT_FIELD_DELIMITER = ',';
7 |
8 | public static final boolean DEFAULT_SKIP_HEADER = false;
9 |
10 | public static final String DEFAULT_NULL_FORMAT = "\\N";
11 |
12 | public static final Integer DEFAULT_BUFFER_SIZE = 8192;
13 | }
14 |
--------------------------------------------------------------------------------
/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/reader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.unstructuredstorage.reader;
2 |
3 | /**
4 | * Created by haiwei.luo on 14-12-5.
5 | */
6 | public class Key {
7 | public static final String COLUMN = "column";
8 |
9 | public static final String ENCODING = "encoding";
10 |
11 | public static final String FIELD_DELIMITER = "fieldDelimiter";
12 |
13 | public static final String SKIP_HEADER = "skipHeader";
14 |
15 | public static final String TYPE = "type";
16 |
17 | public static final String FORMAT = "format";
18 |
19 | public static final String INDEX = "index";
20 |
21 | public static final String VALUE = "value";
22 |
23 | public static final String COMPRESS = "compress";
24 |
25 | public static final String NULL_FORMAT = "nullFormat";
26 |
27 | public static final String BUFFER_SIZE = "bufferSize";
28 |
29 | public static final String CSV_READER_CONFIG = "csvReaderConfig";
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.unstructuredstorage.writer;
2 |
3 | public class Constant {
4 |
5 | public static final String DEFAULT_ENCODING = "UTF-8";
6 |
7 | public static final char DEFAULT_FIELD_DELIMITER = ',';
8 |
9 | public static final String DEFAULT_NULL_FORMAT = "\\N";
10 |
11 | public static final String FILE_FORMAT_CSV = "csv";
12 |
13 | public static final String FILE_FORMAT_TEXT = "text";
14 |
15 | //每个分块10MB，最大10000个分块
16 | public static final Long MAX_FILE_SIZE = 1024 * 1024 * 10 * 10000L;
17 |
18 | public static final String DEFAULT_SUFFIX = "";
19 | }
20 |
--------------------------------------------------------------------------------
/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredStorageWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.unstructuredstorage.writer;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 |
6 | public enum UnstructuredStorageWriterErrorCode implements ErrorCode {
7 | ILLEGAL_VALUE("UnstructuredStorageWriter-00", "您填写的参数值不合法."),
8 | Write_FILE_WITH_CHARSET_ERROR("UnstructuredStorageWriter-01", "您配置的编码未能正常写入."),
9 | Write_FILE_IO_ERROR("UnstructuredStorageWriter-02", "您配置的文件在写入时出现IO异常."),
10 | RUNTIME_EXCEPTION("UnstructuredStorageWriter-03", "出现运行时异常, 请联系我们"),
11 | REQUIRED_VALUE("UnstructuredStorageWriter-04", "您缺失了必须填写的参数值."),;
12 |
13 | private final String code;
14 | private final String description;
15 |
16 | private UnstructuredStorageWriterErrorCode(String code, String description) {
17 | this.code = code;
18 | this.description = description;
19 | }
20 |
21 | @Override
22 | public String getCode() {
23 | return this.code;
24 | }
25 |
26 | @Override
27 | public String getDescription() {
28 | return this.description;
29 | }
30 |
31 | @Override
32 | public String toString() {
33 | return String.format("Code:[%s], Description:[%s].", this.code,
34 | this.description);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredWriter.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.unstructuredstorage.writer;
2 |
3 | import java.io.Closeable;
4 | import java.io.IOException;
5 | import java.util.List;
6 |
7 | public interface UnstructuredWriter extends Closeable {
8 |
9 | public void writeOneRecord(List splitedRows) throws IOException;
10 |
11 | public void flush() throws IOException;
12 |
13 | public void close() throws IOException;
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/postgresqlreader/src/main/java/com/alibaba/datax/plugin/reader/postgresqlreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.postgresqlreader;
2 |
3 | public class Constant {
4 |
5 | public static final int DEFAULT_FETCH_SIZE = 1000;
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/postgresqlreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "postgresqlreader",
3 | "class": "com.alibaba.datax.plugin.reader.postgresqlreader.PostgresqlReader",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/postgresqlreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "postgresqlreader",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "connection": [
7 | {
8 | "table": [],
9 | "jdbcUrl": []
10 | }
11 | ]
12 | }
13 | }
--------------------------------------------------------------------------------
/postgresqlwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "postgresqlwriter",
3 | "class": "com.alibaba.datax.plugin.writer.postgresqlwriter.PostgresqlWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/postgresqlwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "postgresqlwriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "preSql": [],
8 | "connection": [
9 | {
10 | "jdbcUrl": "",
11 | "table": []
12 | }
13 | ],
14 | "preSql": [],
15 | "postSql": []
16 | }
17 | }
--------------------------------------------------------------------------------
/rdbmsreader/src/main/java/com/alibaba/datax/plugin/reader/rdbmsreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.rdbmsreader;
2 |
3 | public class Constant {
4 |
5 | public static final int DEFAULT_FETCH_SIZE = 1000;
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/rdbmsreader/src/main/libs/Dm7JdbcDriver16.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmsreader/src/main/libs/Dm7JdbcDriver16.jar
--------------------------------------------------------------------------------
/rdbmsreader/src/main/libs/db2jcc4.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmsreader/src/main/libs/db2jcc4.jar
--------------------------------------------------------------------------------
/rdbmsreader/src/main/libs/edb-jdbc16.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmsreader/src/main/libs/edb-jdbc16.jar
--------------------------------------------------------------------------------
/rdbmsreader/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmsreader/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar
--------------------------------------------------------------------------------
/rdbmsreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "rdbmsreader",
3 | "class": "com.alibaba.datax.plugin.reader.rdbmsreader.RdbmsReader",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba",
6 | "drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.informix.jdbc.IfxDriver", "com.edb.Driver"]
7 | }
8 |
--------------------------------------------------------------------------------
/rdbmsreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "rdbmsreader",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "connection": [
8 | {
9 | "jdbcUrl": [],
10 | "table": []
11 | }
12 | ],
13 | "where": ""
14 | }
15 | }
--------------------------------------------------------------------------------
/rdbmswriter/src/main/libs/Dm7JdbcDriver16.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmswriter/src/main/libs/Dm7JdbcDriver16.jar
--------------------------------------------------------------------------------
/rdbmswriter/src/main/libs/db2jcc4.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmswriter/src/main/libs/db2jcc4.jar
--------------------------------------------------------------------------------
/rdbmswriter/src/main/libs/edb-jdbc16.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmswriter/src/main/libs/edb-jdbc16.jar
--------------------------------------------------------------------------------
/rdbmswriter/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/rdbmswriter/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar
--------------------------------------------------------------------------------
/rdbmswriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "rdbmswriter",
3 | "class": "com.alibaba.datax.plugin.reader.rdbmswriter.RdbmsWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba",
6 | "drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver"]
7 | }
8 |
--------------------------------------------------------------------------------
/rdbmswriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "rdbmswriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "writeMode": "",
7 | "column": [],
8 | "session": [],
9 | "preSql": [],
10 | "connection": [
11 | {
12 | "jdbcUrl": "",
13 | "table": []
14 | }
15 | ]
16 | }
17 | }
--------------------------------------------------------------------------------
/rpm/t_dp_dw_datax_3_core_all-build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | export PATH=/home/tops/bin/:${PATH}
3 | export temppath=$1
4 | cd $temppath/rpm
5 | sed -i "s/^Release:.*$/Release: "$4"/" $2.spec
6 | sed -i "s/^Version:.*$/Version: "$3"/" $2.spec
7 | sed -i "s/UNKNOWN_DATAX_VERSION/$3-$4/g" ../core/src/main/bin/datax.py
8 | sed -i "s/UNKNOWN_DATAX_VERSION/$3-$4/g" ../core/src/main/bin/perftrace.py
9 | export TAGS=TAG:`svn info|grep "URL"|cut -d ":" -f 2-|sed "s/^ //g"|awk -F "trunk|tags|branche" '{print $1}'`tags/$2_A_`echo $3|tr "." "_"`_$4
10 | sed -i "s#%description#%description \n $TAGS#g" $2.spec
11 | /usr/local/bin/rpm_create -p /home/admin -v $3 -r $4 $2.spec -k
12 | mv `find . -name $2-$3-$4*rpm` .
13 |
--------------------------------------------------------------------------------
/rpm/t_dp_dw_datax_3_hook_dqc-build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | export PATH=/home/tops/bin/:${PATH}
3 | export temppath=$1
4 | cd $temppath/rpm
5 | sed -i "s/^Release:.*$/Release: "$4"/" $2.spec
6 | sed -i "s/^Version:.*$/Version: "$3"/" $2.spec
7 | export TAGS=TAG:`svn info|grep "URL"|cut -d ":" -f 2-|sed "s/^ //g"|awk -F "trunk|tags|branche" '{print $1}'`tags/$2_A_`echo $3|tr "." "_"`_$4
8 | sed -i "s#%description#%description \n $TAGS#g" $2.spec
9 | /usr/local/bin/rpm_create -p /home/admin -v $3 -r $4 $2.spec -k
10 | mv `find . -name $2-$3-$4*rpm` .
11 |
--------------------------------------------------------------------------------
/sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.sqlserverreader;
2 |
3 | public class Constant {
4 |
5 | public static final int DEFAULT_FETCH_SIZE = 1024;
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.sqlserverreader;
2 |
3 | public class Key {
4 |
5 | public static final String FETCH_SIZE = "fetchSize";
6 | }
7 |
--------------------------------------------------------------------------------
/sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/SqlServerReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.sqlserverreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum SqlServerReaderErrorCode implements ErrorCode {
6 | ;
7 |
8 | private String code;
9 | private String description;
10 |
11 | private SqlServerReaderErrorCode(String code, String description) {
12 | this.code = code;
13 | this.description = description;
14 | }
15 |
16 | @Override
17 | public String getCode() {
18 | return this.code;
19 | }
20 |
21 | @Override
22 | public String getDescription() {
23 | return this.description;
24 | }
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/sqlserverreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sqlserverreader",
3 | "class": "com.alibaba.datax.plugin.reader.sqlserverreader.SqlServerReader",
4 | "description": "useScene: test. mechanism: use datax framework to transport data from SQL Server. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
7 |
8 |
--------------------------------------------------------------------------------
/sqlserverreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sqlserverreader",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "connection": [
7 | {
8 | "table": [],
9 | "jdbcUrl": []
10 | }
11 | ]
12 | }
13 | }
--------------------------------------------------------------------------------
/sqlserverwriter/src/main/java/com/alibaba/datax/plugin/writer/sqlserverwriter/SqlServerWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.sqlserverwriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum SqlServerWriterErrorCode implements ErrorCode {
6 | ;
7 |
8 | private final String code;
9 | private final String describe;
10 |
11 | private SqlServerWriterErrorCode(String code, String describe) {
12 | this.code = code;
13 | this.describe = describe;
14 | }
15 |
16 | @Override
17 | public String getCode() {
18 | return this.code;
19 | }
20 |
21 | @Override
22 | public String getDescription() {
23 | return this.describe;
24 | }
25 |
26 | @Override
27 | public String toString() {
28 | return String.format("Code:[%s], Describe:[%s]. ", this.code,
29 | this.describe);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/sqlserverwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sqlserverwriter",
3 | "class": "com.alibaba.datax.plugin.writer.sqlserverwriter.SqlServerWriter",
4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/sqlserverwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sqlserverwriter",
3 | "parameter": {
4 | "username": "",
5 | "password": "",
6 | "column": [],
7 | "preSql": [],
8 | "connection": [
9 | {
10 | "jdbcUrl": "",
11 | "table": []
12 | }
13 | ],
14 | "preSql": [],
15 | "postSql": []
16 | }
17 | }
--------------------------------------------------------------------------------
/streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.streamreader;
2 |
3 | public class Constant {
4 |
5 | public static final String TYPE = "type";
6 |
7 | public static final String VALUE = "value";
8 |
9 | public static final String RANDOM = "random";
10 |
11 |
12 |
13 | public static final String DATE_FORMAT_MARK = "dateFormat";
14 |
15 | public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
16 |
17 | public static final String HAVE_MIXUP_FUNCTION = "hasMixupFunction";
18 | public static final String MIXUP_FUNCTION_PATTERN = "\\s*(.*)\\s*,\\s*(.*)\\s*";
19 | public static final String MIXUP_FUNCTION_PARAM1 = "mixupParam1";
20 | public static final String MIXUP_FUNCTION_PARAM2 = "mixupParam2";
21 |
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.streamreader;
2 |
3 | public class Key {
4 |
5 | /**
6 | * should look like:[{"value":"123","type":"int"},{"value":"hello","type":"string"}]
7 | */
8 | public static final String COLUMN = "column";
9 |
10 | public static final String SLICE_RECORD_COUNT = "sliceRecordCount";
11 |
12 | }
13 |
--------------------------------------------------------------------------------
/streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/StreamReaderErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.streamreader;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum StreamReaderErrorCode implements ErrorCode {
6 | REQUIRED_VALUE("StreamReader-00", "缺失必要的值"),
7 | ILLEGAL_VALUE("StreamReader-01", "值非法"),
8 | NOT_SUPPORT_TYPE("StreamReader-02", "不支持的column类型"),;
9 |
10 |
11 | private final String code;
12 | private final String description;
13 |
14 | private StreamReaderErrorCode(String code, String description) {
15 | this.code = code;
16 | this.description = description;
17 | }
18 |
19 | @Override
20 | public String getCode() {
21 | return this.code;
22 | }
23 |
24 | @Override
25 | public String getDescription() {
26 | return this.description;
27 | }
28 |
29 | @Override
30 | public String toString() {
31 | return String.format("Code:[%s], Description:[%s]. ", this.code,
32 | this.description);
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/streamreader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "streamreader",
3 | "class": "com.alibaba.datax.plugin.reader.streamreader.StreamReader",
4 | "description": {
5 | "useScene": "only for developer test.",
6 | "mechanism": "use datax framework to transport data from stream.",
7 | "warn": "Never use it in your real job."
8 | },
9 | "developer": "alibaba"
10 | }
--------------------------------------------------------------------------------
/streamreader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "streamreader",
3 | "parameter": {
4 | "sliceRecordCount": "",
5 | "column": []
6 | }
7 | }
--------------------------------------------------------------------------------
/streamwriter/src/main/java/com/alibaba/datax/plugin/writer/streamwriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.streamwriter;
2 |
3 | public class Key {
4 | public static final String FIELD_DELIMITER = "fieldDelimiter";
5 |
6 | public static final String PRINT = "print";
7 |
8 | public static final String PATH = "path";
9 |
10 | public static final String FILE_NAME = "fileName";
11 |
12 | public static final String RECORD_NUM_BEFORE_SLEEP = "recordNumBeforeSleep";
13 |
14 | public static final String SLEEP_TIME = "sleepTime";
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/streamwriter/src/main/java/com/alibaba/datax/plugin/writer/streamwriter/StreamWriterErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.streamwriter;
2 |
3 | import com.alibaba.datax.common.spi.ErrorCode;
4 |
5 | public enum StreamWriterErrorCode implements ErrorCode {
6 | RUNTIME_EXCEPTION("StreamWriter-00", "运行时异常"),
7 | ILLEGAL_VALUE("StreamWriter-01", "您填写的参数值不合法."),
8 | CONFIG_INVALID_EXCEPTION("StreamWriter-02", "您的参数配置错误."),
9 | SECURITY_NOT_ENOUGH("TxtFileWriter-03", "您缺少权限执行相应的文件写入操作.");
10 |
11 |
12 |
13 | private final String code;
14 | private final String description;
15 |
16 | private StreamWriterErrorCode(String code, String description) {
17 | this.code = code;
18 | this.description = description;
19 | }
20 |
21 | @Override
22 | public String getCode() {
23 | return this.code;
24 | }
25 |
26 | @Override
27 | public String getDescription() {
28 | return this.description;
29 | }
30 |
31 | @Override
32 | public String toString() {
33 | return String.format("Code:[%s], Description:[%s]. ", this.code,
34 | this.description);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/streamwriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "streamwriter",
3 | "class": "com.alibaba.datax.plugin.writer.streamwriter.StreamWriter",
4 | "description": {
5 | "useScene": "only for developer test.",
6 | "mechanism": "use datax framework to transport data to stream.",
7 | "warn": "Never use it in your real job."
8 | },
9 | "developer": "alibaba"
10 | }
--------------------------------------------------------------------------------
/streamwriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "streamwriter",
3 | "parameter": {
4 | "encoding": "",
5 | "print": true
6 | }
7 | }
--------------------------------------------------------------------------------
/transformer/doc/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HashDataInc/DataX/c944b5dd425abc36f0dbcf7a9a677839063722d5/transformer/doc/.gitkeep
--------------------------------------------------------------------------------
/transformer/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | target/
13 |
14 | datax-transformer-${version}.jar
15 |
16 | /lib
17 |
18 |
19 |
20 |
21 |
22 | false
23 | /lib
24 | runtime
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/transformer/src/main/java/com/alibaba/datax/transformer/ComplexTransformer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.transformer;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 | import java.util.Map;
6 |
7 | /**
8 | * no comments.
9 | * Created by liqiang on 16/3/3.
10 | */
11 | public abstract class ComplexTransformer {
12 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。
13 | private String transformerName;
14 |
15 |
16 | public String getTransformerName() {
17 | return transformerName;
18 | }
19 |
20 | public void setTransformerName(String transformerName) {
21 | this.transformerName = transformerName;
22 | }
23 |
24 | /**
25 | * @param record 行记录，UDF进行record的处理后，更新相应的record
26 | * @param tContext transformer运行的配置项
27 | * @param paras transformer函数参数
28 | */
29 | abstract public Record evaluate(Record record, Map tContext, Object... paras);
30 | }
31 |
--------------------------------------------------------------------------------
/transformer/src/main/java/com/alibaba/datax/transformer/Transformer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.transformer;
2 |
3 | import com.alibaba.datax.common.element.Record;
4 |
5 |
6 | /**
7 | * no comments.
8 | * Created by liqiang on 16/3/3.
9 | */
10 | public abstract class Transformer {
11 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。
12 | private String transformerName;
13 |
14 |
15 | public String getTransformerName() {
16 | return transformerName;
17 | }
18 |
19 | public void setTransformerName(String transformerName) {
20 | this.transformerName = transformerName;
21 | }
22 |
23 | /**
24 | * @param record 行记录，UDF进行record的处理后，更新相应的record
25 | * @param paras transformer函数参数
26 | */
27 | abstract public Record evaluate(Record record, Object... paras);
28 | }
29 |
--------------------------------------------------------------------------------
/txtfilereader/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/reader/txtfilereader
18 |
19 |
20 | target/
21 |
22 | txtfilereader-${version}.jar
23 |
24 | plugin/reader/txtfilereader
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/reader/txtfilereader/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/txtfilereader/src/main/java/com/alibaba/datax/plugin/reader/txtfilereader/Constant.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.txtfilereader;
2 |
3 | /**
4 | * Created by haiwei.luo on 14-9-20.
5 | */
6 | public class Constant {
7 | public static final String SOURCE_FILES = "sourceFiles";
8 |
9 | }
10 |
--------------------------------------------------------------------------------
/txtfilereader/src/main/java/com/alibaba/datax/plugin/reader/txtfilereader/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.reader.txtfilereader;
2 |
3 | /**
4 | * Created by haiwei.luo on 14-9-20.
5 | */
6 | public class Key {
7 | public static final String PATH = "path";
8 | }
9 |
--------------------------------------------------------------------------------
/txtfilereader/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "txtfilereader",
3 | "class": "com.alibaba.datax.plugin.reader.txtfilereader.TxtFileReader",
4 | "description": "useScene: test. mechanism: use datax framework to transport data from txt file. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/txtfilereader/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "txtfilereader",
3 | "parameter": {
4 | "path": [],
5 | "encoding": "",
6 | "column": [],
7 | "fieldDelimiter": ""
8 | }
9 | }
--------------------------------------------------------------------------------
/txtfilewriter/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | dir
8 |
9 | false
10 |
11 |
12 | src/main/resources
13 |
14 | plugin.json
15 | plugin_job_template.json
16 |
17 | plugin/writer/txtfilewriter
18 |
19 |
20 | target/
21 |
22 | txtfilewriter-${version}.jar
23 |
24 | plugin/writer/txtfilewriter
25 |
26 |
27 |
28 |
29 |
30 | false
31 | plugin/writer/txtfilewriter/libs
32 | runtime
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/txtfilewriter/src/main/java/com/alibaba/datax/plugin/writer/txtfilewriter/Key.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.datax.plugin.writer.txtfilewriter;
2 |
3 | /**
4 | * Created by haiwei.luo on 14-9-17.
5 | */
6 | public class Key {
7 | // must have
8 | public static final String PATH = "path";
9 | }
10 |
--------------------------------------------------------------------------------
/txtfilewriter/src/main/resources/plugin.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "txtfilewriter",
3 | "class": "com.alibaba.datax.plugin.writer.txtfilewriter.TxtFileWriter",
4 | "description": "useScene: test. mechanism: use datax framework to transport data to txt file. warn: The more you know about the data, the less problems you encounter.",
5 | "developer": "alibaba"
6 | }
--------------------------------------------------------------------------------
/txtfilewriter/src/main/resources/plugin_job_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "txtfilewriter",
3 | "parameter": {
4 | "path": "",
5 | "fileName": "",
6 | "writeMode": "",
7 | "fieldDelimiter":"",
8 | "dateFormat": ""
9 | }
10 | }
--------------------------------------------------------------------------------