├── src └── main │ ├── resources │ ├── plugin_job_template.json │ └── transformer.json │ └── java │ └── com │ └── alibaba │ └── datax │ └── transformer │ ├── support │ ├── AnonyMasker.java │ └── Hiding.java │ ├── Transformer.java │ ├── ComplexTransformer.java │ ├── TransformerErrorCode.java │ └── HidingTransformer.java ├── README.md ├── .gitignore ├── LICENSE ├── package.xml └── pom.xml /src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "parameter": 4 | { 5 | "columnIndex":1, 6 | "paras":["",""] 7 | } 8 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DIY-DataX-Transformer 2 | 早该知道的一种更高级的 transformer 自定义实现方法。 3 | 4 | 让 DataX 运行时加载自定义 transformer 插件。 5 | 6 | 详情链接：[我的 CSDN 博客](https://blog.csdn.net/landstream/article/details/88878172) -------------------------------------------------------------------------------- /src/main/resources/transformer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hiding_transformer", 3 | "class": "com.alibaba.datax.transformer.HidingTransformer", 4 | "description": "Hiding transformer setting any value as default", 5 | "developer": "build2last@github.com" 6 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/support/AnonyMasker.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer.support; 2 | 3 | import java.util.Date; 4 | 5 | /** 6 | * Created by Liu Kun on 2018/5/8. 7 | */ 8 | public interface AnonyMasker { 9 | String mask(String origin) throws Exception; 10 | int mask(int origin) throws Exception; 11 | long mask(long origin) throws Exception; 12 | double mask(double origin) throws Exception; 13 | boolean mask(boolean origin) throws Exception; 14 | Date mask(Date origin) throws Exception; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/Transformer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | 6 | /** 7 | * no comments. 8 | * Created by liqiang on 16/3/3. 9 | */ 10 | public abstract class Transformer { 11 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。 12 | private String transformerName; 13 | 14 | 15 | public String getTransformerName() { 16 | return transformerName; 17 | } 18 | 19 | public void setTransformerName(String transformerName) { 20 | this.transformerName = transformerName; 21 | } 22 | 23 | /** 24 | * @param record 行记录，UDF进行record的处理后，更新相应的record 25 | * @param paras transformer函数参数 26 | */ 27 | abstract public Record evaluate(Record record, Object... paras); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/ComplexTransformer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * no comments. 9 | * Created by liqiang on 16/3/3. 10 | */ 11 | public abstract class ComplexTransformer { 12 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。 13 | private String transformerName; 14 | 15 | 16 | public String getTransformerName() { 17 | return transformerName; 18 | } 19 | 20 | public void setTransformerName(String transformerName) { 21 | this.transformerName = transformerName; 22 | } 23 | 24 | /** 25 | * @param record 行记录，UDF进行record的处理后，更新相应的record 26 | * @param tContext transformer运行的配置项 27 | * @param paras transformer函数参数 28 | */ 29 | abstract public Record evaluate(Record record, Map tContext, Object... paras); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/support/Hiding.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer.support; 2 | 3 | import java.util.Date; 4 | 5 | /** 6 | * Created by Liu Kun on 2018/5/8. 7 | */ 8 | 9 | /* 10 | * 将数据替换成一个常量，常用作处理不需要的敏感字段。 11 | * */ 12 | public class Hiding implements AnonyMasker { 13 | public boolean mask(boolean origin) throws Exception{ 14 | return true; 15 | } 16 | public Date mask(Date origin)throws Exception{ 17 | Date date = new Date(); 18 | return date; 19 | } 20 | 21 | public double mask(double origin) throws Exception{ 22 | return 0; 23 | } 24 | 25 | public int mask(int origin) throws Exception{ 26 | return 0; 27 | } 28 | 29 | public long mask(long origin)throws Exception{ 30 | return 0; 31 | } 32 | 33 | public String mask(String origin)throws Exception{ 34 | return "0"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 liu kun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | dir 8 | 9 | false 10 | 11 | 12 | src/main/resources 13 | 14 | transformer.json 15 | plugin_job_template.json 16 | 17 | plugin/transformer/hiding_transformer 18 | 19 | 20 | target/ 21 | 22 | hidingtransformer-1.0-SNAPSHOT.jar 23 | 24 | plugin/transformer/hiding_transformer 25 | 26 | 27 | 28 | 29 | 30 | false 31 | plugin/transformer/hiding_transformer/libs 32 | runtime 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/TransformerErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum TransformerErrorCode implements ErrorCode { 6 | //重复命名 7 | TRANSFORMER_NAME_ERROR("TransformerErrorCode-01","Transformer name illegal"), 8 | TRANSFORMER_DUPLICATE_ERROR("TransformerErrorCode-02","Transformer name has existed"), 9 | TRANSFORMER_NOTFOUND_ERROR("TransformerErrorCode-03","Transformer name not found"), 10 | TRANSFORMER_CONFIGURATION_ERROR("TransformerErrorCode-04","Transformer configuration error"), 11 | TRANSFORMER_ILLEGAL_PARAMETER("TransformerErrorCode-05","Transformer parameter illegal"), 12 | TRANSFORMER_RUN_EXCEPTION("TransformerErrorCode-06","Transformer run exception"), 13 | TRANSFORMER_GROOVY_INIT_EXCEPTION("TransformerErrorCode-07","Transformer Groovy init exception"), 14 | ; 15 | 16 | private final String code; 17 | 18 | private final String description; 19 | 20 | private TransformerErrorCode(String code, String description) { 21 | this.code = code; 22 | this.description = description; 23 | } 24 | 25 | public String getCode() { 26 | return this.code; 27 | } 28 | 29 | public String getDescription() { 30 | return this.description; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return String.format("Code:[%s], Description:[%s]. ", this.code, 36 | this.description); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/datax/transformer/HidingTransformer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.element.*; 4 | import com.alibaba.datax.common.exception.DataXException; 5 | import com.alibaba.datax.transformer.support.Hiding; 6 | 7 | import java.util.Arrays; 8 | import java.util.Date; 9 | import java.util.Map; 10 | 11 | /** 12 | * Created by Liu Kun on 2019/3/14. 13 | */ 14 | public class HidingTransformer extends ComplexTransformer{ 15 | private Object masker; 16 | String maskMethodId = ""; 17 | String key; 18 | int columnIndex; 19 | 20 | public HidingTransformer(){ 21 | super.setTransformerName("hiding_transformer"); 22 | } 23 | 24 | 25 | public Record evaluate(Record record, Map tContext, Object... paras){ 26 | try { 27 | if (paras.length < 1) { 28 | throw new RuntimeException("Hiding transformer 缺少参数"); 29 | } 30 | columnIndex = (Integer) paras[0]; 31 | } catch (Exception e) { 32 | throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "paras:" + Arrays.asList(paras).toString() + " => " + e.getMessage()); 33 | } 34 | Column column = record.getColumn(columnIndex); 35 | try{ 36 | String oriValue = column.asString(); 37 | if(oriValue == null){ 38 | return record; 39 | } 40 | Hiding masker = new Hiding(); 41 | if(column.getType() == Column.Type.STRING){ 42 | String newValue = masker.mask(column.asString()); 43 | record.setColumn(columnIndex, new StringColumn(newValue)); 44 | } 45 | else if(column.getType() == Column.Type.DATE){ 46 | Date newValue = masker.mask(column.asDate()); 47 | record.setColumn(columnIndex, new DateColumn(newValue)); 48 | } 49 | else if(column.getType() == Column.Type.LONG || column.getType()==Column.Type.INT){ 50 | long newValue = masker.mask(column.asLong()); 51 | record.setColumn(columnIndex, new LongColumn(newValue)); 52 | } 53 | else if(column.getType() == Column.Type.BOOL){ 54 | boolean newValue = ((Hiding) masker).mask(column.asBoolean()); 55 | record.setColumn(columnIndex, new BoolColumn(newValue)); 56 | } 57 | else if(column.getType() == Column.Type.DOUBLE){ 58 | double newValue = masker.mask(column.asDouble()); 59 | record.setColumn(columnIndex, new DoubleColumn(newValue)); 60 | } 61 | } catch (Exception e) { 62 | throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(),e); 63 | } 64 | return record; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.ecnu.ecnu1x 8 | hidingtransformer 9 | 1.0-SNAPSHOT 10 | 11 | 12 | UTF-8 13 | UTF-8 14 | UTF-8 15 | UTF-8 16 | 17 | 18 | 19 | 20 | com.alibaba.datax 21 | datax-core 22 | 0.0.1-SNAPSHOT 23 | 24 | 25 | com.alibaba.datax 26 | datax-common 27 | 0.0.1-SNAPSHOT 28 | 29 | 30 | slf4j-log4j12 31 | org.slf4j 32 | 33 | 34 | 35 | 36 | 37 | org.apache.commons 38 | commons-math3 39 | 3.6.1 40 | 41 | 42 | commons-logging 43 | commons-logging 44 | 1.1.1 45 | 46 | 47 | junit 48 | junit 49 | 4.11 50 | 51 | 52 | 53 | 54 | 55 | 56 | maven-assembly-plugin 57 | 58 | datax 59 | 60 | package.xml 61 | 62 | 63 | 64 | 65 | make-assembly 66 | package 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-compiler-plugin 73 | 2.3.2 74 | 75 | 1.6 76 | 1.6 77 | ${project-sourceEncoding} 78 | 79 | 80 | 81 | 82 | 83 | --------------------------------------------------------------------------------