├── .gitignore ├── README.md ├── src ├── main │ └── java │ │ └── org │ │ └── keedio │ │ └── flume │ │ ├── metrics │ │ ├── SqlSourceCounterMBean.java │ │ └── SqlSourceCounter.java │ │ └── source │ │ ├── SQLServerCustomDialect.java │ │ ├── HibernateHelper.java │ │ ├── SQLSource.java │ │ └── SQLSourceHelper.java └── test │ └── java │ └── org │ └── keedio │ └── flume │ └── source │ └── SQLSourceHelperTest.java ├── pom.xml └── dependency-reduced-pom.xml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yucy/flume-ng-sql-source-json/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flume-ng-sql-source-json 2 | 这个项目是改造于https://github.com/keedio/flume-ng-sql-source.git 3 | 因keedio的插件flume-ng-sql-source只支持csv的格式，如果开始同步之后，数据库表需要增减字段，则会给开发者造成很大的困扰。所以我添加了一个分支版本，用来将数据以JSON的格式，同步到kafka，字段语义更加清晰。 4 | 5 | 部署方式和flume-ng-sql-source的一致，参照：http://www.cnblogs.com/yucy/p/7845105.html 6 | -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/metrics/SqlSourceCounterMBean.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | *******************************************************************************/ 19 | 20 | 21 | package org.keedio.flume.metrics; 22 | 23 | /** 24 | * 25 | * @author Luis Lazaro 26 | * @author Marcelo Valle 27 | */ 28 | public interface SqlSourceCounterMBean { 29 | public long getEventCount(); 30 | public void incrementEventCount(int value); 31 | public long getAverageThroughput(); 32 | public long getCurrentThroughput(); 33 | public long getMaxThroughput(); 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/source/SQLServerCustomDialect.java: -------------------------------------------------------------------------------- 1 | package org.keedio.flume.source; 2 | 3 | import org.hibernate.dialect.SQLServerDialect; 4 | import org.hibernate.type.StandardBasicTypes; 5 | 6 | import java.sql.Types; 7 | 8 | public class SQLServerCustomDialect extends SQLServerDialect { 9 | 10 | /** 11 | * Initializes a new instance of the {@link SQLServerDialect} class. 12 | */ 13 | public SQLServerCustomDialect(){ 14 | registerHibernateType(Types.ARRAY, StandardBasicTypes.STRING.getName()); 15 | registerHibernateType(Types.BIGINT, StandardBasicTypes.STRING.getName()); 16 | registerHibernateType(Types.BINARY, StandardBasicTypes.STRING.getName()); 17 | registerHibernateType(Types.BIT, StandardBasicTypes.STRING.getName()); 18 | registerHibernateType(Types.BLOB, StandardBasicTypes.STRING.getName()); 19 | registerHibernateType(Types.BOOLEAN, StandardBasicTypes.STRING.getName()); 20 | registerHibernateType(Types.CHAR, StandardBasicTypes.STRING.getName()); 21 | registerHibernateType(Types.CLOB, StandardBasicTypes.STRING.getName()); 22 | registerHibernateType(Types.DATALINK, StandardBasicTypes.STRING.getName()); 23 | registerHibernateType(Types.DATE, StandardBasicTypes.STRING.getName()); 24 | registerHibernateType(Types.DECIMAL, StandardBasicTypes.STRING.getName()); 25 | registerHibernateType(Types.DISTINCT, StandardBasicTypes.STRING.getName()); 26 | registerHibernateType(Types.DOUBLE, StandardBasicTypes.STRING.getName()); 27 | registerHibernateType(Types.FLOAT, StandardBasicTypes.STRING.getName()); 28 | registerHibernateType(Types.INTEGER, StandardBasicTypes.STRING.getName()); 29 | registerHibernateType(Types.JAVA_OBJECT, StandardBasicTypes.STRING.getName()); 30 | registerHibernateType(Types.LONGNVARCHAR, StandardBasicTypes.STRING.getName()); 31 | registerHibernateType(Types.LONGVARBINARY, StandardBasicTypes.STRING.getName()); 32 | registerHibernateType(Types.LONGVARCHAR, StandardBasicTypes.STRING.getName()); 33 | registerHibernateType(Types.NCHAR, StandardBasicTypes.STRING.getName()); 34 | registerHibernateType(Types.NCLOB, StandardBasicTypes.STRING.getName()); 35 | registerHibernateType(Types.NULL, StandardBasicTypes.STRING.getName()); 36 | registerHibernateType(Types.NUMERIC, StandardBasicTypes.STRING.getName()); 37 | registerHibernateType(Types.NVARCHAR, StandardBasicTypes.STRING.getName()); 38 | registerHibernateType(Types.OTHER, StandardBasicTypes.STRING.getName()); 39 | registerHibernateType(Types.REAL, StandardBasicTypes.STRING.getName()); 40 | registerHibernateType(Types.REF, StandardBasicTypes.STRING.getName()); 41 | registerHibernateType(Types.ROWID, StandardBasicTypes.STRING.getName()); 42 | registerHibernateType(Types.SMALLINT, StandardBasicTypes.STRING.getName()); 43 | registerHibernateType(Types.SQLXML, StandardBasicTypes.STRING.getName()); 44 | registerHibernateType(Types.STRUCT, StandardBasicTypes.STRING.getName()); 45 | registerHibernateType(Types.TIME, StandardBasicTypes.STRING.getName()); 46 | registerHibernateType(Types.TIMESTAMP, StandardBasicTypes.STRING.getName()); 47 | registerHibernateType(Types.TINYINT, StandardBasicTypes.STRING.getName()); 48 | registerHibernateType(Types.VARBINARY, StandardBasicTypes.STRING.getName()); 49 | registerHibernateType(Types.VARCHAR, StandardBasicTypes.STRING.getName()); 50 | } 51 | } -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/metrics/SqlSourceCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ***************************************************************************** 3 | * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work 4 | * for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 10 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations 11 | * under the License. ***************************************************************************** 12 | */ 13 | package org.keedio.flume.metrics; 14 | 15 | import java.util.concurrent.TimeUnit; 16 | 17 | import org.apache.flume.instrumentation.MonitoredCounterGroup; 18 | 19 | /** 20 | * 21 | * @author Luis Lazaro 22 | * @author Marcelo Valle 23 | */ 24 | public class SqlSourceCounter extends MonitoredCounterGroup implements SqlSourceCounterMBean { 25 | 26 | private long startProcessTime; 27 | 28 | private static final String AVERAGE_THROUGHPUT = "average_throughput"; 29 | private static final String CURRENT_THROUGHPUT = "current_throughput"; 30 | private static final String MAX_THROUGHPUT = "max_throughput"; 31 | private static final String EVENT_COUNT = "events_count"; 32 | 33 | private static final String[] ATTRIBUTES = {AVERAGE_THROUGHPUT, CURRENT_THROUGHPUT, MAX_THROUGHPUT, EVENT_COUNT}; 34 | 35 | public SqlSourceCounter(String name) { 36 | super(MonitoredCounterGroup.Type.SOURCE, name, ATTRIBUTES); 37 | } 38 | 39 | @Override 40 | public void incrementEventCount(int value) { 41 | addAndGet(EVENT_COUNT, value); 42 | } 43 | 44 | @Override 45 | public long getEventCount() { 46 | return get(EVENT_COUNT); 47 | } 48 | 49 | @Override 50 | public long getAverageThroughput() { 51 | return get(AVERAGE_THROUGHPUT); 52 | } 53 | 54 | @Override 55 | public long getCurrentThroughput() { 56 | return get(CURRENT_THROUGHPUT); 57 | } 58 | 59 | @Override 60 | public long getMaxThroughput() { 61 | return get(MAX_THROUGHPUT); 62 | } 63 | 64 | 65 | public void startProcess(){ 66 | startProcessTime = System.currentTimeMillis(); 67 | } 68 | 69 | public void endProcess(int events){ 70 | 71 | long runningTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - getStartTime()); 72 | long processTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startProcessTime); 73 | long throughput = 0L; 74 | 75 | if (events > 0 && processTime > 0) 76 | throughput = events/processTime; 77 | if (getMaxThroughput() < throughput) 78 | set(MAX_THROUGHPUT,throughput); 79 | 80 | if (runningTime > 0 && getEventCount() > 0) 81 | set(AVERAGE_THROUGHPUT, (getEventCount()/runningTime)); 82 | 83 | set(CURRENT_THROUGHPUT,throughput); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | org.keedio.flume.flume-ng-sources 5 | flume-ng-sql-source-json 6 | 1.0 7 | 8 | jar 9 | 10 | Flume SQL Source 11 | 12 | This project is used for flume-ng to import data from SQL databases. 13 | 14 | https://github.com/keedio/flume-ng-sql-source 15 | 16 | 17 | 18 | The Apache License, Version 2.0 19 | http://www.apache.org/licenses/LICENSE-2.0.txt 20 | 21 | 22 | 23 | 24 | 25 | Luis Alfonso Lázaro Medina 26 | lalazaro@keedio.org 27 | Keedio 28 | http://www.keedio.com 29 | 30 | 31 | Marcelo Valle Ávila 32 | mvalle@keedio.org 33 | Keedio 34 | http://www.keedio.com 35 | 36 | 37 | Luca Rosellini 38 | lrosellini@keedio.org 39 | Keedio 40 | http://www.keedio.com 41 | 42 | 43 | 44 | 45 | scm:git:git@github.com:keedio/flume-ng-sql-source.git 46 | scm:git:git@github.com:keedio/flume-ng-sql-source.git 47 | https://github.com/keedio/flume-ng-sql-source.git 48 | 49 | 50 | 51 | 52 | org.apache.flume 53 | flume-ng-core 54 | 1.8.0 55 | provided 56 | 57 | 58 | com.opencsv 59 | opencsv 60 | 3.6 61 | 62 | 63 | mysql 64 | mysql-connector-java 65 | 5.1.43 66 | provided 67 | 68 | 69 | org.hibernate 70 | hibernate-core 71 | 4.3.10.Final 72 | 73 | 74 | org.hibernate 75 | hibernate-c3p0 76 | 4.3.11.Final 77 | 78 | 79 | org.mockito 80 | mockito-core 81 | 1.10.19 82 | test 83 | 84 | 85 | org.powermock 86 | powermock-module-junit4 87 | 1.6.2 88 | test 89 | 90 | 91 | org.powermock 92 | powermock-api-mockito 93 | 1.6.2 94 | test 95 | 96 | 97 | com.googlecode.json-simple 98 | json-simple 99 | 1.1.1 100 | 101 | 102 | 103 | 104 | 105 | org.apache.maven.plugins 106 | maven-compiler-plugin 107 | 3.1 108 | 109 | 1.7 110 | 1.7 111 | 112 | 113 | 114 | org.apache.maven.plugins 115 | maven-shade-plugin 116 | 2.3 117 | 118 | 119 | package 120 | 121 | shade 122 | 123 | 124 | 125 | 126 | 127 | org.apache.maven.plugins 128 | maven-surefire-plugin 129 | 2.18.1 130 | 131 | 132 | org.apache.maven.plugins 133 | maven-source-plugin 134 | 2.2.1 135 | 136 | 137 | attach-sources 138 | 139 | jar 140 | 141 | 142 | 143 | 144 | 145 | org.apache.maven.plugins 146 | maven-javadoc-plugin 147 | 2.9.1 148 | 149 | 150 | attach-javadocs 151 | 152 | jar 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | Keedio 161 | www.keedio.org 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/source/HibernateHelper.java: -------------------------------------------------------------------------------- 1 | package org.keedio.flume.source; 2 | 3 | import org.apache.flume.Context; 4 | import org.hibernate.CacheMode; 5 | import org.hibernate.Query; 6 | import org.hibernate.Session; 7 | import org.hibernate.SessionFactory; 8 | import org.hibernate.boot.registry.StandardServiceRegistryBuilder; 9 | import org.hibernate.cfg.Configuration; 10 | import org.hibernate.service.ServiceRegistry; 11 | import org.hibernate.transform.Transformers; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Iterator; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | /** 21 | * Helper class to manage hibernate sessions and perform queries 22 | * 23 | * @author Marcelo Valle 24 | * 25 | */ 26 | public class HibernateHelper { 27 | 28 | private static final Logger LOG = LoggerFactory 29 | .getLogger(HibernateHelper.class); 30 | 31 | private static SessionFactory factory; 32 | private Session session; 33 | private ServiceRegistry serviceRegistry; 34 | private Configuration config; 35 | private SQLSourceHelper sqlSourceHelper; 36 | 37 | /** 38 | * Constructor to initialize hibernate configuration parameters 39 | * @param sqlSourceHelper Contains the configuration parameters from flume config file 40 | */ 41 | public HibernateHelper(SQLSourceHelper sqlSourceHelper) { 42 | 43 | this.sqlSourceHelper = sqlSourceHelper; 44 | Context context = sqlSourceHelper.getContext(); 45 | 46 | Map hibernateProperties = context.getSubProperties("hibernate."); 47 | Iterator> it = hibernateProperties.entrySet().iterator(); 48 | 49 | config = new Configuration(); 50 | Map.Entry e; 51 | 52 | while (it.hasNext()){ 53 | e = it.next(); 54 | config.setProperty("hibernate." + e.getKey(), e.getValue()); 55 | } 56 | } 57 | 58 | /** 59 | * Connect to database using hibernate 60 | */ 61 | public void establishSession() { 62 | 63 | LOG.info("Opening hibernate session"); 64 | 65 | serviceRegistry = new StandardServiceRegistryBuilder() 66 | .applySettings(config.getProperties()).build(); 67 | factory = config.buildSessionFactory(serviceRegistry); 68 | session = factory.openSession(); 69 | session.setCacheMode(CacheMode.IGNORE); 70 | 71 | session.setDefaultReadOnly(sqlSourceHelper.isReadOnlySession()); 72 | } 73 | 74 | /** 75 | * Close database connection 76 | */ 77 | public void closeSession() { 78 | 79 | LOG.info("Closing hibernate session"); 80 | 81 | session.close(); 82 | factory.close(); 83 | } 84 | 85 | /** 86 | * Execute the selection query in the database 87 | * @return The query result. Each Object is a cell content.

88 | * The cell contents use database types (date,int,string...), 89 | * keep in mind in case of future conversions/castings. 90 | * @throws InterruptedException 91 | */ 92 | @SuppressWarnings("unchecked") 93 | public List> executeQuery() throws InterruptedException { 94 | 95 | List> rowsList = new ArrayList>() ; 96 | Query query; 97 | 98 | if (!session.isConnected()){ 99 | resetConnection(); 100 | } 101 | 102 | if (sqlSourceHelper.isCustomQuerySet()){ 103 | 104 | query = session.createSQLQuery(sqlSourceHelper.buildQuery()); 105 | 106 | if (sqlSourceHelper.getMaxRows() != 0){ 107 | query = query.setMaxResults(sqlSourceHelper.getMaxRows()); 108 | } 109 | } 110 | else 111 | { 112 | query = session 113 | .createSQLQuery(sqlSourceHelper.getQuery()) 114 | .setFirstResult(Integer.parseInt(sqlSourceHelper.getCurrentIndex())); 115 | 116 | if (sqlSourceHelper.getMaxRows() != 0){ 117 | query = query.setMaxResults(sqlSourceHelper.getMaxRows()); 118 | } 119 | } 120 | 121 | try { 122 | rowsList = query.setFetchSize(sqlSourceHelper.getMaxRows()).setResultTransformer(Transformers.TO_LIST).list(); 123 | }catch (Exception e){ 124 | LOG.error("Exception thrown, resetting connection.",e); 125 | resetConnection(); 126 | } 127 | 128 | if (!rowsList.isEmpty()){ 129 | if (sqlSourceHelper.isCustomQuerySet()){ 130 | sqlSourceHelper.setCurrentIndex(rowsList.get(rowsList.size()-1).get(0).toString()); 131 | } 132 | else 133 | { 134 | sqlSourceHelper.setCurrentIndex(Integer.toString((Integer.parseInt(sqlSourceHelper.getCurrentIndex()) 135 | + rowsList.size()))); 136 | } 137 | } 138 | 139 | return rowsList; 140 | } 141 | 142 | /** 143 | * Execute the selection query in the database 144 | * @return The query result. Each Object is a cell content.

145 | * The cell contents use database types (date,int,string...), 146 | * keep in mind in case of future conversions/castings. 147 | * @throws InterruptedException 148 | */ 149 | public List> executeQueryForJson() throws InterruptedException { 150 | 151 | List> rowsList = new ArrayList>() ; 152 | Query query; 153 | 154 | if (!session.isConnected()){ 155 | resetConnection(); 156 | } 157 | 158 | if (sqlSourceHelper.isCustomQuerySet()){ 159 | 160 | query = session.createSQLQuery(sqlSourceHelper.buildQuery()); 161 | 162 | if (sqlSourceHelper.getMaxRows() != 0){ 163 | query = query.setMaxResults(sqlSourceHelper.getMaxRows()); 164 | } 165 | } 166 | else 167 | { 168 | query = session 169 | .createSQLQuery(sqlSourceHelper.getQuery()) 170 | .setFirstResult(Integer.parseInt(sqlSourceHelper.getCurrentIndex())); 171 | 172 | if (sqlSourceHelper.getMaxRows() != 0){ 173 | query = query.setMaxResults(sqlSourceHelper.getMaxRows()); 174 | } 175 | } 176 | 177 | try { 178 | rowsList = query.setFetchSize(sqlSourceHelper.getMaxRows()).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP).list(); 179 | }catch (Exception e){ 180 | LOG.error("Exception thrown, resetting connection.",e); 181 | resetConnection(); 182 | } 183 | 184 | if (!rowsList.isEmpty()){ 185 | if (sqlSourceHelper.isCustomQuerySet()){ 186 | sqlSourceHelper.setCurrentIndex(rowsList.get(rowsList.size()-1).get(sqlSourceHelper.getOrderBy()).toString()); 187 | } 188 | else 189 | { 190 | sqlSourceHelper.setCurrentIndex(Integer.toString((Integer.parseInt(sqlSourceHelper.getCurrentIndex()) 191 | + rowsList.size()))); 192 | } 193 | } 194 | 195 | return rowsList; 196 | } 197 | 198 | private void resetConnection() throws InterruptedException{ 199 | session.close(); 200 | factory.close(); 201 | establishSession(); 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/source/SQLSource.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | *******************************************************************************/ 19 | package org.keedio.flume.source; 20 | 21 | import org.apache.flume.Context; 22 | import org.apache.flume.Event; 23 | import org.apache.flume.EventDeliveryException; 24 | import org.apache.flume.PollableSource; 25 | import org.apache.flume.conf.Configurable; 26 | import org.apache.flume.event.SimpleEvent; 27 | import org.apache.flume.source.AbstractSource; 28 | import org.keedio.flume.metrics.SqlSourceCounter; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.io.IOException; 33 | import java.io.PrintWriter; 34 | import java.io.Writer; 35 | import java.util.ArrayList; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | import java.util.Map; 39 | 40 | 41 | /** 42 | * A Source to read data from a SQL database. This source ask for new data in a table each configured time.

43 | * 44 | * @author Marcelo Valle 45 | */ 46 | public class SQLSource extends AbstractSource implements Configurable, PollableSource { 47 | 48 | private static final Logger LOG = LoggerFactory.getLogger(SQLSource.class); 49 | protected SQLSourceHelper sqlSourceHelper; 50 | private SqlSourceCounter sqlSourceCounter; 51 | // private CSVWriter csvWriter; 52 | private PrintWriter printWriter ; 53 | private HibernateHelper hibernateHelper; 54 | 55 | /** 56 | * Configure the source, load configuration properties and establish connection with database 57 | */ 58 | @Override 59 | public void configure(Context context) { 60 | 61 | LOG.getName(); 62 | 63 | LOG.info("Reading and processing configuration values for source " + getName()); 64 | 65 | /* Initialize configuration parameters */ 66 | sqlSourceHelper = new SQLSourceHelper(context, this.getName()); 67 | 68 | /* Initialize metric counters */ 69 | sqlSourceCounter = new SqlSourceCounter("SOURCESQL." + this.getName()); 70 | 71 | /* Establish connection with database */ 72 | hibernateHelper = new HibernateHelper(sqlSourceHelper); 73 | hibernateHelper.establishSession(); 74 | 75 | /* Instantiate the CSV Writer */ 76 | // csvWriter = new CSVWriter(new ChannelWriter(),sqlSourceHelper.getDelimiterEntry().charAt(0)); 77 | printWriter = new PrintWriter(new ChannelWriter()); 78 | } 79 | 80 | /** 81 | * Process a batch of events performing SQL Queries 82 | */ 83 | @Override 84 | public Status process() throws EventDeliveryException { 85 | 86 | try { 87 | sqlSourceCounter.startProcess(); 88 | 89 | // List> result = hibernateHelper.executeQue 90 | List> result = hibernateHelper.executeQueryForJson(); 91 | if (!result.isEmpty()) 92 | { 93 | sqlSourceHelper.writeAllRows(result,printWriter); 94 | printWriter.flush(); 95 | // csvWriter.writeAll(sqlSourceHelper.getAllRows(result),sqlSourceHelper.encloseByQuotes()); 96 | // csvWriter.flush(); 97 | sqlSourceCounter.incrementEventCount(result.size()); 98 | sqlSourceHelper.updateStatusFile(); 99 | } 100 | 101 | sqlSourceCounter.endProcess(result.size()); 102 | 103 | if (result.size() < sqlSourceHelper.getMaxRows()){ 104 | Thread.sleep(sqlSourceHelper.getRunQueryDelay()); 105 | } 106 | return Status.READY; 107 | 108 | } catch (InterruptedException e) { 109 | LOG.error("Error procesing row", e); 110 | return Status.BACKOFF; 111 | } 112 | } 113 | 114 | @Override 115 | public long getBackOffSleepIncrement() { 116 | return 1000; 117 | } 118 | 119 | @Override 120 | public long getMaxBackOffSleepInterval() { 121 | return 5000; 122 | } 123 | 124 | /** 125 | * Starts the source. Starts the metrics counter. 126 | */ 127 | @Override 128 | public void start() { 129 | 130 | LOG.info("Starting sql source {} ...", getName()); 131 | sqlSourceCounter.start(); 132 | super.start(); 133 | } 134 | 135 | /** 136 | * Stop the source. Close database connection and stop metrics counter. 137 | */ 138 | @Override 139 | public void stop() { 140 | 141 | LOG.info("Stopping sql source {} ...", getName()); 142 | 143 | try 144 | { 145 | hibernateHelper.closeSession(); 146 | // csvWriter.close(); 147 | printWriter.close(); 148 | } catch (Exception e) { 149 | LOG.warn("Error CSVWriter object ", e); 150 | } finally { 151 | this.sqlSourceCounter.stop(); 152 | super.stop(); 153 | } 154 | } 155 | 156 | private class ChannelWriter extends Writer{ 157 | private List events = new ArrayList<>(); 158 | 159 | @Override 160 | public void write(char[] cbuf, int off, int len) throws IOException { 161 | Event event = new SimpleEvent(); 162 | 163 | String s = new String(cbuf); 164 | // The reason for this number [len] minus one is that the source string adds the line_end 165 | event.setBody(s.substring(off, len-1).getBytes()); 166 | // LOG.info("event body:{}||",new String(event.getBody())); 167 | 168 | Map headers; 169 | headers = new HashMap(); 170 | headers.put("timestamp", String.valueOf(System.currentTimeMillis())); 171 | event.setHeaders(headers); 172 | events.add(event); 173 | if (events.size() >= sqlSourceHelper.getBatchSize()) { 174 | flush(); 175 | } 176 | } 177 | 178 | @Override 179 | public void flush() throws IOException { 180 | getChannelProcessor().processEventBatch(events); 181 | events.clear(); 182 | } 183 | 184 | @Override 185 | public void close() throws IOException { 186 | flush(); 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | org.keedio.flume.flume-ng-sources 5 | flume-ng-sql-source-json 6 | Flume SQL Source 7 | 1.0 8 | This project is used for flume-ng to import data from SQL databases. 9 | https://github.com/keedio/flume-ng-sql-source 10 | 11 | 12 | Luis Alfonso Lázaro Medina 13 | lalazaro@keedio.org 14 | Keedio 15 | http://www.keedio.com 16 | 17 | 18 | Marcelo Valle Ávila 19 | mvalle@keedio.org 20 | Keedio 21 | http://www.keedio.com 22 | 23 | 24 | Luca Rosellini 25 | lrosellini@keedio.org 26 | Keedio 27 | http://www.keedio.com 28 | 29 | 30 | 31 | 32 | The Apache License, Version 2.0 33 | http://www.apache.org/licenses/LICENSE-2.0.txt 34 | 35 | 36 | 37 | scm:git:git@github.com:keedio/flume-ng-sql-source.git 38 | scm:git:git@github.com:keedio/flume-ng-sql-source.git 39 | https://github.com/keedio/flume-ng-sql-source.git 40 | 41 | 42 | Keedio 43 | www.keedio.org 44 | 45 | 46 | 47 | 48 | maven-compiler-plugin 49 | 3.1 50 | 51 | 1.7 52 | 1.7 53 | 54 | 55 | 56 | maven-shade-plugin 57 | 2.3 58 | 59 | 60 | package 61 | 62 | shade 63 | 64 | 65 | 66 | 67 | 68 | maven-surefire-plugin 69 | 2.18.1 70 | 71 | 72 | maven-source-plugin 73 | 2.2.1 74 | 75 | 76 | attach-sources 77 | 78 | jar 79 | 80 | 81 | 82 | 83 | 84 | maven-javadoc-plugin 85 | 2.9.1 86 | 87 | 88 | attach-javadocs 89 | 90 | jar 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | org.apache.flume 100 | flume-ng-core 101 | 1.8.0 102 | provided 103 | 104 | 105 | flume-ng-sdk 106 | org.apache.flume 107 | 108 | 109 | flume-ng-configuration 110 | org.apache.flume 111 | 112 | 113 | flume-ng-auth 114 | org.apache.flume 115 | 116 | 117 | slf4j-api 118 | org.slf4j 119 | 120 | 121 | guava 122 | com.google.guava 123 | 124 | 125 | commons-io 126 | commons-io 127 | 128 | 129 | commons-codec 130 | commons-codec 131 | 132 | 133 | log4j 134 | log4j 135 | 136 | 137 | slf4j-log4j12 138 | org.slf4j 139 | 140 | 141 | commons-cli 142 | commons-cli 143 | 144 | 145 | commons-lang 146 | commons-lang 147 | 148 | 149 | avro 150 | org.apache.avro 151 | 152 | 153 | avro-ipc 154 | org.apache.avro 155 | 156 | 157 | netty 158 | io.netty 159 | 160 | 161 | joda-time 162 | joda-time 163 | 164 | 165 | servlet-api 166 | org.mortbay.jetty 167 | 168 | 169 | jetty-util 170 | org.mortbay.jetty 171 | 172 | 173 | jetty 174 | org.mortbay.jetty 175 | 176 | 177 | gson 178 | com.google.code.gson 179 | 180 | 181 | libthrift 182 | org.apache.thrift 183 | 184 | 185 | mina-core 186 | org.apache.mina 187 | 188 | 189 | 190 | 191 | mysql 192 | mysql-connector-java 193 | 5.1.43 194 | provided 195 | 196 | 197 | org.mockito 198 | mockito-core 199 | 1.10.19 200 | test 201 | 202 | 203 | objenesis 204 | org.objenesis 205 | 206 | 207 | 208 | 209 | org.powermock 210 | powermock-module-junit4 211 | 1.6.2 212 | test 213 | 214 | 215 | powermock-module-junit4-common 216 | org.powermock 217 | 218 | 219 | 220 | 221 | org.powermock 222 | powermock-api-mockito 223 | 1.6.2 224 | test 225 | 226 | 227 | mockito-all 228 | org.mockito 229 | 230 | 231 | powermock-api-support 232 | org.powermock 233 | 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /src/test/java/org/keedio/flume/source/SQLSourceHelperTest.java: -------------------------------------------------------------------------------- 1 | package org.keedio.flume.source; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import com.google.common.collect.Maps; 5 | import com.google.gson.Gson; 6 | import com.google.gson.reflect.TypeToken; 7 | import com.google.gson.stream.JsonWriter; 8 | import org.apache.commons.io.FileUtils; 9 | import org.apache.flume.Context; 10 | import org.apache.flume.Event; 11 | import org.apache.flume.conf.ConfigurationException; 12 | import org.apache.flume.event.SimpleEvent; 13 | import org.junit.After; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | 17 | import java.io.*; 18 | import java.lang.reflect.Type; 19 | import java.sql.Date; 20 | import java.text.SimpleDateFormat; 21 | import java.util.ArrayList; 22 | import java.util.HashMap; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | import static org.junit.Assert.assertArrayEquals; 27 | import static org.junit.Assert.assertEquals; 28 | import static org.mockito.Mockito.mock; 29 | import static org.mockito.Mockito.when; 30 | 31 | /** 32 | * @author Marcelo Valle https://github.com/mvalleavila 33 | */ 34 | 35 | //@RunWith(PowerMockRunner.class) 36 | public class SQLSourceHelperTest { 37 | 38 | Context context = mock(Context.class); 39 | 40 | @Before 41 | public void setup() { 42 | 43 | when(context.getString("status.file.name")).thenReturn("statusFileName.txt"); 44 | when(context.getString("hibernate.connection.url")).thenReturn("jdbc:mysql://localhost:3306/database"); 45 | when(context.getString("table")).thenReturn("my_table"); 46 | when(context.getString("incremental.column.name")).thenReturn("incrementalColumName"); 47 | when(context.getString("status.file.path", "/var/lib/flume")).thenReturn("/tmp/flume"); 48 | when(context.getString("columns.to.select", "*")).thenReturn("*"); 49 | when(context.getInteger("run.query.delay", 10000)).thenReturn(10000); 50 | when(context.getInteger("batch.size", 100)).thenReturn(100); 51 | when(context.getInteger("max.rows", 100)).thenReturn(100); 52 | when(context.getString("incremental.value", "0")).thenReturn("0"); 53 | when(context.getString("start.from", "0")).thenReturn("0"); 54 | } 55 | 56 | /* 57 | @Test 58 | public void checkNotCreatedDirectory() throws Exception { 59 | 60 | SQLSourceHelper sqlSourceUtils = new SQLSourceHelper(context,"Source Name"); 61 | SQLSourceHelper sqlSourceUtilsSpy = PowerMockito.spy(sqlSourceUtils); 62 | 63 | PowerMockito.verifyPrivate(sqlSourceUtilsSpy, Mockito.times(1)).invoke("createDirectory"); 64 | }*/ 65 | 66 | @Test 67 | public void getConnectionURL() { 68 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 69 | assertEquals("jdbc:mysql://host:3306/database", sqlSourceHelper.getConnectionURL()); 70 | } 71 | 72 | @Test 73 | public void getCurrentIndex() { 74 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 75 | assertEquals("0",sqlSourceHelper.getCurrentIndex()); 76 | } 77 | 78 | @Test 79 | public void setCurrentIndex() { 80 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 81 | sqlSourceHelper.setCurrentIndex("10"); 82 | assertEquals("10",sqlSourceHelper.getCurrentIndex()); 83 | } 84 | 85 | @Test 86 | public void getRunQueryDelay() { 87 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 88 | assertEquals(10000,sqlSourceHelper.getRunQueryDelay()); 89 | } 90 | 91 | @Test 92 | public void getBatchSize() { 93 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 94 | assertEquals(100,sqlSourceHelper.getBatchSize()); 95 | } 96 | 97 | @Test 98 | public void getQuery() { 99 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 100 | assertEquals("SELECT * FROM table",sqlSourceHelper.getQuery()); 101 | } 102 | 103 | @Test 104 | public void execQuery() { 105 | Map hibernate = Maps.newHashMap(); 106 | hibernate.put("connection.url", "jdbc:mysql://localhost/mydatabase?useSSL=false"); 107 | hibernate.put("connection.user", "test"); 108 | hibernate.put("connection.password", "test"); 109 | hibernate.put("connection.autocommit", "true"); 110 | hibernate.put("dialect", "org.hibernate.dialect.MySQL5Dialect"); 111 | hibernate.put("connection.driver_class", "com.mysql.jdbc.Driver"); 112 | hibernate.put("connection.provider_class", "org.hibernate.connection.C3P0ConnectionProvider"); 113 | hibernate.put("c3p0.min_size","5"); 114 | hibernate.put("c3p0.max_size","20"); 115 | when(context.getSubProperties("hibernate.")).thenReturn(ImmutableMap.copyOf(hibernate)); 116 | 117 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 118 | HibernateHelper hibernateHelper = new HibernateHelper(sqlSourceHelper); 119 | hibernateHelper.establishSession(); 120 | try { 121 | List> lists = hibernateHelper.executeQueryForJson(); 122 | // testPrintWriter(lists); 123 | SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 124 | for (Map item :lists 125 | ) { 126 | for (Map.Entry entry:item.entrySet() 127 | ) { 128 | Object value = entry.getValue(); 129 | if (value instanceof java.util.Date){ 130 | entry.setValue(SDF.format(value)); 131 | } 132 | } 133 | System.out.println(item); 134 | } 135 | } catch (InterruptedException e) { 136 | e.printStackTrace(); 137 | } 138 | } 139 | 140 | 141 | public void testJsonWriter(List> lists) throws IOException { 142 | File file = new File("d:\\gson.txt");// 把json保存文本 143 | OutputStream out = new FileOutputStream(file); 144 | JsonWriter writer = new JsonWriter(new OutputStreamWriter(out, "UTF-8"));//设置编码 145 | Gson gson = new Gson(); 146 | Type type = new TypeToken(){}.getType(); 147 | for (Map item :lists 148 | ) { 149 | String json = gson.toJson(item); 150 | gson.toJson(json, type, writer);//把值写进去 151 | } 152 | writer.flush(); 153 | writer.close(); 154 | } 155 | 156 | public void testPrintWriter(List> lists) throws IOException { 157 | File file = new File("d:\\gson.txt");// 把json保存文本 158 | OutputStream out = new FileOutputStream(file); 159 | PrintWriter writer = new PrintWriter(new ChannelWriter());//设置编码 160 | Gson gson = new Gson(); 161 | for (Map item :lists 162 | ) { 163 | writer.write(gson.toJson(item)); 164 | } 165 | writer.flush(); 166 | writer.close(); 167 | } 168 | 169 | @Test 170 | public void getCustomQuery() { 171 | when(context.getString("custom.query")).thenReturn("SELECT column FROM table"); 172 | when(context.getString("incremental.column")).thenReturn("incremental"); 173 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 174 | assertEquals("SELECT column FROM table",sqlSourceHelper.getQuery()); 175 | } 176 | 177 | @Test 178 | public void chekGetAllRowsWithNullParam() { 179 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 180 | assertEquals(new ArrayList(),sqlSourceHelper.getAllRows(null)); 181 | } 182 | 183 | @Test(expected = ConfigurationException.class) 184 | public void checkStatusFileNameNotSet() { 185 | when(context.getString("status.file.name")).thenReturn(null); 186 | new SQLSourceHelper(context,"Source Name"); 187 | } 188 | 189 | @Test(expected = ConfigurationException.class) 190 | public void connectionURLNotSet() { 191 | when(context.getString("hibernate.connection.url")).thenReturn(null); 192 | new SQLSourceHelper(context,"Source Name"); 193 | } 194 | 195 | @Test(expected = ConfigurationException.class) 196 | public void tableNotSet() { 197 | when(context.getString("table")).thenReturn(null); 198 | new SQLSourceHelper(context,"Source Name"); 199 | } 200 | 201 | @Test 202 | public void chekGetAllRowsWithEmptyParam() { 203 | 204 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 205 | assertEquals(new ArrayList(),sqlSourceHelper.getAllRows(new ArrayList>())); 206 | } 207 | 208 | @SuppressWarnings("deprecation") 209 | @Test 210 | public void chekGetAllRows() { 211 | 212 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 213 | List> queryResult = new ArrayList>(2); 214 | List expectedResult = new ArrayList(2); 215 | String string1 = "string1"; 216 | String string2 = "string2"; 217 | int int1 = 1; 218 | int int2 = 2; 219 | Date date1 = new Date(115,0,1); 220 | Date date2 = new Date(115,1,2); 221 | 222 | List row1 = new ArrayList(3); 223 | String[] expectedRow1 = new String[3]; 224 | row1.add(string1); 225 | expectedRow1[0] = string1; 226 | row1.add(int1); 227 | expectedRow1[1] = Integer.toString(int1); 228 | row1.add(date1); 229 | expectedRow1[2] = date1.toString(); 230 | queryResult.add(row1); 231 | expectedResult.add(expectedRow1); 232 | 233 | List row2 = new ArrayList(3); 234 | String[] expectedRow2 = new String[3]; 235 | row2.add(string2); 236 | expectedRow2[0] = string2; 237 | row2.add(int2); 238 | expectedRow2[1] = Integer.toString(int2); 239 | row2.add(date2); 240 | expectedRow2[2] = date2.toString(); 241 | queryResult.add(row2); 242 | expectedResult.add(expectedRow2); 243 | 244 | assertArrayEquals(expectedResult.get(0),sqlSourceHelper.getAllRows(queryResult).get(0)); 245 | assertArrayEquals(expectedResult.get(1),sqlSourceHelper.getAllRows(queryResult).get(1)); 246 | } 247 | 248 | @SuppressWarnings("unused") 249 | @Test 250 | public void createDirectory() { 251 | 252 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 253 | File file = new File("/tmp/flume"); 254 | assertEquals(true, file.exists()); 255 | assertEquals(true, file.isDirectory()); 256 | if (file.exists()){ 257 | file.delete(); 258 | } 259 | } 260 | 261 | @Test 262 | public void checkStatusFileCorrectlyCreated() { 263 | 264 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 265 | //sqlSourceHelper.setCurrentIndex(10); 266 | 267 | sqlSourceHelper.updateStatusFile(); 268 | 269 | File file = new File("/tmp/flume/statusFileName.txt"); 270 | assertEquals(true, file.exists()); 271 | if (file.exists()){ 272 | file.delete(); 273 | file.getParentFile().delete(); 274 | } 275 | } 276 | 277 | @Test 278 | public void checkStatusFileCorrectlyUpdated() throws Exception { 279 | 280 | //File file = File.createTempFile("statusFileName", ".txt"); 281 | 282 | when(context.getString("status.file.path")).thenReturn("/var/lib/flume"); 283 | when(context.getString("hibernate.connection.url")).thenReturn("jdbc:mysql://host:3306/database"); 284 | when(context.getString("table")).thenReturn("table"); 285 | when(context.getString("status.file.name")).thenReturn("statusFileName"); 286 | 287 | SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context,"Source Name"); 288 | sqlSourceHelper.createStatusFile(); 289 | sqlSourceHelper.setCurrentIndex("10"); 290 | 291 | sqlSourceHelper.updateStatusFile(); 292 | 293 | SQLSourceHelper sqlSourceHelper2 = new SQLSourceHelper(context,"Source Name"); 294 | assertEquals("10", sqlSourceHelper2.getCurrentIndex()); 295 | } 296 | 297 | 298 | 299 | @After 300 | public void deleteDirectory(){ 301 | try { 302 | 303 | File file = new File("/tmp/flume"); 304 | if (file.exists()) 305 | FileUtils.deleteDirectory(file); 306 | 307 | } catch (IOException e) { 308 | e.printStackTrace(); 309 | } 310 | } 311 | 312 | private class ChannelWriter extends Writer{ 313 | private List events = new ArrayList<>(); 314 | 315 | @Override 316 | public void write(char[] cbuf, int off, int len) throws IOException { 317 | Event event = new SimpleEvent(); 318 | 319 | String s = new String(cbuf); 320 | // s.substring(off, len-1).getBytes() 321 | event.setBody(s.substring(off, len).getBytes()); 322 | System.out.println("111111111111111 event:"+s+"|||||||"); 323 | 324 | Map headers; 325 | headers = new HashMap(); 326 | headers.put("timestamp", String.valueOf(System.currentTimeMillis())); 327 | event.setHeaders(headers); 328 | 329 | events.add(event); 330 | 331 | } 332 | 333 | @Override 334 | public void flush() throws IOException { 335 | events.clear(); 336 | } 337 | 338 | @Override 339 | public void close() throws IOException { 340 | flush(); 341 | } 342 | } 343 | 344 | } 345 | -------------------------------------------------------------------------------- /src/main/java/org/keedio/flume/source/SQLSourceHelper.java: -------------------------------------------------------------------------------- 1 | package org.keedio.flume.source; 2 | 3 | 4 | import com.google.gson.Gson; 5 | import com.google.gson.reflect.TypeToken; 6 | import org.apache.flume.Context; 7 | import org.apache.flume.conf.ConfigurationException; 8 | import org.json.simple.JSONValue; 9 | import org.json.simple.parser.JSONParser; 10 | import org.json.simple.parser.ParseException; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.io.*; 15 | import java.lang.reflect.Type; 16 | import java.text.SimpleDateFormat; 17 | import java.util.ArrayList; 18 | import java.util.LinkedHashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import static org.json.simple.parser.ParseException.ERROR_UNEXPECTED_EXCEPTION; 23 | 24 | 25 | /** 26 | * Helper to manage configuration parameters and utility methods

27 | * 28 | * Configuration parameters readed from flume configuration file: 29 | * type: org.keedio.flume.source.SQLSource

30 | * table: table to read from

31 | * columns.to.select: columns to select for import data (* will import all)

32 | * run.query.delay: delay time to execute each query to database

33 | * status.file.path: Directory to save status file

34 | * status.file.name: Name for status file (saves last row index processed)

35 | * batch.size: Batch size to send events from flume source to flume channel

36 | * max.rows: Max rows to import from DB in one query

37 | * custom.query: Custom query to execute to database (be careful)

38 | * 39 | * @author Marcelo Valle 40 | * @author Luis Lazaro 41 | */ 42 | 43 | public class SQLSourceHelper { 44 | 45 | private static final Logger LOG = LoggerFactory.getLogger(SQLSourceHelper.class); 46 | 47 | private File file,directory; 48 | private int runQueryDelay, batchSize, maxRows; 49 | private String startFrom, currentIndex,orderBy; 50 | private String statusFilePath, statusFileName, connectionURL, table, 51 | columnsToSelect, customQuery, query, sourceName, delimiterEntry; 52 | private Boolean encloseByQuotes; 53 | 54 | private Context context; 55 | 56 | private Map statusFileJsonMap = new LinkedHashMap(); 57 | 58 | private boolean readOnlySession; 59 | 60 | private static final String DEFAULT_STATUS_DIRECTORY = "/var/lib/flume"; 61 | private static final int DEFAULT_QUERY_DELAY = 10000; 62 | private static final int DEFAULT_BATCH_SIZE = 100; 63 | private static final int DEFAULT_MAX_ROWS = 10000; 64 | private static final String DEFAULT_INCREMENTAL_VALUE = "0"; 65 | private static final String DEFAULT_DELIMITER_ENTRY = ","; 66 | private static final Boolean DEFAULT_ENCLOSE_BY_QUOTES = true; 67 | 68 | private static final String SOURCE_NAME_STATUS_FILE = "SourceName"; 69 | private static final String URL_STATUS_FILE = "URL"; 70 | private static final String COLUMNS_TO_SELECT_STATUS_FILE = "ColumnsToSelect"; 71 | private static final String TABLE_STATUS_FILE = "Table"; 72 | private static final String LAST_INDEX_STATUS_FILE = "LastIndex"; 73 | private static final String QUERY_STATUS_FILE = "Query"; 74 | private static final String DEFAULT_ORDER_BY_VALUE = "id"; 75 | 76 | private final SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 77 | private final String JSON_LINE_END = " "; 78 | 79 | 80 | /** 81 | * Builds an SQLSourceHelper containing the configuration parameters and 82 | * usefull utils for SQL Source 83 | * @param context Flume source context, contains the properties from configuration file 84 | * @param sourceName source file name for store status 85 | */ 86 | public SQLSourceHelper(Context context, String sourceName){ 87 | 88 | this.context = context; 89 | 90 | statusFilePath = context.getString("status.file.path", DEFAULT_STATUS_DIRECTORY); 91 | statusFileName = context.getString("status.file.name"); 92 | table = context.getString("table"); 93 | columnsToSelect = context.getString("columns.to.select","*"); 94 | runQueryDelay = context.getInteger("run.query.delay",DEFAULT_QUERY_DELAY); 95 | directory = new File(statusFilePath); 96 | customQuery = context.getString("custom.query"); 97 | batchSize = context.getInteger("batch.size",DEFAULT_BATCH_SIZE); 98 | maxRows = context.getInteger("max.rows",DEFAULT_MAX_ROWS); 99 | connectionURL = context.getString("hibernate.connection.url"); 100 | readOnlySession = context.getBoolean("read.only",false); 101 | 102 | this.sourceName = sourceName; 103 | startFrom = context.getString("start.from",DEFAULT_INCREMENTAL_VALUE); 104 | orderBy = context.getString("order.by",DEFAULT_ORDER_BY_VALUE); 105 | delimiterEntry = context.getString("delimiter.entry",DEFAULT_DELIMITER_ENTRY); 106 | encloseByQuotes = context.getBoolean("enclose.by.quotes", DEFAULT_ENCLOSE_BY_QUOTES); 107 | statusFileJsonMap = new LinkedHashMap(); 108 | 109 | checkMandatoryProperties(); 110 | 111 | if (!(isStatusDirectoryCreated())) { 112 | createDirectory(); 113 | } 114 | 115 | file = new File(statusFilePath + "/" + statusFileName); 116 | 117 | if (!isStatusFileCreated()){ 118 | LOG.warn("file:{} is not exists.",file.getPath()); 119 | currentIndex = startFrom; 120 | createStatusFile(); 121 | } 122 | else 123 | currentIndex = getStatusFileIndex(startFrom); 124 | LOG.warn("currentIndex:{} .",currentIndex); 125 | query = buildQuery(); 126 | } 127 | 128 | 129 | public String buildQuery() { 130 | 131 | if (customQuery == null){ 132 | return "SELECT " + columnsToSelect + " FROM " + table; 133 | } 134 | else { 135 | if (customQuery.contains("$@$") || customQuery.contains("$OB$")){ 136 | return customQuery.replace("$@$", currentIndex) 137 | .replace("$OB$", orderBy); 138 | } 139 | else{ 140 | return customQuery; 141 | } 142 | } 143 | } 144 | 145 | 146 | private boolean isStatusFileCreated(){ 147 | return file.exists() && !file.isDirectory() ? true: false; 148 | } 149 | 150 | private boolean isStatusDirectoryCreated() { 151 | return directory.exists() && !directory.isFile() ? true: false; 152 | } 153 | 154 | /** 155 | * Converter from a List of Object List to a List of String arrays

156 | * Useful for csvWriter 157 | * @param queryResult Query Result from hibernate executeQuery method 158 | * @return A list of String arrays, ready for csvWriter.writeall method 159 | */ 160 | public List getAllRows(List> queryResult){ 161 | 162 | List allRows = new ArrayList(); 163 | 164 | if (queryResult == null || queryResult.isEmpty()) 165 | return allRows; 166 | 167 | String[] row=null; 168 | 169 | for (int i=0; i rawRow = queryResult.get(i); 172 | row = new String[rawRow.size()]; 173 | for (int j=0; j< rawRow.size(); j++){ 174 | if (rawRow.get(j) != null) 175 | row[j] = rawRow.get(j).toString(); 176 | else 177 | row[j] = ""; 178 | } 179 | allRows.add(row); 180 | } 181 | 182 | return allRows; 183 | } 184 | 185 | /** 186 | * write the query result to the output stream 187 | * @param queryResult Query Result from hibernate executeQuery method 188 | * @param printWriter output stream object 189 | */ 190 | public void writeAllRows(List> queryResult,PrintWriter printWriter){ 191 | 192 | if (queryResult == null || queryResult.isEmpty()) { 193 | return ; 194 | } 195 | Gson gson = new Gson(); 196 | Type type = new TypeToken>(){}.getType(); 197 | for (Map item :queryResult) { 198 | for (Map.Entry entry:item.entrySet()) { 199 | Object value = entry.getValue(); 200 | // initialize null field 201 | if (value == null){ 202 | entry.setValue(""); 203 | } 204 | // format time 205 | if (value instanceof java.util.Date){ 206 | entry.setValue(SDF.format(value)); 207 | } 208 | } 209 | String json = gson.toJson(item, type); 210 | // write to output stream 211 | printWriter.print(json+JSON_LINE_END); 212 | } 213 | } 214 | 215 | /** 216 | * Create status file 217 | */ 218 | public void createStatusFile(){ 219 | 220 | statusFileJsonMap.put(SOURCE_NAME_STATUS_FILE, sourceName); 221 | statusFileJsonMap.put(URL_STATUS_FILE, connectionURL); 222 | statusFileJsonMap.put(LAST_INDEX_STATUS_FILE, currentIndex); 223 | 224 | if (isCustomQuerySet()){ 225 | statusFileJsonMap.put(QUERY_STATUS_FILE,customQuery); 226 | }else{ 227 | statusFileJsonMap.put(COLUMNS_TO_SELECT_STATUS_FILE, columnsToSelect); 228 | statusFileJsonMap.put(TABLE_STATUS_FILE, table); 229 | } 230 | 231 | 232 | try { 233 | Writer fileWriter = new FileWriter(file,false); 234 | JSONValue.writeJSONString(statusFileJsonMap, fileWriter); 235 | fileWriter.close(); 236 | } catch (IOException e) { 237 | LOG.error("Error creating value to status file!!!",e); 238 | } 239 | } 240 | 241 | /** 242 | * Update status file with last read row index 243 | */ 244 | public void updateStatusFile() { 245 | 246 | statusFileJsonMap.put(LAST_INDEX_STATUS_FILE, currentIndex); 247 | 248 | try { 249 | Writer fileWriter = new FileWriter(file,false); 250 | JSONValue.writeJSONString(statusFileJsonMap, fileWriter); 251 | fileWriter.close(); 252 | } catch (IOException e) { 253 | LOG.error("Error writing incremental value to status file!!!",e); 254 | } 255 | } 256 | 257 | private String getStatusFileIndex(String configuredStartValue) { 258 | 259 | if (!isStatusFileCreated()) { 260 | LOG.info("Status file not created, using start value from config file and creating file"); 261 | return configuredStartValue; 262 | } 263 | else{ 264 | try { 265 | FileReader fileReader = new FileReader(file); 266 | 267 | JSONParser jsonParser = new JSONParser(); 268 | statusFileJsonMap = (Map)jsonParser.parse(fileReader); 269 | checkJsonValues(); 270 | return statusFileJsonMap.get(LAST_INDEX_STATUS_FILE); 271 | 272 | } catch (Exception e) { 273 | LOG.error("Exception reading status file, doing back up and creating new status file", e); 274 | backupStatusFile(); 275 | return configuredStartValue; 276 | } 277 | } 278 | } 279 | 280 | private void checkJsonValues() throws ParseException { 281 | 282 | // Check commons values to default and custom query 283 | if (!statusFileJsonMap.containsKey(SOURCE_NAME_STATUS_FILE) || !statusFileJsonMap.containsKey(URL_STATUS_FILE) || 284 | !statusFileJsonMap.containsKey(LAST_INDEX_STATUS_FILE)) { 285 | LOG.error("Status file doesn't contains all required values"); 286 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 287 | } 288 | if (!statusFileJsonMap.get(URL_STATUS_FILE).equals(connectionURL)){ 289 | LOG.error("Connection url in status file doesn't match with configured in properties file"); 290 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 291 | }else if (!statusFileJsonMap.get(SOURCE_NAME_STATUS_FILE).equals(sourceName)){ 292 | LOG.error("Source name in status file doesn't match with configured in properties file"); 293 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 294 | } 295 | 296 | // Check default query values 297 | if (customQuery == null) 298 | { 299 | if (!statusFileJsonMap.containsKey(COLUMNS_TO_SELECT_STATUS_FILE) || !statusFileJsonMap.containsKey(TABLE_STATUS_FILE)){ 300 | LOG.error("Expected ColumsToSelect and Table fields in status file"); 301 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 302 | } 303 | if (!statusFileJsonMap.get(COLUMNS_TO_SELECT_STATUS_FILE).equals(columnsToSelect)){ 304 | LOG.error("ColumsToSelect value in status file doesn't match with configured in properties file"); 305 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 306 | } 307 | if (!statusFileJsonMap.get(TABLE_STATUS_FILE).equals(table)){ 308 | LOG.error("Table value in status file doesn't match with configured in properties file"); 309 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 310 | } 311 | return; 312 | } 313 | 314 | // Check custom query values 315 | if (customQuery != null){ 316 | if (!statusFileJsonMap.containsKey(QUERY_STATUS_FILE)){ 317 | LOG.error("Expected Query field in status file"); 318 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 319 | } 320 | if (!statusFileJsonMap.get(QUERY_STATUS_FILE).equals(customQuery)){ 321 | LOG.error("Query value in status file doesn't match with configured in properties file"); 322 | throw new ParseException(ERROR_UNEXPECTED_EXCEPTION); 323 | } 324 | return; 325 | } 326 | } 327 | 328 | private void backupStatusFile() { 329 | file.renameTo(new File(statusFilePath + "/" + statusFileName + ".bak." + System.currentTimeMillis())); 330 | } 331 | 332 | private void checkMandatoryProperties() { 333 | 334 | if (connectionURL == null){ 335 | throw new ConfigurationException("hibernate.connection.url property not set"); 336 | } 337 | if (statusFileName == null){ 338 | throw new ConfigurationException("status.file.name property not set"); 339 | } 340 | if (table == null && customQuery == null){ 341 | throw new ConfigurationException("property table not set"); 342 | } 343 | } 344 | 345 | /* 346 | * @return boolean pathname into directory 347 | */ 348 | private boolean createDirectory() { 349 | return directory.mkdir(); 350 | } 351 | 352 | /* 353 | * @return long incremental value as parameter from this 354 | */ 355 | String getCurrentIndex() { 356 | return currentIndex; 357 | } 358 | 359 | String getOrderBy() { 360 | return orderBy; 361 | } 362 | 363 | /* 364 | * @void set incrementValue 365 | */ 366 | void setCurrentIndex(String newValue) { 367 | currentIndex = newValue; 368 | } 369 | 370 | /* 371 | * @return int delay in ms 372 | */ 373 | int getRunQueryDelay() { 374 | return runQueryDelay; 375 | } 376 | 377 | int getBatchSize() { 378 | return batchSize; 379 | } 380 | 381 | int getMaxRows() { 382 | return maxRows; 383 | } 384 | 385 | String getQuery() { 386 | return query; 387 | } 388 | 389 | String getConnectionURL() { 390 | return connectionURL; 391 | } 392 | 393 | boolean isCustomQuerySet() { 394 | return (customQuery != null); 395 | } 396 | 397 | Context getContext() { 398 | return context; 399 | } 400 | 401 | 402 | boolean isReadOnlySession() { 403 | return readOnlySession; 404 | } 405 | 406 | boolean encloseByQuotes() { 407 | return encloseByQuotes; 408 | } 409 | 410 | String getDelimiterEntry() {return delimiterEntry;} 411 | 412 | } 413 | --------------------------------------------------------------------------------