├── .gitignore ├── .travis.yml ├── FURTHER_EXAMPLES.md ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── graphaware │ └── importer │ ├── BatchImporter.java │ ├── FileBatchImporter.java │ ├── cache │ ├── BaseCaches.java │ ├── Cache.java │ ├── Caches.java │ ├── InjectCache.java │ ├── MapDBCache.java │ └── MapDBCaches.java │ ├── cli │ ├── BaseCommandLineParser.java │ ├── CommandLineParser.java │ ├── CsvCommandLineParser.java │ ├── DbCommandLineParser.java │ └── FileCommandLineParser.java │ ├── config │ ├── BaseImportConfig.java │ ├── CsvImportConfig.java │ ├── DbImportConfig.java │ ├── FileImportConfig.java │ └── ImportConfig.java │ ├── context │ ├── BaseImportContext.java │ ├── CacheAwareImportContext.java │ ├── ImportContext.java │ └── SimpleImportContext.java │ ├── data │ ├── Data.java │ ├── DynamicData.java │ ├── access │ │ ├── BaseTabularDataReader.java │ │ ├── CacheDataReader.java │ │ ├── CacheEntryMapper.java │ │ ├── CsvDataReader.java │ │ ├── DataAccess.java │ │ ├── DataReader.java │ │ ├── DataWriter.java │ │ ├── DbDataReader.java │ │ ├── FileDataWriter.java │ │ ├── QueueDbDataReader.java │ │ └── TabularDataReader.java │ └── location │ │ ├── DataLocator.java │ │ ├── FileLocator.java │ │ ├── InputFileLocator.java │ │ └── SimpleDataLocator.java │ ├── domain │ ├── DirectedRelationship.java │ ├── GenericRelationship.java │ ├── Neo4jProperty.java │ ├── Neo4jPropertyContainer.java │ ├── Neo4jRelationship.java │ └── UndirectedRelationship.java │ ├── importer │ ├── BaseImporter.java │ ├── Importer.java │ └── TabularImporter.java │ ├── inserter │ └── SynchronizedBatchInserter.java │ ├── plan │ ├── DefaultExecutionPlan.java │ └── ExecutionPlan.java │ ├── stats │ ├── LoggingStatisticsCollector.java │ ├── StatisticsCollector.java │ └── StopWatch.java │ └── util │ ├── BlockingArrayBlockingQueue.java │ └── ReflectionUtils.java └── test ├── java └── com │ └── graphaware │ └── importer │ ├── cache │ └── CachesTest.java │ ├── cli │ └── FileCommandLineParserTest.java │ ├── domain │ └── Neo4jPropertyContainerTest.java │ ├── integration │ ├── BatchImporterIntegrationTest.java │ ├── TestBatchImporter.java │ ├── TestBatchImporter2.java │ ├── domain │ │ └── Person.java │ └── inserter │ │ ├── FriendsImporter.java │ │ ├── JobsImporter.java │ │ ├── LastRoleImporter.java │ │ ├── LocationImporter.java │ │ └── PersonImporter.java │ ├── plan │ └── DefaultExecutionPlanTest.java │ └── stats │ └── StopWatchTest.java └── resources ├── friends.csv ├── jobs.csv ├── locations.csv ├── log4j.properties ├── logback.xml ├── neo4j.properties └── people.csv /.gitignore: -------------------------------------------------------------------------------- 1 | .buildpath 2 | .project 3 | .settings 4 | target/ 5 | .idea/ 6 | *.class 7 | *.iml 8 | *.iws 9 | *.ipr 10 | *.db 11 | *.log 12 | .DS_Store 13 | .classpath -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | jdk: 4 | - oraclejdk8 5 | 6 | sudo: false 7 | 8 | branches: 9 | only: 10 | - master 11 | 12 | after_success: 13 | - echo "ossrh\${env.OSSRH_USER}\${env.OSSRH_PASS}" > ~/settings.xml 14 | - mvn deploy --settings ~/settings.xml -------------------------------------------------------------------------------- /FURTHER_EXAMPLES.md: -------------------------------------------------------------------------------- 1 | Further Examples 2 | ================ 3 | 4 | This document showcases some further examples of reading data from alternate data sources. To successfully set up and customize additional database import tools, the `DataReader`, `ImportConfig` and `CommandLineParser` classes must be implemented. 5 | 6 | # MySQL Data Source 7 | 8 | ### `MySqlDataReader` Implementation 9 | 10 | The following libraries are required for the `DataReader`. 11 | 12 | ```java 13 | import com.graphaware.importer.data.access.QueueDbDataReader; 14 | import org.apache.commons.dbcp2.BasicDataSource; 15 | import org.springframework.jdbc.core.JdbcTemplate; 16 | import javax.sql.DataSource; 17 | ``` 18 | 19 | The class itself is similar to that of the Oracle example in [README.md](../master/README.md). `QueueDbDataReader` is still extended although the MySQL JDBC driver is alternatively used. Note that variable `db` is added to represent the name of the database. The variable `prefetchsize` found in the `OracleDataReader` example is defunct in a MySQL environment and is therefore not included. 20 | 21 | ```java 22 | public class MySqlDataReader extends QueueDbDataReader { 23 | 24 | private final String db; 25 | private final int fetchSize; 26 | 27 | public MySqlDataReader(String dbHost, String dbPort, String user, String password, String db, int fetchSize) { 28 | super (dbHost, dbPort, user, password); 29 | this.db = db; 30 | this.fetchSize = fetchSize; 31 | try { 32 | Class.forName("com.mysql.jdbc.Driver"); 33 | } catch (ClassNotFoundException e) { 34 | e.printStackTrace(); 35 | } 36 | } 37 | 38 | @Override 39 | protected String getDriverClassName() { 40 | return "com.mysql.jdbc.Driver"; 41 | } 42 | 43 | @Override 44 | protected String getUrl(String host, String port) { 45 | return "jdbc:mysql://" + host + ":" + port + "/" + db; 46 | } 47 | 48 | @Override 49 | protected void additionalConfig(JdbcTemplate template) { 50 | template.setFetchSize(fetchSize); 51 | } 52 | 53 | @Override 54 | protected void additionalConfig(DataSource dataSource) { 55 | ((BasicDataSource) dataSource).setInitialSize(1000); 56 | } 57 | } 58 | ``` 59 | 60 | ### `MySqlImportConfig` Implementation 61 | 62 | The following libraries are required for the `ImportConfig`. 63 | 64 | ```java 65 | import com.graphaware.importer.config.DbImportConfig; 66 | import com.graphaware.importer.data.access.DataReader; 67 | ``` 68 | 69 | Again, the class dismissed the `prefetchsize` variable found in the `OracleDataReader` example. The `db` database name variable is added in a similar way. 70 | 71 | ```java 72 | public class MySqlImportConfig extends DbImportConfig { 73 | 74 | private final String db; 75 | private final int fetchSize; 76 | 77 | public MySqlImportConfig(String graphDir, String outputDir, String props, String dbHost, String dbPort, String user, String password, String db, int fetchSize) { 78 | super(graphDir, outputDir, props, dbHost, dbPort, user, password); 79 | this.db = db; 80 | this.fetchSize = fetchSize; 81 | } 82 | 83 | @Override 84 | public DataReader createReader() { 85 | return new MySqlDataReader(getDbHost(), getDbPort(), getUser(), getPassword(), getDb(), fetchSize); 86 | } 87 | 88 | public String getDb() { 89 | return this.db; 90 | } 91 | } 92 | ``` 93 | 94 | ### `MySqlCommandLineParser` Implementation 95 | 96 | The following libraries are required for the `CommandLineParser`. 97 | 98 | ```java 99 | import com.graphaware.importer.cli.DbCommandLineParser; 100 | import org.apache.commons.cli.CommandLine; 101 | import org.apache.commons.cli.Option; 102 | import org.apache.commons.cli.Options; 103 | import org.apache.commons.cli.ParseException; 104 | ``` 105 | 106 | The `CommandLineParser` is the first point of interaction between user input and the program. It therefore has been slightly modified to idiomatically interpret a MySQL data source. 107 | 108 | ```java 109 | public class MySqlCommandLineParser extends DbCommandLineParser { 110 | 111 | @Override 112 | protected MySqlImportConfig doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String host, String port, String user, String password) throws ParseException { 113 | String db = String.valueOf(getOptionalValue(line, "db", "")); 114 | int fetchSize = Integer.valueOf(getOptionalValue(line, "fs", "10000")); 115 | 116 | return new MySqlImportConfig(graphDir, outputDir, props, host, port, user, password, db, fetchSize); 117 | } 118 | 119 | @Override 120 | protected void addOptions(Options options) { 121 | super.addOptions(options); 122 | options.addOption(new Option("db", "db", true, "database name")); 123 | options.addOption(new Option("fs", "fetchSize", true, "JDBC driver row fetch size (default 10000)")); 124 | } 125 | } 126 | ``` -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 15 | 16 | 17 | 4.0.0 18 | 19 | 20 | com.graphaware.neo4j 21 | module-parent 22 | 2.3.3.37 23 | 24 | 25 | programmatic-importer 26 | 2.3.3.37.4-SNAPSHOT 27 | 28 | GraphAware Programmatic Neo4j Importer 29 | Importer high-performance multi-threaded initial data load into Neo4j from CSV/SQL with custom logic 30 | http://graphaware.com 31 | 32 | 33 | 34 | GNU General Public License, version 3 35 | http://www.gnu.org/licenses/gpl-3.0.txt 36 | repo 37 | 38 | 39 | 40 | 41 | scm:git:git@github.com:graphaware/neo4j-importer.git 42 | scm:git:git@github.com:graphaware/neo4j-importer.git 43 | git@github.com:graphaware/neo4j-importer.git 44 | HEAD 45 | 46 | 47 | 48 | 49 | bachmanm 50 | Michal Bachman 51 | neo4j-importer@graphaware.com 52 | 53 | 54 | 55 | 56 | https://travis-ci.org/graphaware/neo4j-importer 57 | Travis CI 58 | 59 | 60 | 2015 61 | 62 | 63 | GitHub 64 | https://github.com/graphaware/neo4j-importer/issues 65 | 66 | 67 | 68 | Graph Aware Limited 69 | http://graphaware.com 70 | 71 | 72 | 73 | 74 | 75 | 76 | org.neo4j 77 | neo4j 78 | compile 79 | 80 | 81 | 82 | commons-io 83 | commons-io 84 | compile 85 | 86 | 87 | 88 | com.graphaware.neo4j 89 | runtime 90 | test 91 | 92 | 93 | 94 | ch.qos.logback 95 | logback-classic 96 | compile 97 | 1.1.5 98 | 99 | 100 | 101 | org.springframework 102 | spring-orm 103 | ${spring.version} 104 | compile 105 | 106 | 107 | 108 | org.apache.commons 109 | commons-dbcp2 110 | 2.0.1 111 | compile 112 | 113 | 114 | 115 | commons-cli 116 | commons-cli 117 | 1.2 118 | compile 119 | 120 | 121 | 122 | com.graphaware.neo4j 123 | tests 124 | 125 | 126 | 127 | org.apache.commons 128 | commons-csv 129 | 1.1 130 | 131 | 132 | 133 | org.mapdb 134 | mapdb 135 | 3.0.2 136 | compile 137 | 138 | 139 | 140 | com.google.guava 141 | guava 142 | compile 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/FileBatchImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer; 17 | 18 | import com.graphaware.importer.cli.CommandLineParser; 19 | import com.graphaware.importer.cli.CsvCommandLineParser; 20 | import com.graphaware.importer.config.FileImportConfig; 21 | import com.graphaware.importer.data.Data; 22 | import com.graphaware.importer.data.DynamicData; 23 | import com.graphaware.importer.data.location.DataLocator; 24 | import com.graphaware.importer.data.location.InputFileLocator; 25 | import com.graphaware.importer.importer.Importer; 26 | 27 | import java.util.HashSet; 28 | import java.util.Map; 29 | import java.util.Set; 30 | 31 | /** 32 | * {@link com.graphaware.importer.BatchImporter} for file-based imports. 33 | */ 34 | public abstract class FileBatchImporter extends BatchImporter { 35 | 36 | /** 37 | * {@inheritDoc} 38 | */ 39 | @Override 40 | protected CommandLineParser commandLineParser() { 41 | return new CsvCommandLineParser(); 42 | } 43 | 44 | /** 45 | * {@inheritDoc} 46 | */ 47 | @Override 48 | protected DataLocator createInputDataLocator(FileImportConfig config) { 49 | return new InputFileLocator(config.getInputDir(), input()); 50 | } 51 | 52 | /** 53 | * Get input data to logical file name mapping. 54 | * 55 | * @return mapping. One-to-one mapping of all importers' input data names by default. 56 | */ 57 | protected Map input() { 58 | Set inputs = new HashSet<>(); 59 | 60 | for (Importer importer : importers()) { 61 | inputs.add(importer.inputData().name()); 62 | } 63 | 64 | return DynamicData.oneToOne(inputs.toArray(new String[inputs.size()])); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/BaseCaches.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import com.graphaware.importer.importer.Importer; 19 | import com.graphaware.importer.util.ReflectionUtils; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.util.Assert; 23 | 24 | import java.lang.reflect.Field; 25 | import java.util.*; 26 | 27 | /** 28 | * Abstract base-class for {@link com.graphaware.importer.cache.Caches} implementations. 29 | */ 30 | public abstract class BaseCaches implements Caches { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(BaseCaches.class); 33 | 34 | private final Map caches = new HashMap<>(); 35 | 36 | /** 37 | * {@inheritDoc} 38 | */ 39 | @Override 40 | public void createCache(String cacheName, Class keyType, Class valueType) { 41 | Assert.hasLength(cacheName); 42 | Assert.notNull(keyType); 43 | Assert.notNull(valueType); 44 | 45 | if (caches.containsKey(cacheName)) { 46 | throw new IllegalStateException("Cache " + cacheName + " has already been created"); 47 | } 48 | 49 | caches.put(cacheName, doCreateCache(cacheName, keyType, valueType)); 50 | } 51 | 52 | /** 53 | * Create a cache. 54 | * 55 | * @param cacheName name of the cache. Never null or empty. 56 | * @param keyType Java type of the key stored in the cache. Never null. 57 | * @param valueType Java type of the value stored in the cache. Typically a {@link java.lang.Long}. Never null. 58 | */ 59 | public abstract Cache doCreateCache(String cacheName, Class keyType, Class valueType); 60 | 61 | /** 62 | * {@inheritDoc} 63 | */ 64 | @Override 65 | public Cache getCache(String cacheName) { 66 | Assert.hasLength(cacheName); 67 | 68 | if (!caches.containsKey(cacheName)) { 69 | throw new IllegalStateException("Cache " + cacheName + " has not been created"); 70 | } 71 | 72 | return caches.get(cacheName); 73 | } 74 | 75 | /** 76 | * {@inheritDoc} 77 | */ 78 | @Override 79 | public void inject(Importer importer) { 80 | Assert.notNull(importer); 81 | 82 | for (Field field : ReflectionUtils.getAllFields(importer.getClass())) { 83 | if (field.getAnnotation(InjectCache.class) != null) { 84 | field.setAccessible(true); 85 | try { 86 | field.set(importer, getCache(field.getAnnotation(InjectCache.class).name())); 87 | } catch (IllegalAccessException e) { 88 | throw new RuntimeException(e); 89 | } 90 | } 91 | } 92 | } 93 | 94 | /** 95 | * {@inheritDoc} 96 | */ 97 | @Override 98 | public void cleanup(Collection unfinished) { 99 | Assert.notNull(unfinished); 100 | 101 | if (unfinished.isEmpty()) { 102 | LOG.info("No need to clear caches, the import is finished."); 103 | return; 104 | } 105 | 106 | Set neededCaches = new HashSet<>(); 107 | for (Importer importer : unfinished) { 108 | neededCaches.addAll(neededCaches(importer)); 109 | } 110 | 111 | boolean cacheCleared = false; 112 | 113 | for (String candidate : caches.keySet()) { 114 | if (!neededCaches.contains(candidate) && !getCache(candidate).isEmpty()) { 115 | LOG.info("Clearing cache: " + candidate); 116 | getCache(candidate).clear(); 117 | cacheCleared = true; 118 | } 119 | } 120 | 121 | if (cacheCleared) { 122 | compact(); 123 | } 124 | } 125 | 126 | /** 127 | * Compact caches after some of them have been cleared, if needed. No-op by default. 128 | */ 129 | protected void compact() { 130 | 131 | } 132 | 133 | /** 134 | * {@inheritDoc} 135 | */ 136 | @Override 137 | public Set neededCaches(Importer importer) { 138 | Assert.notNull(importer); 139 | 140 | Set result = new HashSet<>(); 141 | 142 | for (Field field : ReflectionUtils.getAllFields(importer.getClass())) { 143 | if (field.getAnnotation(InjectCache.class) != null) { 144 | result.add(field.getAnnotation(InjectCache.class).name()); 145 | } 146 | } 147 | 148 | return result; 149 | } 150 | 151 | /** 152 | * {@inheritDoc} 153 | */ 154 | @Override 155 | public Set createdCaches(Importer importer) { 156 | Assert.notNull(importer); 157 | 158 | Set result = new HashSet<>(); 159 | 160 | for (Field field : ReflectionUtils.getAllFields(importer.getClass())) { 161 | if (field.getAnnotation(InjectCache.class) != null && field.getAnnotation(InjectCache.class).creator()) { 162 | result.add(field.getAnnotation(InjectCache.class).name()); 163 | } 164 | } 165 | 166 | return result; 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/Cache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import java.util.Map; 19 | import java.util.Set; 20 | 21 | /** 22 | * A cache, like a {@link java.util.Map} with {@link #get(Object)} and {@link #containsKey(Object)} supporting null keys, 23 | * returning null and false, respectively. 24 | *

25 | * Caches are needed to map identifiers from custom data sources to Neo4j node IDs assigned to the nodes upon insertion. 26 | * 27 | * @param key type. 28 | * @param value type. 29 | */ 30 | public interface Cache { 31 | 32 | int size(); 33 | 34 | boolean isEmpty(); 35 | 36 | boolean containsKey(K key); 37 | 38 | V get(K key); 39 | 40 | void put(K key, V value); 41 | 42 | void clear(); 43 | 44 | Set> entrySet(); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/Caches.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import com.graphaware.importer.importer.Importer; 19 | 20 | import java.util.Collection; 21 | import java.util.Set; 22 | 23 | /** 24 | * {@link com.graphaware.importer.cache.Cache}s used by all {@link com.graphaware.importer.importer.Importer}s. 25 | */ 26 | public interface Caches { 27 | 28 | /** 29 | * Create a cache. Cache has to be created before it can be used and cache names must be unique. 30 | * 31 | * @param cacheName name of the cache. Must not be null or empty and must be unique. 32 | * @param keyType Java type of the key stored in the cache. Must not be null. 33 | * @param valueType Java type of the value stored in the cache. Typically a {@link java.lang.Long}. Must not be null. 34 | * @throws java.lang.IllegalStateException in case a cache with the given name already exists. 35 | */ 36 | void createCache(String cacheName, Class keyType, Class valueType); 37 | 38 | /** 39 | * Get cache with the given name. 40 | * 41 | * @param cacheName name of the cache to locate. Must not be null or empty. 42 | * @return cache, never null. 43 | * @throws java.lang.IllegalStateException in case no such cache exists. 44 | */ 45 | Cache getCache(String cacheName); 46 | 47 | /** 48 | * Get names of caches needed by the given inserter. 49 | * Found from {@link com.graphaware.importer.cache.InjectCache} annotations on fields. 50 | * 51 | * @param importer for which to find needed caches. Must not be null. 52 | * @return names of caches needed by the inserter. 53 | */ 54 | Set neededCaches(Importer importer); 55 | 56 | /** 57 | * Get names of caches created by the given inserter. 58 | * Found from {@link com.graphaware.importer.cache.InjectCache} annotations on fields, 59 | * where {@link InjectCache#creator()} equals true. 60 | * 61 | * @param importer for which to find created caches. Must not be null. 62 | * @return names of created by the inserter. 63 | */ 64 | Set createdCaches(Importer importer); 65 | 66 | /** 67 | * Inject the right caches to the given inserter. Will populate fields annotated with 68 | * {@link com.graphaware.importer.cache.InjectCache}. 69 | * 70 | * @param importer to inject caches to. Must not be null. 71 | */ 72 | void inject(Importer importer); 73 | 74 | /** 75 | * Cleanup caches that will no longer be used. 76 | * 77 | * @param unfinished inserters that are not yet finished. Must not be null. 78 | */ 79 | void cleanup(Collection unfinished); 80 | 81 | /** 82 | * Destroy all caches. 83 | */ 84 | void destroy(); 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/InjectCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import java.lang.annotation.Documented; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.Target; 21 | 22 | import static java.lang.annotation.ElementType.FIELD; 23 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 24 | 25 | /** 26 | * An annotation indicating that a field should have a named cache injected. 27 | */ 28 | @Target({FIELD}) 29 | @Retention(RUNTIME) 30 | @Documented 31 | public @interface InjectCache { 32 | 33 | /** 34 | * @return Name of the cache to inject. 35 | */ 36 | String name(); 37 | 38 | /** 39 | * @return Whether the class whose field is annotated is the creator of the cache. 40 | */ 41 | boolean creator() default false; 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/MapDBCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import org.mapdb.DB; 19 | import org.mapdb.Serializer; 20 | 21 | import java.util.Map; 22 | import java.util.Set; 23 | 24 | /** 25 | * {@link com.graphaware.importer.cache.Cache} backed by {@link org.mapdb.DB}. 26 | * 27 | * @param key type. 28 | * @param value type. 29 | */ 30 | public class MapDBCache implements Cache { 31 | 32 | private final Map map; 33 | 34 | public MapDBCache(DB db, String name, Serializer keySerializer, Serializer valueSerializer) { 35 | map = db.hashMap(name).keySerializer(keySerializer).valueSerializer(valueSerializer).create(); 36 | } 37 | 38 | @Override 39 | public int size() { 40 | return map.size(); 41 | } 42 | 43 | @Override 44 | public boolean isEmpty() { 45 | return map.isEmpty(); 46 | } 47 | 48 | @Override 49 | public boolean containsKey(K key) { 50 | return map.containsKey(key); 51 | } 52 | 53 | @Override 54 | public V get(K key) { 55 | return map.get(key); 56 | } 57 | 58 | @Override 59 | public void put(K key, V value) { 60 | map.put(key, value); 61 | } 62 | 63 | @Override 64 | public void clear() { 65 | map.clear(); 66 | } 67 | 68 | @Override 69 | public Set> entrySet() { 70 | return map.entrySet(); 71 | } 72 | } 73 | 74 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cache/MapDBCaches.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import com.graphaware.importer.config.ImportConfig; 19 | import org.mapdb.DB; 20 | import org.mapdb.DBMaker; 21 | import org.mapdb.Serializer; 22 | import org.springframework.util.Assert; 23 | 24 | /** 25 | * {@link com.graphaware.importer.cache.Caches} of type {@link com.graphaware.importer.cache.MapDBCache}. 26 | */ 27 | public class MapDBCaches extends BaseCaches { 28 | 29 | private final DB db; 30 | 31 | public MapDBCaches(ImportConfig config) { 32 | db = DBMaker 33 | .fileDB(config.getCacheFile()) 34 | .fileMmapEnable() 35 | .fileMmapEnableIfSupported() 36 | .fileMmapPreclearDisable() 37 | .cleanerHackEnable() 38 | .fileDeleteAfterClose() 39 | .closeOnJvmShutdown() 40 | .make(); 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | */ 46 | @Override 47 | public final Cache doCreateCache(String cacheName, Class keyType, Class valueType) { 48 | return doCreateCache(db, cacheName, keyType, valueType); 49 | } 50 | 51 | /** 52 | * Create a cache backed by the given {@link org.mapdb.DB}. 53 | * 54 | * @param db backing the new cache. 55 | * @param cacheName name of the cache. 56 | * @param keyType Java type of the key stored in the cache. 57 | * @param valueType Java type of the value stored in the cache. Typically a {@link java.lang.Long}. 58 | * @return cache. 59 | */ 60 | protected MapDBCache doCreateCache(DB db, String cacheName, Class keyType, Class valueType) { 61 | return new MapDBCache(db, cacheName, getSerializer(keyType), getSerializer(valueType)); 62 | } 63 | 64 | /** 65 | * {@inheritDoc} 66 | */ 67 | @Override 68 | public void destroy() { 69 | db.close(); 70 | } 71 | 72 | /** 73 | * Resolve a serializer for a given key or value type. 74 | * 75 | * @param type type. Must not be null. 76 | * @return serializer for the given type. 77 | */ 78 | protected Serializer getSerializer(Class type) { 79 | Assert.notNull(type); 80 | 81 | if (String.class.isAssignableFrom(type)) { 82 | return Serializer.STRING; 83 | } 84 | 85 | if (Long.class.isAssignableFrom(type)) { 86 | return Serializer.LONG; 87 | } 88 | 89 | if (Integer.class.isAssignableFrom(type)) { 90 | return Serializer.INTEGER; 91 | } 92 | 93 | return defaultSerializer(type); 94 | } 95 | 96 | /** 97 | * Get a serializer for a type, for which a serializer hasn't been resolved by {@link #getSerializer(Class)}. 98 | * 99 | * @param type type. Never null. 100 | * @return serializer for the given type. 101 | */ 102 | protected Serializer defaultSerializer(Class type) { 103 | Assert.notNull(type); 104 | 105 | return Serializer.JAVA; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cli/BaseCommandLineParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.ImportConfig; 19 | import org.apache.commons.cli.*; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | /** 24 | * Abstract base-class for implementations of {@link com.graphaware.importer.cli.CommandLineParser}. 25 | * 26 | * @param type of the produced config. 27 | */ 28 | public abstract class BaseCommandLineParser implements CommandLineParser { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(BaseCommandLineParser.class); 31 | 32 | /** 33 | * {@inheritDoc} 34 | */ 35 | @Override 36 | public final T parseArgs(String[] args) { 37 | Options options = produceOptions(); 38 | 39 | try { 40 | return produceConfig(args, options); 41 | } catch (ParseException exp) { 42 | System.err.println("Parsing failed. Reason: " + exp.getMessage()); 43 | HelpFormatter formatter = new HelpFormatter(); 44 | formatter.printHelp("importer", options); 45 | return null; 46 | } 47 | } 48 | 49 | private T produceConfig(String[] args, Options options) throws ParseException { 50 | CommandLine line = new BasicParser().parse(options, args); 51 | 52 | String graphDir = getMandatoryValue(line, "g"); 53 | String outputDir = getMandatoryValue(line, "o"); 54 | String props = getMandatoryValue(line, "r"); 55 | String cacheFile = getMandatoryValue(line, "c"); 56 | 57 | LOG.info("Producing import config:"); 58 | LOG.info("\tGraph: " + graphDir); 59 | LOG.info("\tOutput: " + outputDir); 60 | LOG.info("\tProps: " + props); 61 | LOG.info("\tCache File: " + cacheFile); 62 | 63 | return doProduceConfig(line, graphDir, outputDir, props, cacheFile); 64 | } 65 | 66 | /** 67 | * Produce a config. 68 | * 69 | * @param line the command line. 70 | * @param graphDir graph directory, already extracted from the command line. 71 | * @param outputDir output directory, already extracted from the command line. 72 | * @param props path to Neo4j properties, already extracted from the command line. 73 | * @return import configuration. 74 | * @throws ParseException 75 | */ 76 | protected abstract T doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile) throws ParseException; 77 | 78 | /** 79 | * Produce default/essential options. 80 | * 81 | * @return command line options. 82 | */ 83 | private Options produceOptions() { 84 | Options options = new Options(); 85 | options.addOption(new Option("g", "graph", true, "use given directory to output the graph")); 86 | options.addOption(new Option("o", "output", true, "use given directory to output auxiliary files, such as statistics")); 87 | options.addOption(new Option("r", "properties", true, "use given file as neo4j properties")); 88 | options.addOption(new Option("c", "cachefile", true, "use given file as temporary cache on disk")); 89 | 90 | addOptions(options); 91 | 92 | return options; 93 | } 94 | 95 | /** 96 | * Add extra options to the default ones. No-op by default, intended to be overridden. 97 | * 98 | * @param options default options. 99 | */ 100 | protected void addOptions(Options options) { 101 | 102 | } 103 | 104 | /** 105 | * Convenience method for getting an optional command line value. 106 | * 107 | * @param line command line. 108 | * @param opt option. 109 | * @param defaultValue if the option isn't present. 110 | * @return value. 111 | */ 112 | protected final String getOptionalValue(CommandLine line, String opt, String defaultValue) { 113 | return line.getOptionValue(opt, defaultValue); 114 | } 115 | 116 | /** 117 | * Convenience method for getting a mandatory command line value. 118 | * 119 | * @param line command line. 120 | * @param opt option. 121 | * @return value. 122 | * @throws org.apache.commons.cli.ParseException if the option is missing. 123 | */ 124 | protected final String getMandatoryValue(CommandLine line, String opt) throws ParseException { 125 | String result = line.getOptionValue(opt); 126 | if (result == null) { 127 | throw new ParseException("Missing option: " + opt); 128 | } 129 | return result; 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cli/CommandLineParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.ImportConfig; 19 | 20 | /** 21 | * A parser of command line arguments, producing an {@link ImportConfig}. 22 | * 23 | * @param type of the produced config. 24 | */ 25 | public interface CommandLineParser { 26 | 27 | /** 28 | * Produce an {@link ImportConfig} from command line arguments. 29 | * 30 | * @param args args. 31 | * @return import context. null if the config could not be produced for whatever reason. 32 | */ 33 | T parseArgs(String[] args); 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cli/CsvCommandLineParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.CsvImportConfig; 19 | import com.graphaware.importer.config.FileImportConfig; 20 | import org.apache.commons.cli.CommandLine; 21 | 22 | /** 23 | * Implementation of {@link com.graphaware.importer.cli.CommandLineParser} for CSV imports. 24 | */ 25 | public class CsvCommandLineParser extends FileCommandLineParser { 26 | 27 | /** 28 | * {@inheritDoc} 29 | */ 30 | @Override 31 | protected FileImportConfig doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile, String inputDir) { 32 | return new CsvImportConfig(graphDir, outputDir, props, cacheFile, inputDir); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cli/DbCommandLineParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.DbImportConfig; 19 | import org.apache.commons.cli.CommandLine; 20 | import org.apache.commons.cli.Option; 21 | import org.apache.commons.cli.Options; 22 | import org.apache.commons.cli.ParseException; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * Implementation of {@link com.graphaware.importer.cli.CommandLineParser} for database imports. 28 | */ 29 | public abstract class DbCommandLineParser extends BaseCommandLineParser { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(DbCommandLineParser.class); 32 | 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | protected DbImportConfig doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile) throws ParseException { 38 | String host = getMandatoryValue(line, "h"); 39 | String port = getMandatoryValue(line, "t"); 40 | String user = getMandatoryValue(line, "u"); 41 | String password = getMandatoryValue(line, "p"); 42 | 43 | LOG.info("\tHost:" + host); 44 | LOG.info("\tPort:" + port); 45 | LOG.info("\tUsername:" + user); 46 | LOG.info("\tPassword: **********"); 47 | 48 | return doProduceConfig(line, graphDir, outputDir, props, cacheFile, host, port, user, password); 49 | } 50 | 51 | /** 52 | * Produce a config. 53 | * 54 | * @param line the command line. 55 | * @param graphDir graph directory, already extracted from the command line. 56 | * @param outputDir output directory, already extracted from the command line. 57 | * @param props path to Neo4j properties, already extracted from the command line. 58 | * @param host db host, already extracted from the command line. 59 | * @param port db port, already extracted from the command line. 60 | * @param user db user, already extracted from the command line. 61 | * @param password db password, already extracted from the command line. 62 | * @return import configuration. 63 | * @throws ParseException 64 | */ 65 | protected abstract DbImportConfig doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile, String host, String port, String user, String password) throws ParseException; 66 | 67 | /** 68 | * {@inheritDoc} 69 | */ 70 | @Override 71 | protected void addOptions(Options options) { 72 | super.addOptions(options); 73 | 74 | options.addOption(new Option("h", "host", true, "database host")); 75 | options.addOption(new Option("t", "port", true, "database port")); 76 | options.addOption(new Option("u", "username", true, "database username")); 77 | options.addOption(new Option("p", "password", true, "database encrypted password")); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/cli/FileCommandLineParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.FileImportConfig; 19 | import org.apache.commons.cli.CommandLine; 20 | import org.apache.commons.cli.Option; 21 | import org.apache.commons.cli.Options; 22 | import org.apache.commons.cli.ParseException; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * Implementation of {@link com.graphaware.importer.cli.CommandLineParser} for file imports. 28 | */ 29 | public abstract class FileCommandLineParser extends BaseCommandLineParser { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(FileCommandLineParser.class); 32 | 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | protected T doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile) throws ParseException { 38 | String inputDir = getMandatoryValue(line, "i"); 39 | LOG.info("\tInput: " + inputDir); 40 | 41 | return doProduceConfig(line, graphDir, outputDir, props, cacheFile, inputDir); 42 | } 43 | 44 | /** 45 | * Produce a config. 46 | * 47 | * @param line the command line. 48 | * @param graphDir graph directory, already extracted from the command line. 49 | * @param outputDir output directory, already extracted from the command line. 50 | * @param props path to Neo4j properties, already extracted from the command line. 51 | * @param inputDir directory for input files, already extracted from the command line. 52 | * @return import configuration. 53 | */ 54 | protected abstract T doProduceConfig(CommandLine line, String graphDir, String outputDir, String props, String cacheFile, String inputDir); 55 | 56 | /** 57 | * {@inheritDoc} 58 | */ 59 | @Override 60 | protected void addOptions(Options options) { 61 | options.addOption(new Option("i", "input", true, "use given directory to find input files")); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/config/BaseImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.config; 17 | 18 | import com.graphaware.importer.data.access.DataReader; 19 | import org.springframework.util.Assert; 20 | 21 | /** 22 | * Base-class for {@link com.graphaware.importer.config.ImportConfig} implementations. 23 | */ 24 | public abstract class BaseImportConfig implements ImportConfig { 25 | 26 | private final String graphDir; 27 | private final String outputDir; 28 | private final String props; 29 | private final String cacheFile; 30 | 31 | /** 32 | * Construct a new config. 33 | * 34 | * @param graphDir directory where the database will be stored. Must not be null or empty. 35 | * @param outputDir directory where other files produced by the import will be stored. Must not be null or empty. 36 | * @param props path to Neo4j properties used during the import. Must not be null or empty. 37 | * @param cacheFile full path to file on disk that will be used as a cache. 38 | */ 39 | protected BaseImportConfig(String graphDir, String outputDir, String props, String cacheFile) { 40 | Assert.hasLength(graphDir); 41 | Assert.hasLength(outputDir); 42 | Assert.hasLength(props); 43 | Assert.hasLength(cacheFile); 44 | 45 | this.graphDir = graphDir; 46 | this.outputDir = outputDir; 47 | this.props = props; 48 | this.cacheFile = cacheFile; 49 | } 50 | 51 | /** 52 | * {@inheritDoc} 53 | */ 54 | @Override 55 | public String getGraphDir() { 56 | return graphDir; 57 | } 58 | 59 | /** 60 | * {@inheritDoc} 61 | */ 62 | @Override 63 | public String getOutputDir() { 64 | return outputDir; 65 | } 66 | 67 | /** 68 | * {@inheritDoc} 69 | */ 70 | @Override 71 | public String getProps() { 72 | return props; 73 | } 74 | 75 | /** 76 | * {@inheritDoc} 77 | */ 78 | @Override 79 | public String getCacheFile() { 80 | return cacheFile; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/config/CsvImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.config; 17 | 18 | import com.graphaware.importer.data.access.CsvDataReader; 19 | import com.graphaware.importer.data.access.DataReader; 20 | 21 | /** 22 | * {@link com.graphaware.importer.config.ImportConfig} implementation for CSV file-based import. 23 | */ 24 | public class CsvImportConfig extends FileImportConfig { 25 | 26 | private final char delimiter; 27 | private final char quote; 28 | 29 | /** 30 | * Construct a new config. 31 | * 32 | * @param graphDir directory where the database will be stored. Must not be null or empty. 33 | * @param outputDir directory where other files produced by the import will be stored. Must not be null or empty. 34 | * @param props path to Neo4j properties used during the import. Must not be null or empty. 35 | * @param cacheFile full path to file on disk that will be used as a cache. 36 | * @param inputDir directory where input files will be searched. Must not be null or empty. 37 | * @param delimiter CSV file delimiter. 38 | * @param quote CSV file quote character. 39 | */ 40 | public CsvImportConfig(String graphDir, String outputDir, String props, String cacheFile, String inputDir, char delimiter, char quote) { 41 | super(graphDir, outputDir, props, cacheFile, inputDir); 42 | this.delimiter = delimiter; 43 | this.quote = quote; 44 | } 45 | 46 | /** 47 | * Construct a new config with comma as delimiter and a double quote as quote. 48 | * 49 | * @param graphDir directory where the database will be stored. Must not be null or empty. 50 | * @param outputDir directory where other files produced by the import will be stored. Must not be null or empty. 51 | * @param props path to Neo4j properties used during the import. Must not be null or empty. 52 | * @param cacheFile full path to file on disk that will be used as a cache. 53 | * @param inputDir directory where input files will be searched. Must not be null or empty. 54 | */ 55 | public CsvImportConfig(String graphDir, String outputDir, String props, String cacheFile, String inputDir) { 56 | this(graphDir, outputDir, props, cacheFile, inputDir, ',', '\"'); 57 | } 58 | 59 | /** 60 | * {@inheritDoc} 61 | */ 62 | @Override 63 | public DataReader createReader() { 64 | return new CsvDataReader(delimiter, quote); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/config/DbImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.config; 17 | 18 | /** 19 | * Base class for {@link com.graphaware.importer.config.ImportConfig} implementations for database-based import. 20 | */ 21 | public abstract class DbImportConfig extends BaseImportConfig { 22 | 23 | private final String dbHost; 24 | private final String dbPort; 25 | private final String user; 26 | private final String password; 27 | 28 | protected DbImportConfig(String graphDir, String outputDir, String props, String cacheFile, String dbHost, String dbPort, String user, String password) { 29 | super(graphDir, outputDir, props, cacheFile); 30 | this.dbHost = dbHost; 31 | this.dbPort = dbPort; 32 | this.user = user; 33 | this.password = password; 34 | } 35 | 36 | public String getDbHost() { 37 | return dbHost; 38 | } 39 | 40 | public String getDbPort() { 41 | return dbPort; 42 | } 43 | 44 | public String getUser() { 45 | return user; 46 | } 47 | 48 | public String getPassword() { 49 | return password; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/config/FileImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.config; 17 | 18 | /** 19 | * Base class for {@link com.graphaware.importer.config.ImportConfig} implementations for file-based import. 20 | */ 21 | public abstract class FileImportConfig extends BaseImportConfig { 22 | 23 | private final String inputDir; 24 | 25 | /** 26 | * Construct a new config. 27 | * 28 | * @param graphDir directory where the database will be stored. Must not be null or empty. 29 | * @param outputDir directory where other files produced by the import will be stored. Must not be null or empty. 30 | * @param props path to Neo4j properties used during the import. Must not be null or empty. 31 | * @param cacheFile full path to file on disk that will be used as a cache. 32 | * @param inputDir directory where the input files will be read from. 33 | */ 34 | protected FileImportConfig(String graphDir, String outputDir, String props, String cacheFile, String inputDir) { 35 | super(graphDir, outputDir, props, cacheFile); 36 | this.inputDir = inputDir; 37 | } 38 | 39 | public String getInputDir() { 40 | return inputDir; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/config/ImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.config; 17 | 18 | import com.graphaware.importer.data.access.DataReader; 19 | 20 | /** 21 | * Configuration of an import. 22 | */ 23 | public interface ImportConfig { 24 | 25 | /** 26 | * @return directory where the database will be stored. 27 | */ 28 | String getGraphDir(); 29 | 30 | /** 31 | * @return directory where other files produced by the import will be stored. 32 | */ 33 | String getOutputDir(); 34 | 35 | /** 36 | * @return path to Neo4j properties used during the import. 37 | */ 38 | String getProps(); 39 | 40 | /** 41 | * Get the data reader that will provide data for this import. 42 | * 43 | * @return data reader. 44 | */ 45 | DataReader createReader(); 46 | 47 | /** 48 | * Get full path to the file on disk that will be used as cache. 49 | * 50 | * @return path to file. 51 | */ 52 | String getCacheFile(); 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/context/BaseImportContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.context; 17 | 18 | import com.graphaware.importer.config.ImportConfig; 19 | import com.graphaware.importer.inserter.SynchronizedBatchInserter; 20 | import com.graphaware.importer.stats.LoggingStatisticsCollector; 21 | import com.graphaware.importer.stats.StatisticsCollector; 22 | import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProviderNewImpl; 23 | import org.neo4j.unsafe.batchinsert.BatchInserter; 24 | import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider; 25 | import org.neo4j.unsafe.batchinsert.BatchInserters; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | import org.springframework.core.io.ClassPathResource; 29 | import org.springframework.util.Assert; 30 | 31 | import java.io.File; 32 | import java.io.IOException; 33 | import java.util.HashMap; 34 | import java.util.Map; 35 | import java.util.Properties; 36 | import java.util.concurrent.Executors; 37 | import java.util.concurrent.ScheduledExecutorService; 38 | import java.util.concurrent.TimeUnit; 39 | 40 | /** 41 | * Bare-bones base-class for {@link ImportContext} implementations. 42 | */ 43 | abstract class BaseImportContext implements ImportContext { 44 | 45 | private static final Logger LOG = LoggerFactory.getLogger(BaseImportContext.class); 46 | 47 | protected final ImportConfig config; 48 | 49 | private BatchInserter inserter; 50 | private BatchInserterIndexProvider indexProvider; 51 | 52 | /** 53 | * Create a new import context with the given config. 54 | * 55 | * @param config import config. Must not be null. 56 | */ 57 | public BaseImportContext(ImportConfig config) { 58 | Assert.notNull(config); 59 | 60 | this.config = config; 61 | } 62 | 63 | /** 64 | * {@inheritDoc} 65 | */ 66 | @Override 67 | public final BatchInserter inserter() { 68 | return inserter; 69 | } 70 | 71 | /** 72 | * {@inheritDoc} 73 | */ 74 | @Override 75 | public final BatchInserterIndexProvider indexProvider() { 76 | return indexProvider; 77 | } 78 | 79 | /** 80 | * {@inheritDoc} 81 | */ 82 | @Override 83 | public final void essentialBootstrap() { 84 | inserter = createBatchInserter(); 85 | indexProvider = createIndexProvider(); 86 | } 87 | 88 | /** 89 | * {@inheritDoc} 90 | */ 91 | @Override 92 | public final void fullBootstrap() { 93 | preBootstrap(); 94 | 95 | essentialBootstrap(); 96 | 97 | postBootstrap(); 98 | } 99 | 100 | /** 101 | * Perform preliminary actions before context bootstrapping. No-op by default, intended to be overridden. 102 | */ 103 | protected void preBootstrap() { 104 | } 105 | 106 | /** 107 | * Perform additional context bootstrapping. No-op by default, intended to be overridden. 108 | */ 109 | protected void postBootstrap() { 110 | } 111 | 112 | /** 113 | * {@inheritDoc} 114 | */ 115 | @Override 116 | public final void check() { 117 | doCheck(); 118 | } 119 | 120 | /** 121 | * Perform additional checks. No-op by default, intended to be overridden. 122 | * 123 | * @throws java.lang.IllegalStateException if not OK. 124 | */ 125 | protected void doCheck() { 126 | } 127 | 128 | /** 129 | * {@inheritDoc} 130 | */ 131 | @Override 132 | public StatisticsCollector createStatistics(String name) { 133 | return new LoggingStatisticsCollector(name); 134 | } 135 | 136 | /** 137 | * {@inheritDoc} 138 | */ 139 | @Override 140 | public final void shutdown() { 141 | ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); 142 | executor.scheduleAtFixedRate(new Runnable() { 143 | @Override 144 | public void run() { 145 | LOG.info("I am still alive!"); 146 | } 147 | }, 1, 1, TimeUnit.MINUTES); 148 | 149 | preShutdown(); 150 | 151 | indexProvider().shutdown(); 152 | 153 | inserter().shutdown(); 154 | 155 | postShutdown(); 156 | 157 | executor.shutdownNow(); 158 | } 159 | 160 | /** 161 | * Perform actions before the context is shutdown. No-op by default, intended to be overridden. 162 | */ 163 | protected void preShutdown() { 164 | } 165 | 166 | /** 167 | * Perform actions after the context has been shutdown. No-op by default, intended to be overridden. 168 | *

169 | * Note that by this time, you should only perform logging and the like, everything else in the context (especially inserters and index providers) has been shut down! 170 | */ 171 | protected void postShutdown() { 172 | } 173 | 174 | /** 175 | * Create a {@link org.neo4j.unsafe.batchinsert.BatchInserter}. 176 | * 177 | * @return batch inserter. 178 | */ 179 | protected final BatchInserter createBatchInserter() { 180 | try { 181 | return new SynchronizedBatchInserter(BatchInserters.inserter(new File(config.getGraphDir()), new HashMap((Map) getProperties()))); 182 | } catch (IOException e) { 183 | throw new RuntimeException(e); 184 | } 185 | } 186 | 187 | /** 188 | * Get the properties passed in to the batch inserter. 189 | * 190 | * @return properties. 191 | */ 192 | protected Properties getProperties() { 193 | final Properties properties = new Properties(); 194 | try { 195 | properties.load(new ClassPathResource(config.getProps()).getInputStream()); 196 | } catch (IOException e) { 197 | LOG.warn("Could not read properties"); 198 | throw new RuntimeException(e); 199 | } 200 | return properties; 201 | } 202 | 203 | /** 204 | * Create a {@link org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider}. 205 | * 206 | * @return index provider. 207 | */ 208 | protected final BatchInserterIndexProvider createIndexProvider() { 209 | return new LuceneBatchInserterIndexProviderNewImpl(((SynchronizedBatchInserter) inserter).getBatchInserter()); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/context/CacheAwareImportContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.context; 17 | 18 | import com.graphaware.importer.cache.Caches; 19 | import com.graphaware.importer.config.ImportConfig; 20 | import com.graphaware.importer.data.Data; 21 | import com.graphaware.importer.data.access.CacheDataReader; 22 | import com.graphaware.importer.data.access.CacheEntryMapper; 23 | import com.graphaware.importer.data.access.DataReader; 24 | import com.graphaware.importer.data.location.DataLocator; 25 | import org.springframework.util.Assert; 26 | 27 | import java.util.Map; 28 | 29 | /** 30 | * {@link com.graphaware.importer.context.ImportContext} useful for imports where {@link Caches} themselves can serve 31 | * as data input. In such cases, a {@link com.graphaware.importer.data.location.DataLocator} for the cached data needs 32 | * to be provided to the constructor, along with a {@link java.util.Map} of {@link com.graphaware.importer.data.access.CacheEntryMapper}s. 33 | */ 34 | public class CacheAwareImportContext extends SimpleImportContext { 35 | 36 | private final DataLocator cacheInputLocator; 37 | private final Map cacheEntryMappers; 38 | 39 | /** 40 | * Create a new context. 41 | * 42 | * @param config import config. 43 | * @param caches caches to be used throughout the import. 44 | * @param inputLocator component capable of locating input data. 45 | * @param outputLocator component capable of locating (creating) output data. 46 | * @param cacheInputLocator component capable of locating cached input data. Must not be null. 47 | * @param cacheEntryMappers mapper for cached data as if they were columns. Must not be null. 48 | */ 49 | public CacheAwareImportContext(ImportConfig config, Caches caches, DataLocator inputLocator, DataLocator outputLocator, DataLocator cacheInputLocator, Map cacheEntryMappers) { 50 | super(config, caches, inputLocator, outputLocator); 51 | Assert.notNull(cacheInputLocator); 52 | Assert.notNull(cacheEntryMappers); 53 | this.cacheInputLocator = cacheInputLocator; 54 | this.cacheEntryMappers = cacheEntryMappers; 55 | } 56 | 57 | /** 58 | * {@inheritDoc} 59 | */ 60 | @Override 61 | protected DataReader doCreateReader(Data data) { 62 | if (cacheInputLocator.canLocate(data)) { 63 | return new CacheDataReader(caches(), cacheEntryMappers); 64 | } 65 | 66 | return super.doCreateReader(data); 67 | } 68 | 69 | /** 70 | * {@inheritDoc} 71 | */ 72 | @Override 73 | protected String locate(Data data) { 74 | if (cacheInputLocator.canLocate(data)) { 75 | return cacheInputLocator.locate(data); 76 | } 77 | 78 | return super.locate(data); 79 | } 80 | 81 | /** 82 | * {@inheritDoc} 83 | */ 84 | @Override 85 | protected void doCheck() { 86 | super.doCheck(); 87 | 88 | cacheInputLocator.check(); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/context/ImportContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.context; 17 | 18 | import com.graphaware.importer.cache.Caches; 19 | import com.graphaware.importer.data.Data; 20 | import com.graphaware.importer.data.access.DataReader; 21 | import com.graphaware.importer.stats.StatisticsCollector; 22 | import org.neo4j.unsafe.batchinsert.BatchInserter; 23 | import org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider; 24 | 25 | /** 26 | * Context for an import. 27 | */ 28 | public interface ImportContext { 29 | 30 | /** 31 | * Perform essential bootstrap of the context, i.e., create {@link org.neo4j.unsafe.batchinsert.BatchInserter} and 32 | * {@link org.neo4j.unsafe.batchinsert.BatchInserterIndexProvider}. 33 | */ 34 | void essentialBootstrap(); 35 | 36 | /** 37 | * Bootstrap the context, i.e. prepare for import. 38 | */ 39 | void fullBootstrap(); 40 | 41 | /** 42 | * Check that the context is OK. 43 | * 44 | * @throws java.lang.IllegalStateException if not OK. 45 | */ 46 | void check(); 47 | 48 | /** 49 | * Create a new statistics collector for stats with the given name. 50 | * 51 | * @param name stats name. Must not be null or empty. 52 | * @return collector. 53 | */ 54 | StatisticsCollector createStatistics(String name); 55 | 56 | /** 57 | * Get caches used throughout the import. 58 | * 59 | * @return caches. 60 | */ 61 | Caches caches(); 62 | 63 | /** 64 | * Get the batch inserter associated with this import. 65 | * 66 | * @return batch inserter. 67 | */ 68 | BatchInserter inserter(); 69 | 70 | /** 71 | * Get the index provider associated with this import. 72 | * 73 | * @return batch inserter index provider. 74 | */ 75 | BatchInserterIndexProvider indexProvider(); 76 | 77 | /** 78 | * Create a reader for the given data. 79 | * 80 | * @param data for which to create a reader. Must not be null. 81 | * @return data reader. Null if no data reader for given input data can be created. 82 | */ 83 | DataReader createReader(Data data); 84 | 85 | /** 86 | * Shutdown the context after import has finished. 87 | */ 88 | void shutdown(); 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/context/SimpleImportContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.context; 17 | 18 | import com.graphaware.importer.cache.Caches; 19 | import com.graphaware.importer.config.ImportConfig; 20 | import com.graphaware.importer.data.Data; 21 | import com.graphaware.importer.data.access.DataReader; 22 | import com.graphaware.importer.data.location.DataLocator; 23 | import org.apache.commons.io.FileUtils; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.springframework.util.Assert; 27 | 28 | import java.io.File; 29 | import java.io.IOException; 30 | 31 | /** 32 | * Simplest usable {@link com.graphaware.importer.context.ImportContext} that is aware of caches and I/O. 33 | */ 34 | public class SimpleImportContext extends BaseImportContext { 35 | 36 | private static final Logger LOG = LoggerFactory.getLogger(SimpleImportContext.class); 37 | 38 | private final Caches caches; 39 | private final DataLocator inputLocator; 40 | private final DataLocator outputLocator; 41 | 42 | /** 43 | * Create a new context. 44 | * 45 | * @param config import config. Must not be null. 46 | * @param caches caches to be used throughout the import. Must not be null. 47 | * @param inputLocator component capable of locating input data. Must not be null. 48 | * @param outputLocator component capable of locating (creating) output data. Must not be null. 49 | */ 50 | public SimpleImportContext(ImportConfig config, Caches caches, DataLocator inputLocator, DataLocator outputLocator) { 51 | super(config); 52 | this.caches = caches; 53 | this.inputLocator = inputLocator; 54 | this.outputLocator = outputLocator; 55 | } 56 | 57 | /** 58 | * {@inheritDoc} 59 | */ 60 | @Override 61 | public final Caches caches() { 62 | return caches; 63 | } 64 | 65 | /** 66 | * {@inheritDoc} 67 | */ 68 | @Override 69 | public final DataReader createReader(Data data) { 70 | Assert.notNull(data); 71 | 72 | DataReader dataReader = doCreateReader(data); 73 | dataReader.initialize(); 74 | dataReader.read(locate(data), data.name()); 75 | return dataReader; 76 | } 77 | 78 | /** 79 | * Create a brand new data reader. 80 | * 81 | * @param data for which to create a reader. Never null. 82 | * @return data reader. 83 | */ 84 | protected DataReader doCreateReader(Data data) { 85 | return config.createReader(); 86 | } 87 | 88 | /** 89 | * Locate data. 90 | * 91 | * @param data to locate. 92 | * @return logical data location. 93 | */ 94 | protected String locate(Data data) { 95 | return inputLocator.locate(data); 96 | } 97 | 98 | /** 99 | * {@inheritDoc} 100 | */ 101 | @Override 102 | protected void preBootstrap() { 103 | super.preBootstrap(); 104 | 105 | deleteGraphDirectory(); 106 | } 107 | 108 | /** 109 | * {@inheritDoc} 110 | */ 111 | @Override 112 | protected void doCheck() { 113 | super.doCheck(); 114 | 115 | inputLocator.check(); 116 | outputLocator.check(); 117 | } 118 | 119 | /** 120 | * Delete the directory (if exists) where the resulting graph will be stored after the import. 121 | */ 122 | protected void deleteGraphDirectory() { 123 | try { 124 | LOG.info("Deleting " + config.getGraphDir() + "..."); 125 | FileUtils.deleteDirectory(new File(config.getGraphDir())); 126 | } catch (IOException e) { 127 | LOG.warn("Could not delete graph directory", e); 128 | throw new RuntimeException(e); 129 | } 130 | } 131 | 132 | protected DataLocator getInputLocator() { 133 | return inputLocator; 134 | } 135 | 136 | protected DataLocator getOutputLocator() { 137 | return outputLocator; 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/Data.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data; 17 | 18 | /** 19 | * Input or output data for importer. 20 | */ 21 | public interface Data { 22 | 23 | /** 24 | * @return name of the data. 25 | */ 26 | String name(); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/DynamicData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data; 17 | 18 | import org.springframework.util.Assert; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | /** 24 | * Named {@link com.graphaware.importer.data.Data} programmatically created on-the-fly. 25 | */ 26 | public class DynamicData implements Data { 27 | 28 | private final String name; 29 | 30 | /** 31 | * Create new data. 32 | * 33 | * @param name name of the data. Must not be null or empty. 34 | * @return data object. 35 | */ 36 | public static Data withName(final String name) { 37 | return new DynamicData(name); 38 | } 39 | 40 | /** 41 | * Create a one-to-one {@link com.graphaware.importer.data.Data} to name map. Convenient when data names and logical 42 | * (file) names of the data is the same. 43 | * 44 | * @param names of data. 45 | * @return map of data-name pairs (where the name of the data in the key and the value are the same). 46 | */ 47 | public static Map oneToOne(String... names) { 48 | Map result = new HashMap<>(); 49 | for (String name : names) { 50 | result.put(withName(name), name); 51 | } 52 | return result; 53 | } 54 | 55 | /** 56 | * Create new data. 57 | * 58 | * @param name name of the data. Must not be null or empty. 59 | */ 60 | protected DynamicData(String name) { 61 | Assert.hasLength(name); 62 | 63 | this.name = name; 64 | } 65 | 66 | /** 67 | * {@inheritDoc} 68 | */ 69 | @Override 70 | public String name() { 71 | return name; 72 | } 73 | 74 | /** 75 | * {@inheritDoc} 76 | */ 77 | @Override 78 | public boolean equals(Object o) { 79 | if (this == o) return true; 80 | if (o == null || getClass() != o.getClass()) return false; 81 | 82 | DynamicData dynamicData = (DynamicData) o; 83 | 84 | if (!name.equals(dynamicData.name)) return false; 85 | 86 | return true; 87 | } 88 | 89 | /** 90 | * {@inheritDoc} 91 | */ 92 | @Override 93 | public int hashCode() { 94 | return name.hashCode(); 95 | } 96 | 97 | /** 98 | * {@inheritDoc} 99 | */ 100 | @Override 101 | public String toString() { 102 | return name; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/BaseTabularDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | import org.springframework.util.StringUtils; 21 | 22 | import java.text.ParseException; 23 | import java.text.SimpleDateFormat; 24 | import java.util.TimeZone; 25 | 26 | /** 27 | * Abstract base-class for {@link TabularDataReader} implementations. 28 | */ 29 | public abstract class BaseTabularDataReader implements TabularDataReader { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(BaseTabularDataReader.class); 32 | 33 | protected SimpleDateFormat dateFormat; 34 | 35 | /** 36 | * {@inheritDoc} 37 | */ 38 | @Override 39 | public void initialize() { 40 | this.dateFormat = dateFormat(); 41 | } 42 | 43 | /** 44 | * Create a date format for date conversions. 45 | * 46 | * @return date format. By default, it is "dd/MM/yyyy" in GMT+1. Override for different format. 47 | */ 48 | protected SimpleDateFormat dateFormat() { 49 | SimpleDateFormat format = new SimpleDateFormat("dd/MM/yyyy"); 50 | format.setTimeZone(TimeZone.getTimeZone("GMT+1")); 51 | return format; 52 | } 53 | 54 | /** 55 | * {@inheritDoc} 56 | */ 57 | @Override 58 | public Long readLong(String columnName) { 59 | String value = readStringForConversion(columnName); 60 | 61 | if (value == null) { 62 | return null; 63 | } 64 | 65 | try { 66 | return Long.parseLong(value); 67 | } catch (NumberFormatException e) { 68 | LOG.warn("Value " + value + " in column " + columnName + " is not a long"); 69 | return null; 70 | } 71 | } 72 | 73 | /** 74 | * {@inheritDoc} 75 | */ 76 | @Override 77 | public Integer readInt(String columnName) { 78 | String value = readStringForConversion(columnName); 79 | 80 | if (value == null) { 81 | return null; 82 | } 83 | 84 | try { 85 | return Integer.parseInt(value); 86 | } catch (NumberFormatException e) { 87 | LOG.warn("Value " + value + " in column " + columnName + " is not an int"); 88 | return null; 89 | } 90 | } 91 | 92 | /** 93 | * {@inheritDoc} 94 | */ 95 | @Override 96 | public Long readDate(String columnName) { 97 | String value = readStringForConversion(columnName); 98 | 99 | if (value == null) { 100 | return null; 101 | } 102 | 103 | try { 104 | return dateFormat.parse(value).getTime(); 105 | } catch (ParseException e) { 106 | LOG.warn("Value " + value + " in column " + columnName + " is not a date"); 107 | return null; 108 | } 109 | } 110 | 111 | /** 112 | * {@inheritDoc} 113 | */ 114 | @Override 115 | public final String readObject(String columnName) { 116 | return doReadObject(columnName); 117 | } 118 | 119 | /** 120 | * Read a String that is intended to be converted to another type. 121 | * 122 | * @param columnName column name. 123 | * @return the String, or null if the value was null, or an empty String. 124 | */ 125 | protected final String readStringForConversion(String columnName) { 126 | String value = doReadObject(columnName); 127 | 128 | if (value == null) { 129 | return null; 130 | } 131 | 132 | if (StringUtils.isEmpty(value.trim())) { 133 | return null; 134 | } 135 | 136 | return value.trim(); 137 | } 138 | 139 | /** 140 | * Read a String from a column. 141 | * 142 | * @param columnName column name. 143 | * @return String, can be null. 144 | */ 145 | protected abstract String doReadObject(String columnName); 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/CacheDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import com.graphaware.importer.cache.Caches; 19 | import org.springframework.util.Assert; 20 | 21 | import java.util.Iterator; 22 | import java.util.Map; 23 | 24 | /** 25 | * A {@link com.graphaware.importer.data.access.TabularDataReader} that reads data from a {@link com.graphaware.importer.cache.Cache}. 26 | */ 27 | public class CacheDataReader implements TabularDataReader { 28 | 29 | private final Caches caches; 30 | private final Map mappers; 31 | private Iterator entryIterator; 32 | private Map.Entry entry; 33 | private CacheEntryMapper mapper; 34 | private int entryNumber = 0; 35 | 36 | /** 37 | * Create a new reader. 38 | * 39 | * @param caches to read from. Must not be null. 40 | * @param mappers of cached data to "columns". Must not be null. 41 | */ 42 | public CacheDataReader(Caches caches, Map mappers) { 43 | Assert.notNull(caches); 44 | Assert.notNull(mappers); 45 | 46 | this.caches = caches; 47 | this.mappers = mappers; 48 | } 49 | 50 | /** 51 | * {@inheritDoc} 52 | */ 53 | @Override 54 | public void initialize() { 55 | //no-op 56 | } 57 | 58 | /** 59 | * {@inheritDoc} 60 | */ 61 | @Override 62 | public void read(String connectionString, String hint) { 63 | if (entryIterator != null || entry != null) { 64 | throw new IllegalStateException("Previous reader hasn't been closed"); 65 | } 66 | entryIterator = caches.getCache(connectionString).entrySet().iterator(); 67 | mapper = mappers.get(connectionString); 68 | } 69 | 70 | /** 71 | * {@inheritDoc} 72 | */ 73 | @Override 74 | public void close() { 75 | entryIterator = null; 76 | entry = null; 77 | entryNumber = 0; 78 | } 79 | 80 | /** 81 | * {@inheritDoc} 82 | */ 83 | @Override 84 | public Long readLong(String columnName) { 85 | Long value = (Long) mapper.getValue(entry, columnName); 86 | 87 | if (value == null) { 88 | return null; 89 | } 90 | 91 | return value; 92 | } 93 | 94 | /** 95 | * {@inheritDoc} 96 | */ 97 | @Override 98 | public Integer readInt(String columnName) { 99 | Integer value = (Integer) mapper.getValue(entry, columnName); 100 | 101 | if (value == null) { 102 | return null; 103 | } 104 | 105 | return value; 106 | } 107 | 108 | /** 109 | * {@inheritDoc} 110 | */ 111 | @Override 112 | public Long readDate(String columnName) { 113 | throw new UnsupportedOperationException("Not yet implemented"); //probably never needed 114 | } 115 | 116 | /** 117 | * {@inheritDoc} 118 | */ 119 | @Override 120 | public String readObject(String columnName) { 121 | String value = (String) mapper.getValue(entry, columnName); 122 | 123 | if (value == null) { 124 | return null; 125 | } 126 | 127 | return value; 128 | } 129 | 130 | /** 131 | * {@inheritDoc} 132 | */ 133 | @Override 134 | public int getRow() { 135 | return entryNumber; 136 | } 137 | 138 | /** 139 | * {@inheritDoc} 140 | */ 141 | @Override 142 | public boolean readRecord() { 143 | if (!entryIterator.hasNext()) { 144 | return false; 145 | } 146 | 147 | entry = entryIterator.next(); 148 | entryNumber++; 149 | 150 | return true; 151 | } 152 | 153 | /** 154 | * {@inheritDoc} 155 | */ 156 | @Override 157 | public String getRawRecord() { 158 | return entry.toString(); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/CacheEntryMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * A component that maps a {@link com.graphaware.importer.cache.Cache} entry into a Java object as if it was read from 22 | * a tabular source. 23 | */ 24 | public interface CacheEntryMapper { 25 | 26 | /** 27 | * Get a cached value. 28 | * 29 | * @param entry cache entry. Must not be null. 30 | * @param columnName name of the "column" (as if reading from a tabular source). Must not be null. 31 | * @return mapped value. 32 | */ 33 | Object getValue(Map.Entry entry, String columnName); 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/CsvDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import org.apache.commons.csv.CSVFormat; 19 | import org.apache.commons.csv.CSVRecord; 20 | 21 | import java.io.FileReader; 22 | import java.io.IOException; 23 | import java.io.Reader; 24 | import java.util.Iterator; 25 | 26 | /** 27 | * {@link TabularDataReader} for CSV files. 28 | */ 29 | public class CsvDataReader extends BaseTabularDataReader { 30 | 31 | private Iterator records; 32 | private CSVRecord record; 33 | private Reader in; 34 | 35 | private final char delimiter; 36 | private final char quote; 37 | 38 | /** 39 | * Create a new reader. 40 | * 41 | * @param delimiter delimiter. 42 | * @param quote quote character. 43 | */ 44 | public CsvDataReader(char delimiter, char quote) { 45 | this.delimiter = delimiter; 46 | this.quote = quote; 47 | } 48 | 49 | /** 50 | * {@inheritDoc} 51 | */ 52 | @Override 53 | public void read(String connectionString, String hint) { 54 | if (records != null) { 55 | throw new IllegalStateException("Previous reader hasn't been closed"); 56 | } 57 | 58 | try { 59 | in = new FileReader(connectionString); 60 | records = CSVFormat.DEFAULT 61 | .withDelimiter(delimiter) 62 | .withQuote(quote) 63 | .withHeader() 64 | .parse(in).iterator(); 65 | 66 | } catch (Exception e) { 67 | throw new RuntimeException(e); 68 | } 69 | } 70 | 71 | /** 72 | * {@inheritDoc} 73 | */ 74 | @Override 75 | public void close() { 76 | try { 77 | in.close(); 78 | } catch (IOException e) { 79 | throw new RuntimeException(e); 80 | } 81 | 82 | in = null; 83 | records = null; 84 | record = null; 85 | } 86 | 87 | /** 88 | * {@inheritDoc} 89 | */ 90 | @Override 91 | protected String doReadObject(String columnName) { 92 | String s = record.get(columnName); 93 | 94 | if (s != null) { 95 | s = s.replaceAll(System.getProperty("line.separator"), "").trim(); 96 | } 97 | 98 | return s; 99 | } 100 | 101 | /** 102 | * {@inheritDoc} 103 | */ 104 | @Override 105 | public int getRow() { 106 | return Long.valueOf(record.getRecordNumber()).intValue(); 107 | } 108 | 109 | /** 110 | * {@inheritDoc} 111 | */ 112 | @Override 113 | public boolean readRecord() { 114 | if (!records.hasNext()) { 115 | return false; 116 | } 117 | record = records.next(); 118 | 119 | return true; 120 | } 121 | 122 | /** 123 | * {@inheritDoc} 124 | */ 125 | @Override 126 | public String getRawRecord() { 127 | return record.toString(); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/DataAccess.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | /** 19 | * A component accessing data. 20 | */ 21 | public interface DataAccess extends AutoCloseable { 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/DataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | /** 19 | * A reader of data. Does not have to be (and usually isn't) thread-safe. Therefore, this class should be accessed 20 | * in a single thread. 21 | * 22 | * @param type of object that is retrieved from cells. 23 | */ 24 | public interface DataReader extends DataAccess { 25 | 26 | /** 27 | * Initialize the reader. 28 | */ 29 | void initialize(); 30 | 31 | /** 32 | * Open the reader. 33 | * 34 | * @param connectionString identification of the data source (e.g. SQL query, file name, etc.) 35 | * @param hint human-readable representation of the connection for naming threads, logging, etc. 36 | */ 37 | void read(String connectionString, String hint); 38 | 39 | /** 40 | * Close the reader and release all resources. 41 | *

42 | * {@inheritDoc} 43 | */ 44 | @Override 45 | void close(); 46 | 47 | /** 48 | * Read an object from a column (family,...). 49 | * 50 | * @param columnName name of the column. 51 | * @return an object. 52 | */ 53 | O readObject(String columnName); 54 | 55 | /** 56 | * Get current row number. 57 | * 58 | * @return current row number. 59 | */ 60 | int getRow(); 61 | 62 | /** 63 | * Read the next record, i.e. advance the cursor by 1. 64 | * 65 | * @return true iff there was a next record. 66 | */ 67 | boolean readRecord(); 68 | 69 | /** 70 | * Get the current record in its raw form. 71 | * 72 | * @return raw record. 73 | */ 74 | String getRawRecord(); 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/DataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | /** 19 | * A data writer. 20 | */ 21 | public interface DataWriter extends DataAccess { 22 | 23 | /** 24 | * Append a string to the writer. 25 | * 26 | * @param s to append. Must not be null. 27 | * @return this instance. 28 | */ 29 | DataWriter append(String s); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/DbDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import org.apache.commons.dbcp2.BasicDataSource; 19 | import org.springframework.jdbc.core.JdbcTemplate; 20 | import org.springframework.util.Assert; 21 | 22 | import javax.sql.DataSource; 23 | 24 | /** 25 | * {@link com.graphaware.importer.data.access.TabularDataReader} for databases. 26 | */ 27 | public abstract class DbDataReader extends BaseTabularDataReader { 28 | 29 | private final String dbHost; 30 | private final String dbPort; 31 | private final String user; 32 | private final String password; 33 | protected JdbcTemplate jdbcTemplate; 34 | 35 | /** 36 | * Construct a new reader. 37 | * 38 | * @param dbHost db host. Must not be null or empty. 39 | * @param dbPort db port. Must not be null or empty. 40 | * @param user db user. Must not be null. 41 | * @param password db password. Must not be null. 42 | */ 43 | public DbDataReader(String dbHost, String dbPort, String user, String password) { 44 | Assert.hasLength(dbHost); 45 | Assert.hasLength(dbPort); 46 | Assert.notNull(user); 47 | Assert.notNull(password); 48 | 49 | this.dbHost = dbHost; 50 | this.dbPort = dbPort; 51 | this.user = user; 52 | this.password = password; 53 | } 54 | 55 | /** 56 | * {@inheritDoc} 57 | */ 58 | @Override 59 | public void initialize() { 60 | super.initialize(); 61 | this.jdbcTemplate = createJdbcTemplate(); 62 | } 63 | 64 | /** 65 | * Create a {@link org.springframework.jdbc.core.JdbcTemplate} used for talking to the database. 66 | * 67 | * @return jdbc template. 68 | */ 69 | protected JdbcTemplate createJdbcTemplate() { 70 | DataSource dataSource = createDataSource(); 71 | 72 | JdbcTemplate result = new JdbcTemplate(dataSource); 73 | 74 | additionalConfig(result); 75 | 76 | return result; 77 | } 78 | 79 | /** 80 | * Create a {@link javax.sql.DataSource} used for talking to the database. 81 | * 82 | * @return data source. 83 | */ 84 | protected DataSource createDataSource() { 85 | BasicDataSource dataSource = new BasicDataSource(); 86 | 87 | dataSource.setUrl(getUrl(dbHost, dbPort)); 88 | dataSource.setUsername(user); 89 | dataSource.setPassword(password); 90 | dataSource.setDefaultReadOnly(true); 91 | dataSource.setDefaultAutoCommit(false); 92 | 93 | dataSource.setDriverClassName(getDriverClassName()); 94 | 95 | additionalConfig(dataSource); 96 | 97 | return dataSource; 98 | } 99 | 100 | /** 101 | * Perform additional configuration on the JDBC template. No-op by default, to be overridden. 102 | * 103 | * @param template to perform additional config on. 104 | */ 105 | protected void additionalConfig(JdbcTemplate template) { 106 | } 107 | 108 | /** 109 | * Perform additional configuration on the data source. No-op by default, to be overridden. 110 | * 111 | * @param dataSource to perform additional config on. 112 | */ 113 | protected void additionalConfig(DataSource dataSource) { 114 | } 115 | 116 | /** 117 | * Get the class name of the JDBC driver. Must be present on the classpath. 118 | * 119 | * @return driver name. 120 | */ 121 | protected abstract String getDriverClassName(); 122 | 123 | /** 124 | * Create the JDBC connection string (url). 125 | * 126 | * @param host host to connect to. 127 | * @param port port to connect on. 128 | * @return JDBC connection string. 129 | */ 130 | protected abstract String getUrl(String host, String port); 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/FileDataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import java.io.FileWriter; 19 | import java.io.IOException; 20 | 21 | /** 22 | * A {@link com.graphaware.importer.data.access.DataWriter} that extends a {@link java.io.FileWriter}. 23 | */ 24 | public class FileDataWriter extends FileWriter implements DataWriter { 25 | 26 | private final String fileName; 27 | 28 | /** 29 | * Constructs a FileWriter object given a file name. 30 | * 31 | * @param fileName String The system-dependent filename. Must not be null. 32 | * @throws IOException if the named file exists but is a directory rather 33 | * than a regular file, does not exist but cannot be 34 | * created, or cannot be opened for any other reason 35 | */ 36 | public FileDataWriter(String fileName) throws IOException { 37 | super(fileName); 38 | this.fileName = fileName; 39 | } 40 | 41 | /** 42 | * {@inheritDoc} 43 | *

44 | * Translates {@link java.io.IOException} to {@link java.lang.RuntimeException}. 45 | */ 46 | @Override 47 | public DataWriter append(String s) { 48 | try { 49 | super.append(s); 50 | } catch (IOException e) { 51 | throw new RuntimeException(e); 52 | } 53 | 54 | return this; 55 | } 56 | 57 | /** 58 | * {@inheritDoc} 59 | */ 60 | @Override 61 | public String toString() { 62 | return "FileDataWriter to " + fileName; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/QueueDbDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | import org.springframework.dao.DataAccessException; 21 | import org.springframework.jdbc.core.ResultSetExtractor; 22 | 23 | import java.sql.ResultSet; 24 | import java.sql.ResultSetMetaData; 25 | import java.sql.SQLException; 26 | import java.util.Calendar; 27 | import java.util.Date; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import java.util.concurrent.ArrayBlockingQueue; 31 | import java.util.concurrent.BlockingQueue; 32 | import java.util.concurrent.TimeUnit; 33 | 34 | /** 35 | * {@link com.graphaware.importer.data.access.DbDataReader} that uses a queue to temporally decouple reading from the 36 | * database and processing the records. 37 | *

38 | * One thread fills the queue with records, another thread reads from the queue. The queue is blocking. This means if it 39 | * is full, the database reading thread will block on insert. If it is empty, calls to {@link #readRecord()} will block. 40 | */ 41 | public abstract class QueueDbDataReader extends DbDataReader { 42 | 43 | private static final Logger LOG = LoggerFactory.getLogger(QueueDbDataReader.class); 44 | public static final String ROW = "row"; 45 | 46 | private volatile BlockingQueue> records; 47 | private volatile Map record; 48 | 49 | private volatile boolean noMoreRecords = false; 50 | 51 | /** 52 | * Construct a new reader. 53 | * 54 | * @param dbHost db host. Must not be null or empty. 55 | * @param dbPort db port. Must not be null or empty. 56 | * @param user db user. Must not be null. 57 | * @param password db password. Must not be null. 58 | */ 59 | public QueueDbDataReader(String dbHost, String dbPort, String user, String password) { 60 | super(dbHost, dbPort, user, password); 61 | } 62 | 63 | /** 64 | * {@inheritDoc} 65 | */ 66 | @Override 67 | public final void read(final String query, final String hint) { 68 | if (records != null) { 69 | throw new IllegalStateException("Previous reader hasn't been closed"); 70 | } 71 | 72 | LOG.info("Start query: \n" + query); 73 | 74 | if (query.startsWith("alter")) { 75 | jdbcTemplate.execute(query); 76 | noMoreRecords = true; 77 | return; 78 | } 79 | 80 | records = new ArrayBlockingQueue<>(queueCapacity()); 81 | 82 | new Thread(new Runnable() { 83 | @Override 84 | public void run() { 85 | Date d1 = Calendar.getInstance().getTime(); 86 | 87 | try { 88 | jdbcTemplate.query(query, new ResultSetExtractor() { 89 | @Override 90 | public Void extractData(ResultSet rs) throws SQLException, DataAccessException { 91 | ResultSetMetaData metaData = rs.getMetaData(); 92 | int colCount = metaData.getColumnCount(); 93 | 94 | while (rs.next()) { 95 | Map columns = new HashMap<>(); 96 | for (int i = 1; i <= colCount; i++) { 97 | columns.put(metaData.getColumnLabel(i), rs.getString(i)); 98 | } 99 | columns.put(ROW, String.valueOf(rs.getRow())); 100 | 101 | try { 102 | records.offer(columns, 1, TimeUnit.HOURS); 103 | } catch (InterruptedException e) { 104 | LOG.warn("Was waiting for more than 1 hour to insert a record for processing, had to drop it"); 105 | } 106 | } 107 | 108 | return null; 109 | } 110 | }); 111 | } 112 | finally { 113 | noMoreRecords = true; 114 | } 115 | 116 | 117 | long diffInSeconds = TimeUnit.MILLISECONDS.toSeconds(Calendar.getInstance().getTime().getTime() - d1.getTime()); 118 | 119 | LOG.info("Finished querying for " + hint + " in " + diffInSeconds + " seconds"); 120 | } 121 | }, "DB READER - " + hint).start(); 122 | } 123 | 124 | /** 125 | * Get the capacity of the queue for DB records. Defaults to 100,000. 126 | * @return queue capacity. 127 | */ 128 | protected int queueCapacity() { 129 | return 100_000; 130 | } 131 | 132 | /** 133 | * {@inheritDoc} 134 | */ 135 | @Override 136 | public final void close() { 137 | records = null; 138 | record = null; 139 | noMoreRecords = false; 140 | } 141 | 142 | /** 143 | * {@inheritDoc} 144 | */ 145 | @Override 146 | protected String doReadObject(String columnName) { 147 | return record.get(columnName); 148 | } 149 | 150 | /** 151 | * {@inheritDoc} 152 | */ 153 | @Override 154 | public final int getRow() { 155 | return Integer.valueOf(record.get(ROW)); 156 | } 157 | 158 | /** 159 | * {@inheritDoc} 160 | */ 161 | @Override 162 | public final boolean readRecord() { 163 | while (!noMoreRecords || !records.isEmpty()) { 164 | try { 165 | record = records.poll(1, TimeUnit.SECONDS); 166 | } catch (InterruptedException e) { 167 | record = null; 168 | } 169 | 170 | if (record != null) { 171 | return true; 172 | } 173 | } 174 | 175 | return false; 176 | } 177 | 178 | /** 179 | * {@inheritDoc} 180 | */ 181 | @Override 182 | public String getRawRecord() { 183 | StringBuilder row = new StringBuilder(); 184 | for (String columnName : record.keySet()) { 185 | row.append(record.get(columnName)).append(";"); 186 | } 187 | return row.toString(); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/access/TabularDataReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.access; 17 | 18 | /** 19 | * A {@link DataReader} of tabular data. Does not have to be (and usually isn't) thread-safe. Therefore, this class should be accessed 20 | * in a single thread. 21 | */ 22 | public interface TabularDataReader extends DataReader { 23 | 24 | /** 25 | * Read a long number from a column. 26 | * 27 | * @param columnName name of the column. 28 | * @return a long number, null if it is empty or not a long. 29 | */ 30 | Long readLong(String columnName); 31 | 32 | /** 33 | * Read an int number from a column. 34 | * 35 | * @param columnName name of the column. 36 | * @return a int number, null if it is empty or not an int. 37 | */ 38 | Integer readInt(String columnName); 39 | 40 | /** 41 | * Read a long representation of a date from a column. 42 | * 43 | * @param columnName name of the column. 44 | * @return a date as ms since epoch, null if it is empty or not a date. 45 | */ 46 | Long readDate(String columnName); 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/location/DataLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.location; 17 | 18 | import com.graphaware.importer.data.Data; 19 | 20 | import java.util.Set; 21 | 22 | /** 23 | * A locator of {@link com.graphaware.importer.data.Data}, i.e. files, queries, etc. 24 | */ 25 | public interface DataLocator { 26 | 27 | /** 28 | * Check that data is available and correct. This method is intended to catch all data inconsistencies early in 29 | * the import process. Check data on best effort basis. 30 | * 31 | * @throws java.lang.IllegalStateException in case there is something wrong with the data this locator is supposed to locate. 32 | */ 33 | void check(); 34 | 35 | /** 36 | * Get all data this locator is able to locate. 37 | * 38 | * @return set of locatable data. 39 | */ 40 | Set allData(); 41 | 42 | /** 43 | * Check if this locator can locate given data. 44 | * 45 | * @param data to check. 46 | * @return true iff this locator can locate the data. 47 | */ 48 | boolean canLocate(Data data); 49 | 50 | /** 51 | * Locate the given data. Depending on the implementation, the location could be a file name, an SQL query, etc. 52 | * 53 | * @param data to locate. 54 | * @return data location. 55 | * @throws java.lang.IllegalStateException iff {@link #canLocate(com.graphaware.importer.data.Data)} returns false. 56 | */ 57 | String locate(Data data); 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/location/FileLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.location; 17 | 18 | import com.graphaware.importer.data.Data; 19 | import org.apache.commons.io.FileUtils; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.util.Map; 24 | 25 | /** 26 | * A {@link DataLocator} that translates data locations to absolute paths to files. 27 | */ 28 | public class FileLocator extends SimpleDataLocator { 29 | 30 | private final String dir; 31 | 32 | /** 33 | * Construct a new locator. 34 | * 35 | * @param dir directory in which to locate files. 36 | * @param fileNames map of data to logical file names (i.e. without suffix). 37 | */ 38 | public FileLocator(String dir, Map fileNames) { 39 | super(fileNames); 40 | this.dir = dir; 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | * 46 | * @return absolute path to a file. 47 | */ 48 | @Override 49 | public String locate(Data data) { 50 | String location = super.locate(data); 51 | 52 | try { 53 | FileUtils.forceMkdir(new File(dir)); 54 | } catch (IOException e) { 55 | throw new RuntimeException(e); 56 | } 57 | 58 | return new File(dir + File.separator + location + suffix()).getAbsolutePath(); 59 | } 60 | 61 | /** 62 | * Get the suffix appended to logical file names. 63 | * 64 | * @return suffix. ".csv" by default. 65 | */ 66 | protected String suffix() { 67 | return ".csv"; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/location/InputFileLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.location; 17 | 18 | import com.graphaware.importer.data.Data; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.io.File; 23 | import java.util.Map; 24 | 25 | /** 26 | * A {@link FileLocator} that also checks whether the files are actually present. Intended for files (like input files) 27 | * that must be present for the import, i.e. cannot be created by the importer. 28 | */ 29 | public class InputFileLocator extends FileLocator { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(InputFileLocator.class); 32 | 33 | public InputFileLocator(String inputDir, Map fileNames) { 34 | super(inputDir, fileNames); 35 | } 36 | 37 | /** 38 | * {@inheritDoc} 39 | *

40 | * Checks for presence of the files and performs additional checks, which subclasses should override. 41 | */ 42 | @Override 43 | public void check() { 44 | LOG.info("Checking files..."); 45 | 46 | checkFiles(); 47 | 48 | LOG.info("Checking data..."); 49 | 50 | checkData(); 51 | } 52 | 53 | /** 54 | * Check that all files exist. 55 | * 56 | * @throws java.lang.IllegalStateException if some files do not exist. 57 | */ 58 | protected void checkFiles() { 59 | boolean valid = true; 60 | 61 | for (Data data : allData()) { 62 | String file = locate(data); 63 | if (!new File(file).exists()) { 64 | LOG.error("File does not exist: " + file); 65 | valid = false; 66 | } 67 | } 68 | 69 | if (!valid) { 70 | LOG.error("There are missing files."); 71 | throw new IllegalStateException("There are missing files."); 72 | } 73 | } 74 | 75 | /** 76 | * Check contents of the files. No checks performed by default, to be overridden. 77 | */ 78 | protected void checkData() { 79 | LOG.info("No data checks."); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/data/location/SimpleDataLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.data.location; 17 | 18 | import com.graphaware.importer.data.Data; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import org.springframework.util.Assert; 22 | 23 | import java.util.Collections; 24 | import java.util.Map; 25 | import java.util.Set; 26 | 27 | /** 28 | * A primitive {@link com.graphaware.importer.data.location.DataLocator} that gets the data locations passed in at 29 | * construction time. 30 | */ 31 | public class SimpleDataLocator implements DataLocator { 32 | private static final Logger LOG = LoggerFactory.getLogger(SimpleDataLocator.class); 33 | 34 | private final Map locations; 35 | 36 | /** 37 | * Create a new locator. 38 | * 39 | * @param locations data locations. 40 | */ 41 | public SimpleDataLocator(Map locations) { 42 | this.locations = locations; 43 | } 44 | 45 | /** 46 | * {@inheritDoc} 47 | *

48 | * Performs no checks by default, should be overridden. 49 | */ 50 | @Override 51 | public void check() { 52 | //to be overridden 53 | } 54 | 55 | /** 56 | * {@inheritDoc} 57 | */ 58 | @Override 59 | public final Set allData() { 60 | return Collections.unmodifiableSet(locations.keySet()); 61 | } 62 | 63 | /** 64 | * {@inheritDoc} 65 | */ 66 | @Override 67 | public final boolean canLocate(Data data) { 68 | Assert.notNull(data); 69 | 70 | return locations.containsKey(data); 71 | } 72 | 73 | /** 74 | * {@inheritDoc} 75 | */ 76 | @Override 77 | public String locate(Data data) { 78 | LOG.info("Locating " + data + "..."); 79 | 80 | if (!canLocate(data)) { 81 | LOG.error("Cannot locate " + data); 82 | throw new IllegalStateException("Cannot locate " + data); 83 | } 84 | 85 | return this.locations.get(data); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/DirectedRelationship.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | public class DirectedRelationship extends GenericRelationship { 19 | 20 | @Override 21 | public boolean directed() { 22 | return true; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/GenericRelationship.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | public abstract class GenericRelationship implements Neo4jRelationship { 22 | 23 | private Long sourceNodeId; 24 | private Long targetNodeId; 25 | private final Map properties = new HashMap<>(); 26 | 27 | @Override 28 | public Long sourceKey() { 29 | return sourceNodeId; 30 | } 31 | 32 | @Override 33 | public Long targetKey() { 34 | return targetNodeId; 35 | } 36 | 37 | @Override 38 | public Map getProperties() { 39 | return properties; 40 | } 41 | 42 | public void setSourceKey(Long sourceNodeId) { 43 | this.sourceNodeId = sourceNodeId; 44 | } 45 | 46 | public void setTargetKey(Long targetNodeId) { 47 | this.targetNodeId = targetNodeId; 48 | } 49 | 50 | public void setProperty(String key, Object value) { 51 | this.properties.put(key, value); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/Neo4jProperty.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | import java.lang.annotation.Documented; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.Target; 21 | 22 | import static java.lang.annotation.ElementType.FIELD; 23 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 24 | 25 | @Target({FIELD}) 26 | @Retention(RUNTIME) 27 | @Documented 28 | public @interface Neo4jProperty { 29 | 30 | String name() default ""; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/Neo4jPropertyContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | import com.graphaware.importer.util.ReflectionUtils; 19 | import org.neo4j.graphdb.Node; 20 | import org.neo4j.helpers.Pair; 21 | import org.springframework.util.StringUtils; 22 | 23 | import java.lang.reflect.Field; 24 | import java.util.HashMap; 25 | import java.util.HashSet; 26 | import java.util.Map; 27 | import java.util.Set; 28 | import java.util.concurrent.ConcurrentHashMap; 29 | 30 | public abstract class Neo4jPropertyContainer { 31 | 32 | private static final Map>> CACHE = new ConcurrentHashMap<>(); 33 | 34 | public void populateNode(Node node) { 35 | Map properties = getProperties(); 36 | 37 | for (String key : properties.keySet()) { 38 | Object value = properties.get(key); 39 | if (value != null) { 40 | node.setProperty(key, value); 41 | } 42 | } 43 | } 44 | 45 | public Map getProperties() { 46 | Map result = new HashMap<>(); 47 | 48 | if (!CACHE.containsKey(this.getClass().getCanonicalName())) { 49 | initializeForClass(this.getClass()); 50 | } 51 | 52 | for (Pair fieldAndName : CACHE.get(this.getClass().getCanonicalName())) { 53 | try { 54 | Object value = fieldAndName.first().get(this); 55 | if (value != null) { 56 | result.put(fieldAndName.other(), value); 57 | } 58 | } catch (IllegalAccessException e) { 59 | throw new RuntimeException(e); 60 | } 61 | } 62 | 63 | return result; 64 | } 65 | 66 | private synchronized void initializeForClass(Class clazz) { 67 | if (CACHE.containsKey(clazz.getCanonicalName())) { 68 | return; 69 | } 70 | 71 | Set> fieldsAndNames = new HashSet<>(); 72 | 73 | for (Field field : ReflectionUtils.getAllFields(clazz)) { 74 | Neo4jProperty annotation = field.getAnnotation(Neo4jProperty.class); 75 | if (annotation == null) { 76 | continue; 77 | } 78 | 79 | String fieldName = StringUtils.isEmpty(annotation.name()) ? field.getName() : annotation.name(); 80 | fieldsAndNames.add(Pair.of(field, fieldName)); 81 | field.setAccessible(true); 82 | } 83 | 84 | CACHE.put(clazz.getCanonicalName(), fieldsAndNames); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/Neo4jRelationship.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * A relationship for the purposes of batch import. 22 | */ 23 | public interface Neo4jRelationship { 24 | 25 | /** 26 | * Get an identifier of the source node, typically a cache key. 27 | * 28 | * @return key. 29 | */ 30 | Long sourceKey(); 31 | 32 | /** 33 | * Get an identifier of the source node, typically a cache key. 34 | * 35 | * @return key. 36 | */ 37 | Long targetKey(); 38 | 39 | /** 40 | * Get the props of the relationship. 41 | * 42 | * @return props. 43 | */ 44 | Map getProperties(); 45 | 46 | /** 47 | * @return true iff directed. 48 | */ 49 | boolean directed(); 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/domain/UndirectedRelationship.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | public class UndirectedRelationship extends GenericRelationship { 19 | 20 | @Override 21 | public boolean directed() { 22 | return false; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/importer/Importer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.importer; 17 | 18 | import com.graphaware.importer.context.ImportContext; 19 | import com.graphaware.importer.data.Data; 20 | 21 | /** 22 | * A component responsible for importing a single concern. Typically, a "concern" is a single file or SQL query. 23 | * Implementations must be thread-safe. 24 | */ 25 | public interface Importer { 26 | 27 | /** 28 | * Importer state. 29 | */ 30 | public enum State { 31 | NOT_STARTED, 32 | RUNNING, 33 | FINISHED 34 | } 35 | 36 | /** 37 | * Get the name of this importer. Useful for logging, thread naming, etc. 38 | * 39 | * @return importer name. 40 | */ 41 | String name(); 42 | 43 | /** 44 | * Get the data this importer works with. 45 | * 46 | * @return input data. 47 | */ 48 | Data inputData(); 49 | 50 | /** 51 | * Prepare this importer for populating the database. Guaranteed to be called before {@link #performImport()}. 52 | * 53 | * @param importContext context. 54 | */ 55 | void prepare(ImportContext importContext); 56 | 57 | /** 58 | * Perform the actual import, i.e., create nodes and relationships. 59 | */ 60 | void performImport(); 61 | 62 | /** 63 | * Create indices (and constraints). 64 | */ 65 | void createIndices(); 66 | 67 | /** 68 | * Get the current state of this importer. 69 | * 70 | * @return state. 71 | */ 72 | State getState(); 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/importer/TabularImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.importer; 17 | 18 | import com.graphaware.importer.data.access.TabularDataReader; 19 | 20 | /** 21 | * A {@link BaseImporter} working with {@link TabularDataReader}. 22 | */ 23 | public abstract class TabularImporter extends BaseImporter { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/inserter/SynchronizedBatchInserter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.inserter; 17 | 18 | import org.neo4j.graphdb.Label; 19 | import org.neo4j.graphdb.RelationshipType; 20 | import org.neo4j.graphdb.schema.ConstraintCreator; 21 | import org.neo4j.graphdb.schema.IndexCreator; 22 | import org.neo4j.unsafe.batchinsert.BatchInserter; 23 | import org.neo4j.unsafe.batchinsert.BatchRelationship; 24 | 25 | import java.util.Map; 26 | 27 | /** 28 | * A {@link BatchInserter} that has all methods synchronized and delegates to a wrapped {@link BatchInserter}. 29 | */ 30 | public class SynchronizedBatchInserter implements BatchInserter { 31 | 32 | private final BatchInserter batchInserter; 33 | 34 | public SynchronizedBatchInserter(BatchInserter batchInserter) { 35 | this.batchInserter = batchInserter; 36 | } 37 | 38 | public BatchInserter getBatchInserter() { 39 | return batchInserter; 40 | } 41 | 42 | @Override 43 | public synchronized long createNode(Map properties, Label... labels) { 44 | return batchInserter.createNode(properties, labels); 45 | } 46 | 47 | @Override 48 | public synchronized void createNode(long id, Map properties, Label... labels) { 49 | batchInserter.createNode(id, properties, labels); 50 | } 51 | 52 | @Override 53 | public synchronized boolean nodeExists(long nodeId) { 54 | return batchInserter.nodeExists(nodeId); 55 | } 56 | 57 | @Override 58 | public synchronized void setNodeProperties(long node, Map properties) { 59 | batchInserter.setNodeProperties(node, properties); 60 | } 61 | 62 | @Override 63 | public synchronized boolean nodeHasProperty(long node, String propertyName) { 64 | return batchInserter.nodeHasProperty(node, propertyName); 65 | } 66 | 67 | @Override 68 | public synchronized void setNodeLabels(long node, Label... labels) { 69 | batchInserter.setNodeLabels(node, labels); 70 | } 71 | 72 | @Override 73 | public synchronized Iterable

29 | * Implementations must be thread-safe. 30 | */ 31 | public interface ExecutionPlan { 32 | 33 | /** 34 | * Get the importers ordered such that if any importer A depends on importer B (usually by means of using a {@link com.graphaware.importer.cache.Cache} 35 | * created by B), the A will appear in the list after B. This is useful for operations that need to be carried out 36 | * sequentially on each importer, such as creating and injecting {@link com.graphaware.importer.cache.Cache}s, or 37 | * to get a predictable order of actions that need be single-threaded, such as creating indices. 38 | * 39 | * @return a list of importers. 40 | */ 41 | List getOrderedImporters(); 42 | 43 | /** 44 | * Can the given importer be run? 45 | * 46 | * @param importer to check. 47 | * @return true iff all importers that it depends on (if any) have finished. 48 | */ 49 | boolean canRun(Importer importer); 50 | 51 | /** 52 | * Clear all {@link com.graphaware.importer.cache.Caches} that will not be needed any more. 53 | */ 54 | void clearCaches(); 55 | 56 | /** 57 | * @return true iff all {@link com.graphaware.importer.importer.Importer}s have finished. 58 | */ 59 | boolean allFinished(); 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/stats/LoggingStatisticsCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.stats; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | import org.springframework.util.Assert; 21 | 22 | import java.util.Map; 23 | import java.util.concurrent.ConcurrentHashMap; 24 | import java.util.concurrent.atomic.AtomicInteger; 25 | 26 | /** 27 | * Default production implementation of {@link StatisticsCollector}. 28 | */ 29 | public class LoggingStatisticsCollector implements StatisticsCollector { 30 | 31 | private static final Logger LOG = LoggerFactory.getLogger(LoggingStatisticsCollector.class); 32 | public static final String DIVIDER = "========================================="; 33 | 34 | private final StopWatch sw = new StopWatch(); 35 | private final ConcurrentHashMap> counters = new ConcurrentHashMap<>(); 36 | private final String name; 37 | 38 | /** 39 | * Create a new statistics collector. 40 | * 41 | * @param name name of the stats, must not be null or empty. 42 | */ 43 | public LoggingStatisticsCollector(String name) { 44 | Assert.hasLength(name); 45 | 46 | this.name = name; 47 | } 48 | 49 | /** 50 | * {@inheritDoc} 51 | */ 52 | @Override 53 | public void startTiming() { 54 | sw.start(); 55 | } 56 | 57 | /** 58 | * {@inheritDoc} 59 | */ 60 | @Override 61 | public void printTiming() { 62 | LOG.info(name + " took " + sw.getElapsedTimeSecs() + " seconds (" + Math.round(sw.getElapsedTimeSecs() / 60.0) + " minutes)"); 63 | } 64 | 65 | /** 66 | * {@inheritDoc} 67 | */ 68 | @Override 69 | public void incrementsStats(String category, String name) { 70 | getCounter(category, name).incrementAndGet(); 71 | } 72 | 73 | /** 74 | * {@inheritDoc} 75 | */ 76 | @Override 77 | public void setStats(String category, String name, int number) { 78 | getCounter(category, name).set(number); 79 | } 80 | 81 | protected ConcurrentHashMap> getCounters() { 82 | return counters; 83 | } 84 | 85 | private AtomicInteger getCounter(String category, String name) { 86 | if (name == null) { 87 | name = "null"; 88 | } 89 | 90 | ConcurrentHashMap counter = counters.get(category); 91 | 92 | if (counter == null) { 93 | counters.putIfAbsent(category, new ConcurrentHashMap()); 94 | counter = counters.get(category); 95 | } 96 | 97 | AtomicInteger count = counter.get(name); 98 | 99 | if (count == null) { 100 | counter.putIfAbsent(name, new AtomicInteger(0)); 101 | count = counter.get(name); 102 | } 103 | 104 | return count; 105 | } 106 | 107 | /** 108 | * {@inheritDoc} 109 | */ 110 | @Override 111 | public void printStats() { 112 | for (String category : counters.keySet()) { 113 | LOG.info(DIVIDER); 114 | LOG.info(name + ": " + category); 115 | LOG.info(DIVIDER); 116 | 117 | printCategory(category); 118 | 119 | LOG.info(DIVIDER); 120 | } 121 | } 122 | 123 | /** 124 | * Print (log) all the stats for a given category. 125 | * 126 | * @param category to print. 127 | */ 128 | protected void printCategory(String category) { 129 | Map stats = counters.get(category); 130 | for (String statistic : stats.keySet()) { 131 | StringBuilder spaces = new StringBuilder(); 132 | AtomicInteger value = stats.get(statistic); 133 | int length = statistic.length() + value.toString().length(); 134 | while (length + spaces.length() < DIVIDER.length() - 1) { 135 | spaces.append(" "); 136 | } 137 | LOG.info(statistic + ":" + spaces + value.get()); 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/stats/StatisticsCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.stats; 17 | 18 | /** 19 | * A component collecting statistics about import. 20 | */ 21 | public interface StatisticsCollector { 22 | 23 | /** 24 | * Start timing of the import. 25 | */ 26 | void startTiming(); 27 | 28 | /** 29 | * Print time elapsed so far.. 30 | */ 31 | void printTiming(); 32 | 33 | /** 34 | * Increment a statistic by 1. 35 | * 36 | * @param category category of the statistic, e.g. "errors", "warnings", "validation problems", etc. 37 | * @param name name of the statistic, e.g. "missing property", ... 38 | */ 39 | void incrementsStats(String category, String name); 40 | 41 | /** 42 | * Increment a statistic by a number. 43 | * 44 | * @param category category of the statistic, e.g. "errors", "warnings", "validation problems", etc. 45 | * @param name name of the statistic, e.g. "missing property", ... 46 | * @param number to increment by. 47 | */ 48 | void setStats(String category, String name, int number); 49 | 50 | /** 51 | * Print all the stats collected so far. 52 | */ 53 | void printStats(); 54 | } 55 | 56 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/stats/StopWatch.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.stats; 17 | 18 | public class StopWatch { 19 | 20 | private long startTime = 0; 21 | 22 | public void start() { 23 | this.startTime = System.currentTimeMillis(); 24 | } 25 | 26 | public long getElapsedTimeSecs() { 27 | return ((System.currentTimeMillis() - startTime) / 1000); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/util/BlockingArrayBlockingQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.util; 17 | 18 | import java.util.concurrent.ArrayBlockingQueue; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | public class BlockingArrayBlockingQueue extends ArrayBlockingQueue { 22 | 23 | public BlockingArrayBlockingQueue(int capacity) { 24 | super(capacity); 25 | } 26 | 27 | @Override 28 | public boolean offer(E e) { 29 | try { 30 | return offer(e, 30, TimeUnit.MINUTES); 31 | } catch (InterruptedException ex) { 32 | throw new RuntimeException(ex); 33 | } 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/com/graphaware/importer/util/ReflectionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.util; 17 | 18 | import java.lang.reflect.Field; 19 | import java.util.Arrays; 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | 23 | public final class ReflectionUtils { 24 | 25 | public static List getAllFields(Class clazz) { 26 | return getAllFields(clazz, new LinkedList()); 27 | } 28 | 29 | public static List getAllFields(Class clazz, List fields) { 30 | Class superClazz = clazz.getSuperclass(); 31 | if (superClazz != null) { 32 | getAllFields(superClazz, fields); 33 | } 34 | fields.addAll(Arrays.asList(clazz.getDeclaredFields())); 35 | return fields; 36 | } 37 | 38 | private ReflectionUtils() { 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/cache/CachesTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cache; 17 | 18 | import com.graphaware.importer.config.ImportConfig; 19 | import com.graphaware.importer.data.Data; 20 | import com.graphaware.importer.data.access.TabularDataReader; 21 | import com.graphaware.importer.importer.Importer; 22 | import com.graphaware.importer.importer.TabularImporter; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | 26 | import java.util.Arrays; 27 | import java.util.Collections; 28 | import java.util.Set; 29 | 30 | import static junit.framework.Assert.assertEquals; 31 | import static junit.framework.Assert.assertTrue; 32 | import static org.mockito.Mockito.mock; 33 | import static org.mockito.Mockito.when; 34 | 35 | /** 36 | * Unit test for {@link com.graphaware.importer.cache.BaseCaches}. 37 | */ 38 | public class CachesTest { 39 | 40 | private ImportConfig config; 41 | 42 | @Before 43 | public void setUp() { 44 | config = mock(ImportConfig.class); 45 | 46 | when(config.getCacheFile()).thenReturn("/tmp/cache"); 47 | } 48 | 49 | @Test 50 | public void shouldCorrectlyReportNeededCaches() { 51 | TestImporter inserter = new TestImporter(); 52 | 53 | MapDBCaches mapDBCaches = new MapDBCaches(config); 54 | 55 | Set neededCaches = mapDBCaches.neededCaches(inserter); 56 | 57 | assertEquals(2, neededCaches.size()); 58 | assertTrue(neededCaches.contains("C1")); 59 | assertTrue(neededCaches.contains("C2")); 60 | 61 | mapDBCaches.destroy(); 62 | } 63 | 64 | @Test 65 | public void shouldInjectCaches() { 66 | Caches caches = new MapDBCaches(config) {{ 67 | createCache("C1", Long.class, Long.class); 68 | createCache("C2", String.class, Long.class); 69 | }}; 70 | 71 | Cache c1 = caches.getCache("C1"); 72 | Cache c2 = caches.getCache("C2"); 73 | 74 | TestImporter inserter = new TestImporter(); 75 | 76 | caches.inject(inserter); 77 | 78 | assertEquals(c1, inserter.candidates); 79 | assertEquals(c2, inserter.companies); 80 | 81 | caches.destroy(); 82 | } 83 | 84 | @Test 85 | public void shouldClearUnusedCaches() { 86 | Caches caches = new MapDBCaches(config) {{ 87 | createCache("C1", Long.class, Long.class); 88 | createCache("C2", String.class, Long.class); 89 | createCache("C3", String.class, Long.class); 90 | }}; 91 | 92 | Cache c1 = caches.getCache("C1"); 93 | Cache c2 = caches.getCache("C2"); 94 | Cache c3 = caches.getCache("C3"); 95 | 96 | c1.put(0L, 1L); 97 | c2.put("test", 1L); 98 | c3.put("test", 1L); 99 | 100 | caches.cleanup(Arrays.asList(new TestImporter2(), new TestImporter())); 101 | 102 | assertEquals(1, c1.size()); 103 | assertEquals(1, c2.size()); 104 | assertEquals(1, c3.size()); 105 | 106 | caches.cleanup(Arrays.asList(new TestImporter())); 107 | 108 | assertEquals(1, c1.size()); 109 | assertEquals(1, c2.size()); 110 | assertEquals(0, c3.size()); 111 | 112 | caches.cleanup((Collections.emptySet())); 113 | 114 | //no need to cleanup 115 | assertEquals(1, c1.size()); 116 | assertEquals(1, c2.size()); 117 | assertEquals(0, c3.size()); 118 | 119 | caches.destroy(); 120 | } 121 | 122 | private class TestImporter extends TabularImporter { 123 | 124 | @InjectCache(name = "C1") 125 | private Cache candidates; 126 | 127 | @InjectCache(name = "C2") 128 | private Cache companies; 129 | 130 | @Override 131 | public Data inputData() { 132 | throw new UnsupportedOperationException(); 133 | } 134 | 135 | @Override 136 | public Object produceObject(TabularDataReader record) { 137 | throw new UnsupportedOperationException(); 138 | } 139 | 140 | @Override 141 | public void processObject(Object object) { 142 | throw new UnsupportedOperationException(); 143 | } 144 | } 145 | 146 | private class TestImporter2 extends TabularImporter { 147 | 148 | @InjectCache(name = "C3") 149 | private Cache candidates; 150 | 151 | @Override 152 | public Data inputData() { 153 | throw new UnsupportedOperationException(); 154 | } 155 | 156 | @Override 157 | public Object produceObject(TabularDataReader record) { 158 | throw new UnsupportedOperationException(); 159 | } 160 | 161 | @Override 162 | public void processObject(Object object) { 163 | throw new UnsupportedOperationException(); 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/cli/FileCommandLineParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.cli; 17 | 18 | import com.graphaware.importer.config.FileImportConfig; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | 22 | import static org.junit.Assert.*; 23 | 24 | /** 25 | * Unit test for {@link BaseCommandLineParser}. 26 | */ 27 | public class FileCommandLineParserTest { 28 | 29 | private CommandLineParser parser; 30 | 31 | @Before 32 | public void setUp() { 33 | parser = new CsvCommandLineParser(); 34 | } 35 | 36 | @Test 37 | public void shouldProduceContext() { 38 | FileImportConfig context = parser.parseArgs(new String[]{ 39 | "-g", "/tmp/graph", 40 | "-i", "/tmp/input", 41 | "-o", "/tmp/output", 42 | "-c", "/tmp/cache", 43 | "-r", "neo4j.props" 44 | }); 45 | 46 | assertTrue(context != null); 47 | assertEquals("/tmp/graph", context.getGraphDir()); 48 | assertEquals("/tmp/input", context.getInputDir()); 49 | assertEquals("/tmp/output", context.getOutputDir()); 50 | assertEquals("/tmp/cache", context.getCacheFile()); 51 | assertEquals("neo4j.props", context.getProps()); 52 | } 53 | 54 | @Test 55 | public void shouldFailWithInvalidArgs() { 56 | assertNull(parser.parseArgs(new String[]{ 57 | "-g", "/tmp/graph", 58 | "-i", "/tmp/input", 59 | "-o", "/tmp/output", 60 | "-c", "/tmp/cache", 61 | "-r", "neo4j.props", 62 | "-m", "invalid" 63 | })); 64 | } 65 | 66 | @Test 67 | public void shouldFailWithInvalidArgs2() { 68 | assertNull(parser.parseArgs(new String[]{ 69 | "-i", "/tmp/input", 70 | "-o", "/tmp/output", 71 | "-c", "/tmp/cache", 72 | "-p", "neo4j.props" 73 | })); 74 | } 75 | 76 | @Test 77 | public void shouldFailWithInvalidArgs3() { 78 | assertNull(parser.parseArgs(new String[]{ 79 | "-g", "/tmp/graph", 80 | "-o", "/tmp/output", 81 | "-c", "/tmp/cache", 82 | "-r", "neo4j.props" 83 | })); 84 | } 85 | 86 | @Test 87 | public void shouldFailWithInvalidArgs4() { 88 | assertNull(parser.parseArgs(new String[]{ 89 | "-g", "/tmp/graph", 90 | "-i", "/tmp/input", 91 | "-c", "/tmp/cache", 92 | "-r", "neo4j.props" 93 | })); 94 | } 95 | 96 | @Test 97 | public void shouldFailWithInvalidArgs5() { 98 | assertNull(parser.parseArgs(new String[]{ 99 | "-g", "/tmp/graph", 100 | "-i", "/tmp/input", 101 | "-c", "/tmp/cache", 102 | "-o", "/tmp/output" 103 | })); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/domain/Neo4jPropertyContainerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.domain; 17 | 18 | import org.junit.Test; 19 | 20 | import java.util.Map; 21 | import java.util.concurrent.ExecutorService; 22 | import java.util.concurrent.Executors; 23 | import java.util.concurrent.atomic.AtomicBoolean; 24 | 25 | import static org.junit.Assert.*; 26 | 27 | /** 28 | * Unit test for {@link Neo4jPropertyContainer}. 29 | */ 30 | public class Neo4jPropertyContainerTest { 31 | 32 | @Test 33 | public void classWithNoAnnotationsShouldProduceNoProperties() { 34 | assertTrue(new NoAnnotations().getProperties().isEmpty()); 35 | } 36 | 37 | @Test 38 | public void classWithNoFieldsShouldProduceNoProperties() { 39 | assertTrue(new NoFields().getProperties().isEmpty()); 40 | } 41 | 42 | @Test 43 | public void classWithAnnotationsShouldProduceProperties() { 44 | Map result = new WithFields().getProperties(); 45 | 46 | assertEquals(2, result.size()); 47 | assertEquals("test", result.get("stringField")); 48 | assertEquals(2, result.get("testField")); 49 | } 50 | 51 | @Test 52 | public void subClassShouldProduceProperties() { 53 | Map result = new WithInheritedFields().getProperties(); 54 | 55 | assertEquals(2, result.size()); 56 | assertEquals("test", result.get("stringField")); 57 | assertEquals(2, result.get("testField")); 58 | } 59 | 60 | @Test 61 | public void classWithAnnotationsShouldProducePropertiesMultiThreaded() { 62 | ExecutorService executor = Executors.newFixedThreadPool(10); 63 | 64 | final AtomicBoolean failed = new AtomicBoolean(false); 65 | 66 | for (int i = 0; i < 1000; i++) { 67 | executor.submit(new Runnable() { 68 | @Override 69 | public void run() { 70 | try { 71 | Map result = new WithFields().getProperties(); 72 | 73 | assertEquals(2, result.size()); 74 | assertEquals("test", result.get("stringField")); 75 | assertEquals(2, result.get("testField")); 76 | } catch (Throwable throwable) { 77 | failed.set(true); 78 | } 79 | } 80 | }); 81 | } 82 | 83 | assertFalse(failed.get()); 84 | } 85 | 86 | private class NoAnnotations extends Neo4jPropertyContainer { 87 | private String field1; 88 | } 89 | 90 | private class NoFields extends Neo4jPropertyContainer { 91 | } 92 | 93 | private class WithFields extends Neo4jPropertyContainer { 94 | 95 | @Neo4jProperty 96 | private String stringField = "test"; 97 | 98 | @Neo4jProperty(name = "testField") 99 | private int intField = 2; 100 | 101 | @Neo4jProperty 102 | private Long longField = null; 103 | } 104 | 105 | private class WithInheritedFields extends WithFields { 106 | 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/BatchImporterIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration; 17 | 18 | import com.graphaware.test.unit.GraphUnit; 19 | import org.junit.Test; 20 | import org.junit.rules.TemporaryFolder; 21 | import org.neo4j.graphdb.GraphDatabaseService; 22 | import org.neo4j.graphdb.factory.GraphDatabaseFactory; 23 | import org.springframework.core.io.ClassPathResource; 24 | 25 | import java.io.File; 26 | import java.io.IOException; 27 | 28 | import static org.junit.Assert.fail; 29 | 30 | /** 31 | * An integration test for (CSV) batch importer. 32 | */ 33 | public class BatchImporterIntegrationTest { 34 | 35 | @Test 36 | public void testImport() throws IOException, InterruptedException { 37 | TemporaryFolder temporaryFolder = new TemporaryFolder(); 38 | temporaryFolder.create(); 39 | String tmpFolder = temporaryFolder.getRoot().getAbsolutePath(); 40 | 41 | String cp = new ClassPathResource("people.csv").getFile().getAbsolutePath(); 42 | String path = cp.substring(0, cp.length() - "people.csv".length()); 43 | 44 | try { 45 | TestBatchImporter.main(new String[]{"-g", tmpFolder + "/graph.db", "-i", path, "-o", tmpFolder, "-r", "neo4j.properties", "-c", tmpFolder + "/cache"}); 46 | } catch (Throwable t) { 47 | fail(); 48 | } 49 | 50 | GraphDatabaseService database = new GraphDatabaseFactory().newEmbeddedDatabase(new File(tmpFolder + "/graph.db")); 51 | 52 | GraphUnit.assertSameGraph(database, "CREATE " + 53 | "(p1:Person {id: 1, name: 'Michal Bachman', role: 'MD at GraphAware', age:30})," + 54 | "(p2:Person {id: 2, name: 'Adam George', role: 'Consultant at GraphAware', age:29})," + 55 | "(l1:Location {id: 1, name: 'London'})," + 56 | "(l2:Location {id: 2, name: 'Watnall'})," + 57 | "(l3:Location {id: 3, name: 'Prague'})," + 58 | "(c1:Company {name: 'K+N'})," + 59 | "(c2:Company {name: 'GraphAware'})," + 60 | "(p1)-[:LIVES_IN]->(l1)," + 61 | "(p2)-[:LIVES_IN]->(l2)," + 62 | "(p1)-[:FRIEND_OF {since:1281654000000}]->(p2)," + 63 | "(p1)-[:WORKS_FOR {role: 'Developer'}]->(c1)," + 64 | "(p1)-[:WORKS_FOR {role: 'MD'}]->(c2)," + 65 | "(p2)-[:WORKS_FOR {role: 'Developer'}]->(c1)," + 66 | "(p2)-[:WORKS_FOR {role: 'Consultant'}]->(c2)" 67 | ); 68 | 69 | database.shutdown(); 70 | temporaryFolder.delete(); 71 | } 72 | 73 | @Test 74 | public void testImport2() throws IOException, InterruptedException { 75 | TemporaryFolder temporaryFolder = new TemporaryFolder(); 76 | temporaryFolder.create(); 77 | String tmpFolder = temporaryFolder.getRoot().getAbsolutePath(); 78 | 79 | String cp = new ClassPathResource("people.csv").getFile().getAbsolutePath(); 80 | String path = cp.substring(0, cp.length() - "people.csv".length()); 81 | 82 | try { 83 | TestBatchImporter2.main(new String[]{"-g", tmpFolder + "/graph.db", "-i", path, "-o", tmpFolder, "-r", "neo4j.properties", "-c", tmpFolder + "/cache"}); 84 | } catch (Throwable t) { 85 | fail(); 86 | } 87 | 88 | GraphDatabaseService database = new GraphDatabaseFactory().newEmbeddedDatabase(new File(tmpFolder + "/graph.db")); 89 | 90 | GraphUnit.assertSameGraph(database, "CREATE " + 91 | "(p1:Person {id: 1, name: 'Michal Bachman', age:30})," + 92 | "(p2:Person {id: 2, name: 'Adam George', age:29})," + 93 | "(l1:Location {id: 1, name: 'London'})," + 94 | "(l2:Location {id: 2, name: 'Watnall'})," + 95 | "(l3:Location {id: 3, name: 'Prague'})," + 96 | "(p1)-[:LIVES_IN]->(l1)," + 97 | "(p2)-[:LIVES_IN]->(l2)," + 98 | "(p1)-[:FRIEND_OF {since:1281654000000}]->(p2)" 99 | ); 100 | 101 | database.shutdown(); 102 | temporaryFolder.delete(); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/TestBatchImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration; 17 | 18 | import com.graphaware.importer.FileBatchImporter; 19 | import com.graphaware.importer.config.FileImportConfig; 20 | import com.graphaware.importer.context.CacheAwareImportContext; 21 | import com.graphaware.importer.context.ImportContext; 22 | import com.graphaware.importer.data.Data; 23 | import com.graphaware.importer.data.DynamicData; 24 | import com.graphaware.importer.data.access.CacheEntryMapper; 25 | import com.graphaware.importer.data.location.DataLocator; 26 | import com.graphaware.importer.data.location.SimpleDataLocator; 27 | import com.graphaware.importer.importer.Importer; 28 | import com.graphaware.importer.integration.inserter.*; 29 | 30 | import java.util.*; 31 | 32 | public class TestBatchImporter extends FileBatchImporter { 33 | 34 | public static void main(String[] args) { 35 | new TestBatchImporter().run(args); 36 | } 37 | 38 | @Override 39 | protected Set createImporters() { 40 | return new HashSet<>(Arrays.asList( 41 | new PersonImporter(), 42 | new FriendsImporter(), 43 | new LocationImporter(), 44 | new JobsImporter(), 45 | new LastRoleImporter() 46 | )); 47 | } 48 | 49 | @Override 50 | protected Map input() { 51 | return DynamicData.oneToOne("people", "jobs", "friends", "locations"); 52 | } 53 | 54 | @Override 55 | protected ImportContext createContext(FileImportConfig config) { 56 | return new CacheAwareImportContext(config, createCaches(config), createInputDataLocator(config), createOutputDataLocator(config), createCacheInputLocator(), createMapper()); 57 | } 58 | 59 | private DataLocator createCacheInputLocator() { 60 | return new SimpleDataLocator(Collections.singletonMap(DynamicData.withName("roles"), "roles")); 61 | } 62 | 63 | private Map createMapper() { 64 | return Collections.singletonMap("roles", new CacheEntryMapper() { 65 | @Override 66 | public Object getValue(Map.Entry entry, String columnName) { 67 | switch (columnName) { 68 | case "personId": 69 | return entry.getKey(); 70 | case "position": 71 | String[] value = (String[]) entry.getValue(); 72 | return value[0] + " at " + value[1]; 73 | default: 74 | throw new IllegalStateException("Unknown column " + columnName); 75 | } 76 | } 77 | }); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/TestBatchImporter2.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration; 17 | 18 | import com.graphaware.importer.FileBatchImporter; 19 | import com.graphaware.importer.importer.Importer; 20 | import com.graphaware.importer.integration.inserter.FriendsImporter; 21 | import com.graphaware.importer.integration.inserter.LocationImporter; 22 | import com.graphaware.importer.integration.inserter.PersonImporter; 23 | 24 | import java.util.Arrays; 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | public class TestBatchImporter2 extends FileBatchImporter { 29 | 30 | public static void main(String[] args) { 31 | new TestBatchImporter2().run(args); 32 | } 33 | 34 | @Override 35 | protected Set createImporters() { 36 | return new HashSet<>(Arrays.asList( 37 | new LocationImporter(), 38 | new PersonImporter(), 39 | new FriendsImporter() 40 | )); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/domain/Person.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.domain; 17 | 18 | import com.graphaware.importer.domain.Neo4jProperty; 19 | import com.graphaware.importer.domain.Neo4jPropertyContainer; 20 | 21 | public class Person extends Neo4jPropertyContainer { 22 | 23 | @Neo4jProperty 24 | private final Long id; 25 | @Neo4jProperty 26 | private final String name; 27 | @Neo4jProperty 28 | private final Integer age; 29 | 30 | private final Long location; 31 | 32 | public Person(Long id, String name, Integer age, Long location) { 33 | this.id = id; 34 | this.name = name; 35 | this.age = age; 36 | this.location = location; 37 | } 38 | 39 | public Long getId() { 40 | return id; 41 | } 42 | 43 | public String getName() { 44 | return name; 45 | } 46 | 47 | public Integer getAge() { 48 | return age; 49 | } 50 | 51 | public Long getLocation() { 52 | return location; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/inserter/FriendsImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.inserter; 17 | 18 | import com.graphaware.importer.cache.Cache; 19 | import com.graphaware.importer.cache.InjectCache; 20 | import com.graphaware.importer.data.Data; 21 | import com.graphaware.importer.data.DynamicData; 22 | import com.graphaware.importer.data.access.TabularDataReader; 23 | import com.graphaware.importer.importer.TabularImporter; 24 | 25 | import java.util.Collections; 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | 29 | import static org.neo4j.graphdb.DynamicRelationshipType.withName; 30 | 31 | public class FriendsImporter extends TabularImporter> { 32 | 33 | @InjectCache(name = "people") 34 | private Cache personCache; 35 | 36 | @Override 37 | public Data inputData() { 38 | return DynamicData.withName("friends"); 39 | } 40 | 41 | @Override 42 | public Map produceObject(TabularDataReader record) { 43 | Map result = new HashMap<>(); 44 | 45 | result.put("id1", record.readLong("id1")); 46 | result.put("id2", record.readLong("id2")); 47 | result.put("since", record.readDate("since")); 48 | 49 | return result; 50 | } 51 | 52 | @Override 53 | public void processObject(Map object) { 54 | context.inserter().createRelationship( 55 | personCache.get((long) object.get("id1")), 56 | personCache.get((long) object.get("id2")), 57 | withName("FRIEND_OF"), 58 | Collections.singletonMap("since", object.get("since"))); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/inserter/JobsImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.inserter; 17 | 18 | import com.graphaware.importer.cache.Cache; 19 | import com.graphaware.importer.cache.Caches; 20 | import com.graphaware.importer.cache.InjectCache; 21 | import com.graphaware.importer.data.Data; 22 | import com.graphaware.importer.data.DynamicData; 23 | import com.graphaware.importer.data.access.TabularDataReader; 24 | import com.graphaware.importer.importer.TabularImporter; 25 | import org.neo4j.graphdb.DynamicLabel; 26 | import org.neo4j.graphdb.DynamicRelationshipType; 27 | 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | 32 | public class JobsImporter extends TabularImporter> { 33 | 34 | @InjectCache(name = "people") 35 | private Cache personCache; 36 | 37 | @InjectCache(name = "companies", creator = true) 38 | private Cache companyCache; 39 | 40 | @InjectCache(name = "roles", creator = true) 41 | private Cache lastRoleCache; 42 | 43 | @Override 44 | public Data inputData() { 45 | return DynamicData.withName("jobs"); 46 | } 47 | 48 | @Override 49 | public Map produceObject(TabularDataReader record) { 50 | Map result = new HashMap<>(); 51 | 52 | result.put("personId", record.readLong("person_id")); 53 | result.put("job", record.readObject("job")); 54 | result.put("company", record.readObject("company")); 55 | 56 | return result; 57 | } 58 | 59 | @Override 60 | public void processObject(Map object) { 61 | long personId = (long) object.get("personId"); 62 | String company = (String) object.get("company"); 63 | 64 | if (!companyCache.containsKey(company)) { 65 | long nodeId = context.inserter().createNode(Collections.singletonMap("name", company), DynamicLabel.label("Company")); 66 | companyCache.put(company, nodeId); 67 | } 68 | 69 | String jobTitle = (String) object.get("job"); 70 | Long personNodeId = personCache.get(personId); 71 | context.inserter().createRelationship(personNodeId, companyCache.get(company), DynamicRelationshipType.withName("WORKS_FOR"), Collections.singletonMap("role", jobTitle)); 72 | 73 | lastRoleCache.put(personNodeId, new String[]{jobTitle, company}); 74 | } 75 | 76 | @Override 77 | protected void createCache(Caches caches, String name) { 78 | if ("companies".equals(name)) { 79 | caches.createCache(name, String.class, Long.class); 80 | return; 81 | } 82 | if ("roles".equals(name)) { 83 | caches.createCache(name, Long.class, String[].class); 84 | return; 85 | } 86 | 87 | super.createCache(caches, name); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/inserter/LastRoleImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.inserter; 17 | 18 | import com.graphaware.common.util.Pair; 19 | import com.graphaware.importer.cache.Cache; 20 | import com.graphaware.importer.cache.InjectCache; 21 | import com.graphaware.importer.data.Data; 22 | import com.graphaware.importer.data.DynamicData; 23 | import com.graphaware.importer.data.access.TabularDataReader; 24 | import com.graphaware.importer.importer.TabularImporter; 25 | 26 | public class LastRoleImporter extends TabularImporter> { 27 | 28 | @InjectCache(name = "roles") 29 | private Cache lastRoleCache; 30 | 31 | @Override 32 | public Data inputData() { 33 | return DynamicData.withName("roles"); 34 | } 35 | 36 | @Override 37 | public Pair produceObject(TabularDataReader record) { 38 | Long personId = record.readLong("personId"); 39 | String position = record.readObject("position"); 40 | 41 | return new Pair<>(personId, position); 42 | } 43 | 44 | @Override 45 | public void processObject(Pair object) { 46 | context.inserter().setNodeProperty(object.first(), "role", object.second()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/inserter/LocationImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.inserter; 17 | 18 | import com.graphaware.importer.cache.Cache; 19 | import com.graphaware.importer.cache.Caches; 20 | import com.graphaware.importer.cache.InjectCache; 21 | import com.graphaware.importer.data.Data; 22 | import com.graphaware.importer.data.DynamicData; 23 | import com.graphaware.importer.data.access.TabularDataReader; 24 | import com.graphaware.importer.importer.TabularImporter; 25 | 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | 29 | import static org.neo4j.graphdb.DynamicLabel.label; 30 | 31 | public class LocationImporter extends TabularImporter> { 32 | 33 | @InjectCache(name = "locations", creator = true) 34 | private Cache locationCache; 35 | 36 | @Override 37 | public Data inputData() { 38 | return DynamicData.withName("locations"); 39 | } 40 | 41 | @Override 42 | public Map produceObject(TabularDataReader record) { 43 | Map result = new HashMap<>(); 44 | 45 | result.put("id", record.readLong("id")); 46 | result.put("name", record.readObject("name")); 47 | 48 | return result; 49 | } 50 | 51 | @Override 52 | public void processObject(Map object) { 53 | locationCache.put((Long) object.get("id"), context.inserter().createNode(object, label("Location"))); 54 | } 55 | 56 | @Override 57 | protected void createCache(Caches caches, String name) { 58 | if ("locations".equals(name)) { 59 | caches.createCache(name, Long.class, Long.class); 60 | } 61 | else { 62 | super.createCache(caches, name); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/integration/inserter/PersonImporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.integration.inserter; 17 | 18 | import com.graphaware.importer.cache.Cache; 19 | import com.graphaware.importer.cache.Caches; 20 | import com.graphaware.importer.cache.InjectCache; 21 | import com.graphaware.importer.data.Data; 22 | import com.graphaware.importer.data.DynamicData; 23 | import com.graphaware.importer.data.access.TabularDataReader; 24 | import com.graphaware.importer.importer.TabularImporter; 25 | import com.graphaware.importer.integration.domain.Person; 26 | import org.springframework.util.StringUtils; 27 | 28 | import java.util.Collections; 29 | 30 | import static org.neo4j.graphdb.DynamicLabel.label; 31 | import static org.neo4j.graphdb.DynamicRelationshipType.withName; 32 | 33 | public class PersonImporter extends TabularImporter { 34 | 35 | @InjectCache(name = "people", creator = true) 36 | private Cache personCache; 37 | 38 | @InjectCache(name = "locations") 39 | private Cache locationCache; 40 | 41 | @Override 42 | public Data inputData() { 43 | return DynamicData.withName("people"); 44 | } 45 | 46 | @Override 47 | public Person produceObject(TabularDataReader record) { 48 | //for testing purposes, let's say we can't construct a person without ID 49 | if (record.readLong("id") == null) { 50 | return null; 51 | } 52 | return new Person(record.readLong("id"), record.readObject("name"), record.readInt("age"), record.readLong("location")); 53 | } 54 | 55 | @Override 56 | public void processObject(Person person) { 57 | //for testing purposes, let's say people with empty names are invalid. 58 | if (StringUtils.isEmpty(person.getName())) { 59 | throw new RuntimeException("Person has empty name"); 60 | } 61 | 62 | personCache.put(person.getId(), context.inserter().createNode(person.getProperties(), label("Person"))); 63 | context.inserter().createRelationship(personCache.get(person.getId()), locationCache.get(person.getLocation()), withName("LIVES_IN"), Collections.emptyMap()); 64 | } 65 | 66 | @Override 67 | protected void createCache(Caches caches, String name) { 68 | if ("people".equals(name)) { 69 | caches.createCache(name, Long.class, Long.class); 70 | } else { 71 | super.createCache(caches, name); 72 | } 73 | } 74 | 75 | @Override 76 | public void createIndices() { 77 | createIndex(label("Person"), "name"); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/plan/DefaultExecutionPlanTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.plan; 17 | 18 | import com.graphaware.importer.cache.Cache; 19 | import com.graphaware.importer.cache.Caches; 20 | import com.graphaware.importer.cache.InjectCache; 21 | import com.graphaware.importer.cache.MapDBCaches; 22 | import com.graphaware.importer.config.ImportConfig; 23 | import com.graphaware.importer.context.ImportContext; 24 | import com.graphaware.importer.data.Data; 25 | import com.graphaware.importer.data.DynamicData; 26 | import com.graphaware.importer.data.access.TabularDataReader; 27 | import com.graphaware.importer.importer.Importer; 28 | import com.graphaware.importer.importer.TabularImporter; 29 | import org.junit.After; 30 | import org.junit.Before; 31 | import org.junit.Test; 32 | 33 | import java.util.Arrays; 34 | import java.util.HashSet; 35 | import java.util.List; 36 | import java.util.Set; 37 | 38 | import static org.junit.Assert.*; 39 | import static org.mockito.Mockito.mock; 40 | import static org.mockito.Mockito.when; 41 | 42 | /** 43 | * Test for {@link com.graphaware.importer.plan.DefaultExecutionPlan}. 44 | */ 45 | public class DefaultExecutionPlanTest { 46 | 47 | private ImportContext context; 48 | private Caches caches; 49 | 50 | @Before 51 | public void setUp() { 52 | ImportConfig config = mock(ImportConfig.class); 53 | 54 | when(config.getCacheFile()).thenReturn("/tmp/cache"); 55 | 56 | caches = new MapDBCaches(config); 57 | context = mock(ImportContext.class); 58 | when(context.caches()).thenReturn(caches); 59 | } 60 | 61 | @After 62 | public void tearDown() { 63 | if (caches != null) { 64 | caches.destroy(); 65 | } 66 | } 67 | 68 | private Set importers(Importer... importers) { 69 | return new HashSet<>(Arrays.asList(importers)); 70 | } 71 | 72 | @Test(expected = IllegalStateException.class) 73 | public void shouldNotBeAbleToOrderWhenCreatorIsMissing() { 74 | Set importers = importers(new TestImporter()); 75 | ExecutionPlan plan = new DefaultExecutionPlan(importers, context); 76 | plan.getOrderedImporters(); 77 | } 78 | 79 | @Test(expected = IllegalStateException.class) 80 | public void shouldNotBeAbleToOrderWhenCreatorIsMissing2() { 81 | Set importers = importers(new TestImporter(), new TestImporter2()); 82 | ExecutionPlan plan = new DefaultExecutionPlan(importers, context); 83 | plan.getOrderedImporters(); 84 | } 85 | 86 | @Test(expected = IllegalStateException.class) 87 | public void shouldNotBeAbleToOrderWhenThereIsACycle() { 88 | Set importers = importers(new TestImporter(), new TestImporter2(), new TestImporter3()); 89 | ExecutionPlan plan = new DefaultExecutionPlan(importers, context); 90 | plan.getOrderedImporters(); 91 | } 92 | 93 | @Test 94 | public void shouldOrderImporters() { 95 | Set importers = importers(new TestImporter(), new TestImporter2(), new TestImporter4()); 96 | ExecutionPlan plan = new DefaultExecutionPlan(importers, context); 97 | List result = plan.getOrderedImporters(); 98 | 99 | assertEquals("TEST4", result.get(0).name()); 100 | assertEquals("TEST2", result.get(1).name()); 101 | assertEquals("TEST1", result.get(2).name()); 102 | } 103 | 104 | @Test 105 | public void shouldCorrectlyJudgeWhenImporterCanRunAndWhenTheyAreFinished() { 106 | TestImporter i1 = new TestImporter(); 107 | TestImporter2 i2 = new TestImporter2(); 108 | TestImporter4 i4 = new TestImporter4(); 109 | 110 | Set importers = importers(i1, i2, i4); 111 | ExecutionPlan plan = new DefaultExecutionPlan(importers, context); 112 | 113 | assertFalse(plan.allFinished()); 114 | assertTrue(plan.canRun(i4)); 115 | assertFalse(plan.canRun(i1)); 116 | assertFalse(plan.canRun(i2)); 117 | 118 | try { 119 | i4.prepare(context); 120 | } catch (Exception e) { 121 | //ok 122 | } 123 | 124 | try { 125 | i4.performImport(); 126 | } catch (Exception e) { 127 | //ok 128 | } 129 | 130 | try { 131 | plan.canRun(i4); 132 | fail(); 133 | } catch (IllegalStateException e) { 134 | //ok 135 | } 136 | 137 | assertFalse(plan.allFinished()); 138 | assertFalse(plan.canRun(i1)); 139 | assertTrue(plan.canRun(i2)); 140 | 141 | try { 142 | i2.prepare(context); 143 | } catch (Exception e) { 144 | //ok 145 | } 146 | 147 | try { 148 | i2.performImport(); 149 | } catch (Exception e) { 150 | //ok 151 | } 152 | 153 | assertFalse(plan.allFinished()); 154 | 155 | try { 156 | i1.prepare(context); 157 | } catch (Exception e) { 158 | //ok 159 | } 160 | 161 | try { 162 | i1.performImport(); 163 | } catch (Exception e) { 164 | //ok 165 | } 166 | 167 | assertTrue(plan.allFinished()); 168 | } 169 | 170 | private class TestImporter extends TabularImporter { 171 | 172 | @InjectCache(name = "C1", creator = true) 173 | private Cache c1; 174 | 175 | @InjectCache(name = "C2") 176 | private Cache c2; 177 | 178 | @Override 179 | public Data inputData() { 180 | return DynamicData.withName("TEST1"); 181 | } 182 | 183 | @Override 184 | public Object produceObject(TabularDataReader record) { 185 | throw new UnsupportedOperationException(); 186 | } 187 | 188 | @Override 189 | public void processObject(Object object) { 190 | throw new UnsupportedOperationException(); 191 | } 192 | } 193 | 194 | private class TestImporter2 extends TabularImporter { 195 | 196 | @InjectCache(name = "C2", creator = true) 197 | private Cache c2; 198 | 199 | @InjectCache(name = "C3") 200 | private Cache c3; 201 | 202 | @Override 203 | public Data inputData() { 204 | return DynamicData.withName("TEST2"); 205 | } 206 | 207 | @Override 208 | public Object produceObject(TabularDataReader record) { 209 | throw new UnsupportedOperationException(); 210 | } 211 | 212 | @Override 213 | public void processObject(Object object) { 214 | throw new UnsupportedOperationException(); 215 | } 216 | } 217 | 218 | private class TestImporter3 extends TabularImporter { 219 | 220 | @InjectCache(name = "C2") 221 | private Cache c2; 222 | 223 | @InjectCache(name = "C3", creator = true) 224 | private Cache c3; 225 | 226 | @Override 227 | public Data inputData() { 228 | return DynamicData.withName("TEST3"); 229 | } 230 | 231 | @Override 232 | public Object produceObject(TabularDataReader record) { 233 | throw new UnsupportedOperationException(); 234 | } 235 | 236 | @Override 237 | public void processObject(Object object) { 238 | throw new UnsupportedOperationException(); 239 | } 240 | } 241 | 242 | private class TestImporter4 extends TabularImporter { 243 | 244 | @InjectCache(name = "C3", creator = true) 245 | private Cache c3; 246 | 247 | @Override 248 | public Data inputData() { 249 | return DynamicData.withName("TEST4"); 250 | } 251 | 252 | @Override 253 | public Object produceObject(TabularDataReader record) { 254 | throw new UnsupportedOperationException(); 255 | } 256 | 257 | @Override 258 | public void processObject(Object object) { 259 | throw new UnsupportedOperationException(); 260 | } 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/test/java/com/graphaware/importer/stats/StopWatchTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 GraphAware 3 | * 4 | * This file is part of the GraphAware Framework. 5 | * 6 | * GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | * of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | * or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | * See the GNU General Public License for more details. You should have received a copy of the 13 | * GNU General Public License along with this program. If not, see . 14 | */ 15 | 16 | package com.graphaware.importer.stats; 17 | 18 | import org.junit.Test; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | /** 23 | * Unit test for {@link StopWatch} 24 | */ 25 | public class StopWatchTest { 26 | 27 | @Test 28 | public void shouldCorrectlyMeasureTime() { 29 | long start = System.currentTimeMillis(); 30 | 31 | StopWatch stopWatch = new StopWatch(); 32 | stopWatch.start(); 33 | 34 | try { 35 | Thread.sleep(1000); 36 | } catch (InterruptedException e) { 37 | throw new RuntimeException(e); 38 | } 39 | 40 | long expected = (System.currentTimeMillis() - start) / 1000; 41 | long actual = stopWatch.getElapsedTimeSecs(); 42 | 43 | assertEquals(expected, actual); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/resources/friends.csv: -------------------------------------------------------------------------------- 1 | "id1","id2","since" 2 | "1","2","13/8/2010" 3 | -------------------------------------------------------------------------------- /src/test/resources/jobs.csv: -------------------------------------------------------------------------------- 1 | person_id,job,company 2 | 1,"Developer","K+N" 3 | 2,"Developer","K+N" 4 | 1,"MD","GraphAware" 5 | 2,"Consultant","GraphAware" 6 | -------------------------------------------------------------------------------- /src/test/resources/locations.csv: -------------------------------------------------------------------------------- 1 | "id","name" 2 | "1","London" 3 | "2","Watnall" 4 | "3","Prague" 5 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2013-2016 GraphAware 3 | # 4 | # This file is part of the GraphAware Framework. 5 | # 6 | # GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 7 | # of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 8 | # or (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 11 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | # See the GNU General Public License for more details. You should have received a copy of the 13 | # GNU General Public License along with this program. If not, see . 14 | # 15 | 16 | log4j.appender.file=org.apache.log4j.FileAppender 17 | log4j.appender.file.File=importer.log 18 | log4j.appender.file.Append=false 19 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%5L - %m%n 21 | 22 | log4j.appender.console=org.apache.log4j.ConsoleAppender 23 | log4j.appender.console.Target=System.out 24 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.console.layout.ConversionPattern=%d{ABSOLUTE} %5p %m%n 26 | 27 | ### Root logger ### 28 | log4j.rootLogger=info, file, console 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | %d{yyyy-MM-dd HH:mm:ss.SSSZ} %-5level %msg%n 22 | 23 | 24 | 25 | 26 | 27 | importer.log 28 | 29 | 30 | %d{yyyy-MM-dd HH:mm:ss.SSSZ} %-5level %msg%n 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /src/test/resources/neo4j.properties: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Copyright (c) 2013-2016 GraphAware 4 | # 5 | # This file is part of the GraphAware Framework. 6 | # 7 | # GraphAware Framework is free software: you can redistribute it and/or modify it under the terms 8 | # of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, 9 | # or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 12 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 13 | # See the GNU General Public License for more details. You should have received a copy of the 14 | # GNU General Public License along with this program. If not, see . 15 | # 16 | 17 | ################################################################ 18 | # Neo4j 19 | # 20 | # neo4j.properties - database tuning parameters 21 | # 22 | ################################################################ 23 | 24 | # Enable this to be able to upgrade a store from an older version. 25 | allow_store_upgrade=false 26 | 27 | # The amount of memory to use for mapping the store files, in bytes (or 28 | # kilobytes with the 'k' suffix, megabytes with 'm' and gigabytes with 'g'). 29 | # If Neo4j is running on a dedicated server, then it is generally recommended 30 | # to leave about 2-4 gigabytes for the operating system, give the JVM enough 31 | # heap to hold all your transaction state and query context, and then leave the 32 | # rest for the page cache. 33 | # The default page cache memory assumes the machine is dedicated to running 34 | # Neo4j, and is heuristically set to 75% of RAM minus the max Java heap size. 35 | dbms.pagecache.memory=32m 36 | 37 | # Enable this to specify a parser other than the default one. 38 | #cypher_parser_version=2.0 39 | 40 | # Keep logical logs, helps debugging but uses more disk space, enabled for 41 | # legacy reasons To limit space needed to store historical logs use values such 42 | # as: "7 days" or "100M size" instead of "true". 43 | #keep_logical_logs=7 days 44 | 45 | # Autoindexing 46 | 47 | # Enable auto-indexing for nodes, default is false. 48 | #node_auto_indexing=true 49 | 50 | # The node property keys to be auto-indexed, if enabled. 51 | #node_keys_indexable=name,age 52 | 53 | # Enable auto-indexing for relationships, default is false. 54 | #relationship_auto_indexing=true 55 | 56 | # The relationship property keys to be auto-indexed, if enabled. 57 | #relationship_keys_indexable=name,age 58 | 59 | # Enable shell server so that remote clients can connect via Neo4j shell. 60 | remote_shell_enabled=true 61 | # The network interface IP the shell will listen on (use 0.0.0 for all interfaces). 62 | #remote_shell_host=127.0.0.1 63 | # The port the shell will listen on, default is 1337. 64 | #remote_shell_port=1337 65 | 66 | # The type of cache to use for nodes and relationships. 67 | #cache_type=soft 68 | -------------------------------------------------------------------------------- /src/test/resources/people.csv: -------------------------------------------------------------------------------- 1 | "id","name","location","age" 2 | "1","Michal Bachman","1",30 3 | "2","Adam George","2",29 4 | "","PersonWithNoId","2",99 5 | "4"," ","2",100 6 | --------------------------------------------------------------------------------