├── .gitignore ├── examples └── in.csv ├── ideas.md ├── import.csv ├── pom.xml ├── readme.md ├── src ├── main │ ├── java │ │ └── org │ │ │ └── neo4j │ │ │ └── shell │ │ │ ├── kernel │ │ │ └── apps │ │ │ │ ├── Eval.java │ │ │ │ └── ScriptEngineViaReflection.java │ │ │ └── tools │ │ │ └── imp │ │ │ ├── AutoIndexApp.java │ │ │ ├── BinaryApp.java │ │ │ ├── ExportBinaryApp.java │ │ │ ├── ExportCypherApp.java │ │ │ ├── ExportGraphMLApp.java │ │ │ ├── ImportBinaryApp.java │ │ │ ├── ImportCsvApp.java │ │ │ ├── ImportCypherApp.java │ │ │ ├── ImportGeoffApp.java │ │ │ ├── ImportGraphMLApp.java │ │ │ ├── format │ │ │ ├── CsvFormat.java │ │ │ ├── CypherFormat.java │ │ │ ├── CypherSingleFormat.java │ │ │ ├── Format.java │ │ │ ├── GeoffExportService.java │ │ │ ├── GeoffFormat.java │ │ │ ├── GraphVizFormat.java │ │ │ ├── MultiStatementCypherSubGraphExporter.java │ │ │ ├── SimpleGraphMLFormat.java │ │ │ ├── XmlGraphMLFormat.java │ │ │ ├── graphml │ │ │ │ ├── SimpleGraphMLWriter.java │ │ │ │ ├── XmlGraphMLReader.java │ │ │ │ └── XmlGraphMLWriter.java │ │ │ └── kryo │ │ │ │ ├── KryoReader.java │ │ │ │ ├── KryoSerializationTypes.java │ │ │ │ └── KryoWriter.java │ │ │ └── util │ │ │ ├── BatchTransaction.java │ │ │ ├── Config.java │ │ │ ├── CountingReader.java │ │ │ ├── CountingWriter.java │ │ │ ├── ElementCounter.java │ │ │ ├── FileInputWrapper.java │ │ │ ├── FileUtils.java │ │ │ ├── FormatUtils.java │ │ │ ├── Json.java │ │ │ ├── MapNodeCache.java │ │ │ ├── MetaInformation.java │ │ │ ├── NodeCache.java │ │ │ ├── ProgressReporter.java │ │ │ ├── QueryParameterExtractor.java │ │ │ ├── Reporter.java │ │ │ ├── SizeCounter.java │ │ │ ├── Type.java │ │ │ └── WriterOutputStream.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.neo4j.shell.App └── test │ ├── java │ └── org │ │ └── neo4j │ │ └── shell │ │ ├── CypherAppTest.java │ │ └── tools │ │ ├── Asserts.java │ │ └── imp │ │ ├── AutoIndexAppTest.java │ │ ├── ExportBinaryAppTest.java │ │ ├── ExportCypherAppTest.java │ │ ├── ExportGraphMLAppTest.java │ │ ├── ExportXmlGraphMLWriterTest.java │ │ ├── GraphMLReaderPerformanceTest.java │ │ ├── GraphMLReaderTest.java │ │ ├── ImportBinaryAppTest.java │ │ ├── ImportCypherAppTest.java │ │ ├── ImportCypherPerformanceTest.java │ │ ├── ImportGeoffAppTest.java │ │ ├── ImportGraphMLAppTest.java │ │ ├── TestCtrlCHandler.java │ │ ├── format │ │ ├── CsvFormatTest.java │ │ ├── CypherFormatTest.java │ │ ├── MultiStatementCypherSubGraphExporterTest.java │ │ ├── ScannerTest.java │ │ └── TestReporter.java │ │ └── util │ │ └── QueryParameterExtractorTest.java │ └── resources │ ├── graphml-with-attributes.xml │ ├── graphml-with-labels.xml │ ├── kryo-export.bin │ ├── tiny-kryo-export.bin │ └── yed.xml └── zip.sh /.gitignore: -------------------------------------------------------------------------------- 1 | test-data 2 | test.xml 3 | *.db 4 | *.log 5 | *.zip 6 | *.properties 7 | *.iml 8 | .shell_history 9 | *.csv 10 | target 11 | .idea 12 | .DS_Store 13 | *.geoff 14 | .classpath 15 | .project 16 | .settings/ 17 | -------------------------------------------------------------------------------- /examples/in.csv: -------------------------------------------------------------------------------- 1 | name age 2 | Michael 38 3 | Selina 15 4 | Rana 8 5 | Selma 5 6 | -------------------------------------------------------------------------------- /ideas.md: -------------------------------------------------------------------------------- 1 | ## Generic Import / Export Tooling 2 | 3 | `load, dump -f format [cypher statement]` 4 | 5 | format would be: 6 | 7 | * csv 8 | * geoff 9 | * cypher-single 10 | * cypher 11 | * graphml 12 | * graphviz 13 | * d3 14 | 15 | 16 | ### Support Shell Variables 17 | 18 | * BATCH_SIZE 19 | * QUIET 20 | 21 | 22 | ### GraphML 23 | 24 | * use java xml streaming also for quoting etc. -------------------------------------------------------------------------------- /import.csv: -------------------------------------------------------------------------------- 1 | Listener Time Trackmbid 2 | STA 1145357598 5151ffce-8617-443f-959a-82692c717cbf 3 | STA 1145358137 5f5dc9ef-a2be-4c32-bfda-6213c0e2f462 4 | STA 1145359359 839cd758-da64-43ac-993d-596f88c0fb96 5 | STA 1145359707 7b8af741-9d72-44c6-91da-4d5bd9e15e12 6 | STA 1145360442 72305f72-e1a1-4a6f-9a27-a9a45fca7fcb 7 | STA 1160215182 44135db1-bf64-4da0-bc24-6e68260c6e60 8 | STA 1160215717 52f859f3-63fa-4054-929d-19dbff813ab6 9 | STA 1160215823 073618a8-b4da-4d0e-8cb8-14aa47dccd0e 10 | STA 1160215975 4eb6c976-e58f-4d83-addd-cdfd2552a6cb 11 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 3.0.3 5 | 6 | 7 | 4.0.0 8 | org.neo4j.shell 9 | import-tools 10 | jar 11 | 3.0.3 12 | neo4j-shell-import-tools 13 | 14 | 15 | 16 | 17 | maven-compiler-plugin 18 | 3.1 19 | 20 | 1.8 21 | 1.8 22 | UTF-8 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | neo4j 31 | http://m2.neo4j.org/content/repositories/snapshots/ 32 | true 33 | 34 | 35 | neo4j-contrib-releases 36 | https://raw.github.com/neo4j-contrib/m2/master/releases 37 | 38 | true 39 | 40 | 41 | false 42 | 43 | 44 | 45 | neo4j-contrib-snapshots 46 | https://raw.github.com/neo4j-contrib/m2/master/snapshots 47 | 48 | false 49 | 50 | 51 | true 52 | 53 | 54 | 55 | 56 | 57 | com.esotericsoftware 58 | kryo 59 | 3.0.3 60 | 61 | 62 | org.ow2.asm 63 | asm 64 | 65 | 69 | 70 | 71 | 72 | org.mapdb 73 | mapdb 74 | 0.9.3 75 | 76 | 77 | net.sf.opencsv 78 | opencsv 79 | 2.3 80 | 81 | 82 | org.neo4j 83 | neo4j 84 | ${neo4j.version} 85 | 86 | 87 | org.neo4j 88 | neo4j-shell 89 | ${neo4j.version} 90 | 91 | 92 | org.neo4j 93 | neo4j-shell 94 | ${neo4j.version} 95 | test-jar 96 | test 97 | 98 | 99 | org.neo4j 100 | neo4j-graphviz 101 | ${neo4j.version} 102 | 103 | 104 | commons-lang 105 | commons-lang 106 | 2.4 107 | 108 | 109 | org.codehaus.jackson 110 | jackson-mapper-asl 111 | 1.9.7 112 | 113 | 114 | junit 115 | junit 116 | 4.11 117 | test 118 | 119 | 120 | org.neo4j 121 | neo4j-kernel 122 | ${neo4j.version} 123 | test-jar 124 | test 125 | 126 | 127 | org.neo4j 128 | neo4j-io 129 | ${neo4j.version} 130 | test-jar 131 | test 132 | 133 | 134 | com.nigelsmall 135 | geoff 136 | 0.5.0 137 | 138 | 139 | org.neo4j 140 | neo4j 141 | 142 | 143 | org.neo4j 144 | neo4j-kernel 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/kernel/apps/Eval.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2002-2014 "Neo Technology," 3 | * Network Engine for Objects in Lund AB [http://neotechnology.com] 4 | * 5 | * This file is part of Neo4j. 6 | * 7 | * Neo4j is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU General Public License 18 | * along with this program. If not, see . 19 | */ 20 | package org.neo4j.shell.kernel.apps; 21 | 22 | import static org.neo4j.shell.kernel.apps.ScriptEngineViaReflection.decorateWithImports; 23 | 24 | import org.neo4j.shell.AppCommandParser; 25 | import org.neo4j.shell.Continuation; 26 | import org.neo4j.shell.Output; 27 | import org.neo4j.shell.Session; 28 | 29 | /** 30 | * Exposes the javax.script.ScriptEngine as a shell app. It's purely via reflection so 31 | * it's OK even if the script engine isn't on the classpath. 32 | */ 33 | public class Eval extends TransactionProvidingApp 34 | { 35 | private ScriptEngineViaReflection scripting; 36 | 37 | @Override 38 | public String getDescription() 39 | { 40 | return "Pass JavaScript to be executed on the shell server, directly on the database. " + 41 | "There are predefined variables you can use:\n" + 42 | " db : the GraphDatabaseService on the server\n" + 43 | " out : output back to you (the shell client)\n" + 44 | " current : current node or relationship you stand on\n\n" + 45 | "Usage:\n" + 46 | " eval db.getReferenceNode().getProperty(\"name\")\n" + 47 | " \n" + 48 | " eval\n" + 49 | " > nodes = db.getAllNodes().iterator();\n" + 50 | " > while ( nodes.hasNext() )\n" + 51 | " > out.println( \"\" + nodes.next() );\n" + 52 | " >\n" + 53 | "So either a one-liner or type 'eval' to enter multi-line mode, where an empty line denotes the end"; 54 | } 55 | 56 | @Override 57 | protected Continuation exec( AppCommandParser parser, Session session, Output out ) throws Exception 58 | { 59 | // satisfied if: 60 | // * it ends with \n 61 | // * there's stuff after eval and no \n on the line 62 | boolean satisfied = 63 | parser.getLine().endsWith( "\n" ) || 64 | (parser.getLineWithoutApp().length() > 0 && parser.getLine().indexOf( '\n' ) == -1); 65 | if ( !satisfied ) return Continuation.INPUT_INCOMPLETE; 66 | scripting = scripting != null ? scripting : new ScriptEngineViaReflection( getServer() ); 67 | String javascriptCode = parser.getLineWithoutApp(); 68 | javascriptCode = decorateWithImports( javascriptCode, STANDARD_EVAL_IMPORTS ); 69 | Object scriptEngine = scripting.getJavascriptEngine(); 70 | scripting.addDefaultContext( scriptEngine, session, out ); 71 | Object result = scripting.interpret( scriptEngine, javascriptCode ); 72 | if ( result != null ) 73 | { 74 | out.println( result.toString() ); 75 | } 76 | return Continuation.INPUT_COMPLETE; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/kernel/apps/ScriptEngineViaReflection.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2002-2014 "Neo Technology," 3 | * Network Engine for Objects in Lund AB [http://neotechnology.com] 4 | * 5 | * This file is part of Neo4j. 6 | * 7 | * Neo4j is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU General Public License 18 | * along with this program. If not, see . 19 | */ 20 | package org.neo4j.shell.kernel.apps; 21 | 22 | import java.lang.reflect.Method; 23 | 24 | import org.neo4j.shell.Output; 25 | import org.neo4j.shell.Session; 26 | import org.neo4j.shell.ShellException; 27 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 28 | 29 | import static org.neo4j.shell.kernel.apps.TransactionProvidingApp.getCurrent; 30 | 31 | public class ScriptEngineViaReflection 32 | { 33 | private static final String JAVAX_SCRIPT_SCRIPT_ENGINE_MANAGER = "javax.script.ScriptEngineManager"; 34 | private static final String JAVAX_SCRIPT_SCRIPT_CONTEXT = "javax.script.ScriptContext"; 35 | 36 | private final Object scriptEngineManager; 37 | private final int engineScopeValue; 38 | private final GraphDatabaseShellServer server; 39 | 40 | public ScriptEngineViaReflection( GraphDatabaseShellServer server ) 41 | { 42 | this.server = server; 43 | Object scriptEngineManager = null; 44 | int engineScopeValue = 0; 45 | try 46 | { 47 | Class scriptEngineManagerClass = Class.forName( JAVAX_SCRIPT_SCRIPT_ENGINE_MANAGER ); 48 | scriptEngineManager = scriptEngineManagerClass.newInstance(); 49 | engineScopeValue = Class.forName( "javax.script.ScriptContext" ).getField( "ENGINE_SCOPE" ).getInt( null ); 50 | } 51 | catch ( Exception e ) 52 | { 53 | scriptEngineManager = null; 54 | } 55 | this.scriptEngineManager = scriptEngineManager; 56 | this.engineScopeValue = engineScopeValue; 57 | } 58 | 59 | public Object getJavascriptEngine() throws Exception 60 | { 61 | return getEngineByName( "javascript" ); 62 | } 63 | 64 | public Object getEngineByName( String name ) throws Exception 65 | { 66 | if ( scriptEngineManager == null ) 67 | { 68 | throw new ShellException( "Scripting not available, make sure javax.script.* is available on " + 69 | "the classpath in the JVM the shell server runs in" ); 70 | } 71 | return scriptEngineManager.getClass().getMethod( "getEngineByName", String.class ).invoke( scriptEngineManager, name ); 72 | } 73 | 74 | public Object interpret( Object scriptEngine, String code ) throws Exception 75 | { 76 | return scriptEngine.getClass().getMethod( "eval", String.class ).invoke( scriptEngine, code ); 77 | } 78 | 79 | public void addDefaultContext( Object scriptEngine, Session session, Output out ) 80 | throws Exception, ShellException 81 | { 82 | addToContext( scriptEngine, 83 | "db", server.getDb(), 84 | "out", out, 85 | "current", session.getCurrent() == null ? null : getCurrent( server, session ).asPropertyContainer() ); 86 | } 87 | 88 | public Object compile( Object scriptEngine, String code ) throws Exception 89 | { 90 | return scriptEngine.getClass().getMethod( "compile", String.class ).invoke( scriptEngine, code ); 91 | } 92 | 93 | public static String decorateWithImports( String code, String... imports ) 94 | { 95 | String importCode = ""; 96 | for ( String oneImport : imports ) 97 | { 98 | importCode += importStatement( oneImport ); 99 | } 100 | return importCode + code; 101 | } 102 | 103 | public Object newContext() throws Exception 104 | { 105 | return Class.forName( "javax.script.SimpleScriptContext" ).getConstructor().newInstance(); 106 | } 107 | 108 | public Object executeCompiledScript( Object compiledScript, Object context ) throws Exception 109 | { 110 | Method method = compiledScript.getClass().getMethod( "eval", Class.forName( JAVAX_SCRIPT_SCRIPT_CONTEXT ) ); 111 | method.setAccessible( true ); 112 | return method.invoke( compiledScript, context ); 113 | } 114 | 115 | public void addToContext( Object scriptEngine, Object... keyValuePairs ) throws Exception 116 | { 117 | Object context = scriptEngine.getClass().getMethod( "getContext" ).invoke( scriptEngine ); 118 | Method setAttributeMethod = context.getClass().getMethod( "setAttribute", String.class, Object.class, Integer.TYPE ); 119 | for ( int i = 0; i < keyValuePairs.length; i++ ) 120 | { 121 | setAttributeMethod.invoke( context, keyValuePairs[i++], keyValuePairs[i], engineScopeValue ); 122 | } 123 | } 124 | 125 | public void setContextAttribute( Object context, String key, Object value ) throws Exception 126 | { 127 | Method setAttributeMethod = context.getClass().getMethod( "setAttribute", String.class, 128 | Object.class, Integer.TYPE ); 129 | setAttributeMethod.invoke( context, key, value, engineScopeValue ); 130 | } 131 | 132 | private static String importStatement( String thePackage ) 133 | { 134 | return "importPackage(Packages." + thePackage + ");"; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/AutoIndexApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import org.neo4j.graphdb.PropertyContainer; 4 | import org.neo4j.graphdb.index.AutoIndexer; 5 | import org.neo4j.graphdb.index.IndexManager; 6 | import org.neo4j.shell.*; 7 | import org.neo4j.shell.kernel.apps.TransactionProvidingApp; 8 | 9 | /** 10 | * Created by mh on 04.07.13. 11 | */ 12 | public class AutoIndexApp extends TransactionProvidingApp { 13 | 14 | { 15 | addOptionDefinition( "t", new OptionDefinition( OptionValueType.MUST, 16 | "The type of index, either Node or Relationship" ) ); 17 | addOptionDefinition( "r", new OptionDefinition( OptionValueType.NONE, 18 | "Removes removes given properties from auto-indexing." ) ); 19 | } 20 | 21 | enum Type { Node, Relationship } 22 | @Override 23 | protected Continuation exec(AppCommandParser parser, Session session, Output out) throws Exception { 24 | Type type = Type.valueOf(parser.option("t", "Node")); 25 | boolean remove = parser.options().containsKey("r"); 26 | 27 | IndexManager index = getServer().getDb().index(); 28 | AutoIndexer autoIndexer = type==Type.Node ? index.getNodeAutoIndexer() : index.getRelationshipAutoIndexer(); 29 | autoIndexer.setEnabled(true); 30 | 31 | out.println((remove ? "Disabling" : "Enabling") + " auto-indexing of " + type + " properties: " + parser.arguments()); 32 | for (String argument : parser.arguments()) { 33 | if (remove) { 34 | autoIndexer.stopAutoIndexingProperty(argument); 35 | } 36 | else { 37 | autoIndexer.startAutoIndexingProperty(argument); 38 | } 39 | } 40 | return Continuation.INPUT_COMPLETE; 41 | } 42 | 43 | @Override 44 | public String getName() { 45 | return "auto-index"; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/BinaryApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import org.neo4j.shell.impl.AbstractApp; 4 | 5 | import java.util.Arrays; 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | 9 | /** 10 | * Created by efulton on 6/6/16. 11 | */ 12 | public abstract class BinaryApp extends AbstractApp { 13 | protected static final String KRYO = "kryo"; 14 | protected static final String CBOR = "cbor"; 15 | protected static final String PACKSTREAM = "packstream"; 16 | protected static final Set BINARY_FORMATS = new HashSet<>(Arrays.asList(KRYO, CBOR, PACKSTREAM)); 17 | protected static final int MAX_FORMAT_STRING_SIZE = 10; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/ExportBinaryApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import org.neo4j.cypher.export.CypherResultSubGraph; 4 | import org.neo4j.cypher.export.DatabaseSubGraph; 5 | import org.neo4j.cypher.export.SubGraph; 6 | import org.neo4j.graphdb.GraphDatabaseService; 7 | import org.neo4j.graphdb.Result; 8 | import org.neo4j.graphdb.Transaction; 9 | import org.neo4j.kernel.internal.GraphDatabaseAPI; 10 | import org.neo4j.shell.*; 11 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 12 | import org.neo4j.shell.tools.imp.format.kryo.KryoWriter; 13 | import org.neo4j.shell.tools.imp.util.Config; 14 | import org.neo4j.shell.tools.imp.util.FileUtils; 15 | import org.neo4j.shell.tools.imp.util.ProgressReporter; 16 | 17 | import java.io.*; 18 | import java.util.zip.DeflaterOutputStream; 19 | 20 | /** 21 | * Created by efulton on 5/5/16. 22 | */ 23 | public class ExportBinaryApp extends BinaryApp { 24 | 25 | { 26 | addOptionDefinition( "o", new OptionDefinition( OptionValueType.MUST, "Output Binary file" ) ); 27 | addOptionDefinition( "f", new OptionDefinition( OptionValueType.MAY, "Binary format (kryo is the default/only option right now)" ) ); 28 | } 29 | 30 | @Override 31 | public String getName() { 32 | return "export-binary"; 33 | } 34 | 35 | @Override 36 | public GraphDatabaseShellServer getServer() { 37 | return (GraphDatabaseShellServer) super.getServer(); 38 | } 39 | 40 | @Override 41 | public Continuation execute(AppCommandParser parser, Session session, Output out) throws Exception { 42 | Config config = Config.fromOptions(parser); 43 | String fileName = parser.option("o", null); 44 | String format = parser.option("f", "kryo"); 45 | ProgressReporter reporter = new ProgressReporter(null, out); 46 | GraphDatabaseService db = getServer().getDb(); 47 | SubGraph graph = new DatabaseSubGraph(db); 48 | 49 | if (fileName == null) { 50 | throw new RuntimeException("Output file option is required"); 51 | } 52 | 53 | switch (format) { 54 | case KRYO: 55 | writeOutputType(KRYO, fileName); 56 | writeKryo(fileName, db, graph, reporter, config); 57 | break; 58 | case CBOR: 59 | throw new RuntimeException(String.format("%s is not supported yet", CBOR)); 60 | case PACKSTREAM: 61 | throw new RuntimeException(String.format("%s is not supported yet", PACKSTREAM)); 62 | } 63 | 64 | reporter.progress("Wrote to binary file " + fileName); 65 | return Continuation.INPUT_COMPLETE; 66 | } 67 | 68 | private void writeOutputType(String type, String fileName) throws IOException { 69 | try (FileWriter fileWriter = new FileWriter(fileName)) { 70 | fileWriter.write(type); 71 | fileWriter.flush(); 72 | fileWriter.close(); 73 | } 74 | } 75 | 76 | private void writeKryo(String fileName, GraphDatabaseService db, SubGraph graph, ProgressReporter reporter, Config config) throws Exception { 77 | OutputStream outputStream = new BufferedOutputStream(new DeflaterOutputStream(new FileOutputStream(fileName, true), false), FileUtils.MEGABYTE); 78 | com.esotericsoftware.kryo.io.Output output = new com.esotericsoftware.kryo.io.Output(outputStream, FileUtils.MEGABYTE); 79 | try (Transaction tx = db.beginTx()) { 80 | KryoWriter kryoWriter = new KryoWriter(); 81 | kryoWriter.write(graph, output, reporter, config); 82 | tx.success(); 83 | } finally { 84 | output.close(); 85 | } 86 | } 87 | 88 | private SubGraph cypherResultSubGraph(String query, boolean relsBetween) { 89 | GraphDatabaseAPI db = getServer().getDb(); 90 | try (Transaction tx = db.beginTx()) { 91 | Result result = db.execute(query); 92 | SubGraph subGraph = CypherResultSubGraph.from(result, db, relsBetween); 93 | tx.success(); 94 | return subGraph; 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/ExportCypherApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import org.neo4j.cypher.export.CypherResultSubGraph; 4 | import org.neo4j.cypher.export.DatabaseSubGraph; 5 | import org.neo4j.cypher.export.SubGraph; 6 | import org.neo4j.graphdb.GraphDatabaseService; 7 | import org.neo4j.graphdb.Result; 8 | import org.neo4j.graphdb.Transaction; 9 | import org.neo4j.shell.*; 10 | import org.neo4j.shell.impl.AbstractApp; 11 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 12 | import org.neo4j.shell.tools.imp.format.MultiStatementCypherSubGraphExporter; 13 | import org.neo4j.shell.tools.imp.util.Config; 14 | import org.neo4j.shell.tools.imp.util.ProgressReporter; 15 | 16 | import java.io.*; 17 | 18 | import static org.neo4j.shell.tools.imp.util.FileUtils.getPrintWriter; 19 | 20 | public class ExportCypherApp extends AbstractApp { 21 | 22 | { 23 | addOptionDefinition( "o", new OptionDefinition( OptionValueType.MUST, 24 | "Output Cypher file" ) ); 25 | addOptionDefinition("b", new OptionDefinition(OptionValueType.MUST, 26 | "Batch Size default " + Config.DEFAULT_BATCH_SIZE)); 27 | addOptionDefinition( "r", new OptionDefinition( OptionValueType.MAY, 28 | "Add all nodes of selected relationships" ) ); 29 | } 30 | 31 | @Override 32 | public String getName() { 33 | return "export-cypher"; 34 | } 35 | 36 | 37 | @Override 38 | public GraphDatabaseShellServer getServer() { 39 | return (GraphDatabaseShellServer) super.getServer(); 40 | } 41 | 42 | private SubGraph cypherResultSubGraph(String query, boolean relsBetween) { 43 | GraphDatabaseService db = getServer().getDb(); 44 | try (Transaction tx = db.beginTx()) { 45 | Result result = db.execute(query); 46 | SubGraph subGraph = CypherResultSubGraph.from(result, db, relsBetween); 47 | tx.success(); 48 | return subGraph; 49 | } 50 | } 51 | 52 | 53 | @Override 54 | public Continuation execute(AppCommandParser parser, Session session, Output out) throws Exception { 55 | Config config = Config.fromOptions(parser); 56 | 57 | boolean relsBetween = parser.options().containsKey("r"); 58 | 59 | 60 | ProgressReporter reporter = new ProgressReporter(null, out); 61 | 62 | GraphDatabaseService db = getServer().getDb(); 63 | 64 | String query = Config.extractQuery(parser); 65 | SubGraph graph = query.isEmpty() ? new DatabaseSubGraph(db) : cypherResultSubGraph(query,relsBetween); 66 | 67 | String fileName = parser.option("o", null); 68 | try (Transaction tx = db.beginTx();PrintWriter printWriter = getPrintWriter(fileName, out)) { 69 | MultiStatementCypherSubGraphExporter exporter = new MultiStatementCypherSubGraphExporter(graph); 70 | exporter.export(printWriter, config.getBatchSize(), reporter); 71 | tx.success(); 72 | } 73 | reporter.progress("Wrote to Cypher-file " + fileName); 74 | return Continuation.INPUT_COMPLETE; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/ExportGraphMLApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import org.neo4j.cypher.export.CypherResultSubGraph; 4 | import org.neo4j.cypher.export.DatabaseSubGraph; 5 | import org.neo4j.cypher.export.SubGraph; 6 | import org.neo4j.graphdb.GraphDatabaseService; 7 | import org.neo4j.graphdb.Result; 8 | import org.neo4j.graphdb.Transaction; 9 | import org.neo4j.kernel.internal.GraphDatabaseAPI; 10 | import org.neo4j.shell.*; 11 | import org.neo4j.shell.impl.AbstractApp; 12 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 13 | import org.neo4j.shell.tools.imp.format.Format; 14 | import org.neo4j.shell.tools.imp.format.XmlGraphMLFormat; 15 | import org.neo4j.shell.tools.imp.util.Config; 16 | import org.neo4j.shell.tools.imp.util.FileUtils; 17 | import org.neo4j.shell.tools.imp.util.ProgressReporter; 18 | 19 | import java.io.BufferedWriter; 20 | import java.io.FileWriter; 21 | import java.io.Writer; 22 | 23 | /** 24 | * TODO: arrays, labels, rel-types, key-types 25 | * @author mh 26 | * @since 17.01.14 27 | */ 28 | public class ExportGraphMLApp extends AbstractApp { 29 | 30 | { 31 | addOptionDefinition( "o", new OptionDefinition( OptionValueType.MUST, 32 | "Output GraphML file" ) ); 33 | addOptionDefinition( "t", new OptionDefinition( OptionValueType.MAY, 34 | "Write key types upfront (double pass)" ) ); 35 | addOptionDefinition( "r", new OptionDefinition( OptionValueType.MAY, 36 | "Add all nodes of selected relationships" ) ); 37 | } 38 | 39 | @Override 40 | public String getName() { 41 | return "export-graphml"; 42 | } 43 | 44 | 45 | @Override 46 | public GraphDatabaseShellServer getServer() { 47 | return (GraphDatabaseShellServer) super.getServer(); 48 | } 49 | 50 | private SubGraph cypherResultSubGraph(String query, boolean relsBetween) { 51 | GraphDatabaseAPI db = getServer().getDb(); 52 | try (Transaction tx = db.beginTx()) { 53 | Result result = db.execute(query); 54 | SubGraph subGraph = CypherResultSubGraph.from(result, db, relsBetween); 55 | tx.success(); 56 | return subGraph; 57 | } 58 | } 59 | 60 | 61 | @Override 62 | public Continuation execute(AppCommandParser parser, Session session, Output out) throws Exception { 63 | Config config = Config.fromOptions(parser); 64 | 65 | String fileName = parser.option("o", null); 66 | boolean relsBetween = parser.options().containsKey("r"); 67 | Writer writer = FileUtils.getPrintWriter(fileName,out); 68 | 69 | ProgressReporter reporter = new ProgressReporter(null, out); 70 | 71 | GraphDatabaseService db = getServer().getDb(); 72 | 73 | Format exportFormat = new XmlGraphMLFormat(db); 74 | String query = Config.extractQuery(parser); 75 | SubGraph graph = query.isEmpty() ? new DatabaseSubGraph(db) : cypherResultSubGraph(query,relsBetween); 76 | exportFormat.dump(graph, writer, reporter, config); 77 | writer.close(); 78 | reporter.progress("Wrote to GraphML-file " + fileName); 79 | return Continuation.INPUT_COMPLETE; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/ImportBinaryApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import com.esotericsoftware.kryo.io.Input; 4 | import org.neo4j.graphdb.GraphDatabaseService; 5 | import org.neo4j.shell.*; 6 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 7 | import org.neo4j.shell.tools.imp.format.kryo.KryoReader; 8 | import org.neo4j.shell.tools.imp.util.*; 9 | 10 | import java.io.BufferedInputStream; 11 | import java.io.FileInputStream; 12 | import java.io.IOException; 13 | import java.util.zip.InflaterInputStream; 14 | 15 | /** 16 | * Created by efulton on 5/5/16. 17 | */ 18 | public class ImportBinaryApp extends BinaryApp { 19 | 20 | private static final int DEFAULT_BATCH_SIZE = 10000; 21 | private static final String DEFAULT_REL_TYPE = "RELATED_TO"; 22 | 23 | { 24 | addOptionDefinition("i", new OptionDefinition(OptionValueType.MUST, "Input Binary file")); 25 | addOptionDefinition("b", new OptionDefinition(OptionValueType.MAY, "Batch Size. Default is: " + DEFAULT_BATCH_SIZE)); 26 | addOptionDefinition("r", new OptionDefinition(OptionValueType.MAY, "Default Relationship-Type otherwise edge attribute 'label' " + DEFAULT_REL_TYPE)); 27 | addOptionDefinition("c", new OptionDefinition(OptionValueType.NONE, "Use a node-cache that overflows to disk, necessary for very large imports")); 28 | } 29 | 30 | @Override 31 | public String getName() { 32 | return "import-binary"; 33 | } 34 | 35 | @Override 36 | public GraphDatabaseShellServer getServer() { 37 | return (GraphDatabaseShellServer) super.getServer(); 38 | } 39 | 40 | @Override 41 | public Continuation execute(AppCommandParser parser, Session session, Output out) throws Exception { 42 | String fileName = parser.option("i", null); 43 | String relType = parser.option("r", DEFAULT_REL_TYPE); 44 | int batchSize = Integer.parseInt( parser.option("b", String.valueOf(DEFAULT_BATCH_SIZE))); 45 | boolean diskSpillCache = parser.options().containsKey("c"); 46 | 47 | try (FileInputStream fileInputStream = new FileInputStream(fileName)) { 48 | String type = loadType(fileInputStream); 49 | switch (type) { 50 | case KRYO: 51 | readKryo(fileName, fileInputStream, out, diskSpillCache, batchSize, relType); 52 | break; 53 | case CBOR: 54 | throw new RuntimeException(String.format("%s is not supported yet", CBOR)); 55 | case PACKSTREAM: 56 | throw new RuntimeException(String.format("%s is not supported yet", PACKSTREAM)); 57 | } 58 | } 59 | 60 | return Continuation.INPUT_COMPLETE; 61 | } 62 | 63 | private String loadType(FileInputStream fileInputStream) throws IOException { 64 | StringBuilder type = new StringBuilder(); 65 | String result = null; 66 | for (int i = 0; i < MAX_FORMAT_STRING_SIZE && fileInputStream.available() > 0; i++) { 67 | char character = (char) fileInputStream.read(); 68 | type.append(character); 69 | if (BINARY_FORMATS.contains(type.toString())) { 70 | result = type.toString(); 71 | break; 72 | } 73 | } 74 | return result; 75 | } 76 | 77 | private void readKryo(String fileName, FileInputStream fileInputStream, Output out, boolean diskSpillCache, int batchSize, String relType) throws IOException { 78 | Input input = new Input(new InflaterInputStream(new BufferedInputStream(fileInputStream,FileUtils.MEGABYTE)),FileUtils.MEGABYTE); 79 | FileInputWrapper fileInputWrapper = new FileInputWrapper(fileInputStream); 80 | out.println(String.format("Binary import file %s rel-type %s batch-size %d use disk-cache %s", fileName, relType, batchSize, diskSpillCache)); 81 | ProgressReporter reporter = new ProgressReporter(fileInputWrapper, out); 82 | GraphDatabaseService db = getServer().getDb(); 83 | NodeCache cache = diskSpillCache ? MapNodeCache.usingMapDb() : MapNodeCache.usingHashMap(); 84 | try(BatchTransaction tx = new BatchTransaction(db, batchSize, reporter)) { 85 | KryoReader importReader = new KryoReader(db, relType, reporter, out); 86 | long count = importReader.readBinaryDump(input, tx, cache); 87 | out.println("Binary import created " + count + " entities."); 88 | } finally { 89 | input.close(); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/org/neo4j/shell/tools/imp/ImportCsvApp.java: -------------------------------------------------------------------------------- 1 | package org.neo4j.shell.tools.imp; 2 | 3 | import au.com.bytecode.opencsv.CSVReader; 4 | import au.com.bytecode.opencsv.CSVWriter; 5 | import org.neo4j.graphdb.GraphDatabaseService; 6 | import org.neo4j.graphdb.Label; 7 | import org.neo4j.graphdb.Node; 8 | import org.neo4j.graphdb.ResourceIterable; 9 | import org.neo4j.helpers.collection.Iterables; 10 | import org.neo4j.shell.*; 11 | import org.neo4j.shell.impl.AbstractApp; 12 | import org.neo4j.shell.kernel.GraphDatabaseShellServer; 13 | import org.neo4j.shell.tools.imp.util.*; 14 | 15 | import java.io.FileNotFoundException; 16 | import java.io.IOException; 17 | import java.io.Writer; 18 | import java.util.HashMap; 19 | import java.util.LinkedHashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * @author mh 24 | * @since 04.07.13 25 | */ 26 | public class ImportCsvApp extends AbstractApp { 27 | 28 | 29 | { 30 | addOptionDefinition( "i", new OptionDefinition( OptionValueType.MUST, 31 | "Input CSV/TSV file" ) ); 32 | addOptionDefinition( "o", new OptionDefinition( OptionValueType.MUST, 33 | "Output CSV/TSV file" ) ); 34 | addOptionDefinition("b", new OptionDefinition(OptionValueType.MUST, 35 | "Batch Size default " + Config.DEFAULT_BATCH_SIZE)); 36 | addOptionDefinition( "d", new OptionDefinition( OptionValueType.MUST, 37 | "Delimeter, default is comma ',' " ) ); 38 | addOptionDefinition( "l", new OptionDefinition( OptionValueType.MAY, 39 | "Label to export" ) ); 40 | addOptionDefinition( "q", new OptionDefinition( OptionValueType.NONE, 41 | "Quoted Strings in file" ) ); 42 | addOptionDefinition( "s", new OptionDefinition( OptionValueType.NONE, 43 | "Silent Operations" ) ); 44 | 45 | } 46 | 47 | @Override 48 | public String getName() { 49 | return "import-csv"; 50 | } 51 | 52 | 53 | @Override 54 | public GraphDatabaseShellServer getServer() { 55 | return (GraphDatabaseShellServer) super.getServer(); 56 | } 57 | 58 | @Override 59 | public Continuation execute(AppCommandParser parser, Session session, Output out) throws Exception { 60 | Config config = Config.fromOptions(parser); 61 | char delim = delim(parser.option("d", ",")); 62 | int batchSize = Integer.parseInt(parser.option("b", String.valueOf(Config.DEFAULT_BATCH_SIZE))); 63 | boolean quotes = parser.options().containsKey("q"); 64 | boolean silent = parser.options().containsKey("s"); 65 | String inputFileName = parser.option("i", null); 66 | CountingReader inputFile = FileUtils.readerFor(inputFileName); 67 | String fileName = parser.option("o", null); 68 | Writer outputFile = FileUtils.getPrintWriter(fileName, out); 69 | 70 | if (!silent) { 71 | out.println(String.format("Infile %s delim '%s' quoted %s outfile %s batch-size %d", 72 | name(inputFileName),delim,quotes,name(fileName),batchSize)); 73 | } 74 | 75 | CSVReader reader = createReader(inputFile, config); 76 | 77 | CSVWriter writer = createWriter(outputFile, config); 78 | 79 | int count; 80 | if (reader==null) { 81 | count = execute(writer); 82 | } else { 83 | count = executeOnInput(reader, writer, config, new ProgressReporter(inputFile,out)); 84 | } 85 | if (!silent) { 86 | out.println("Import statement execution created " + count + " rows of output."); 87 | } 88 | if (reader!=null) reader.close(); 89 | if (writer!=null) writer.close(); 90 | return Continuation.INPUT_COMPLETE; 91 | } 92 | 93 | private String name(Object file) { 94 | if (file==null) return "(none)"; 95 | return file.toString(); 96 | } 97 | 98 | private char delim(String value) { 99 | if (value.length()==1) return value.charAt(0); 100 | if (value.contains("\\t")) return '\t'; 101 | if (value.contains(" ")) return ' '; 102 | throw new RuntimeException("Illegal delimiter '"+value+"'"); 103 | } 104 | 105 | private CSVWriter createWriter(Writer file, Config config) throws IOException { 106 | if (file==null) return null; 107 | return config.isQuotes() ? new CSVWriter(file,config.getDelimChar(), Config.QUOTECHAR) : new CSVWriter(file,config.getDelimChar()); 108 | } 109 | 110 | private CSVReader createReader(CountingReader reader, Config config) throws FileNotFoundException { 111 | if (reader==null) return null; 112 | return config.isQuotes() ? new CSVReader(reader,config.getDelimChar(), Config.QUOTECHAR) : new CSVReader(reader,config.getDelimChar()); 113 | } 114 | 115 | private int execute(CSVWriter writer) { 116 | return writeResult(writer,true, getServer().getDb()); 117 | } 118 | 119 | 120 | private int executeOnInput(CSVReader reader, CSVWriter writer, Config config, ProgressReporter reporter) throws IOException { 121 | Map params = createParams(reader); 122 | Map types = extractTypes(params); 123 | String[] input; 124 | boolean first = true; 125 | int outCount = 0; 126 | try (BatchTransaction tx = new BatchTransaction(getServer().getDb(),config.getBatchSize(),reporter)) { 127 | while ((input = reader.readNext()) != null) { 128 | Map queryParams = update(params, types, input); 129 | // String newQuery = applyReplacements(query, replacements, queryParams); 130 | // SubGraph result = getEngine().execute(newQuery, queryParams); 131 | // outCount += writeResult(writer, first, getServer().getDb()); 132 | // first = false; 133 | // ProgressReporter.update(result.getQueryStatistics(), reporter); 134 | tx.increment(); 135 | } 136 | } 137 | return outCount; 138 | } 139 | 140 | private Map extractTypes(Map params) { 141 | Map newParams = new LinkedHashMap<>(); 142 | Map types = new LinkedHashMap<>(); 143 | for (String header : params.keySet()) { 144 | if (header.contains(":")) { 145 | String[] parts = header.split(":"); 146 | newParams.put(parts[0],null); 147 | types.put(parts[0],Type.fromString(parts[1])); 148 | } else { 149 | newParams.put(header,null); 150 | types.put(header,Type.STRING); 151 | } 152 | } 153 | params.clear(); 154 | params.putAll(newParams); 155 | return types; 156 | } 157 | 158 | private String applyReplacements(String query, Map replacements, Map queryParams) { 159 | for (Map.Entry entry : replacements.entrySet()) { 160 | Object value = queryParams.get(entry.getKey()); 161 | query = query.replace(entry.getValue(), String.valueOf(value)); 162 | } 163 | return query; 164 | } 165 | 166 | private Map computeReplacements(Map params, String query) { 167 | Map result = new HashMap<>(); 168 | for (String name : params.keySet()) { 169 | String pattern = "#{" + name + "}"; 170 | if (query.contains(pattern)) result.put(name,pattern); 171 | } 172 | return result; 173 | } 174 | 175 | private int writeResult(CSVWriter writer, boolean first, GraphDatabaseService db) { 176 | if (writer==null) return -1; 177 | 178 | String[] cols = getAllProperties(db); 179 | if (first) { 180 | writer.writeNext(cols); 181 | } 182 | 183 | String[] data = new String[cols.length]; 184 | int count=0; 185 | for (Node node : db.getAllNodes()) { 186 | data[0]=String.valueOf(node.getId()); 187 | data[1]= toLabelString(node); 188 | for (int i = 2; i < cols.length; i++) { 189 | String col = cols[i]; 190 | data[i] = node.getProperty(col,"").toString(); 191 | } 192 | writer.writeNext(data); 193 | count++; 194 | } 195 | return count; 196 | } 197 | 198 | private String toLabelString(Node node) { 199 | Iterable