├── src └── main │ ├── java │ ├── NEO4J.png │ ├── bi │ │ └── know │ │ │ └── kettle │ │ │ └── neo4j │ │ │ ├── steps │ │ │ ├── output │ │ │ │ ├── OperationType.java │ │ │ │ ├── Neo4JOutputData.java │ │ │ │ └── messages │ │ │ │ │ └── messages_en_US.properties │ │ │ ├── BaseNeoStepData.java │ │ │ ├── split │ │ │ │ ├── SplitGraphData.java │ │ │ │ ├── SplitGraph.java │ │ │ │ └── SplitGraphMeta.java │ │ │ ├── graph │ │ │ │ ├── messages │ │ │ │ │ └── messages_en_US.properties │ │ │ │ ├── ModelTargetType.java │ │ │ │ ├── GraphOutputData.java │ │ │ │ ├── CypherParameters.java │ │ │ │ ├── FieldModelMapping.java │ │ │ │ └── TargetParameter.java │ │ │ ├── gencsv │ │ │ │ ├── NodeCollisionListener.java │ │ │ │ ├── RelationshipCollisionListener.java │ │ │ │ ├── UniquenessStrategy.java │ │ │ │ ├── IdType.java │ │ │ │ ├── StreamConsumer.java │ │ │ │ ├── GenerateCsvData.java │ │ │ │ ├── CsvFile.java │ │ │ │ ├── IndexedGraphData.java │ │ │ │ └── GenerateCsvMeta.java │ │ │ ├── importer │ │ │ │ ├── ImporterData.java │ │ │ │ └── Importer.java │ │ │ ├── cypher │ │ │ │ ├── messages │ │ │ │ │ └── messages_en_US.properties │ │ │ │ ├── ParameterMapping.java │ │ │ │ ├── CypherData.java │ │ │ │ ├── ReturnValue.java │ │ │ │ ├── CypherTransactionWork.java │ │ │ │ └── CypherStatement.java │ │ │ └── BaseNeoStep.java │ │ │ ├── perspective │ │ │ ├── messages │ │ │ │ └── messages_en_US.properties │ │ │ ├── Neo4jMenuController.java │ │ │ ├── Neo4jController.java │ │ │ ├── Neo4jSpoonPlugin.java │ │ │ └── Neo4jHelper.java │ │ │ ├── model │ │ │ ├── messages │ │ │ │ └── messages_en_US.properties │ │ │ └── GraphModelUtils.java │ │ │ ├── xp │ │ │ ├── CloseDriverSingletonInTopLevelJobExtensionPoint.java │ │ │ └── CloseDriverSingletonInTopLevelTransformationExtensionPoint.java │ │ │ ├── shared │ │ │ ├── messages │ │ │ │ └── messages_en_US.properties │ │ │ ├── MetaStoreUtil.java │ │ │ └── NeoConnectionUtils.java │ │ │ └── entries │ │ │ ├── check │ │ │ ├── CheckConnections.java │ │ │ └── CheckConnectionsDialog.java │ │ │ └── cypherscript │ │ │ └── CypherScript.java │ ├── neo4j_graph_output.svg │ ├── NEO4J.svg │ ├── neo4j_output.svg │ ├── neo4j_logo.svg │ ├── neo4j_load.svg │ ├── neo4j_check.svg │ ├── neo4j_cypher.svg │ └── neo4j_split.svg │ └── resources │ ├── neo4j_perspective.xul │ ├── neo4j_spoon_overlays.xul │ └── neo4j_logo.svg ├── .gitignore ├── .project ├── README.md ├── pom.xml └── LICENSE /src/main/java/NEO4J.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/knowbi/knowbi-pentaho-pdi-neo4j-output/HEAD/src/main/java/NEO4J.png -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/output/OperationType.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.output; 2 | 3 | public enum OperationType { 4 | MATCH, MERGE, CREATE, NONE 5 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | #.project 3 | .settings/ 4 | target/ 5 | /.apt_generated/ 6 | .idea 7 | .idea/.name 8 | .idea/misc.xml 9 | .idea/workspace.xml 10 | .idea/compiler.xml 11 | .idea/encodings.xml 12 | .idea/vcs.xml 13 | src/main/java/*.png 14 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/perspective/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | Neo4j.Perspective.perspectiveName = Neo4j Browser 2 | 3 | Neo4jPerspective.DeleteGraphModelConfirmation.Title = Delete graph model? 4 | Neo4jPerspective.DeleteGraphModelConfirmation.Message = Are you sure you want to delete graph model {0}? 5 | NeoConnectionUtils.Error.ErrorDeletingConnection.Title = Error 6 | NeoConnectionUtils.Error.ErrorDeletingConnection.Message = Error deleting graph model {0} -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/BaseNeoStepData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps; 2 | 3 | import org.pentaho.di.trans.step.BaseStepData; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | public class BaseNeoStepData extends BaseStepData { 10 | 11 | /** 12 | * A map : GraphUsage / StepName / NodeLabels 13 | */ 14 | 15 | public Map>> usageMap = new HashMap<>(); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/resources/neo4j_perspective.xul: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/split/SplitGraphData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.split; 2 | 3 | import org.pentaho.di.core.row.RowMetaInterface; 4 | import org.pentaho.di.trans.step.BaseStepData; 5 | import org.pentaho.di.trans.step.StepDataInterface; 6 | 7 | import java.util.List; 8 | 9 | public class SplitGraphData extends BaseStepData implements StepDataInterface { 10 | 11 | public RowMetaInterface outputRowMeta; 12 | public int graphFieldIndex; 13 | public String typeField; 14 | public String idField; 15 | public String propertySetField; 16 | } 17 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Neo4JOutput 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/graph/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | GraphOutput.Injection.CONNECTION = Neo4j Connection name 2 | GraphOutput.Injection.MODEL = Graph model name 3 | GraphOutput.Injection.BATCH_SIZE = Batch size 4 | GraphOutput.Injection.CREATE_INDEXES = Create indexes flag 5 | GraphOutput.Injection.MAPPINGS = Mappings list 6 | GraphOutput.Injection.MAPPING_SOURCE_FIELD = Mapping Source field 7 | GraphOutput.Injection.MAPPING_TARGET_TYPE = Mapping target type (Node, Relationship) 8 | GraphOutput.Injection.MAPPING_TARGET_NAME = Mapping target name 9 | GraphOutput.Injection.MAPPING_TARGET_PROPERTY = Mapping target property 10 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/NodeCollisionListener.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.neo4j.kettle.core.data.GraphNodeData; 4 | import org.pentaho.di.core.exception.KettleException; 5 | 6 | public interface NodeCollisionListener { 7 | /** 8 | * When a node gets added and an existing node already exists, let the system know what to do with the existing node. 9 | * This is used to update or aggregate properties in the existing data. 10 | * 11 | * @param existing 12 | * @param added 13 | * 14 | * @throws KettleException 15 | */ 16 | void handleCollission( GraphNodeData existing, GraphNodeData added) throws KettleException; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/importer/ImporterData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.importer; 2 | 3 | import org.pentaho.di.trans.step.BaseStepData; 4 | import org.pentaho.di.trans.step.StepDataInterface; 5 | 6 | import java.util.List; 7 | 8 | public class ImporterData extends BaseStepData implements StepDataInterface { 9 | 10 | public List nodesFiles; 11 | public List relsFiles; 12 | 13 | public String importFolder; 14 | public String adminCommand; 15 | public String databaseFilename; 16 | public String baseFolder; 17 | public String reportFile; 18 | 19 | 20 | public int filenameFieldIndex; 21 | public int fileTypeFieldIndex; 22 | public String readBufferSize; 23 | public String maxMemory; 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/RelationshipCollisionListener.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.neo4j.kettle.core.data.GraphRelationshipData; 4 | import org.pentaho.di.core.exception.KettleException; 5 | 6 | public interface RelationshipCollisionListener { 7 | /** 8 | * When a relationship gets added and an existing relationship already exists, let the system know what to do with the existing relationship. 9 | * This is used to update or aggregate properties in the existing data. 10 | * 11 | * @param existing 12 | * @param added 13 | * 14 | * @throws KettleException 15 | */ 16 | void handleCollission( GraphRelationshipData existing, GraphRelationshipData added ) throws KettleException; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/UniquenessStrategy.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | public enum UniquenessStrategy { 4 | None, // Don't calculate unique nore or relationship values 5 | First, // Take the first version of the node or relationship 6 | Last, // Take the last version of the node or relationship 7 | // UpdateProperties, // Not supported yet. Update all the available properties 8 | ; 9 | 10 | public static final UniquenessStrategy getStrategyFromName( String name) { 11 | for (UniquenessStrategy strategy : values()) { 12 | if (strategy.name().equalsIgnoreCase( name )) { 13 | return strategy; 14 | } 15 | } 16 | return None; 17 | } 18 | 19 | public static final String[] getNames() { 20 | String[] names = new String[values().length]; 21 | for (int i=0;i> unwindList; 33 | 34 | public List cypherStatements; 35 | 36 | public Map returnSourceTypeMap; 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/perspective/Neo4jMenuController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Hitachi America, Ltd., R&D. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package bi.know.kettle.neo4j.perspective; 18 | 19 | import org.pentaho.di.ui.spoon.ISpoonMenuController; 20 | import org.pentaho.ui.xul.dom.Document; 21 | import org.pentaho.ui.xul.impl.AbstractXulEventHandler; 22 | 23 | public class Neo4jMenuController extends AbstractXulEventHandler implements ISpoonMenuController { 24 | 25 | private static final Class PKG = Neo4jController.class; 26 | 27 | private Neo4jController neo4jController; 28 | 29 | public void setNeo4jController( Neo4jController gitController ) { 30 | this.neo4jController = gitController; 31 | } 32 | 33 | 34 | @Override public void updateMenu( Document document ) { 35 | // TODO 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/graph/GraphOutputData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.graph; 2 | 3 | import bi.know.kettle.neo4j.steps.BaseNeoStepData; 4 | import org.neo4j.driver.Driver; 5 | import org.neo4j.driver.Session; 6 | import org.neo4j.driver.Transaction; 7 | import org.neo4j.kettle.model.GraphModel; 8 | import org.neo4j.kettle.model.GraphProperty; 9 | import org.neo4j.kettle.model.validation.ModelValidator; 10 | import org.neo4j.kettle.shared.NeoConnection; 11 | import org.pentaho.di.core.row.RowMetaInterface; 12 | import org.pentaho.di.trans.step.StepDataInterface; 13 | import org.pentaho.metastore.api.IMetaStore; 14 | 15 | import java.util.HashMap; 16 | import java.util.Map; 17 | 18 | public class GraphOutputData extends BaseNeoStepData implements StepDataInterface { 19 | 20 | public RowMetaInterface outputRowMeta; 21 | public NeoConnection neoConnection; 22 | public String url; 23 | public Session session; 24 | public int[] fieldIndexes; 25 | public long batchSize; 26 | public Transaction transaction; 27 | public long outputCount; 28 | public boolean hasInput; 29 | public GraphModel graphModel; 30 | public int nodeCount; 31 | public IMetaStore metaStore; 32 | public Map cypherMap; 33 | public HashMap> relationshipPropertyIndexMap; 34 | public boolean version4; 35 | public ModelValidator modelValidator;; 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/IdType.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.neo4j.kettle.core.data.GraphPropertyDataType; 4 | 5 | import java.util.Objects; 6 | 7 | public class IdType { 8 | 9 | private String id; 10 | private GraphPropertyDataType type; 11 | 12 | public IdType() { 13 | } 14 | 15 | public IdType( String id, GraphPropertyDataType type ) { 16 | this.id = id; 17 | this.type = type; 18 | } 19 | 20 | @Override 21 | public boolean equals( Object o ) { 22 | if ( this == o ) { 23 | return true; 24 | } 25 | if ( o == null || getClass() != o.getClass() ) { 26 | return false; 27 | } 28 | IdType idType = (IdType) o; 29 | return id.equals( idType.id ); 30 | } 31 | 32 | @Override 33 | public int hashCode() { 34 | return Objects.hash( id ); 35 | } 36 | 37 | /** 38 | * Gets id 39 | * 40 | * @return value of id 41 | */ 42 | public String getId() { 43 | return id; 44 | } 45 | 46 | /** 47 | * @param id The id to set 48 | */ 49 | public void setId( String id ) { 50 | this.id = id; 51 | } 52 | 53 | /** 54 | * Gets type 55 | * 56 | * @return value of type 57 | */ 58 | public GraphPropertyDataType getType() { 59 | return type; 60 | } 61 | 62 | /** 63 | * @param type The type to set 64 | */ 65 | public void setType( GraphPropertyDataType type ) { 66 | this.type = type; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/graph/CypherParameters.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.graph; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | /** 8 | * The class contains a cypher statement and the mapping between each parameter name and the input field 9 | */ 10 | public class CypherParameters { 11 | private String cypher; 12 | private List targetParameters; 13 | 14 | public CypherParameters() { 15 | targetParameters = new ArrayList<>(); 16 | } 17 | 18 | public CypherParameters( String cypher, List targetParameters ) { 19 | this.cypher = cypher; 20 | this.targetParameters = targetParameters; 21 | } 22 | 23 | /** 24 | * Gets cypher 25 | * 26 | * @return value of cypher 27 | */ 28 | public String getCypher() { 29 | return cypher; 30 | } 31 | 32 | /** 33 | * @param cypher The cypher to set 34 | */ 35 | public void setCypher( String cypher ) { 36 | this.cypher = cypher; 37 | } 38 | 39 | /** 40 | * Gets targetParameters 41 | * 42 | * @return value of targetParameters 43 | */ 44 | public List getTargetParameters() { 45 | return targetParameters; 46 | } 47 | 48 | /** 49 | * @param targetParameters The targetParameters to set 50 | */ 51 | public void setTargetParameters( List targetParameters ) { 52 | this.targetParameters = targetParameters; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/cypher/ReturnValue.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.cypher; 2 | 3 | import org.pentaho.di.core.injection.Injection; 4 | 5 | public class ReturnValue { 6 | 7 | @Injection( name = "RETURN_NAME", group = "RETURNS" ) 8 | private String name; 9 | 10 | @Injection( name = "RETURN_TYPE", group = "RETURNS" ) 11 | private String type; 12 | 13 | @Injection (name = "RETURN_SOURCE_TYPE", group = "RETURNS" ) 14 | private String sourceType; 15 | 16 | public ReturnValue() { 17 | } 18 | 19 | public ReturnValue( String name, String type, String sourceType ) { 20 | this.name = name; 21 | this.type = type; 22 | this.sourceType = sourceType; 23 | } 24 | 25 | /** 26 | * Gets name 27 | * 28 | * @return value of name 29 | */ 30 | public String getName() { 31 | return name; 32 | } 33 | 34 | /** 35 | * @param name The name to set 36 | */ 37 | public void setName( String name ) { 38 | this.name = name; 39 | } 40 | 41 | /** 42 | * Gets type 43 | * 44 | * @return value of type 45 | */ 46 | public String getType() { 47 | return type; 48 | } 49 | 50 | /** 51 | * @param type The type to set 52 | */ 53 | public void setType( String type ) { 54 | this.type = type; 55 | } 56 | 57 | /** 58 | * Gets sourceType 59 | * 60 | * @return value of sourceType 61 | */ 62 | public String getSourceType() { 63 | return sourceType; 64 | } 65 | 66 | /** 67 | * @param sourceType The sourceType to set 68 | */ 69 | public void setSourceType( String sourceType ) { 70 | this.sourceType = sourceType; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/resources/neo4j_spoon_overlays.xul: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/StreamConsumer.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.pentaho.di.core.logging.LogChannelInterface; 4 | import org.pentaho.di.core.logging.LogLevel; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.io.InputStreamReader; 10 | 11 | public class StreamConsumer extends Thread { 12 | 13 | private LogChannelInterface log; 14 | private InputStream inputStream; 15 | private final LogLevel logLevel; 16 | 17 | public StreamConsumer( LogChannelInterface log, InputStream inputStream, LogLevel logLevel ) { 18 | super(); 19 | this.log = log; 20 | this.inputStream = inputStream; 21 | this.logLevel = logLevel; 22 | } 23 | 24 | @Override public void run() { 25 | 26 | try { 27 | InputStreamReader inputStreamReader = new InputStreamReader( inputStream ); 28 | BufferedReader bufferedReader = new BufferedReader( inputStreamReader ); 29 | String line = null; 30 | while ((line=bufferedReader.readLine())!=null) { 31 | switch(logLevel){ 32 | case MINIMAL: log.logMinimal(line); break; 33 | case BASIC: log.logBasic(line); break; 34 | case DETAILED: log.logDetailed(line); break; 35 | case DEBUG: log.logDebug(line); break; 36 | case ROWLEVEL: log.logRowlevel(line); break; 37 | case ERROR: 38 | if (!line.contains( "neo4j-import is deprecated" ) && !line.contains( "please use neo4j-admin import" )) { 39 | log.logError( line ); 40 | break; 41 | } 42 | case NOTHING: 43 | default: 44 | break; 45 | } 46 | } 47 | } catch ( IOException e ) { 48 | log.logError("Error consuming thread:", e); 49 | } 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/xp/CloseDriverSingletonInTopLevelJobExtensionPoint.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.xp; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.neo4j.kettle.core.Neo4jDefaults; 5 | import org.neo4j.kettle.shared.DriverSingleton; 6 | import org.pentaho.di.core.exception.KettleException; 7 | import org.pentaho.di.core.extension.ExtensionPoint; 8 | import org.pentaho.di.core.extension.ExtensionPointInterface; 9 | import org.pentaho.di.core.logging.LogChannelInterface; 10 | import org.pentaho.di.job.Job; 11 | import org.pentaho.di.job.JobAdapter; 12 | 13 | @ExtensionPoint( 14 | id = "CloseDriverSingletonInTopLevelJobExtensionPoint", 15 | extensionPointId = "JobStart", 16 | description = "Close the Neo4j driver singleton at the end of a top level job" 17 | ) 18 | public class CloseDriverSingletonInTopLevelJobExtensionPoint implements ExtensionPointInterface { 19 | 20 | @Override public void callExtensionPoint( LogChannelInterface log, Object object ) throws KettleException { 21 | 22 | if ( !( object instanceof Job ) ) { 23 | return; // not for us 24 | } 25 | 26 | Job job = (Job) object; 27 | 28 | job.addJobListener( new JobAdapter() { 29 | @Override public void jobFinished( Job job ) throws KettleException { 30 | // Only close if we're in a top level job and not running on a container 31 | // 32 | if (job.getContainerObjectId()==null && job.getParentJob()==null && job.getParentTrans()==null) { 33 | 34 | String cleanup = job.getVariable( Neo4jDefaults.VARIABLE_NEO4J_CLEANUP_DRIVERS ); 35 | if (!StringUtils.isEmpty( cleanup ) && ( "Y".equalsIgnoreCase( cleanup ) || "Yes".equalsIgnoreCase( cleanup ) || "TRUE".equalsIgnoreCase( cleanup )) ) { 36 | DriverSingleton.closeAll(); 37 | } 38 | } 39 | } 40 | } ); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/BaseNeoStep.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps; 2 | 3 | import org.neo4j.kettle.core.Neo4jDefaults; 4 | import org.pentaho.di.trans.Trans; 5 | import org.pentaho.di.trans.TransMeta; 6 | import org.pentaho.di.trans.step.BaseStep; 7 | import org.pentaho.di.trans.step.StepDataInterface; 8 | import org.pentaho.di.trans.step.StepMeta; 9 | import org.pentaho.di.trans.step.StepMetaInterface; 10 | 11 | import java.util.HashMap; 12 | 13 | public abstract class BaseNeoStep extends BaseStep { 14 | 15 | /** 16 | * This is the base step that forms that basis for all steps. You can derive from this class to implement your own 17 | * steps. 18 | * 19 | * @param stepMeta The StepMeta object to run. 20 | * @param stepDataInterface the data object to store temporary data, database connections, caches, result sets, 21 | * hashtables etc. 22 | * @param copyNr The copynumber for this step. 23 | * @param transMeta The TransInfo of which the step stepMeta is part of. 24 | * @param trans The (running) transformation to obtain information shared among the steps. 25 | */ 26 | public BaseNeoStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, 27 | Trans trans ) { 28 | super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); 29 | } 30 | 31 | @Override public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { 32 | 33 | ( (BaseNeoStepData) sdi ).usageMap = new HashMap<>(); 34 | 35 | return super.init( smi, sdi ); 36 | } 37 | 38 | @Override public void dispose( StepMetaInterface smi, StepDataInterface sdi ) { 39 | 40 | getTrans().getExtensionDataMap().put( Neo4jDefaults.TRANS_NODE_UPDATES_GROUP, ( (BaseNeoStepData) sdi ).usageMap ); 41 | 42 | super.dispose( smi, sdi ); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/xp/CloseDriverSingletonInTopLevelTransformationExtensionPoint.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.xp; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.neo4j.kettle.core.Neo4jDefaults; 5 | import org.neo4j.kettle.shared.DriverSingleton; 6 | import org.pentaho.di.core.exception.KettleException; 7 | import org.pentaho.di.core.extension.ExtensionPoint; 8 | import org.pentaho.di.core.extension.ExtensionPointInterface; 9 | import org.pentaho.di.core.logging.LogChannelInterface; 10 | import org.pentaho.di.trans.Trans; 11 | import org.pentaho.di.trans.TransAdapter; 12 | 13 | @ExtensionPoint( 14 | id = "CloseDriverSingletonInTopLevelTransformationExtensionPoint", 15 | extensionPointId = "TransformationStart", 16 | description = "Close the Neo4j driver singleton at the end of a top level transformation" 17 | ) 18 | public class CloseDriverSingletonInTopLevelTransformationExtensionPoint implements ExtensionPointInterface { 19 | 20 | @Override public void callExtensionPoint( LogChannelInterface log, Object object ) throws KettleException { 21 | 22 | if ( !( object instanceof Trans ) ) { 23 | return; // not for us 24 | } 25 | 26 | Trans trans = (Trans) object; 27 | 28 | trans.addTransListener( new TransAdapter() { 29 | @Override public void transFinished( Trans trans ) throws KettleException { 30 | // Only close if we're in a top level transformation and not running on a container 31 | // 32 | if (trans.getContainerObjectId()==null && trans.getParentJob()==null && trans.getParentTrans()==null) { 33 | String cleanup = trans.getVariable( Neo4jDefaults.VARIABLE_NEO4J_CLEANUP_DRIVERS ); 34 | if (!StringUtils.isEmpty( cleanup ) && ( "Y".equalsIgnoreCase( cleanup ) || "Yes".equalsIgnoreCase( cleanup ) || "TRUE".equalsIgnoreCase( cleanup )) ) { 35 | DriverSingleton.closeAll(); 36 | } 37 | } 38 | } 39 | } ); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/cypher/CypherTransactionWork.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.cypher; 2 | 3 | import org.neo4j.driver.Result; 4 | import org.neo4j.driver.Transaction; 5 | import org.neo4j.driver.TransactionWork; 6 | import org.pentaho.di.core.exception.KettleException; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public class CypherTransactionWork implements TransactionWork { 12 | private final Cypher step; 13 | private final Object[] currentRow; 14 | private final boolean unwind; 15 | private String cypher; 16 | private Map unwindMap; 17 | 18 | public CypherTransactionWork( Cypher step, Object[] currentRow, boolean unwind, String cypher, Map unwindMap ) { 19 | this.step = step; 20 | this.currentRow = currentRow; 21 | this.unwind = unwind; 22 | this.cypher = cypher; 23 | this.unwindMap = unwindMap; 24 | } 25 | 26 | @Override public Void execute( Transaction tx ) { 27 | Result result = tx.run( cypher, unwindMap ); 28 | try { 29 | step.getResultRows( result, currentRow, unwind ); 30 | return null; 31 | } catch ( KettleException e ) { 32 | throw new RuntimeException( "Unable to execute cypher statement '"+cypher+"'", e ); 33 | } 34 | } 35 | 36 | /** 37 | * Gets cypher 38 | * 39 | * @return value of cypher 40 | */ 41 | public String getCypher() { 42 | return cypher; 43 | } 44 | 45 | /** 46 | * @param cypher The cypher to set 47 | */ 48 | public void setCypher( String cypher ) { 49 | this.cypher = cypher; 50 | } 51 | 52 | /** 53 | * Gets unwindMap 54 | * 55 | * @return value of unwindMap 56 | */ 57 | public Map getUnwindMap() { 58 | return unwindMap; 59 | } 60 | 61 | /** 62 | * @param unwindMap The unwindMap to set 63 | */ 64 | public void setUnwindMap( Map unwindMap ) { 65 | this.unwindMap = unwindMap; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/cypher/CypherStatement.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.cypher; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | public class CypherStatement { 8 | 9 | private Object[] row; 10 | private String cypher; 11 | private Map parameters; 12 | private List resultRows; 13 | 14 | public CypherStatement() { 15 | parameters = new HashMap<>(); 16 | } 17 | 18 | public CypherStatement( Object[] row, String cypher, Map parameters ) { 19 | this.row = row; 20 | this.cypher = cypher; 21 | this.parameters = parameters; 22 | } 23 | 24 | /** 25 | * Gets row 26 | * 27 | * @return value of row 28 | */ 29 | public Object[] getRow() { 30 | return row; 31 | } 32 | 33 | /** 34 | * @param row The row to set 35 | */ 36 | public void setRow( Object[] row ) { 37 | this.row = row; 38 | } 39 | 40 | /** 41 | * Gets cypher 42 | * 43 | * @return value of cypher 44 | */ 45 | public String getCypher() { 46 | return cypher; 47 | } 48 | 49 | /** 50 | * @param cypher The cypher to set 51 | */ 52 | public void setCypher( String cypher ) { 53 | this.cypher = cypher; 54 | } 55 | 56 | /** 57 | * Gets parameters 58 | * 59 | * @return value of parameters 60 | */ 61 | public Map getParameters() { 62 | return parameters; 63 | } 64 | 65 | /** 66 | * @param parameters The parameters to set 67 | */ 68 | public void setParameters( Map parameters ) { 69 | this.parameters = parameters; 70 | } 71 | 72 | /** 73 | * Gets resultRows 74 | * 75 | * @return value of resultRows 76 | */ 77 | public List getResultRows() { 78 | return resultRows; 79 | } 80 | 81 | /** 82 | * @param resultRows The resultRows to set 83 | */ 84 | public void setResultRows( List resultRows ) { 85 | this.resultRows = resultRows; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/GenerateCsvData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.pentaho.di.core.row.RowMetaInterface; 5 | import org.pentaho.di.trans.step.BaseStepData; 6 | import org.pentaho.di.trans.step.StepDataInterface; 7 | 8 | import java.util.Map; 9 | 10 | public class GenerateCsvData extends BaseStepData implements StepDataInterface { 11 | 12 | public RowMetaInterface outputRowMeta; 13 | public String importFolder; 14 | public int graphFieldIndex; 15 | public IndexedGraphData indexedGraphData; 16 | public String baseFolder; 17 | 18 | public String filesPrefix; 19 | public String filenameField; 20 | public String fileTypeField; 21 | 22 | public Map fileMap; 23 | 24 | public static String getPropertySetKey( String sourceTransformation, String sourceStep, String propertySetId ) { 25 | StringBuilder key = new StringBuilder(); 26 | if ( StringUtils.isNotEmpty( sourceTransformation ) ) { 27 | key.append( sourceTransformation ); 28 | } 29 | if ( StringUtils.isNotEmpty( sourceStep ) ) { 30 | if ( key.length() > 0 ) { 31 | key.append( "-" ); 32 | } 33 | key.append( sourceStep ); 34 | } 35 | if ( StringUtils.isNotEmpty( propertySetId ) ) { 36 | if ( key.length() > 0 ) { 37 | key.append( "-" ); 38 | } 39 | key.append( propertySetId ); 40 | } 41 | 42 | // Replace troublesome characters from the key for filename purposes 43 | // 44 | String setKey = key.toString(); 45 | setKey = setKey.replace( "*", "" ); 46 | setKey = setKey.replace( ":", "" ); 47 | setKey = setKey.replace( ";", "" ); 48 | setKey = setKey.replace( "[", "" ); 49 | setKey = setKey.replace( "]", "" ); 50 | setKey = setKey.replace( "$", "" ); 51 | setKey = setKey.replace( "/", "" ); 52 | setKey = setKey.replace( "{", "" ); 53 | setKey = setKey.replace( "}", "" ); 54 | 55 | return setKey; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/graph/FieldModelMapping.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.graph; 2 | 3 | import org.pentaho.di.core.injection.Injection; 4 | 5 | public class FieldModelMapping { 6 | 7 | /** 8 | * The Kettle input field where the data is coming from 9 | */ 10 | @Injection( name = "MAPPING_SOURCE_FIELD", group = "MAPPINGS" ) 11 | private String field; 12 | 13 | /** 14 | * Write to a node or a relationship 15 | */ 16 | @Injection( name = "MAPPING_TARGET_TYPE", group = "MAPPINGS" ) 17 | private ModelTargetType targetType; 18 | 19 | /** 20 | * Name of the node or relationship to write to 21 | */ 22 | @Injection( name = "MAPPING_TARGET_NAME", group = "MAPPINGS" ) 23 | private String targetName; 24 | 25 | /** 26 | * Name of the property to write to 27 | */ 28 | @Injection( name = "MAPPING_TARGET_PROPERTY", group = "MAPPINGS" ) 29 | private String targetProperty; 30 | 31 | 32 | public FieldModelMapping() { 33 | targetType = ModelTargetType.Node; 34 | } 35 | 36 | public FieldModelMapping( String field, ModelTargetType targetType, String targetName, String targetProperty ) { 37 | this.field = field; 38 | this.targetType = targetType; 39 | this.targetName = targetName; 40 | this.targetProperty = targetProperty; 41 | } 42 | 43 | public String getField() { 44 | return field; 45 | } 46 | 47 | public void setField( String field ) { 48 | this.field = field; 49 | } 50 | 51 | public ModelTargetType getTargetType() { 52 | return targetType; 53 | } 54 | 55 | public void setTargetType( ModelTargetType targetType ) { 56 | this.targetType = targetType; 57 | } 58 | 59 | public String getTargetName() { 60 | return targetName; 61 | } 62 | 63 | public void setTargetName( String targetName ) { 64 | this.targetName = targetName; 65 | } 66 | 67 | public String getTargetProperty() { 68 | return targetProperty; 69 | } 70 | 71 | public void setTargetProperty( String targetProperty ) { 72 | this.targetProperty = targetProperty; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/perspective/Neo4jController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Hitachi America, Ltd., R&D. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package bi.know.kettle.neo4j.perspective; 18 | 19 | import org.eclipse.jface.resource.JFaceResources; 20 | import org.eclipse.swt.widgets.Text; 21 | import org.pentaho.ui.xul.XulException; 22 | import org.pentaho.ui.xul.binding.Binding; 23 | import org.pentaho.ui.xul.binding.BindingFactory; 24 | import org.pentaho.ui.xul.components.XulTextbox; 25 | import org.pentaho.ui.xul.impl.AbstractXulEventHandler; 26 | import org.pentaho.ui.xul.swt.SwtBindingFactory; 27 | 28 | import java.lang.reflect.InvocationTargetException; 29 | 30 | public class Neo4jController extends AbstractXulEventHandler { 31 | 32 | private static final Class PKG = Neo4jController.class; 33 | 34 | private BindingFactory bf = new SwtBindingFactory(); 35 | private Binding revisionBinding; 36 | private Binding changedBinding; 37 | private Binding branchBinding; 38 | private Binding diffBinding; 39 | 40 | public Neo4jController() { 41 | setName( "neo4jController" ); 42 | } 43 | 44 | public void init() throws IllegalArgumentException, InvocationTargetException, XulException { 45 | XulTextbox diffText = (XulTextbox) document.getElementById( "diff" ); 46 | Text text = (Text) diffText.getManagedObject(); 47 | text.setFont( JFaceResources.getFont( JFaceResources.TEXT_FONT ) ); 48 | } 49 | 50 | private void createBindings() { 51 | // TODO 52 | } 53 | 54 | public void onTabClose() { 55 | // Simply close 56 | } 57 | } -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/shared/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | NeoConnectionDialog.Shell.Title=Neo4j Connection dialog 2 | NeoConnectionDialog.Name.Label=Connection name 3 | NeoConnectionDialog.Server.Label=Server or IP address 4 | NeoConnectionDialog.DatabaseName.Label=Database name (4.0) 5 | NeoConnectionDialog.Version4.Label = Version 4 database? 6 | NeoConnectionDialog.BoltPort.Label=Bolt Port 7 | NeoConnectionDialog.BrowserPort.Label=Browser Port 8 | NeoConnectionDialog.Routing.Label=Use routing, neo4j:// protocol? 9 | NeoConnectionDialog.Policy.Label=Routing Policy 10 | NeoConnectionDialog.UserName.Label=Username 11 | NeoConnectionDialog.Password.Label=Password 12 | NeoConnectionDialog.Encryption.Label=Use encryption? 13 | NeoConnectionDialog.URLs.Label=Manual URLs 14 | NeoConnectionDialog.URLColumn.Label = URL 15 | 16 | NeoConnectionDialog.NoNameDialog.Title=Connection name? 17 | NeoConnectionDialog.NoNameDialog.Message=You need to give your connection a name 18 | NeoConnectionUtils.Error.ConnectionExists.Title=Connection exists 19 | NeoConnectionUtils.Error.ConnectionExists.Message=A Neo4j connection with this name already exists. Do you want to overwrite this connection? 20 | NeoConnectionUtils.Error.ErrorSavingConnection.Title=Error 21 | NeoConnectionUtils.Error.ErrorSavingConnection.Message=Error saving Neo4j connection 22 | NeoConnectionUtils.DeleteConnectionConfirmation.Title=Delete? 23 | NeoConnectionUtils.DeleteConnectionConfirmation.Message=Are you sure you want to delete connection {0}? 24 | NeoConnectionUtils.Error.ErrorDeletingConnection.Title=Error 25 | NeoConnectionUtils.Error.ErrorDeletingConnection.Message=There was an error deleting connection {0} 26 | 27 | NeoConnectionDialog.ConnectionLivenessCheckTimeout.Label = Connection liveness Check Timeout (ms) 28 | NeoConnectionDialog.MaxConnectionLifetime.Label = Maximum connection lifetime (ms) 29 | NeoConnectionDialog.MaxConnectionPoolSize.Label = Maximum pool size 30 | NeoConnectionDialog.ConnectionAcquisitionTimeout.Label = Connection aquisition time (ms) 31 | NeoConnectionDialog.ConnectionTimeout.Label = Connection timeout (ms) 32 | NeoConnectionDialog.MaxTransactionRetryTime.Label = Maximum transaction retry time (ms) 33 | 34 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/graph/TargetParameter.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.graph; 2 | 3 | import org.neo4j.kettle.model.GraphPropertyType; 4 | 5 | public class TargetParameter { 6 | private String inputField; 7 | private int inputFieldIndex; 8 | 9 | private String parameterName; 10 | private GraphPropertyType parameterType; 11 | 12 | public TargetParameter() { 13 | } 14 | 15 | public TargetParameter( String inputField, int inputFieldIndex, String parameterName, GraphPropertyType parameterType ) { 16 | this.inputField = inputField; 17 | this.inputFieldIndex = inputFieldIndex; 18 | this.parameterName = parameterName; 19 | this.parameterType = parameterType; 20 | } 21 | 22 | /** 23 | * Gets inputField 24 | * 25 | * @return value of inputField 26 | */ 27 | public String getInputField() { 28 | return inputField; 29 | } 30 | 31 | /** 32 | * @param inputField The inputField to set 33 | */ 34 | public void setInputField( String inputField ) { 35 | this.inputField = inputField; 36 | } 37 | 38 | /** 39 | * Gets inputFieldIndex 40 | * 41 | * @return value of inputFieldIndex 42 | */ 43 | public int getInputFieldIndex() { 44 | return inputFieldIndex; 45 | } 46 | 47 | /** 48 | * @param inputFieldIndex The inputFieldIndex to set 49 | */ 50 | public void setInputFieldIndex( int inputFieldIndex ) { 51 | this.inputFieldIndex = inputFieldIndex; 52 | } 53 | 54 | /** 55 | * Gets parameterName 56 | * 57 | * @return value of parameterName 58 | */ 59 | public String getParameterName() { 60 | return parameterName; 61 | } 62 | 63 | /** 64 | * @param parameterName The parameterName to set 65 | */ 66 | public void setParameterName( String parameterName ) { 67 | this.parameterName = parameterName; 68 | } 69 | 70 | /** 71 | * Gets parameterType 72 | * 73 | * @return value of parameterType 74 | */ 75 | public GraphPropertyType getParameterType() { 76 | return parameterType; 77 | } 78 | 79 | /** 80 | * @param parameterType The parameterType to set 81 | */ 82 | public void setParameterType( GraphPropertyType parameterType ) { 83 | this.parameterType = parameterType; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/output/Neo4JOutputData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.output; 2 | 3 | import bi.know.kettle.neo4j.steps.BaseNeoStepData; 4 | import org.neo4j.driver.Driver; 5 | import org.neo4j.driver.Session; 6 | import org.neo4j.driver.Transaction; 7 | import org.neo4j.kettle.model.GraphPropertyType; 8 | import org.neo4j.kettle.shared.NeoConnection; 9 | import org.pentaho.di.core.row.RowMetaInterface; 10 | import org.pentaho.di.trans.step.StepDataInterface; 11 | import org.pentaho.metastore.api.IMetaStore; 12 | 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | public class Neo4JOutputData extends BaseNeoStepData implements StepDataInterface { 17 | 18 | public RowMetaInterface outputRowMeta; 19 | 20 | public String[] fieldNames; 21 | 22 | public NeoConnection neoConnection; 23 | public String url; 24 | public Driver driver; 25 | public Session session; 26 | 27 | public long batchSize; 28 | public long outputCount; 29 | 30 | public int[] fromNodePropIndexes; 31 | public int[] fromNodeLabelIndexes; 32 | public int[] toNodePropIndexes; 33 | public int[] toNodeLabelIndexes; 34 | public int[] relPropIndexes; 35 | public int relationshipIndex; 36 | public GraphPropertyType[] fromNodePropTypes; 37 | public GraphPropertyType[] toNodePropTypes; 38 | public GraphPropertyType[] relPropTypes; 39 | 40 | public List> unwindList; 41 | 42 | public String fromLabelsClause; 43 | public String toLabelsClause; 44 | public String[] fromLabelValues; 45 | public String[] toLabelValues; 46 | public String relationshipLabelValue; 47 | 48 | public String previousFromLabelsClause; 49 | public String previousToLabelsClause; 50 | 51 | public IMetaStore metaStore; 52 | public boolean dynamicFromLabels; 53 | public boolean dynamicToLabels; 54 | public boolean dynamicRelLabel; 55 | 56 | public List previousFromLabels; 57 | public List fromLabels; 58 | public List previousToLabels; 59 | public List toLabels; 60 | public String previousRelationshipLabel; 61 | public String relationshipLabel; 62 | 63 | public OperationType fromOperationType; 64 | public OperationType toOperationType; 65 | public OperationType relOperationType; 66 | 67 | public String cypher; 68 | public boolean version4; 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/perspective/Neo4jSpoonPlugin.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.perspective; 2 | 3 | /* 4 | * Copyright 2017 Hitachi America, Ltd., R&D. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | import java.util.ResourceBundle; 20 | 21 | import org.pentaho.di.ui.spoon.SpoonLifecycleListener; 22 | import org.pentaho.di.ui.spoon.SpoonPerspective; 23 | import org.pentaho.di.ui.spoon.SpoonPlugin; 24 | import org.pentaho.di.ui.spoon.SpoonPluginCategories; 25 | import org.pentaho.di.ui.spoon.SpoonPluginInterface; 26 | import org.pentaho.di.ui.spoon.XulSpoonResourceBundle; 27 | import org.pentaho.ui.xul.XulDomContainer; 28 | import org.pentaho.ui.xul.XulException; 29 | 30 | @SpoonPlugin( id = "Neo4jSpoonPlugin", image = "" ) 31 | @SpoonPluginCategories( { "spoon" } ) 32 | public class Neo4jSpoonPlugin implements SpoonPluginInterface, SpoonLifecycleListener { 33 | 34 | private static final Class PKG = Neo4jSpoonPlugin.class; 35 | private ResourceBundle resourceBundle = new XulSpoonResourceBundle( PKG ); 36 | 37 | // private Neo4jPerspective perspective; 38 | 39 | public Neo4jSpoonPlugin() throws XulException { 40 | // this.perspective = new Neo4jPerspective(); 41 | } 42 | 43 | @Override 44 | public void onEvent( SpoonLifeCycleEvent evt ) { 45 | // TODO Auto-generated method stub 46 | } 47 | 48 | @Override 49 | public void applyToContainer( String category, XulDomContainer container ) throws XulException { 50 | container.registerClassLoader( getClass().getClassLoader() ); 51 | if ( category.equals( "spoon" ) ) { 52 | container.loadOverlay( "neo4j_spoon_overlays.xul", resourceBundle ); 53 | container.addEventHandler( Neo4jHelper.getInstance() ); 54 | } 55 | } 56 | 57 | @Override 58 | public SpoonLifecycleListener getLifecycleListener() { 59 | // TODO Auto-generated method stub 60 | return this; 61 | } 62 | 63 | @Override 64 | public SpoonPerspective getPerspective() { 65 | // return perspective; 66 | return null; 67 | } 68 | 69 | } -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/shared/MetaStoreUtil.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.shared; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.pentaho.di.core.logging.LoggingObjectInterface; 5 | import org.pentaho.di.job.Job; 6 | import org.pentaho.di.metastore.MetaStoreConst; 7 | import org.pentaho.di.trans.Trans; 8 | import org.pentaho.di.trans.step.StepInterface; 9 | import org.pentaho.metastore.api.IMetaStore; 10 | import org.pentaho.metastore.api.exceptions.MetaStoreException; 11 | import org.pentaho.metastore.stores.delegate.DelegatingMetaStore; 12 | import org.pentaho.metastore.stores.xml.XmlMetaStore; 13 | 14 | public class MetaStoreUtil { 15 | 16 | public static final IMetaStore findMetaStore( LoggingObjectInterface executor ) throws MetaStoreException { 17 | 18 | if ( executor instanceof StepInterface ) { 19 | StepInterface step = (StepInterface) executor; 20 | if ( step.getMetaStore() != null ) { 21 | return step.getMetaStore(); 22 | } 23 | Trans trans = step.getTrans(); 24 | if ( trans != null ) { 25 | if ( trans.getMetaStore() != null ) { 26 | return trans.getMetaStore(); 27 | } 28 | if ( trans.getTransMeta().getMetaStore() != null ) { 29 | return trans.getTransMeta().getMetaStore(); 30 | } 31 | } 32 | } 33 | 34 | if ( executor instanceof Trans ) { 35 | Trans trans = (Trans) executor; 36 | if ( trans.getMetaStore() != null ) { 37 | return trans.getMetaStore(); 38 | } 39 | if ( trans.getTransMeta().getMetaStore() != null ) { 40 | return trans.getMetaStore(); 41 | } 42 | } 43 | 44 | if ( executor instanceof Job ) { 45 | Job job = (Job) executor; 46 | if ( job.getJobMeta().getMetaStore() != null ) { 47 | return job.getJobMeta().getMetaStore(); 48 | } 49 | } 50 | 51 | LoggingObjectInterface parent = executor.getParent(); 52 | if ( parent != null ) { 53 | IMetaStore metaStore = findMetaStore( parent ); 54 | if ( metaStore != null ) { 55 | return metaStore; 56 | } 57 | } 58 | 59 | // Didn't find it anywhere in the tree above: lazy programmers! 60 | // 61 | System.err.println("METASTORE PROBLEM: Local couldn't be found, force local anyway"); 62 | 63 | return MetaStoreConst.openLocalPentahoMetaStore(); 64 | } 65 | 66 | public static final String getMetaStoreDescription(IMetaStore metaStore) throws MetaStoreException { 67 | String name = metaStore.getName(); 68 | String desc = metaStore.getDescription(); 69 | 70 | String description = ""; 71 | if (metaStore instanceof DelegatingMetaStore ) { 72 | DelegatingMetaStore delegatingMetaStore = (DelegatingMetaStore) metaStore; 73 | boolean first = true; 74 | for (IMetaStore store : delegatingMetaStore.getMetaStoreList()) { 75 | if (first) { 76 | description+="{ "; 77 | } else { 78 | description+=", "; 79 | } 80 | description+=getMetaStoreDescription( store ); 81 | if (first) { 82 | description+=" }"; 83 | first = false; 84 | } 85 | } 86 | } else if (metaStore instanceof XmlMetaStore ) { 87 | String rootFolder = ((XmlMetaStore)metaStore).getRootFolder(); 88 | description += name + "( located in folder: " + rootFolder + " )"; 89 | } else { 90 | if ( StringUtils.isNotEmpty(desc)) { 91 | description += name + "(" + desc + ")"; 92 | } else { 93 | description += name; 94 | } 95 | } 96 | return description; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/neo4j_graph_output.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/CsvFile.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import javax.validation.constraints.NotNull; 4 | import java.io.FileNotFoundException; 5 | import java.io.FileOutputStream; 6 | import java.io.IOException; 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.Objects; 12 | 13 | public class CsvFile { 14 | 15 | @NotNull 16 | private String filename; 17 | 18 | @NotNull 19 | private String shortFilename; 20 | 21 | @NotNull 22 | private String fileType; 23 | 24 | 25 | 26 | private transient List propsList; 27 | 28 | private transient Map propsIndexes; 29 | 30 | private transient FileOutputStream outputStream; 31 | 32 | private transient String idFieldName; 33 | 34 | public CsvFile() { 35 | propsList = new ArrayList<>(); 36 | propsIndexes = new HashMap<>(); 37 | } 38 | 39 | public CsvFile( @NotNull String filename, @NotNull String shortFilename, @NotNull String fileType ) { 40 | this(); 41 | this.filename = filename; 42 | this.shortFilename = shortFilename; 43 | this.fileType = fileType; 44 | } 45 | 46 | public void openFile() throws FileNotFoundException { 47 | outputStream = new FileOutputStream( filename ); 48 | } 49 | 50 | public void closeFile() throws IOException { 51 | if ( outputStream != null ) { 52 | outputStream.flush(); 53 | outputStream.close(); 54 | } 55 | } 56 | 57 | /** 58 | * Gets outputStream 59 | * 60 | * @return value of outputStream 61 | */ 62 | public FileOutputStream getOutputStream() { 63 | return outputStream; 64 | } 65 | 66 | /** 67 | * @param outputStream The outputStream to set 68 | */ 69 | public void setOutputStream( FileOutputStream outputStream ) { 70 | this.outputStream = outputStream; 71 | } 72 | 73 | @Override public boolean equals( Object o ) { 74 | if ( this == o ) { 75 | return true; 76 | } 77 | if ( o == null || getClass() != o.getClass() ) { 78 | return false; 79 | } 80 | CsvFile csvFile = (CsvFile) o; 81 | return filename.equals( csvFile.filename ); 82 | } 83 | 84 | @Override public int hashCode() { 85 | return Objects.hash( filename ); 86 | } 87 | 88 | /** 89 | * Gets filename 90 | * 91 | * @return value of filename 92 | */ 93 | public String getFilename() { 94 | return filename; 95 | } 96 | 97 | /** 98 | * @param filename The filename to set 99 | */ 100 | public void setFilename( String filename ) { 101 | this.filename = filename; 102 | } 103 | 104 | /** 105 | * Gets fileType 106 | * 107 | * @return value of fileType 108 | */ 109 | public String getFileType() { 110 | return fileType; 111 | } 112 | 113 | /** 114 | * @param fileType The fileType to set 115 | */ 116 | public void setFileType( String fileType ) { 117 | this.fileType = fileType; 118 | } 119 | 120 | /** 121 | * Gets shortFilename 122 | * 123 | * @return value of shortFilename 124 | */ 125 | public String getShortFilename() { 126 | return shortFilename; 127 | } 128 | 129 | /** 130 | * @param shortFilename The shortFilename to set 131 | */ 132 | public void setShortFilename( String shortFilename ) { 133 | this.shortFilename = shortFilename; 134 | } 135 | 136 | /** 137 | * Gets propsList 138 | * 139 | * @return value of propsList 140 | */ 141 | public List getPropsList() { 142 | return propsList; 143 | } 144 | 145 | /** 146 | * @param propsList The propsList to set 147 | */ 148 | public void setPropsList( List propsList ) { 149 | this.propsList = propsList; 150 | } 151 | 152 | /** 153 | * Gets propsIndexes 154 | * 155 | * @return value of propsIndexes 156 | */ 157 | public Map getPropsIndexes() { 158 | return propsIndexes; 159 | } 160 | 161 | /** 162 | * @param propsIndexes The propsIndexes to set 163 | */ 164 | public void setPropsIndexes( Map propsIndexes ) { 165 | this.propsIndexes = propsIndexes; 166 | } 167 | 168 | /** 169 | * Gets idFieldName 170 | * 171 | * @return value of idFieldName 172 | */ 173 | public String getIdFieldName() { 174 | return idFieldName; 175 | } 176 | 177 | /** 178 | * @param idFieldName The idFieldName to set 179 | */ 180 | public void setIdFieldName( String idFieldName ) { 181 | this.idFieldName = idFieldName; 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/main/resources/neo4j_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml 27 | 28 | 29 | 46 | 49 | 54 | 59 | 64 | 69 | 74 | 79 | 84 | 89 | 94 | 101 | 108 | 115 | 116 | 119 | 120 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/model/GraphModelUtils.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.model; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.eclipse.swt.SWT; 5 | import org.eclipse.swt.widgets.MessageBox; 6 | import org.eclipse.swt.widgets.Shell; 7 | import org.neo4j.kettle.core.Neo4jDefaults; 8 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 9 | import org.neo4j.kettle.model.GraphModel; 10 | import org.pentaho.di.core.row.RowMetaInterface; 11 | import org.pentaho.di.i18n.BaseMessages; 12 | import org.pentaho.di.ui.core.dialog.ErrorDialog; 13 | import org.pentaho.metastore.api.IMetaStore; 14 | 15 | public class GraphModelUtils { 16 | private static Class PKG = GraphModelUtils.class; // for i18n purposes, needed by Translator2!! 17 | 18 | private static MetaStoreFactory staticFactory; 19 | 20 | public static MetaStoreFactory getModelFactory( IMetaStore metaStore ) { 21 | if ( staticFactory == null ) { 22 | staticFactory = new MetaStoreFactory<>( GraphModel.class, metaStore, Neo4jDefaults.NAMESPACE ); 23 | } 24 | return staticFactory; 25 | } 26 | 27 | public static GraphModel newModel( Shell shell, MetaStoreFactory factory, RowMetaInterface inputRowMeta ) { 28 | GraphModel graphModel = new GraphModel(); 29 | boolean ok = false; 30 | while ( !ok ) { 31 | GraphModelDialog dialog = new GraphModelDialog( shell, graphModel, inputRowMeta ); 32 | if ( dialog.open() ) { 33 | 34 | // write to metastore... 35 | // 36 | try { 37 | if ( factory.loadElement( graphModel.getName() ) != null ) { 38 | MessageBox box = new MessageBox( shell, SWT.YES | SWT.NO | SWT.ICON_ERROR ); 39 | box.setText( BaseMessages.getString( PKG, "GraphModelUtils.Error.ModelExists.Title" ) ); 40 | box.setMessage( BaseMessages.getString( PKG, "GraphModelUtils.Error.ModelExists.Message" ) ); 41 | int answer = box.open(); 42 | if ( ( answer & SWT.YES ) != 0 ) { 43 | factory.saveElement( graphModel ); 44 | ok = true; 45 | } 46 | } else { 47 | factory.saveElement( graphModel ); 48 | ok = true; 49 | } 50 | } catch ( Exception exception ) { 51 | new ErrorDialog( shell, 52 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorSavingModel.Title" ), 53 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorSavingModel.Message" ), 54 | exception ); 55 | return null; 56 | } 57 | } else { 58 | // Cancel 59 | return null; 60 | } 61 | } 62 | return graphModel; 63 | } 64 | 65 | public static void editModel( Shell shell, MetaStoreFactory factory, String modelName, RowMetaInterface inputRowMeta ) { 66 | if ( StringUtils.isEmpty( modelName ) ) { 67 | return; 68 | } 69 | try { 70 | GraphModel GraphModel = factory.loadElement( modelName ); 71 | if ( GraphModel == null ) { 72 | newModel( shell, factory, inputRowMeta ); 73 | } else { 74 | GraphModelDialog GraphModelDialog = new GraphModelDialog( shell, GraphModel, inputRowMeta ); 75 | if ( GraphModelDialog.open() ) { 76 | factory.saveElement( GraphModel ); 77 | } 78 | } 79 | } catch ( Exception exception ) { 80 | new ErrorDialog( shell, 81 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorEditingModel.Title" ), 82 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorEditingModel.Message" ), 83 | exception ); 84 | } 85 | } 86 | 87 | public static void deleteModel( Shell shell, MetaStoreFactory factory, String connectionName ) { 88 | if ( StringUtils.isEmpty( connectionName ) ) { 89 | return; 90 | } 91 | 92 | MessageBox box = new MessageBox( shell, SWT.YES | SWT.NO | SWT.ICON_ERROR ); 93 | box.setText( BaseMessages.getString( PKG, "GraphModelUtils.DeleteModelConfirmation.Title" ) ); 94 | box.setMessage( BaseMessages.getString( PKG, "GraphModelUtils.DeleteModelConfirmation.Message", connectionName ) ); 95 | int answer = box.open(); 96 | if ( ( answer & SWT.YES ) != 0 ) { 97 | try { 98 | factory.deleteElement( connectionName ); 99 | } catch ( Exception exception ) { 100 | new ErrorDialog( shell, 101 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorDeletingModel.Title" ), 102 | BaseMessages.getString( PKG, "GraphModelUtils.Error.ErrorDeletingModel.Message", connectionName ), 103 | exception ); 104 | } 105 | } 106 | } 107 | 108 | } -------------------------------------------------------------------------------- /src/main/java/NEO4J.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml -------------------------------------------------------------------------------- /src/main/java/neo4j_output.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | bi.know.kettle.neo4j.output 6 | Neo4JOutput 7 | 5.0.9 8 | jar 9 | 10 | Neo4JOutput 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 8.2.0.7-719 16 | 4.3.4 17 | 18 | 19 | 20 | install 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-jar-plugin 26 | 2.3.1 27 | 28 | target/Neo4JOutput 29 | 30 | 31 | 32 | org.apache.maven.plugins 33 | maven-dependency-plugin 34 | 2.10 35 | 36 | 37 | copy-dependencies 38 | package 39 | 40 | copy-dependencies 41 | 42 | 43 | target/Neo4JOutput/lib 44 | false 45 | false 46 | true 47 | test 48 | system 49 | runtime 50 | compile 51 | provided 52 | 53 | 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-compiler-plugin 59 | 3.6.1 60 | 61 | 1.8 62 | 1.8 63 | 64 | 65 | 66 | 67 | 68 | 69 | src/main/java 70 | 71 | **/*.properties 72 | **/*.svg 73 | **/*.xul 74 | 75 | 76 | 77 | src/main/resources 78 | 79 | **/*.properties 80 | **/*.svg 81 | **/*.xul 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | neo4j-release-repository 91 | Neo4j Maven 2 release repository 92 | http://m2.neo4j.org/content/repositories/releases/ 93 | 94 | true 95 | 96 | 97 | true 98 | 99 | 100 | 101 | 102 | pentaho-releases 103 | http://nexus.pentaho.org/content/groups/omni 104 | 105 | 106 | 107 | 108 | 109 | 110 | junit 111 | junit 112 | 3.8.1 113 | test 114 | 115 | 116 | 117 | kettle-neo4j 118 | kettle-neo4j-core 119 | 2.0.6 120 | 121 | 122 | 123 | org.neo4j.driver 124 | neo4j-java-driver 125 | ${neo4j.driver.version} 126 | 127 | 128 | 129 | com.google.code.gson 130 | gson 131 | 2.8.5 132 | 133 | 134 | 135 | pentaho-kettle 136 | kettle-core 137 | ${pdi.version} 138 | provided 139 | 140 | 141 | 142 | pentaho-kettle 143 | kettle-engine 144 | ${pdi.version} 145 | provided 146 | 147 | 148 | 149 | pentaho-kettle 150 | kettle-ui-swt 151 | ${pdi.version} 152 | provided 153 | 154 | 155 | 156 | 157 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/split/SplitGraph.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.split; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.neo4j.kettle.core.data.GraphData; 5 | import org.neo4j.kettle.core.data.GraphNodeData; 6 | import org.neo4j.kettle.core.data.GraphRelationshipData; 7 | import org.neo4j.kettle.core.value.ValueMetaGraph; 8 | import org.pentaho.di.core.exception.KettleException; 9 | import org.pentaho.di.core.row.RowDataUtil; 10 | import org.pentaho.di.core.row.ValueMetaInterface; 11 | import org.pentaho.di.trans.Trans; 12 | import org.pentaho.di.trans.TransMeta; 13 | import org.pentaho.di.trans.step.BaseStep; 14 | import org.pentaho.di.trans.step.StepDataInterface; 15 | import org.pentaho.di.trans.step.StepInterface; 16 | import org.pentaho.di.trans.step.StepMeta; 17 | import org.pentaho.di.trans.step.StepMetaInterface; 18 | 19 | public class SplitGraph extends BaseStep implements StepInterface { 20 | 21 | private SplitGraphMeta meta; 22 | private SplitGraphData data; 23 | 24 | /** 25 | * This is the base step that forms that basis for all steps. You can derive from this class to implement your own 26 | * steps. 27 | * 28 | * @param stepMeta The StepMeta object to run. 29 | * @param stepDataInterface the data object to store temporary data, database connections, caches, result sets, 30 | * hashtables etc. 31 | * @param copyNr The copynumber for this step. 32 | * @param transMeta The TransInfo of which the step stepMeta is part of. 33 | * @param trans The (running) transformation to obtain information shared among the steps. 34 | */ 35 | public SplitGraph( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, 36 | Trans trans ) { 37 | super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); 38 | } 39 | 40 | @Override public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { 41 | 42 | meta = (SplitGraphMeta) smi; 43 | data = (SplitGraphData) sdi; 44 | 45 | Object[] row = getRow(); 46 | if ( row == null ) { 47 | setOutputDone(); 48 | return false; 49 | } 50 | 51 | if ( first ) { 52 | first = false; 53 | 54 | data.outputRowMeta = getInputRowMeta().clone(); 55 | meta.getFields( data.outputRowMeta, getStepname(), null, null, getTransMeta(), getRepository(), getMetaStore() ); 56 | 57 | data.graphFieldIndex = getInputRowMeta().indexOfValue( meta.getGraphField() ); 58 | if ( data.graphFieldIndex < 0 ) { 59 | throw new KettleException( "Unable to find graph field " + meta.getGraphField() + "' in the step input" ); 60 | } 61 | ValueMetaInterface valueMeta = getInputRowMeta().getValueMeta( data.graphFieldIndex ); 62 | if ( valueMeta.getType() != ValueMetaGraph.TYPE_GRAPH ) { 63 | throw new KettleException( "Please specify a Graph field to split" ); 64 | } 65 | 66 | data.typeField = null; 67 | if ( StringUtils.isNotEmpty( meta.getTypeField() ) ) { 68 | data.typeField = environmentSubstitute( meta.getTypeField() ); 69 | } 70 | data.idField = null; 71 | if ( StringUtils.isNotEmpty( meta.getIdField() ) ) { 72 | data.idField = environmentSubstitute( meta.getIdField() ); 73 | } 74 | data.propertySetField = null; 75 | if ( StringUtils.isNotEmpty( meta.getPropertySetField() ) ) { 76 | data.propertySetField = environmentSubstitute( meta.getPropertySetField() ); 77 | } 78 | } 79 | 80 | ValueMetaGraph valueMeta = (ValueMetaGraph) getInputRowMeta().getValueMeta( data.graphFieldIndex ); 81 | Object valueData = row[ data.graphFieldIndex ]; 82 | GraphData graphData = valueMeta.getGraphData( valueData ); 83 | 84 | for ( GraphNodeData nodeData : graphData.getNodes() ) { 85 | Object[] outputRowData = RowDataUtil.createResizedCopy( row, data.outputRowMeta.size() ); 86 | int index = getInputRowMeta().size(); 87 | GraphData copy = graphData.createEmptyCopy(); 88 | copy.getNodes().add( nodeData.clone() ); 89 | 90 | outputRowData[ data.graphFieldIndex ] = copy; 91 | if ( data.typeField != null ) { 92 | outputRowData[ index++ ] = "Node"; 93 | } 94 | if ( data.idField != null ) { 95 | outputRowData[ index++ ] = nodeData.getId(); 96 | } 97 | if ( data.propertySetField != null ) { 98 | outputRowData[ index++ ] = nodeData.getPropertySetId(); 99 | } 100 | putRow( data.outputRowMeta, outputRowData ); 101 | } 102 | 103 | for ( GraphRelationshipData relationshipData : graphData.getRelationships() ) { 104 | Object[] outputRowData = RowDataUtil.createResizedCopy( row, data.outputRowMeta.size() ); 105 | int index = getInputRowMeta().size(); 106 | GraphData copy = graphData.createEmptyCopy(); 107 | copy.getRelationships().add( relationshipData.clone() ); 108 | 109 | outputRowData[ data.graphFieldIndex ] = copy; 110 | if ( data.typeField != null ) { 111 | outputRowData[ index++ ] = "Relationship"; 112 | } 113 | if ( data.idField != null ) { 114 | outputRowData[ index++ ] = relationshipData.getId(); 115 | } 116 | if ( data.propertySetField != null ) { 117 | outputRowData[ index++ ] = relationshipData.getPropertySetId(); 118 | } 119 | putRow( data.outputRowMeta, outputRowData ); 120 | } 121 | 122 | return true; 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/neo4j_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 14 | image/svg+xml 15 | 17 | 18 | 19 | 24 | 70 | -------------------------------------------------------------------------------- /src/main/java/neo4j_load.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xmlcsv 164 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/output/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | Neo4JOutput.Step.Name=Neo4J Output 2 | Neo4JOutput.Step.Description=Neo4J Output 3 | Neo4JOutput.Step.Category=Neo4j 4 | Neo4JOutputDialog.Shell.Title=Neo4J Output 5 | Neo4JOutputDialog.StepName.Label=Step name 6 | Neo4JOutputDialog.Protocol.Label=Protocol 7 | Neo4JOutputDialog.Host.Label=Host 8 | Neo4JOutputDialog.Port.Label=Port 9 | Neo4JOutputDialog.Username.Label=Username 10 | Neo4JOutputDialog.Password.Label=Password 11 | Neo4JOutputDialog.ConnectionTest.Success=Connection test worked! 12 | Neo4JOutputDialog.ConnectionTest.Failed=Connection test failed :( 13 | Neo4JOutputDialog.Fields.Label=Fields 14 | Neo4JOutputDialog.FieldsTable.Fields=Node Property Fields 15 | Neo4JOutputDialog.LabelsField.Label=Labels Field 16 | Neo4JOutputDialog.LabelsSeparator.Label=Labels Separated B 17 | Neo4JOutputDialog.KeyField.Label=Unique Key 18 | Neo4JOutputDialog.LabelsTable.Fields=Labels 19 | Neo4JOutputDialog.Fields.Properties=Properties 20 | Neo4JOutputDialog.NodeTab.Label=Node 21 | Neo4JOutputDialog.RelationshipTab.Label=Relationship 22 | Neo4JOutputDialog.Type.Label=Type 23 | Neo4JOutputDialog.RelationshipField.Label = Relationship field 24 | Neo4JOutputDialog.RelationshipValue.Label = Relationship value 25 | Neo4JOutputDialog.RelationshipProperties.Label = Relationship properties 26 | Neo4JOutputDialog.RelationshipTable.FromNodeField=From Node Field 27 | Neo4JOutputDialog.RelationshipTable.FromNodeProperty=From Node Property 28 | Neo4JOutputDialog.RelationshipTable.Relationship=Relationship Field 29 | Neo4JOutputDialog.RelationshipTable.ToNodeField=To Node Field 30 | Neo4JOutputDialog.RelationshipTable.ToNodeProperty=To Node Property 31 | Neo4JOutputDialog.RelPropsTable.PropertiesField=Relationship Properties 32 | Neo4JOutput.addNode=Node added: 33 | Neo4JOutput.addNodeError=Error adding nodes 34 | Neo4JOutput.addNodeLabelError=Error adding label to node 35 | Neo4JOutput.addNodeLabel=Add label to node: 36 | Neo4JOutput.addNodeProperty=Added Property to node: 37 | Neo4JOutput.addNodePropertyError=Error adding property to node 38 | Neo4JOutput.addRelationshipProperty=Add property to relationship 39 | Neo4JOutput.addRelationshipPropertyError=Error adding property to relationship 40 | Neo4JOutput.addRelationshipError=Error adding relationships 41 | Neo4JOutput.addRelationship=Added Relationship 42 | Neo4JOutputDialog.LabelsField.FromLabel=From Label 43 | Neo4JOutputDialog.FromFields.Properties=From Property 44 | Neo4JOutputDialog.FromLabelsTable.FromFields=From Label Fields 45 | Neo4JOutputDialog.FromLabelsTable.FromValues=From Label Values 46 | Neo4JOutputDialog.FromFieldsTable.FromPropFields=From Property Fields 47 | Neo4JOutputDialog.LabelsField.ToLabel=To Node 48 | Neo4JOutputDialog.ToLabelsTable.ToFields=To Label Fields 49 | Neo4JOutputDialog.ToLabelsTable.ToValues=To Label Values 50 | Neo4JOutputDialog.ToFields.Properties=To Property Fields 51 | Neo4JOutputDialog.ToFieldsTable.ToFields=To Property Fields 52 | Neo4JOutputDialog.FromFieldsTable.FromPropFieldsName=Property Name 53 | Neo4JOutputDialog.ToFieldsTable.ToFieldsName=Property Name 54 | Neo4JOutputDialog.Relationship.Label=Relationship field 55 | Neo4JOutputDialog.RelPropsTable.PropertiesFieldName=Relationship property name 56 | Neo4JOutputDialog.FromTab=From Node 57 | Neo4JOutputDialog.ToTab=To Node 58 | Neo4JOutputDialog.RelationshipsTab=Relationship 59 | Neo4JOutputDialog.GetFields.Button=Get Fields 60 | Neo4JOutputDialog.PropType=Property type 61 | Neo4JOutputDialog.PropPrimary=Primary? 62 | 63 | Neo4JOutputDialog.LabelsField.ReadOnlyFromNode = Perform lookups only, do not update the "from" nodes 64 | Neo4JOutputDialog.LabelsField.ReadOnlyToNode = Perform lookups only, do not update the "to" nodes 65 | 66 | Neo4JOutputDialog.Warning.SortDynamicFromLabels = You are using one or more labels defined in an input field in combination with the "Create" option to create the "from" nodes. For performance reasons, make sure the data is sorted on these label fields.{0} 67 | Neo4JOutputDialog.Warning.SortDynamicToLabels = You are using one or more labels defined in an input field in combination with the "Create" option to create the "to" nodes. For performance reasons, make sure the data is sorted on these label fields.{0} 68 | Neo4JOutputDialog.Warning.SortDynamicRelationshipLabel = You are creating relationships using labels defined in fields. For performance reasons make sure the data is sorted on these fields for node(s) label(s) field(s) and the relationship field in that order.{0} 69 | Neo4JOutputDialog.Warning.ReferencedNeo4jConnectionDoesntExist = IMPORTANT: the referenced Neo4j connection ''{0}'' doesn''t exist.{1} 70 | Neo4JOutputDialog.Warning.CreateIndexesIsLimited=When creating indexes for nodes with dynamic labels, this step can only look at the first row. We recommend creating the needed indexes separately.{0} 71 | Neo4JOutputDialog.DynamicLabelsWarning.DialogTitle = Warning! 72 | Neo4JOutputDialog.DynamicLabelsWarning.Understood = Understood! 73 | Neo4JOutputDialog.DynamicLabelsWarning.HideNextTime = Do not show this warning again. 74 | 75 | 76 | Neo4JOutput.Injection.CONNECTION = Neo4j Connection name 77 | Neo4JOutput.Injection.FROM_NODE_PROPS = From node properties 78 | Neo4JOutput.Injection.FROM_NODE_PROPERTY_FIELD = From node property field 79 | Neo4JOutput.Injection.FROM_NODE_PROPERTY_NAME = From node property name 80 | Neo4JOutput.Injection.FROM_NODE_PROPERTY_TYPE = From node property type 81 | Neo4JOutput.Injection.FROM_NODE_PROPERTY_PRIMARY = From node property primary flag 82 | Neo4JOutput.Injection.TO_NODE_PROPS = To node properties 83 | Neo4JOutput.Injection.TO_NODE_PROPERTY_FIELD = To node property field 84 | Neo4JOutput.Injection.TO_NODE_PROPERTY_NAME = To node property name 85 | Neo4JOutput.Injection.TO_NODE_PROPERTY_TYPE = To node property type 86 | Neo4JOutput.Injection.TO_NODE_PROPERTY_PRIMARY = To node property primary flag 87 | Neo4JOutput.Injection.FROM_LABELS = From labels 88 | Neo4JOutput.Injection.FROM_LABEL = From label 89 | Neo4JOutput.Injection.TO_LABELS = To labels 90 | Neo4JOutput.Injection.TO_LABEL = To label 91 | Neo4JOutput.Injection.REL_PROPS = Relationship properties 92 | Neo4JOutput.Injection.REL_PROPERTY_FIELD = Relationship property field 93 | Neo4JOutput.Injection.REL_PROPERTY_NAME = Relationship property name 94 | Neo4JOutput.Injection.REL_PROPERTY_TYPE = Relationship property type 95 | Neo4JOutput.Injection.BATCH_SIZE = Batch size 96 | Neo4JOutput.Injection.CREATE_INDEXES = Create indexes flag 97 | Neo4JOutput.Injection.USE_CREATE = Use create flag 98 | 99 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/entries/check/CheckConnections.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.entries.check; 2 | 3 | import bi.know.kettle.neo4j.shared.MetaStoreUtil; 4 | import org.neo4j.kettle.core.Neo4jDefaults; 5 | import org.neo4j.kettle.shared.NeoConnection; 6 | import org.pentaho.di.cluster.SlaveServer; 7 | import org.pentaho.di.core.Result; 8 | import org.pentaho.di.core.annotations.JobEntry; 9 | import org.pentaho.di.core.database.DatabaseMeta; 10 | import org.pentaho.di.core.exception.KettleException; 11 | import org.pentaho.di.core.exception.KettleXMLException; 12 | import org.pentaho.di.core.xml.XMLHandler; 13 | import org.pentaho.di.job.entry.JobEntryBase; 14 | import org.pentaho.di.job.entry.JobEntryInterface; 15 | import org.pentaho.di.repository.ObjectId; 16 | import org.pentaho.di.repository.Repository; 17 | import org.pentaho.metastore.api.IMetaStore; 18 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 19 | import org.w3c.dom.Node; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | @JobEntry( 25 | id = "NEO4J_CHECK_CONNECTIONS", 26 | name = "Check Neo4j Connections", 27 | description = "Check to see if we can connecto to the listed Neo4j databases", 28 | image = "neo4j_check.svg", 29 | categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Conditions", 30 | documentationUrl = "https://github.com/knowbi/knowbi-pentaho-pdi-neo4j-output/wiki/" 31 | ) 32 | public class CheckConnections extends JobEntryBase implements JobEntryInterface { 33 | 34 | private List connectionNames; 35 | 36 | public CheckConnections() { 37 | this.connectionNames = new ArrayList<>(); 38 | } 39 | 40 | public CheckConnections( String name ) { 41 | this( name, "" ); 42 | } 43 | 44 | public CheckConnections( String name, String description ) { 45 | super( name, description ); 46 | connectionNames = new ArrayList<>(); 47 | } 48 | 49 | @Override public String getXML() { 50 | StringBuilder xml = new StringBuilder(); 51 | // Add entry name, type, ... 52 | // 53 | xml.append( super.getXML() ); 54 | 55 | xml.append( XMLHandler.openTag( "connections" ) ); 56 | 57 | for ( String connectionName : connectionNames ) { 58 | xml.append( XMLHandler.addTagValue( "connection", connectionName ) ); 59 | } 60 | 61 | xml.append( XMLHandler.closeTag( "connections" ) ); 62 | return xml.toString(); 63 | } 64 | 65 | @Override public void loadXML( Node node, List databases, List slaveServers, Repository rep, IMetaStore metaStore ) throws KettleXMLException { 66 | 67 | super.loadXML( node, databases, slaveServers ); 68 | 69 | connectionNames = new ArrayList<>(); 70 | Node connectionsNode = XMLHandler.getSubNode( node, "connections" ); 71 | List connectionNodes = XMLHandler.getNodes( connectionsNode, "connection" ); 72 | for ( Node connectionNode : connectionNodes ) { 73 | String connectionName = XMLHandler.getNodeValue( connectionNode ); 74 | connectionNames.add( connectionName ); 75 | } 76 | } 77 | 78 | @Override public void saveRep( Repository rep, IMetaStore metaStore, ObjectId jobId ) throws KettleException { 79 | for ( int i = 0; i < connectionNames.size(); i++ ) { 80 | rep.saveJobEntryAttribute( jobId, getObjectId(), i, "connection", connectionNames.get( i ) ); 81 | } 82 | } 83 | 84 | @Override public void loadRep( Repository rep, IMetaStore metaStore, ObjectId jobEntryId, List databases, List slaveServers ) throws KettleException { 85 | connectionNames = new ArrayList<>(); 86 | int nrConnections = rep.countNrJobEntryAttributes( jobEntryId, "connection" ); 87 | for ( int i = 0; i < nrConnections; i++ ) { 88 | connectionNames.add( rep.getJobEntryAttributeString( jobEntryId, i, "connection" ) ); 89 | } 90 | } 91 | 92 | @Override public Result execute( Result result, int nr ) throws KettleException { 93 | 94 | try { 95 | metaStore = MetaStoreUtil.findMetaStore( this ); 96 | } catch ( Exception e ) { 97 | throw new KettleException( "Error finding metastore", e ); 98 | } 99 | MetaStoreFactory connectionFactory = new MetaStoreFactory<>( NeoConnection.class, metaStore, Neo4jDefaults.NAMESPACE ); 100 | 101 | // Replace variables & parameters 102 | // 103 | List realConnectionNames = new ArrayList<>(); 104 | for ( String connectionName : connectionNames ) { 105 | realConnectionNames.add( environmentSubstitute( connectionName ) ); 106 | } 107 | 108 | // Check all the connections. If any one fails, fail the step 109 | // Check 'm all though, report on all, nr of errors is nr of failed connections 110 | // 111 | int testCount = 0; 112 | for ( String connectionName : realConnectionNames ) { 113 | testCount++; 114 | try { 115 | NeoConnection connection = connectionFactory.loadElement( connectionName ); 116 | if ( connection == null ) { 117 | throw new KettleException( "Unable to find connection with name '" + connectionName + "'" ); 118 | } 119 | connection.initializeVariablesFrom( this ); 120 | connection.test(); 121 | } catch ( Exception e ) { 122 | // Something bad happened, log the error, flag error 123 | // 124 | result.increaseErrors( 1 ); 125 | result.setResult( false ); 126 | logError( "Error on connection: " + connectionName, e ); 127 | } 128 | } 129 | 130 | if ( result.getNrErrors() == 0 ) { 131 | logBasic( testCount + " Neo4j connections tested without error" ); 132 | } else { 133 | logBasic( testCount + " Neo4j connections tested with " + result.getNrErrors() + " error(s)" ); 134 | } 135 | 136 | return result; 137 | } 138 | 139 | @Override public String getDialogClassName() { 140 | return super.getDialogClassName(); 141 | } 142 | 143 | /** 144 | * Gets connectionNames 145 | * 146 | * @return value of connectionNames 147 | */ 148 | public List getConnectionNames() { 149 | return connectionNames; 150 | } 151 | 152 | /** 153 | * @param connectionNames The connectionNames to set 154 | */ 155 | public void setConnectionNames( List connectionNames ) { 156 | this.connectionNames = connectionNames; 157 | } 158 | 159 | @Override public boolean evaluates() { 160 | return true; 161 | } 162 | 163 | @Override public boolean isUnconditional() { 164 | return false; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/perspective/Neo4jHelper.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho Data Integration 4 | * 5 | * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com 6 | * 7 | ******************************************************************************* 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with 11 | * the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | ******************************************************************************/ 22 | 23 | package bi.know.kettle.neo4j.perspective; 24 | 25 | import bi.know.kettle.neo4j.model.GraphModelUtils; 26 | import bi.know.kettle.neo4j.shared.NeoConnectionUtils; 27 | import org.neo4j.kettle.core.Neo4jDefaults; 28 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 29 | import org.neo4j.kettle.model.GraphModel; 30 | import org.neo4j.kettle.shared.NeoConnection; 31 | import org.pentaho.di.core.variables.VariableSpace; 32 | import org.pentaho.di.core.variables.Variables; 33 | import org.pentaho.di.ui.core.dialog.EnterSelectionDialog; 34 | import org.pentaho.di.ui.core.dialog.ErrorDialog; 35 | import org.pentaho.di.ui.spoon.ISpoonMenuController; 36 | import org.pentaho.di.ui.spoon.Spoon; 37 | import org.pentaho.di.ui.spoon.dialog.MetaStoreExplorerDialog; 38 | import org.pentaho.ui.xul.dom.Document; 39 | import org.pentaho.ui.xul.impl.AbstractXulEventHandler; 40 | 41 | import java.util.Collections; 42 | import java.util.List; 43 | 44 | public class Neo4jHelper extends AbstractXulEventHandler implements ISpoonMenuController { 45 | protected static Class PKG = Neo4jHelper.class; // for i18n 46 | 47 | private static Neo4jHelper instance = null; 48 | 49 | private Spoon spoon; 50 | private MetaStoreFactory connectionFactory; 51 | private MetaStoreFactory modelFactory; 52 | private VariableSpace space; 53 | 54 | private Neo4jHelper() { 55 | spoon = Spoon.getInstance(); 56 | } 57 | 58 | public static Neo4jHelper getInstance() { 59 | if ( instance == null ) { 60 | instance = new Neo4jHelper(); ; 61 | instance.spoon.addSpoonMenuController( instance ); 62 | instance.space = new Variables(); 63 | instance.space.initializeVariablesFrom( null ); 64 | instance.connectionFactory = new MetaStoreFactory( NeoConnection.class, instance.spoon.getMetaStore(), Neo4jDefaults.NAMESPACE ); 65 | instance.modelFactory = new MetaStoreFactory( GraphModel.class, instance.spoon.getMetaStore(), Neo4jDefaults.NAMESPACE ); 66 | } 67 | return instance; 68 | } 69 | 70 | public String getName() { 71 | return "neo4jHelper"; 72 | } 73 | 74 | public void updateMenu( Document doc ) { 75 | // Nothing so far. 76 | } 77 | 78 | public void createConnection() { 79 | NeoConnectionUtils.newConnection( spoon.getShell(), space, connectionFactory ); 80 | } 81 | 82 | public void editConnection() { 83 | try { 84 | List elementNames = connectionFactory.getElementNames(); 85 | Collections.sort(elementNames); 86 | String[] names = elementNames.toArray( new String[ 0 ] ); 87 | 88 | EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Edit Neo4j connection", "Select the connection to edit" ); 89 | String choice = dialog.open(); 90 | if (choice!=null) { 91 | NeoConnectionUtils.editConnection( spoon.getShell(), space, connectionFactory, choice ); 92 | } 93 | } catch(Exception e) { 94 | new ErrorDialog( spoon.getShell(), "Error", "Error editing Neo4j connection", e ); 95 | } 96 | } 97 | 98 | public void deleteConnection() { 99 | try { 100 | List elementNames = connectionFactory.getElementNames(); 101 | Collections.sort( elementNames ); 102 | String[] names = elementNames.toArray( new String[ 0 ] ); 103 | 104 | EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Delete Neo4j connection", "Select the connection to delete" ); 105 | String choice = dialog.open(); 106 | if ( choice != null ) { 107 | NeoConnectionUtils.deleteConnection( spoon.getShell(), connectionFactory, choice ); 108 | } 109 | } catch(Exception e) { 110 | new ErrorDialog( spoon.getShell(), "Error", "Error deleting Neo4j connection", e ); 111 | } 112 | } 113 | 114 | public void createModel() { 115 | GraphModelUtils.newModel( spoon.getShell(), modelFactory, null); 116 | } 117 | 118 | public void editModel() { 119 | try { 120 | List elementNames = modelFactory.getElementNames(); 121 | Collections.sort(elementNames); 122 | String[] names = elementNames.toArray( new String[ 0 ] ); 123 | 124 | EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Edit Neo4j model", "Select the graph model to edit" ); 125 | String choice = dialog.open(); 126 | if (choice!=null) { 127 | GraphModelUtils.editModel( spoon.getShell(), modelFactory, choice, null); 128 | } 129 | } catch(Exception e) { 130 | new ErrorDialog( spoon.getShell(), "Error", "Error editing Neo4j graph model", e ); 131 | } 132 | } 133 | 134 | public void deleteModel() { 135 | try { 136 | List elementNames = modelFactory.getElementNames(); 137 | Collections.sort( elementNames ); 138 | String[] names = elementNames.toArray( new String[ 0 ] ); 139 | 140 | EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Delete Neo4j model", "Select the graph model to delete" ); 141 | String choice = dialog.open(); 142 | if ( choice != null ) { 143 | GraphModelUtils.deleteModel( spoon.getShell(), modelFactory, choice ); 144 | } 145 | } catch(Exception e) { 146 | new ErrorDialog( spoon.getShell(), "Error", "Error deleting Neo4j graph model", e ); 147 | } 148 | } 149 | 150 | public void showMetaStoreBrowser() { 151 | Spoon spoon = Spoon.getInstance(); 152 | new MetaStoreExplorerDialog( spoon.getShell(), spoon.getMetaStore() ).open(); 153 | } 154 | 155 | } 156 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/shared/NeoConnectionUtils.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.shared; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.eclipse.swt.SWT; 5 | import org.eclipse.swt.widgets.MessageBox; 6 | import org.eclipse.swt.widgets.Shell; 7 | import org.neo4j.driver.Session; 8 | import org.neo4j.kettle.core.Neo4jDefaults; 9 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 10 | import org.neo4j.kettle.shared.NeoConnection; 11 | import org.pentaho.di.core.logging.LogChannelInterface; 12 | import org.pentaho.di.core.variables.VariableSpace; 13 | import org.pentaho.di.i18n.BaseMessages; 14 | import org.pentaho.di.ui.core.dialog.ErrorDialog; 15 | import org.pentaho.metastore.api.IMetaStore; 16 | 17 | import java.util.List; 18 | 19 | public class NeoConnectionUtils { 20 | private static Class PKG = NeoConnectionUtils.class; // for i18n purposes, needed by Translator2!! 21 | 22 | private static MetaStoreFactory staticFactory; 23 | 24 | public static MetaStoreFactory getConnectionFactory( IMetaStore metaStore ) { 25 | if ( staticFactory == null ) { 26 | staticFactory = new MetaStoreFactory<>( NeoConnection.class, metaStore, Neo4jDefaults.NAMESPACE ); 27 | } 28 | return staticFactory; 29 | } 30 | 31 | public static NeoConnection newConnection( Shell shell, VariableSpace space, MetaStoreFactory factory ) { 32 | NeoConnection connection = new NeoConnection( space ); 33 | boolean ok = false; 34 | while ( !ok ) { 35 | NeoConnectionDialog dialog = new NeoConnectionDialog( shell, connection ); 36 | if ( dialog.open() ) { 37 | // write to metastore... 38 | try { 39 | if ( factory.loadElement( connection.getName() ) != null ) { 40 | MessageBox box = new MessageBox( shell, SWT.YES | SWT.NO | SWT.ICON_ERROR ); 41 | box.setText( BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ConnectionExists.Title" ) ); 42 | box.setMessage( BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ConnectionExists.Message" ) ); 43 | int answer = box.open(); 44 | if ( ( answer & SWT.YES ) != 0 ) { 45 | factory.saveElement( connection ); 46 | ok = true; 47 | } 48 | } else { 49 | factory.saveElement( connection ); 50 | ok = true; 51 | } 52 | } catch ( Exception exception ) { 53 | new ErrorDialog( shell, 54 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorSavingConnection.Title" ), 55 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorSavingConnection.Message" ), 56 | exception ); 57 | return null; 58 | } 59 | } else { 60 | // Cancel 61 | return null; 62 | } 63 | } 64 | return connection; 65 | } 66 | 67 | public static void editConnection( Shell shell, VariableSpace space, MetaStoreFactory factory, String connectionName ) { 68 | if ( StringUtils.isEmpty( connectionName ) ) { 69 | return; 70 | } 71 | try { 72 | NeoConnection neoConnection = factory.loadElement( connectionName ); 73 | neoConnection.initializeVariablesFrom( space ); 74 | if ( neoConnection == null ) { 75 | newConnection( shell, space, factory ); 76 | } else { 77 | NeoConnectionDialog neoConnectionDialog = new NeoConnectionDialog( shell, neoConnection ); 78 | if ( neoConnectionDialog.open() ) { 79 | factory.saveElement( neoConnection ); 80 | } 81 | } 82 | } catch ( Exception exception ) { 83 | new ErrorDialog( shell, 84 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorEditingConnection.Title" ), 85 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorEditingConnection.Message" ), 86 | exception ); 87 | } 88 | } 89 | 90 | public static void deleteConnection( Shell shell, MetaStoreFactory factory, String connectionName ) { 91 | if ( StringUtils.isEmpty( connectionName ) ) { 92 | return; 93 | } 94 | 95 | MessageBox box = new MessageBox( shell, SWT.YES | SWT.NO | SWT.ICON_ERROR ); 96 | box.setText( BaseMessages.getString( PKG, "NeoConnectionUtils.DeleteConnectionConfirmation.Title" ) ); 97 | box.setMessage( BaseMessages.getString( PKG, "NeoConnectionUtils.DeleteConnectionConfirmation.Message", connectionName ) ); 98 | int answer = box.open(); 99 | if ( ( answer & SWT.YES ) != 0 ) { 100 | try { 101 | factory.deleteElement( connectionName ); 102 | } catch ( Exception exception ) { 103 | new ErrorDialog( shell, 104 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorDeletingConnection.Title" ), 105 | BaseMessages.getString( PKG, "NeoConnectionUtils.Error.ErrorDeletingConnection.Message", connectionName ), 106 | exception ); 107 | } 108 | } 109 | } 110 | 111 | public static final void createNodeIndex( LogChannelInterface log, Session session, List labels, List keyProperties ) { 112 | 113 | // If we have no properties or labels, we have nothing to do here 114 | // 115 | if ( keyProperties.size() == 0 ) { 116 | return; 117 | } 118 | if ( labels.size() == 0 ) { 119 | return; 120 | } 121 | 122 | // We only use the first label for index or constraint 123 | // 124 | String labelsClause = ":" + labels.get( 0 ); 125 | 126 | // CREATE CONSTRAINT ON (n:NodeLabel) ASSERT n.property1 IS UNIQUE 127 | // 128 | if ( keyProperties.size() == 1 ) { 129 | String property = keyProperties.get( 0 ); 130 | String constraintCypher = "CREATE CONSTRAINT ON (n" + labelsClause + ") ASSERT n." + property + " IS UNIQUE;"; 131 | 132 | log.logDetailed( "Creating constraint : " + constraintCypher ); 133 | session.run( constraintCypher ); 134 | 135 | // This creates an index, no need to go further here... 136 | // 137 | return; 138 | } 139 | 140 | // Composite index case... 141 | // 142 | // CREATE INDEX ON :NodeLabel(property, property2, ...); 143 | // 144 | String indexCypher = "CREATE INDEX ON "; 145 | 146 | indexCypher += labelsClause; 147 | indexCypher += "("; 148 | boolean firstProperty = true; 149 | for ( String property : keyProperties ) { 150 | if ( firstProperty ) { 151 | firstProperty = false; 152 | } else { 153 | indexCypher += ", "; 154 | } 155 | indexCypher += property; 156 | } 157 | indexCypher += ")"; 158 | 159 | log.logDetailed( "Creating index : " + indexCypher ); 160 | session.run( indexCypher ); 161 | } 162 | 163 | } 164 | -------------------------------------------------------------------------------- /src/main/java/neo4j_check.svg: -------------------------------------------------------------------------------- 1 | 2 | image/svg+xml -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/split/SplitGraphMeta.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.split; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.pentaho.di.core.annotations.Step; 5 | import org.pentaho.di.core.database.DatabaseMeta; 6 | import org.pentaho.di.core.exception.KettleException; 7 | import org.pentaho.di.core.exception.KettleStepException; 8 | import org.pentaho.di.core.exception.KettleXMLException; 9 | import org.pentaho.di.core.row.RowMetaInterface; 10 | import org.pentaho.di.core.row.value.ValueMetaString; 11 | import org.pentaho.di.core.variables.VariableSpace; 12 | import org.pentaho.di.core.xml.XMLHandler; 13 | import org.pentaho.di.repository.ObjectId; 14 | import org.pentaho.di.repository.Repository; 15 | import org.pentaho.di.trans.Trans; 16 | import org.pentaho.di.trans.TransMeta; 17 | import org.pentaho.di.trans.step.BaseStepMeta; 18 | import org.pentaho.di.trans.step.StepDataInterface; 19 | import org.pentaho.di.trans.step.StepInterface; 20 | import org.pentaho.di.trans.step.StepMeta; 21 | import org.pentaho.di.trans.step.StepMetaInterface; 22 | import org.pentaho.metastore.api.IMetaStore; 23 | import org.w3c.dom.Node; 24 | 25 | import java.util.List; 26 | 27 | @Step( 28 | id = "Neo4jSplitGraph", 29 | name = "Neo4j Split Graph", 30 | description = "Splits the nodes and relationships of a graph data type", 31 | image = "neo4j_split.svg", 32 | categoryDescription = "Neo4j", 33 | documentationUrl = "https://github.com/knowbi/knowbi-pentaho-pdi-neo4j-output/wiki/" 34 | ) 35 | public class SplitGraphMeta extends BaseStepMeta implements StepMetaInterface { 36 | 37 | public static final String GRAPH_FIELD = "graph_field"; 38 | public static final String TYPE_FIELD = "type_field"; 39 | public static final String ID_FIELD = "id_field"; 40 | public static final String PROPERTY_SET_FIELD = "property_set_field"; 41 | 42 | protected String graphField; 43 | protected String typeField; 44 | protected String idField; 45 | protected String propertySetField; 46 | 47 | @Override public void setDefault() { 48 | graphField = "graph"; 49 | typeField = "type"; 50 | idField = "id"; 51 | propertySetField = "propertySet"; 52 | } 53 | 54 | @Override public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) 55 | throws KettleStepException { 56 | if ( StringUtils.isNotEmpty( typeField ) ) { 57 | ValueMetaString typeValueMeta = new ValueMetaString(space.environmentSubstitute( typeField )); 58 | typeValueMeta.setOrigin( name ); 59 | inputRowMeta.addValueMeta( typeValueMeta ); 60 | } 61 | if ( StringUtils.isNotEmpty( idField ) ) { 62 | ValueMetaString idValueMeta = new ValueMetaString(space.environmentSubstitute( idField )); 63 | idValueMeta.setOrigin( name ); 64 | inputRowMeta.addValueMeta( idValueMeta ); 65 | } 66 | if ( StringUtils.isNotEmpty( propertySetField ) ) { 67 | ValueMetaString propertySetValueMeta = new ValueMetaString(space.environmentSubstitute( propertySetField )); 68 | propertySetValueMeta.setOrigin( name ); 69 | inputRowMeta.addValueMeta( propertySetValueMeta ); 70 | } 71 | } 72 | 73 | @Override public String getXML() throws KettleException { 74 | StringBuffer xml = new StringBuffer(); 75 | xml.append( XMLHandler.addTagValue( GRAPH_FIELD, graphField ) ); 76 | xml.append( XMLHandler.addTagValue( TYPE_FIELD, typeField ) ); 77 | xml.append( XMLHandler.addTagValue( ID_FIELD, idField ) ); 78 | xml.append( XMLHandler.addTagValue( PROPERTY_SET_FIELD, propertySetField ) ); 79 | return xml.toString(); 80 | } 81 | 82 | @Override public void loadXML( Node stepnode, List databases, IMetaStore metaStore ) throws KettleXMLException { 83 | graphField = XMLHandler.getTagValue( stepnode, GRAPH_FIELD ); 84 | typeField = XMLHandler.getTagValue( stepnode, TYPE_FIELD ); 85 | idField = XMLHandler.getTagValue( stepnode, ID_FIELD ); 86 | propertySetField = XMLHandler.getTagValue( stepnode, PROPERTY_SET_FIELD); 87 | } 88 | 89 | @Override public void saveRep( Repository rep, IMetaStore metaStore, ObjectId transformationId, ObjectId stepId ) throws KettleException { 90 | rep.saveStepAttribute( transformationId, stepId, GRAPH_FIELD, graphField ); 91 | rep.saveStepAttribute( transformationId, stepId, TYPE_FIELD, typeField ); 92 | rep.saveStepAttribute( transformationId, stepId, ID_FIELD, idField ); 93 | rep.saveStepAttribute( transformationId, stepId, PROPERTY_SET_FIELD, propertySetField ); 94 | } 95 | 96 | @Override public void readRep( Repository rep, IMetaStore metaStore, ObjectId stepId, List databases ) throws KettleException { 97 | graphField = rep.getStepAttributeString( stepId, GRAPH_FIELD ); 98 | typeField = rep.getStepAttributeString( stepId, TYPE_FIELD ); 99 | idField = rep.getStepAttributeString( stepId, ID_FIELD ); 100 | propertySetField = rep.getStepAttributeString( stepId, PROPERTY_SET_FIELD ); 101 | } 102 | 103 | @Override public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans ) { 104 | return new SplitGraph( stepMeta, stepDataInterface, copyNr, transMeta, trans ); 105 | } 106 | 107 | @Override public StepDataInterface getStepData() { 108 | return new SplitGraphData(); 109 | } 110 | 111 | /** 112 | * Gets graphField 113 | * 114 | * @return value of graphField 115 | */ 116 | public String getGraphField() { 117 | return graphField; 118 | } 119 | 120 | /** 121 | * @param graphField The graphField to set 122 | */ 123 | public void setGraphField( String graphField ) { 124 | this.graphField = graphField; 125 | } 126 | 127 | /** 128 | * Gets typeField 129 | * 130 | * @return value of typeField 131 | */ 132 | public String getTypeField() { 133 | return typeField; 134 | } 135 | 136 | /** 137 | * @param typeField The typeField to set 138 | */ 139 | public void setTypeField( String typeField ) { 140 | this.typeField = typeField; 141 | } 142 | 143 | /** 144 | * Gets idField 145 | * 146 | * @return value of idField 147 | */ 148 | public String getIdField() { 149 | return idField; 150 | } 151 | 152 | /** 153 | * @param idField The idField to set 154 | */ 155 | public void setIdField( String idField ) { 156 | this.idField = idField; 157 | } 158 | 159 | /** 160 | * Gets propertySetField 161 | * 162 | * @return value of propertySetField 163 | */ 164 | public String getPropertySetField() { 165 | return propertySetField; 166 | } 167 | 168 | /** 169 | * @param propertySetField The propertySetField to set 170 | */ 171 | public void setPropertySetField( String propertySetField ) { 172 | this.propertySetField = propertySetField; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/IndexedGraphData.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | 4 | import org.neo4j.kettle.core.data.GraphData; 5 | import org.neo4j.kettle.core.data.GraphNodeData; 6 | import org.neo4j.kettle.core.data.GraphPropertyData; 7 | import org.neo4j.kettle.core.data.GraphRelationshipData; 8 | import org.pentaho.di.core.exception.KettleException; 9 | 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.HashSet; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.Set; 16 | 17 | import static bi.know.kettle.neo4j.steps.gencsv.UniquenessStrategy.None; 18 | 19 | /** 20 | * Adds an index to graph data. 21 | * This allows us to quickly lookup nodes and relationships 22 | */ 23 | public class IndexedGraphData extends GraphData { 24 | 25 | protected UniquenessStrategy nodeUniquenessStrategy; 26 | protected UniquenessStrategy relationshipUniquenessStrategy; 27 | 28 | protected Map nodeIdMap; 29 | protected Map relIdMap; 30 | 31 | protected List nodeCollisionListeners; 32 | protected List relCollisionListeners; 33 | protected Set nodePropertiesSet; 34 | protected Set relPropertiesSet; 35 | 36 | public IndexedGraphData() { 37 | super(); 38 | 39 | nodeUniquenessStrategy = None; 40 | relationshipUniquenessStrategy = None; 41 | 42 | nodeIdMap = new HashMap<>(); 43 | relIdMap = new HashMap<>(); 44 | 45 | nodeCollisionListeners = new ArrayList<>(); 46 | relCollisionListeners = new ArrayList<>(); 47 | 48 | nodePropertiesSet = new HashSet<>(); 49 | relPropertiesSet = new HashSet<>(); 50 | } 51 | 52 | public IndexedGraphData( UniquenessStrategy nodeUniquenessStrategy, UniquenessStrategy relationshipUniquenessStrategy ) { 53 | this(); 54 | this.nodeUniquenessStrategy = nodeUniquenessStrategy; 55 | this.relationshipUniquenessStrategy = relationshipUniquenessStrategy; 56 | } 57 | 58 | public Integer addAndIndexNode( GraphNodeData node ) throws KettleException { 59 | 60 | Integer index = nodeIdMap.get( node.getId() ); 61 | if ( index == null ) { 62 | getNodes().add( node ); 63 | nodeIdMap.put( node.getId(), getNodes().size() - 1 ); 64 | } else { 65 | 66 | GraphNodeData existingNode = getNodes().get(index); 67 | GraphNodeData updatingNode = node; 68 | 69 | switch(nodeUniquenessStrategy) { 70 | case Last: // Replace with the updating node 71 | getNodes().set(index, updatingNode); 72 | break; 73 | case First: // Simply keep the existing node 74 | default: 75 | break; 76 | } 77 | 78 | // Also call the listeners 79 | // 80 | for ( NodeCollisionListener listener : nodeCollisionListeners ) { 81 | listener.handleCollission( getNodes().get( index ), node ); 82 | } 83 | } 84 | for ( GraphPropertyData property : node.getProperties() ) { 85 | nodePropertiesSet.add( new IdType( property.getId(), property.getType()) ); 86 | } 87 | return index; 88 | } 89 | 90 | public Integer addAndIndexRelationship( GraphRelationshipData relationship ) throws KettleException { 91 | 92 | // If we don't have a uniqueness strategy: just add the node... 93 | // 94 | Integer index = relIdMap.get( relationship.getId() ); 95 | if ( index == null ) { 96 | getRelationships().add( relationship ); 97 | relIdMap.put( relationship.getId(), getRelationships().size() - 1 ); 98 | } else { 99 | 100 | GraphRelationshipData existingRelationship = getRelationships().get(index); 101 | GraphRelationshipData updatingRelationship = relationship; 102 | 103 | switch(relationshipUniquenessStrategy) { 104 | case Last: // Replace with the updating node 105 | getRelationships().set(index, updatingRelationship); 106 | break; 107 | case First: // Simply keep the existing node 108 | default: 109 | break; 110 | } 111 | 112 | for ( RelationshipCollisionListener listener : relCollisionListeners ) { 113 | listener.handleCollission( getRelationships().get( index ), relationship ); 114 | } 115 | } 116 | for ( GraphPropertyData property : relationship.getProperties() ) { 117 | relPropertiesSet.add( new IdType( property.getId(), property.getType()) ); 118 | } 119 | return index; 120 | } 121 | 122 | public void clearAll() { 123 | nodes.clear(); 124 | relationships.clear(); 125 | nodeIdMap.clear(); 126 | relIdMap.clear(); 127 | } 128 | 129 | /** 130 | * Gets nodeIdMap 131 | * 132 | * @return value of nodeIdMap 133 | */ 134 | public Map getNodeIdMap() { 135 | return nodeIdMap; 136 | } 137 | 138 | /** 139 | * @param nodeIdMap The nodeIdMap to set 140 | */ 141 | public void setNodeIdMap( Map nodeIdMap ) { 142 | this.nodeIdMap = nodeIdMap; 143 | } 144 | 145 | /** 146 | * Gets relIdMap 147 | * 148 | * @return value of relIdMap 149 | */ 150 | public Map getRelIdMap() { 151 | return relIdMap; 152 | } 153 | 154 | /** 155 | * @param relIdMap The relIdMap to set 156 | */ 157 | public void setRelIdMap( Map relIdMap ) { 158 | this.relIdMap = relIdMap; 159 | } 160 | 161 | /** 162 | * Gets nodeCollisionListeners 163 | * 164 | * @return value of nodeCollisionListeners 165 | */ 166 | public List getNodeCollisionListeners() { 167 | return nodeCollisionListeners; 168 | } 169 | 170 | /** 171 | * @param nodeCollisionListeners The nodeCollisionListeners to set 172 | */ 173 | public void setNodeCollisionListeners( List nodeCollisionListeners ) { 174 | this.nodeCollisionListeners = nodeCollisionListeners; 175 | } 176 | 177 | /** 178 | * Gets relCollisionListeners 179 | * 180 | * @return value of relCollisionListeners 181 | */ 182 | public List getRelCollisionListeners() { 183 | return relCollisionListeners; 184 | } 185 | 186 | /** 187 | * @param relCollisionListeners The relCollisionListeners to set 188 | */ 189 | public void setRelCollisionListeners( List relCollisionListeners ) { 190 | this.relCollisionListeners = relCollisionListeners; 191 | } 192 | 193 | /** 194 | * Gets nodePropertiesSet 195 | * 196 | * @return value of nodePropertiesSet 197 | */ 198 | public Set getNodePropertiesSet() { 199 | return nodePropertiesSet; 200 | } 201 | 202 | /** 203 | * @param nodePropertiesSet The nodePropertiesSet to set 204 | */ 205 | public void setNodePropertiesSet( Set nodePropertiesSet ) { 206 | this.nodePropertiesSet = nodePropertiesSet; 207 | } 208 | 209 | /** 210 | * Gets relPropertiesSet 211 | * 212 | * @return value of relPropertiesSet 213 | */ 214 | public Set getRelPropertiesSet() { 215 | return relPropertiesSet; 216 | } 217 | 218 | /** 219 | * @param relPropertiesSet The relPropertiesSet to set 220 | */ 221 | public void setRelPropertiesSet( Set relPropertiesSet ) { 222 | this.relPropertiesSet = relPropertiesSet; 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/importer/Importer.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.importer; 2 | 3 | import bi.know.kettle.neo4j.steps.gencsv.StreamConsumer; 4 | import org.apache.commons.io.FileUtils; 5 | import org.apache.commons.lang.StringUtils; 6 | import org.pentaho.di.core.exception.KettleException; 7 | import org.pentaho.di.core.logging.LogLevel; 8 | import org.pentaho.di.trans.Trans; 9 | import org.pentaho.di.trans.TransMeta; 10 | import org.pentaho.di.trans.step.BaseStep; 11 | import org.pentaho.di.trans.step.StepDataInterface; 12 | import org.pentaho.di.trans.step.StepInterface; 13 | import org.pentaho.di.trans.step.StepMeta; 14 | import org.pentaho.di.trans.step.StepMetaInterface; 15 | 16 | import java.io.File; 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | public class Importer extends BaseStep implements StepInterface { 22 | 23 | private ImporterMeta meta; 24 | private ImporterData data; 25 | 26 | /** 27 | * This is the base step that forms that basis for all steps. You can derive from this class to implement your own 28 | * steps. 29 | * 30 | * @param stepMeta The StepMeta object to run. 31 | * @param stepDataInterface the data object to store temporary data, database connections, caches, result sets, 32 | * hashtables etc. 33 | * @param copyNr The copynumber for this step. 34 | * @param transMeta The TransInfo of which the step stepMeta is part of. 35 | * @param trans The (running) transformation to obtain information shared among the steps. 36 | */ 37 | public Importer( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, 38 | Trans trans ) { 39 | super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); 40 | } 41 | 42 | @Override public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { 43 | 44 | meta = (ImporterMeta) smi; 45 | data = (ImporterData) sdi; 46 | 47 | Object[] row = getRow(); 48 | if ( row == null ) { 49 | if ( ( data.nodesFiles!=null && !data.nodesFiles.isEmpty() ) || ( data.relsFiles!=null && !data.relsFiles.isEmpty() ) ) { 50 | runImport(); 51 | } 52 | setOutputDone(); 53 | return false; 54 | } 55 | 56 | if ( first ) { 57 | first = false; 58 | 59 | data.nodesFiles = new ArrayList<>(); 60 | data.relsFiles = new ArrayList<>(); 61 | 62 | data.filenameFieldIndex = getInputRowMeta().indexOfValue( meta.getFilenameField() ); 63 | if ( data.filenameFieldIndex < 0 ) { 64 | throw new KettleException( "Unable to find filename field " + meta.getFilenameField() + "' in the step input" ); 65 | } 66 | data.fileTypeFieldIndex = getInputRowMeta().indexOfValue( meta.getFileTypeField() ); 67 | if ( data.fileTypeFieldIndex < 0 ) { 68 | throw new KettleException( "Unable to find file type field " + meta.getFileTypeField() + "' in the step input" ); 69 | } 70 | 71 | if ( StringUtils.isEmpty( meta.getAdminCommand() ) ) { 72 | data.adminCommand = "neo4j-admin"; 73 | } else { 74 | data.adminCommand = environmentSubstitute( meta.getAdminCommand() ); 75 | } 76 | 77 | data.databaseFilename = environmentSubstitute( meta.getDatabaseFilename() ); 78 | data.reportFile = environmentSubstitute( meta.getReportFile() ); 79 | 80 | data.baseFolder = environmentSubstitute( meta.getBaseFolder() ); 81 | if ( !data.baseFolder.endsWith( File.separator ) ) { 82 | data.baseFolder += File.separator; 83 | } 84 | 85 | data.importFolder = data.baseFolder + "import/"; 86 | 87 | data.readBufferSize = environmentSubstitute( meta.getReadBufferSize() ); 88 | 89 | data.maxMemory = environmentSubstitute( meta.getMaxMemory() ); 90 | 91 | } 92 | 93 | String filename = getInputRowMeta().getString( row, data.filenameFieldIndex ); 94 | String fileType = getInputRowMeta().getString( row, data.fileTypeFieldIndex ); 95 | 96 | if ( StringUtils.isNotEmpty( filename ) && StringUtils.isNotEmpty( fileType ) ) { 97 | 98 | if ( "Node".equalsIgnoreCase( fileType ) || 99 | "Nodes".equalsIgnoreCase( fileType ) || 100 | "N".equalsIgnoreCase( fileType ) ) { 101 | data.nodesFiles.add( filename ); 102 | } 103 | if ( "Relationship".equalsIgnoreCase( fileType ) || 104 | "Relationships".equalsIgnoreCase( fileType ) || 105 | "Rel".equalsIgnoreCase( fileType ) || 106 | "Rels".equalsIgnoreCase( fileType ) || 107 | "R".equalsIgnoreCase( fileType ) || 108 | "Edge".equalsIgnoreCase( fileType ) || 109 | "Edges".equalsIgnoreCase( fileType ) || 110 | "E".equalsIgnoreCase( fileType ) ) { 111 | data.relsFiles.add( filename ); 112 | } 113 | } 114 | 115 | // Pay it forward 116 | // 117 | putRow( getInputRowMeta(), row ); 118 | 119 | return true; 120 | } 121 | 122 | 123 | private void runImport() throws KettleException { 124 | 125 | // See if we need to delete the existing database folder... 126 | // 127 | String targetDbFolder = data.baseFolder + "data/databases/" + data.databaseFilename; 128 | try { 129 | if ( new File( targetDbFolder ).exists() ) { 130 | log.logBasic( "Removing exsting folder: " + targetDbFolder ); 131 | FileUtils.deleteDirectory( new File( targetDbFolder ) ); 132 | } 133 | } catch ( Exception e ) { 134 | throw new KettleException( "Unable to remove old database files from '" + targetDbFolder + "'", e ); 135 | } 136 | 137 | List arguments = new ArrayList<>(); 138 | 139 | arguments.add( data.adminCommand ); 140 | arguments.add( "--database=" + data.databaseFilename ); 141 | arguments.add( "--into=data/databases/"+data.databaseFilename); 142 | arguments.add( "--id-type=STRING" ); 143 | for ( String nodesFile : data.nodesFiles ) { 144 | arguments.add( "--nodes=" + nodesFile ); 145 | } 146 | for ( String relsFile : data.relsFiles) { 147 | arguments.add( "--relationships=" + relsFile ); 148 | } 149 | arguments.add( "--report-file=" + data.reportFile ); 150 | arguments.add( "--high-io=" + ( meta.isHighIo() ? "true" : "false" ) ); 151 | arguments.add( "--ignore-extra-columns=" + ( meta.isIgnoringExtraColumns() ? "true" : "false" ) ); 152 | arguments.add( "--ignore-duplicate-nodes=" + ( meta.isIgnoringDuplicateNodes() ? "true" : "false" ) ); 153 | arguments.add( "--ignore-missing-nodes=" + ( meta.isIgnoringMissingNodes() ? "true" : "false" ) ); 154 | arguments.add( "--skip-bad-relationships=" + ( meta.isSkippingBadRelationships() ? "true" : "false" ) ); 155 | arguments.add( "--multiline-fields=" + ( meta.isMultiLine() ? "true" : "false" ) ); 156 | if (StringUtils.isNotEmpty( data.readBufferSize )) { 157 | arguments.add("--read-buffer-size="+data.readBufferSize); 158 | } 159 | if (StringUtils.isNotEmpty( data.maxMemory)) { 160 | arguments.add("--max-memory="+data.maxMemory); 161 | } 162 | 163 | StringBuffer command = new StringBuffer(); 164 | for ( String argument : arguments ) { 165 | command.append( argument ).append( " " ); 166 | } 167 | log.logBasic( "Running command : " + command ); 168 | log.logBasic( "Running from base folder: " + data.baseFolder ); 169 | 170 | ProcessBuilder pb = new ProcessBuilder( arguments ); 171 | pb.directory( new File( data.baseFolder ) ); 172 | try { 173 | Process process = pb.start(); 174 | 175 | StreamConsumer errorConsumer = new StreamConsumer( getLogChannel(), process.getErrorStream(), LogLevel.ERROR ); 176 | errorConsumer.start(); 177 | StreamConsumer outputConsumer = new StreamConsumer( getLogChannel(), process.getInputStream(), LogLevel.BASIC ); 178 | outputConsumer.start(); 179 | 180 | boolean exited = process.waitFor( 10, TimeUnit.MILLISECONDS ); 181 | while ( !exited && !isStopped() ) { 182 | exited = process.waitFor( 10, TimeUnit.MILLISECONDS ); 183 | } 184 | if ( !exited && isStopped() ) { 185 | process.destroyForcibly(); 186 | } 187 | } catch ( Exception e ) { 188 | throw new KettleException( "Error running command: " + arguments, e ); 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/entries/check/CheckConnectionsDialog.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.entries.check; 2 | 3 | import org.eclipse.swt.SWT; 4 | import org.eclipse.swt.events.ModifyEvent; 5 | import org.eclipse.swt.events.ModifyListener; 6 | import org.eclipse.swt.layout.FormAttachment; 7 | import org.eclipse.swt.layout.FormData; 8 | import org.eclipse.swt.layout.FormLayout; 9 | import org.eclipse.swt.widgets.Button; 10 | import org.eclipse.swt.widgets.Display; 11 | import org.eclipse.swt.widgets.Label; 12 | import org.eclipse.swt.widgets.MessageBox; 13 | import org.eclipse.swt.widgets.Shell; 14 | import org.eclipse.swt.widgets.Text; 15 | import org.neo4j.kettle.core.Neo4jDefaults; 16 | import org.neo4j.kettle.shared.NeoConnection; 17 | import org.pentaho.di.core.Const; 18 | import org.pentaho.di.core.util.Utils; 19 | import org.pentaho.di.i18n.BaseMessages; 20 | import org.pentaho.di.job.JobMeta; 21 | import org.pentaho.di.job.entry.JobEntryDialogInterface; 22 | import org.pentaho.di.job.entry.JobEntryInterface; 23 | import org.pentaho.di.repository.Repository; 24 | import org.pentaho.di.ui.core.gui.WindowProperty; 25 | import org.pentaho.di.ui.core.widget.ColumnInfo; 26 | import org.pentaho.di.ui.core.widget.TableView; 27 | import org.pentaho.di.ui.job.dialog.JobDialog; 28 | import org.pentaho.di.ui.job.entry.JobEntryDialog; 29 | import org.pentaho.di.ui.spoon.Spoon; 30 | import org.pentaho.di.ui.trans.step.BaseStepDialog; 31 | import org.pentaho.metastore.api.exceptions.MetaStoreException; 32 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 33 | 34 | import java.util.ArrayList; 35 | import java.util.Collections; 36 | import java.util.List; 37 | 38 | public class CheckConnectionsDialog extends JobEntryDialog implements JobEntryDialogInterface { 39 | 40 | public static final String CHECK_CONNECTIONS_DIALOG = "Neo4jCheckConnectionsDialog"; 41 | private static Class PKG = CheckConnectionsDialog.class; // for i18n purposes, needed by Translator2!! 42 | 43 | private Shell shell; 44 | 45 | private CheckConnections jobEntry; 46 | 47 | private boolean changed; 48 | 49 | private Text wName; 50 | private TableView wConnections; 51 | 52 | private Button wOK, wCancel; 53 | 54 | private String[] availableConnectionNames; 55 | private MetaStoreFactory connectionFactory; 56 | 57 | public CheckConnectionsDialog( Shell parent, JobEntryInterface jobEntry, Repository rep, JobMeta jobMeta ) { 58 | super( parent, jobEntry, rep, jobMeta ); 59 | this.jobEntry = (CheckConnections) jobEntry; 60 | connectionFactory = new MetaStoreFactory<>( NeoConnection.class, Spoon.getInstance().getMetaStore(), Neo4jDefaults.NAMESPACE ); 61 | 62 | if ( this.jobEntry.getName() == null ) { 63 | this.jobEntry.setName( "Check Neo4j Connections" ); 64 | } 65 | } 66 | 67 | @Override public JobEntryInterface open() { 68 | 69 | Shell parent = getParent(); 70 | Display display = parent.getDisplay(); 71 | 72 | shell = new Shell( parent, props.getJobsDialogStyle() ); 73 | props.setLook( shell ); 74 | JobDialog.setShellImage( shell, jobEntry ); 75 | 76 | ModifyListener lsMod = new ModifyListener() { 77 | public void modifyText( ModifyEvent e ) { 78 | jobEntry.setChanged(); 79 | } 80 | }; 81 | changed = jobEntry.hasChanged(); 82 | 83 | FormLayout formLayout = new FormLayout(); 84 | formLayout.marginWidth = Const.FORM_MARGIN; 85 | formLayout.marginHeight = Const.FORM_MARGIN; 86 | 87 | shell.setLayout( formLayout ); 88 | shell.setText( "Check Neo4j Connections" ); 89 | 90 | int middle = props.getMiddlePct(); 91 | int margin = Const.MARGIN; 92 | 93 | Label wlName = new Label( shell, SWT.RIGHT ); 94 | wlName.setText( "Job entry name" ); 95 | props.setLook( wlName ); 96 | FormData fdlName = new FormData(); 97 | fdlName.left = new FormAttachment( 0, 0 ); 98 | fdlName.right = new FormAttachment( middle, -margin ); 99 | fdlName.top = new FormAttachment( 0, margin ); 100 | wlName.setLayoutData( fdlName ); 101 | wName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); 102 | props.setLook( wName ); 103 | wName.addModifyListener( lsMod ); 104 | FormData fdName = new FormData(); 105 | fdName.left = new FormAttachment( middle, 0 ); 106 | fdName.top = new FormAttachment( 0, margin ); 107 | fdName.right = new FormAttachment( 100, 0 ); 108 | wName.setLayoutData( fdName ); 109 | 110 | Label wlConnections = new Label( shell, SWT.RIGHT ); 111 | wlConnections.setText( "Neo4j Connections:" ); 112 | props.setLook( wlConnections ); 113 | FormData fdlConnections = new FormData(); 114 | fdlConnections.left = new FormAttachment( 0, 0 ); 115 | fdlConnections.right = new FormAttachment( middle, -margin ); 116 | fdlConnections.top = new FormAttachment( 0, margin ); 117 | wlConnections.setLayoutData( fdlConnections ); 118 | 119 | // Add buttons first, then the list of connections dynamically sizing 120 | // 121 | wOK = new Button( shell, SWT.PUSH ); 122 | wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) ); 123 | wOK.addListener( SWT.Selection, e -> ok() ); 124 | wCancel = new Button( shell, SWT.PUSH ); 125 | wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) ); 126 | wCancel.addListener( SWT.Selection, e -> cancel() ); 127 | 128 | // Put these buttons at the bottom 129 | // 130 | BaseStepDialog.positionBottomButtons( shell, new Button[] { wOK, wCancel, }, margin, null ); 131 | 132 | try { 133 | List names = connectionFactory.getElementNames(); 134 | Collections.sort( names ); 135 | availableConnectionNames = names.toArray( new String[ 0 ] ); 136 | } catch ( MetaStoreException e ) { 137 | availableConnectionNames = new String[] {}; 138 | } 139 | 140 | ColumnInfo[] columns = new ColumnInfo[] { 141 | new ColumnInfo( "Connection name", ColumnInfo.COLUMN_TYPE_CCOMBO, availableConnectionNames, false ), 142 | }; 143 | wConnections = new TableView( jobEntry, shell, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, columns, 144 | jobEntry.getConnectionNames().size(), false, lsMod, props ); 145 | FormData fdConnections = new FormData(); 146 | fdConnections.left = new FormAttachment( 0, 0 ); 147 | fdConnections.top = new FormAttachment( wName, margin ); 148 | fdConnections.right = new FormAttachment( 100, 0 ); 149 | fdConnections.bottom = new FormAttachment( wOK, -margin * 2 ); 150 | wConnections.setLayoutData( fdConnections ); 151 | 152 | // Detect X or ALT-F4 or something that kills this window... 153 | // 154 | shell.addListener( SWT.Close, e -> cancel() ); 155 | wName.addListener( SWT.DefaultSelection, e -> ok() ); 156 | 157 | getData(); 158 | 159 | BaseStepDialog.setSize( shell ); 160 | 161 | shell.open(); 162 | while ( !shell.isDisposed() ) { 163 | if ( !display.readAndDispatch() ) { 164 | display.sleep(); 165 | } 166 | } 167 | 168 | return jobEntry; 169 | } 170 | 171 | private void cancel() { 172 | jobEntry.setChanged( changed ); 173 | jobEntry = null; 174 | dispose(); 175 | } 176 | 177 | private void getData() { 178 | wName.setText( Const.NVL( jobEntry.getName(), "" ) ); 179 | wConnections.removeAll(); 180 | for ( int i = 0; i < jobEntry.getConnectionNames().size(); i++ ) { 181 | wConnections.add( Const.NVL( jobEntry.getConnectionNames().get( i ), "" ) ); 182 | } 183 | wConnections.removeEmptyRows(); 184 | wConnections.setRowNums(); 185 | wConnections.optWidth( true ); 186 | } 187 | 188 | private void ok() { 189 | if ( Utils.isEmpty( wName.getText() ) ) { 190 | MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); 191 | mb.setText( "Warning" ); 192 | mb.setMessage( "The name of the job entry is missing!" ); 193 | mb.open(); 194 | return; 195 | } 196 | jobEntry.setName( wName.getText() ); 197 | 198 | int nrItems = wConnections.nrNonEmpty(); 199 | jobEntry.setConnectionNames( new ArrayList<>() ); 200 | for ( int i = 0; i < nrItems; i++ ) { 201 | String connectionName = wConnections.getNonEmpty( i ).getText( 1 ); 202 | jobEntry.getConnectionNames().add( connectionName ); 203 | } 204 | 205 | dispose(); 206 | } 207 | 208 | public void dispose() { 209 | props.setScreen( new WindowProperty( shell ) ); 210 | shell.dispose(); 211 | } 212 | 213 | public boolean evaluates() { 214 | return true; 215 | } 216 | 217 | public boolean isUnconditional() { 218 | return false; 219 | } 220 | 221 | } 222 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/steps/gencsv/GenerateCsvMeta.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.steps.gencsv; 2 | 3 | import org.pentaho.di.core.annotations.Step; 4 | import org.pentaho.di.core.database.DatabaseMeta; 5 | import org.pentaho.di.core.exception.KettleException; 6 | import org.pentaho.di.core.exception.KettleXMLException; 7 | import org.pentaho.di.core.row.RowMetaInterface; 8 | import org.pentaho.di.core.row.ValueMetaInterface; 9 | import org.pentaho.di.core.row.value.ValueMetaString; 10 | import org.pentaho.di.core.variables.VariableSpace; 11 | import org.pentaho.di.core.xml.XMLHandler; 12 | import org.pentaho.di.repository.ObjectId; 13 | import org.pentaho.di.repository.Repository; 14 | import org.pentaho.di.trans.Trans; 15 | import org.pentaho.di.trans.TransMeta; 16 | import org.pentaho.di.trans.step.BaseStepMeta; 17 | import org.pentaho.di.trans.step.StepDataInterface; 18 | import org.pentaho.di.trans.step.StepInterface; 19 | import org.pentaho.di.trans.step.StepMeta; 20 | import org.pentaho.di.trans.step.StepMetaInterface; 21 | import org.pentaho.metastore.api.IMetaStore; 22 | import org.w3c.dom.Node; 23 | 24 | import java.util.List; 25 | 26 | @Step( 27 | id = "Neo4jLoad", 28 | name = "Neo4j Generate CSVs", 29 | description = "Generate CSV files for nodes and relationships in the import/ folder for use with neo4j-import", 30 | image = "neo4j_load.svg", 31 | categoryDescription = "Neo4j", 32 | documentationUrl = "https://github.com/knowbi/knowbi-pentaho-pdi-neo4j-output/wiki/" 33 | ) 34 | public class GenerateCsvMeta extends BaseStepMeta implements StepMetaInterface { 35 | 36 | public static final String GRAPH_FIELD_NAME = "graph_field_name"; 37 | public static final String BASE_FOLDER = "base_folder"; 38 | public static final String UNIQUENESS_STRATEGY = "uniqueness_strategy"; 39 | public static final String FILES_PREFIX = "files_prefix"; 40 | public static final String FILENAME_FIELD = "filename_field"; 41 | public static final String FILE_TYPE_FIELD = "file_type_field"; 42 | 43 | protected String graphFieldName; 44 | protected String baseFolder; 45 | protected UniquenessStrategy uniquenessStrategy; 46 | 47 | protected String filesPrefix; 48 | protected String filenameField; 49 | protected String fileTypeField; 50 | 51 | @Override public void setDefault() { 52 | baseFolder = "/var/lib/neo4j/"; 53 | uniquenessStrategy = UniquenessStrategy.None; 54 | filesPrefix = "prefix"; 55 | filenameField = "filename"; 56 | fileTypeField = "fileType"; 57 | } 58 | 59 | @Override public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) { 60 | 61 | inputRowMeta.clear(); 62 | 63 | ValueMetaInterface filenameValueMeta = new ValueMetaString( space.environmentSubstitute( filenameField ) ); 64 | filenameValueMeta.setOrigin( name ); 65 | inputRowMeta.addValueMeta( filenameValueMeta ); 66 | 67 | ValueMetaInterface fileTypeValueMeta = new ValueMetaString( space.environmentSubstitute( fileTypeField ) ); 68 | fileTypeValueMeta.setOrigin( name ); 69 | inputRowMeta.addValueMeta( fileTypeValueMeta ); 70 | 71 | 72 | } 73 | 74 | @Override public String getXML() throws KettleException { 75 | StringBuffer xml = new StringBuffer(); 76 | xml.append( XMLHandler.addTagValue( GRAPH_FIELD_NAME, graphFieldName ) ); 77 | xml.append( XMLHandler.addTagValue( BASE_FOLDER, baseFolder ) ); 78 | xml.append( XMLHandler.addTagValue( UNIQUENESS_STRATEGY, uniquenessStrategy != null ? uniquenessStrategy.name() : null ) ); 79 | xml.append( XMLHandler.addTagValue( FILES_PREFIX, filesPrefix ) ); 80 | xml.append( XMLHandler.addTagValue( FILENAME_FIELD, filenameField ) ); 81 | xml.append( XMLHandler.addTagValue( FILE_TYPE_FIELD, fileTypeField ) ); 82 | return xml.toString(); 83 | } 84 | 85 | @Override public void loadXML( Node stepnode, List databases, IMetaStore metaStore ) throws KettleXMLException { 86 | graphFieldName = XMLHandler.getTagValue( stepnode, GRAPH_FIELD_NAME ); 87 | baseFolder = XMLHandler.getTagValue( stepnode, BASE_FOLDER ); 88 | uniquenessStrategy = UniquenessStrategy.getStrategyFromName( XMLHandler.getTagValue( stepnode, UNIQUENESS_STRATEGY ) ); 89 | filesPrefix = XMLHandler.getTagValue( stepnode, FILES_PREFIX ); 90 | filenameField = XMLHandler.getTagValue( stepnode, FILENAME_FIELD ); 91 | fileTypeField = XMLHandler.getTagValue( stepnode, FILE_TYPE_FIELD ); 92 | } 93 | 94 | @Override public void saveRep( Repository rep, IMetaStore metaStore, ObjectId transformationId, ObjectId stepId ) throws KettleException { 95 | rep.saveStepAttribute( transformationId, stepId, GRAPH_FIELD_NAME, graphFieldName ); 96 | rep.saveStepAttribute( transformationId, stepId, BASE_FOLDER, baseFolder ); 97 | rep.saveStepAttribute( transformationId, stepId, UNIQUENESS_STRATEGY, uniquenessStrategy != null ? uniquenessStrategy.name() : null ); 98 | rep.saveStepAttribute( transformationId, stepId, FILES_PREFIX, filesPrefix ); 99 | rep.saveStepAttribute( transformationId, stepId, FILENAME_FIELD, filenameField ); 100 | rep.saveStepAttribute( transformationId, stepId, FILE_TYPE_FIELD, fileTypeField ); 101 | } 102 | 103 | @Override public void readRep( Repository rep, IMetaStore metaStore, ObjectId stepId, List databases ) throws KettleException { 104 | graphFieldName = rep.getStepAttributeString( stepId, GRAPH_FIELD_NAME ); 105 | baseFolder = rep.getStepAttributeString( stepId, BASE_FOLDER ); 106 | uniquenessStrategy = UniquenessStrategy.getStrategyFromName( rep.getStepAttributeString( stepId, UNIQUENESS_STRATEGY ) ); 107 | filesPrefix = rep.getStepAttributeString( stepId, FILES_PREFIX ); 108 | filenameField = rep.getStepAttributeString( stepId, FILENAME_FIELD ); 109 | fileTypeField = rep.getStepAttributeString( stepId, FILE_TYPE_FIELD ); 110 | } 111 | 112 | @Override public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans ) { 113 | return new GenerateCsv( stepMeta, stepDataInterface, copyNr, transMeta, trans ); 114 | } 115 | 116 | @Override public StepDataInterface getStepData() { 117 | return new GenerateCsvData(); 118 | } 119 | 120 | /** 121 | * Gets graphFieldName 122 | * 123 | * @return value of graphFieldName 124 | */ 125 | public String getGraphFieldName() { 126 | return graphFieldName; 127 | } 128 | 129 | /** 130 | * @param graphFieldName The graphFieldName to set 131 | */ 132 | public void setGraphFieldName( String graphFieldName ) { 133 | this.graphFieldName = graphFieldName; 134 | } 135 | 136 | 137 | /** 138 | * Gets baseFolder 139 | * 140 | * @return value of baseFolder 141 | */ 142 | public String getBaseFolder() { 143 | return baseFolder; 144 | } 145 | 146 | /** 147 | * @param baseFolder The baseFolder to set 148 | */ 149 | public void setBaseFolder( String baseFolder ) { 150 | this.baseFolder = baseFolder; 151 | } 152 | 153 | /** 154 | * Gets nodeUniquenessStrategy 155 | * 156 | * @return value of nodeUniquenessStrategy 157 | */ 158 | public UniquenessStrategy getUniquenessStrategy() { 159 | return uniquenessStrategy; 160 | } 161 | 162 | /** 163 | * @param uniquenessStrategy The nodeUniquenessStrategy to set 164 | */ 165 | public void setUniquenessStrategy( UniquenessStrategy uniquenessStrategy ) { 166 | this.uniquenessStrategy = uniquenessStrategy; 167 | } 168 | 169 | /** 170 | * Gets filesPrefix 171 | * 172 | * @return value of filesPrefix 173 | */ 174 | public String getFilesPrefix() { 175 | return filesPrefix; 176 | } 177 | 178 | /** 179 | * @param filesPrefix The filesPrefix to set 180 | */ 181 | public void setFilesPrefix( String filesPrefix ) { 182 | this.filesPrefix = filesPrefix; 183 | } 184 | 185 | /** 186 | * Gets filenameField 187 | * 188 | * @return value of filenameField 189 | */ 190 | public String getFilenameField() { 191 | return filenameField; 192 | } 193 | 194 | /** 195 | * @param filenameField The filenameField to set 196 | */ 197 | public void setFilenameField( String filenameField ) { 198 | this.filenameField = filenameField; 199 | } 200 | 201 | /** 202 | * Gets fileTypeField 203 | * 204 | * @return value of fileTypeField 205 | */ 206 | public String getFileTypeField() { 207 | return fileTypeField; 208 | } 209 | 210 | /** 211 | * @param fileTypeField The fileTypeField to set 212 | */ 213 | public void setFileTypeField( String fileTypeField ) { 214 | this.fileTypeField = fileTypeField; 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/main/java/bi/know/kettle/neo4j/entries/cypherscript/CypherScript.java: -------------------------------------------------------------------------------- 1 | package bi.know.kettle.neo4j.entries.cypherscript; 2 | 3 | import bi.know.kettle.neo4j.shared.MetaStoreUtil; 4 | import org.apache.commons.lang.StringUtils; 5 | import org.neo4j.driver.Session; 6 | import org.neo4j.driver.Transaction; 7 | import org.neo4j.kettle.core.Neo4jDefaults; 8 | import org.neo4j.kettle.core.metastore.MetaStoreFactory; 9 | import org.neo4j.kettle.shared.NeoConnection; 10 | import org.pentaho.di.cluster.SlaveServer; 11 | import org.pentaho.di.core.Result; 12 | import org.pentaho.di.core.annotations.JobEntry; 13 | import org.pentaho.di.core.database.DatabaseMeta; 14 | import org.pentaho.di.core.exception.KettleException; 15 | import org.pentaho.di.core.exception.KettleXMLException; 16 | import org.pentaho.di.core.xml.XMLHandler; 17 | import org.pentaho.di.job.entry.JobEntryBase; 18 | import org.pentaho.di.job.entry.JobEntryInterface; 19 | import org.pentaho.di.repository.ObjectId; 20 | import org.pentaho.di.repository.Repository; 21 | import org.pentaho.metastore.api.IMetaStore; 22 | import org.w3c.dom.Node; 23 | 24 | import java.util.List; 25 | 26 | @JobEntry( 27 | id="NEO4J_CYPHER_SCRIPT", 28 | name="Neo4j Cypher Script", 29 | description = "Execute a Neo4j Cypher script", 30 | image="neo4j_cypher.svg", 31 | categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.Scripting", 32 | documentationUrl = "https://github.com/knowbi/knowbi-pentaho-pdi-neo4j-output/wiki/" 33 | ) 34 | public class CypherScript extends JobEntryBase implements JobEntryInterface { 35 | 36 | private String connectionName; 37 | 38 | private String script; 39 | 40 | private boolean replacingVariables; 41 | 42 | public CypherScript() { 43 | this("", ""); 44 | } 45 | 46 | public CypherScript( String name) { 47 | this( name, "" ); 48 | } 49 | 50 | public CypherScript( String name, String description ) { 51 | super( name, description ); 52 | } 53 | 54 | @Override public String getXML() { 55 | StringBuilder xml = new StringBuilder(); 56 | // Add entry name, type, ... 57 | // 58 | xml.append( super.getXML() ); 59 | 60 | xml.append( XMLHandler.addTagValue( "connection", connectionName ) ); 61 | xml.append( XMLHandler.addTagValue( "script", script) ); 62 | xml.append( XMLHandler.addTagValue( "replace_variables", replacingVariables ? "Y" : "N") ); 63 | 64 | return xml.toString(); 65 | } 66 | 67 | @Override public void loadXML( Node node, List databases, List slaveServers, Repository rep, IMetaStore metaStore ) throws KettleXMLException { 68 | 69 | super.loadXML( node, databases, slaveServers ); 70 | 71 | connectionName = XMLHandler.getTagValue( node, "connection" ); 72 | script = XMLHandler.getTagValue( node, "script" ); 73 | replacingVariables = "Y".equalsIgnoreCase( XMLHandler.getTagValue( node, "replace_variables" ) ); 74 | } 75 | 76 | @Override public void saveRep( Repository rep, IMetaStore metaStore, ObjectId jobId ) throws KettleException { 77 | rep.saveJobEntryAttribute( jobId, getObjectId(), "connection", connectionName ); 78 | rep.saveJobEntryAttribute( jobId, getObjectId(), "script", script ); 79 | rep.saveJobEntryAttribute( jobId, getObjectId(), "replace_variables", replacingVariables); 80 | } 81 | 82 | @Override public void loadRep( Repository rep, IMetaStore metaStore, ObjectId jobEntryId, List databases, List slaveServers ) throws KettleException { 83 | connectionName = rep.getJobEntryAttributeString( jobEntryId, "connection" ); 84 | script = rep.getJobEntryAttributeString( jobEntryId, "script" ); 85 | replacingVariables = rep.getJobEntryAttributeBoolean( jobEntryId, "replace_variables" ); 86 | } 87 | 88 | @Override public Result execute( Result result, int nr ) throws KettleException { 89 | 90 | try { 91 | metaStore = MetaStoreUtil.findMetaStore( this ); 92 | } catch(Exception e) { 93 | throw new KettleException( "Error finding metastore", e ); 94 | } 95 | MetaStoreFactory connectionFactory = new MetaStoreFactory<>( NeoConnection.class, metaStore, Neo4jDefaults.NAMESPACE ); 96 | 97 | // Replace variables & parameters 98 | // 99 | NeoConnection connection; 100 | String realConnectionName = environmentSubstitute( connectionName ); 101 | try { 102 | if (StringUtils.isEmpty( realConnectionName )) { 103 | throw new KettleException( "The Neo4j connection name is not set" ); 104 | } 105 | 106 | connection = connectionFactory.loadElement( realConnectionName ); 107 | if (connection==null) { 108 | throw new KettleException( "Unable to find connection with name '"+realConnectionName+"'" ); 109 | } 110 | } catch(Exception e) { 111 | result.setResult( false ); 112 | result.increaseErrors( 1L ); 113 | throw new KettleException( "Unable to gencsv or find connection with name '"+realConnectionName+"'", e); 114 | } 115 | 116 | String realScript; 117 | if (replacingVariables) { 118 | realScript = environmentSubstitute( script ); 119 | } else { 120 | realScript = script; 121 | } 122 | 123 | // Share variables with the connection metadata 124 | // 125 | connection.initializeVariablesFrom( this ); 126 | 127 | Session session = null; 128 | Transaction transaction = null; 129 | int nrExecuted = 0; 130 | try { 131 | 132 | // Connect to the database 133 | // 134 | session = connection.getSession(log); 135 | transaction = session.beginTransaction(); 136 | 137 | // Split the script into parts : semi-colon at the start of a separate line 138 | // 139 | String[] commands = realScript.split( "\\r?\\n;" ); 140 | for ( String command : commands ) { 141 | // Cleanup command: replace leading and trailing whitespaces and newlines 142 | // 143 | String cypher = command 144 | .replaceFirst( "^\\s+", "" ) 145 | .replaceFirst( "\\s+$", "" ); 146 | 147 | // Only execute if the statement is not empty 148 | // 149 | if ( StringUtils.isNotEmpty( cypher ) ) { 150 | transaction.run( cypher ); 151 | nrExecuted++; 152 | log.logDetailed("Executed cypher statement: "+cypher); 153 | } 154 | } 155 | 156 | // Commit 157 | // 158 | transaction.commit(); 159 | } catch(Exception e) { 160 | // Error connecting or executing 161 | // Roll back 162 | if (transaction!=null) { 163 | transaction.rollback(); 164 | } 165 | result.increaseErrors( 1L ); 166 | result.setResult( false ); 167 | log.logError("Error executing statements:", e); 168 | } finally { 169 | // Clean up transaction, session and driver 170 | // 171 | if (transaction!=null) { 172 | transaction.close(); 173 | } 174 | if (session!=null) { 175 | session.close(); 176 | } 177 | } 178 | 179 | if (result.getNrErrors()==0) { 180 | logBasic("Neo4j script executed "+nrExecuted+" statements without error"); 181 | } else { 182 | logBasic("Neo4j script executed with error(s)"); 183 | } 184 | 185 | return result; 186 | } 187 | 188 | @Override public String getDialogClassName() { 189 | return super.getDialogClassName(); 190 | } 191 | 192 | @Override public boolean evaluates() { 193 | return true; 194 | } 195 | 196 | @Override public boolean isUnconditional() { 197 | return false; 198 | } 199 | 200 | /** 201 | * Gets connectionName 202 | * 203 | * @return value of connectionName 204 | */ 205 | public String getConnectionName() { 206 | return connectionName; 207 | } 208 | 209 | /** 210 | * @param connectionName The connectionName to set 211 | */ 212 | public void setConnectionName( String connectionName ) { 213 | this.connectionName = connectionName; 214 | } 215 | 216 | /** 217 | * Gets script 218 | * 219 | * @return value of script 220 | */ 221 | public String getScript() { 222 | return script; 223 | } 224 | 225 | /** 226 | * @param script The script to set 227 | */ 228 | public void setScript( String script ) { 229 | this.script = script; 230 | } 231 | 232 | /** 233 | * Gets replacingVariables 234 | * 235 | * @return value of replacingVariables 236 | */ 237 | public boolean isReplacingVariables() { 238 | return replacingVariables; 239 | } 240 | 241 | /** 242 | * @param replacingVariables The replacingVariables to set 243 | */ 244 | public void setReplacingVariables( boolean replacingVariables ) { 245 | this.replacingVariables = replacingVariables; 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /src/main/java/neo4j_cypher.svg: -------------------------------------------------------------------------------- 1 | 2 | image/svg+xml -------------------------------------------------------------------------------- /src/main/java/neo4j_split.svg: -------------------------------------------------------------------------------- 1 | 2 | 34 | 36 | image/svg+xml 38 | 40 | 41 | 43 | 51 | 97 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------