├── .github └── CODEOWNERS ├── assembly ├── plugin │ ├── src │ │ ├── main │ │ │ └── resources │ │ │ │ └── version.xml │ │ └── assembly │ │ │ └── assembly.xml │ └── pom.xml └── pom.xml ├── pentaho-mongodb-plugin ├── src │ ├── test │ │ ├── resources │ │ │ ├── test-settings.properties │ │ │ ├── job.properties │ │ │ └── badJob.properties │ │ └── java │ │ │ └── org │ │ │ └── pentaho │ │ │ ├── mongo │ │ │ └── wrapper │ │ │ │ ├── KettleMongoUtilLoggerTest.java │ │ │ │ ├── MongoFieldTest.java │ │ │ │ └── MongoWrapperUtilTest.java │ │ │ ├── di │ │ │ └── trans │ │ │ │ └── steps │ │ │ │ ├── mongodbinput │ │ │ │ ├── MongoDbInputDiscoverFieldsHolderTest.java │ │ │ │ ├── BaseMongoDbStepTest.java │ │ │ │ ├── MongoDbConnectionAnalyzerTest.java │ │ │ │ ├── MongoDbInputMetaTest.java │ │ │ │ ├── MongoDbResourceInfoTest.java │ │ │ │ └── MongoDbInputMetaInjectionTest.java │ │ │ │ └── mongodboutput │ │ │ │ ├── MongoDbOutputMetaTest.java │ │ │ │ └── MongoDbOutputMetaInjectionTest.java │ │ │ └── reporting │ │ │ └── ui │ │ │ └── datasources │ │ │ └── mongodb │ │ │ └── MongoDbDatasourceDialogTest.java │ └── main │ │ ├── resources │ │ ├── org │ │ │ └── pentaho │ │ │ │ └── di │ │ │ │ ├── trans │ │ │ │ └── steps │ │ │ │ │ ├── mongodb │ │ │ │ │ └── messages │ │ │ │ │ │ └── messages_en_US.properties │ │ │ │ │ └── mongodbinput │ │ │ │ │ └── messages │ │ │ │ │ ├── messages_ko_KR.properties │ │ │ │ │ ├── messages_it_IT.properties │ │ │ │ │ ├── messages_fr_FR.properties │ │ │ │ │ └── messages_ja_JP.properties │ │ │ │ └── ui │ │ │ │ └── trans │ │ │ │ └── steps │ │ │ │ └── mongodbinput │ │ │ │ └── xul │ │ │ │ ├── mongodb_input.properties │ │ │ │ ├── mongodb_input_en_US_zh_TW.properties │ │ │ │ └── mongodb_input.xul │ │ ├── mongodb-input.svg │ │ ├── OSGI-INF │ │ │ └── blueprint │ │ │ │ └── blueprint.xml │ │ └── MongoDB.svg │ │ └── java │ │ └── org │ │ └── pentaho │ │ ├── di │ │ ├── trans │ │ │ └── steps │ │ │ │ ├── mongodbinput │ │ │ │ ├── DiscoverFieldsCallback.java │ │ │ │ ├── MongoDbInputExternalResourceConsumer.java │ │ │ │ ├── MongoDbInputDiscoverFields.java │ │ │ │ ├── MongoDbInputDiscoverFieldsHolder.java │ │ │ │ ├── MongoDbConnectionAnalyzer.java │ │ │ │ ├── MongoDbInputStepAnalyzer.java │ │ │ │ ├── MongoDbResourceInfo.java │ │ │ │ └── MongoDbInputData.java │ │ │ │ ├── mongodb │ │ │ │ └── MongoDBHelper.java │ │ │ │ └── mongodboutput │ │ │ │ ├── MongoDbOutputHelper.java │ │ │ │ └── MongoDbOutputUtilHelper.java │ │ ├── lifecycle │ │ │ └── KettleMongoDbPluginLifecycleListener.java │ │ └── ui │ │ │ ├── trans │ │ │ └── steps │ │ │ │ └── mongodbinput │ │ │ │ └── models │ │ │ │ ├── MongoTag.java │ │ │ │ └── MongoDocumentField.java │ │ │ └── swing │ │ │ └── preview │ │ │ └── PreviewRowsSwingDialog.java │ │ └── mongo │ │ └── wrapper │ │ ├── MongoWrapperClientFactory.java │ │ ├── KettleMongoUtilLogger.java │ │ ├── MongoWrapperUtil.java │ │ └── field │ │ └── MongoArrayExpansion.java ├── README.md └── pom.xml ├── .gitignore ├── pom.xml ├── pentaho-mongo.iml └── LICENSE.txt /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @pentaho/sp-branch-write -------------------------------------------------------------------------------- /assembly/plugin/src/main/resources/version.xml: -------------------------------------------------------------------------------- 1 | 2 | ${project.version} 3 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/resources/test-settings.properties: -------------------------------------------------------------------------------- 1 | hostname=hadoop-vm1.pentaho.com 2 | hdfsPort=9000 3 | trackerPort=9001 4 | username=username 5 | password=password -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | dist/ 3 | lib/ 4 | lib-provided/ 5 | lib-codegen/ 6 | stage-pmr/ 7 | test-lib/ 8 | eclipse-bin/ 9 | override.properties 10 | .settings/ 11 | .classpath 12 | /dev-lib 13 | /pdi-null 14 | /pdi-bin 15 | target/ 16 | .idea 17 | pentaho-mongodb-plugin.iml 18 | pentaho-mongodb-plugin/logs/pdi.log -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/trans/steps/mongodb/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | MongoDb.ErrorMessage.MissingConnectionDetails=Some connection/configuration details are missing: {0} 2 | MongoDb.ErrorMessage.MissingConnectionDetails.Title=Missing connection details 3 | MongoDb.ErrorMessage.UnableToConnect=Unable to connect to MongoDB - check connection details 4 | MongoDb.SuccessMessage.SuccessConnectionDetails.Title=Successfully connected 5 | MongoDb.SuccessMessage.SuccessConnectionDetails=Connection Test Successful with Mongo Database -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/trans/steps/mongodbinput/messages/messages_ko_KR.properties: -------------------------------------------------------------------------------- 1 | #File generated by Hitachi Vantara Translator for package 'org.pentaho.di.trans.steps.mongodbinput' in locale 'ko_KR' 2 | # 3 | # 4 | #Tue Apr 12 17:46:12 KST 2011 5 | MongoDbInputDialog.PreviewSize.DialogTitle=\uBBF8\uB9AC\uBCF4\uAE30 \uAC1C\uC218 \uC785\uB825 6 | MongoDbInputDialog.Hostname.Label=\uD638\uC2A4\uD2B8 \uC774\uB984 \uB610\uB294 IP \uC8FC\uC18C 7 | MongoDbInputDialog.Collection.Label=\uCEEC\uB809\uC158 8 | MongoDbInputDialog.Port.Label=\uD3EC\uD2B8 9 | MongoDbInputDialog.Stepname.Label=Step \uC774\uB984 10 | MongoDbInputDialog.DbName.Label=\uB370\uC774\uD130\uBCA0\uC774\uC2A4 11 | MongoDbInputDialog.JsonField.Label=JSON \uCD9C\uB825 \uD544\uB4DC \uC774\uB984 12 | -------------------------------------------------------------------------------- /assembly/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | pentaho 9 | pentaho-mongo-plugin-parent 10 | 11.1.0.0-SNAPSHOT 11 | 12 | 13 | pentaho 14 | pentaho-mongo-plugin-assemblies 15 | pom 16 | 17 | Pentaho Mongo Plugin Assembly Parent 18 | 19 | 20 | plugin 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.pentaho 8 | pentaho-ce-jar-parent-pom 9 | 11.1.0.0-SNAPSHOT 10 | 11 | 12 | pentaho 13 | pentaho-mongo-plugin-parent 14 | 11.1.0.0-SNAPSHOT 15 | pom 16 | 17 | Pentaho Mongo Plugin Parent 18 | 19 | 20 | pentaho-mongodb-plugin 21 | assembly 22 | 23 | 24 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/DiscoverFieldsCallback.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | package org.pentaho.di.trans.steps.mongodbinput; 13 | 14 | import org.pentaho.mongo.wrapper.field.MongoField; 15 | 16 | import java.util.List; 17 | 18 | /** 19 | * Created by brendan on 12/30/14. 20 | */ 21 | public interface DiscoverFieldsCallback { 22 | public void notifyFields( List fields ); 23 | public void notifyException( Exception exception ); 24 | } 25 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/mongo/wrapper/MongoWrapperClientFactory.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | package org.pentaho.mongo.wrapper; 14 | 15 | import org.pentaho.mongo.MongoDbException; 16 | import org.pentaho.mongo.MongoProperties; 17 | import org.pentaho.mongo.MongoUtilLogger; 18 | 19 | /** 20 | * Created by bryan on 8/22/14. 21 | */ 22 | public interface MongoWrapperClientFactory { 23 | MongoClientWrapper createMongoClientWrapper( MongoProperties props, MongoUtilLogger log ) throws MongoDbException; 24 | 25 | MongoClientWrapper createConnectionStringMongoClientWrapper( String connectionString, MongoUtilLogger log ) throws MongoDbException; 26 | } 27 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/resources/job.properties: -------------------------------------------------------------------------------- 1 | # 2 | # ******************************************************************************* 3 | # Pentaho Big Data 4 | # 5 | # Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com 6 | # ******************************************************************************* 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ****************************************************************************** 19 | # 20 | 21 | nameNode=hdfs://localhost:9000 22 | jobTracker=localhost:9001 23 | queueName=default 24 | examplesRoot=examples 25 | 26 | oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/map-reduce 27 | outputDir=map-reduce -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/resources/badJob.properties: -------------------------------------------------------------------------------- 1 | # 2 | # ******************************************************************************* 3 | # Pentaho Big Data 4 | # 5 | # Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com 6 | # ******************************************************************************* 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ****************************************************************************** 19 | # 20 | nameNode=hdfs://localhost:9000 21 | jobTracker=localhost:9001 22 | queueName=default 23 | examplesRoot=examples 24 | 25 | # comment this guy out so it is a bad property config 26 | #oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/map-reduce 27 | 28 | outputDir=map-reduce 29 | -------------------------------------------------------------------------------- /assembly/plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | pentaho 9 | pentaho-mongo-plugin-assemblies 10 | 11.1.0.0-SNAPSHOT 11 | 12 | com.pentaho.di.plugins 13 | mongodb-plugin 14 | pom 15 | 16 | Pentaho Mongo Plugin Distribution 17 | 18 | 19 | 20 | pentaho 21 | pentaho-mongodb-plugin 22 | ${project.version} 23 | runtime 24 | 25 | 26 | 27 | 28 | 29 | 30 | maven-assembly-plugin 31 | 32 | 33 | src/assembly/assembly.xml 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /assembly/plugin/src/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | pentaho-mongo-plugin-assembly 5 | 6 | zip 7 | 8 | mongodb-plugins 9 | 10 | 11 | src/main/resources 12 | . 13 | true 14 | 15 | 16 | 17 | 18 | runtime 19 | . 20 | false 21 | false 22 | 23 | pentaho:pentaho-mongodb-plugin:jar 24 | 25 | 26 | 27 | runtime 28 | false 29 | true 30 | lib 31 | 32 | pentaho:pentaho-mongodb-plugin:* 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/ui/trans/steps/mongodbinput/xul/mongodb_input.properties: -------------------------------------------------------------------------------- 1 | dialog-title=MongoDB Query 2 | simple-message=Simple Edit of a Kettle Transformation Step 3 | edit-query=Enter a new MongoDB query string: 4 | step-name=Step Name 5 | # Configure connection panel 6 | configure-connection=Configure connection 7 | options=Input Options 8 | query=Query 9 | fields=Fields 10 | host-name=Host name(s) or IP address(es) 11 | port=Port 12 | database=Database 13 | collection=Collection 14 | auth-database=Auth Database 15 | auth-user=User 16 | auth-password=Password 17 | connection-timeout=Connection timeout 18 | socket-timeout=Socket timeout 19 | read-preference=Read preference 20 | use-all-replicas=Use all replica set members 21 | use-kerberos=Authenticate using Kerberos 22 | column-tag=Tag Set 23 | max-preview-rows=Max Preview Rows 24 | preview=Preview 25 | get-dbs=Get DBs 26 | get-collections=Get collections 27 | dialog-title=MongoDB Input 28 | # Query panel 29 | query-expression=Query expression (JSON) 30 | fields-expression=Fields expression (JSON) 31 | is-agg-pipeline=Query is aggregation pipeline 32 | # Fields panel 33 | column-name=Name 34 | column-path=Path 35 | column-type=Type 36 | column-indexed-values=Indexed values 37 | column-sample-array=Sample: array min:max index 38 | column-sample-occurences=Sample: # occur/# docs 39 | column-sample-disparate-types=Sample: disparate types 40 | get-fields=Get fields 41 | get-tags=Get tags 42 | join-tags=Join tags 43 | test-tags=Test tag sets 44 | 45 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputExternalResourceConsumer.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.pentaho.metaverse.api.IAnalysisContext; 17 | import org.pentaho.metaverse.api.analyzer.kettle.step.BaseStepExternalResourceConsumer; 18 | import org.pentaho.metaverse.api.model.IExternalResourceInfo; 19 | 20 | import java.util.Collection; 21 | import java.util.HashSet; 22 | import java.util.Set; 23 | 24 | public class MongoDbInputExternalResourceConsumer 25 | extends BaseStepExternalResourceConsumer { 26 | 27 | @Override 28 | public Class getMetaClass() { 29 | return MongoDbInputMeta.class; 30 | } 31 | 32 | @Override 33 | public Collection getResourcesFromMeta( MongoDbInputMeta meta, IAnalysisContext context ) { 34 | Set resources = new HashSet(); 35 | MongoDbResourceInfo mongoDbResourceInfo = new MongoDbResourceInfo( meta ); 36 | mongoDbResourceInfo.setInput( true ); 37 | resources.add( mongoDbResourceInfo ); 38 | return resources; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputDiscoverFields.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.pentaho.di.core.exception.KettleException; 17 | import org.pentaho.di.core.variables.VariableSpace; 18 | import org.pentaho.mongo.MongoProperties; 19 | import org.pentaho.mongo.wrapper.field.MongoField; 20 | 21 | import java.util.List; 22 | 23 | /** 24 | * Created by brendan on 11/4/14. 25 | */ 26 | public interface MongoDbInputDiscoverFields { 27 | List discoverFields( MongoProperties.Builder properties, String db, String collection, 28 | String query, String fields, 29 | boolean isPipeline, int docsToSample, MongoDbInputMeta step, VariableSpace vars ) 30 | throws KettleException; 31 | 32 | void discoverFields( MongoProperties.Builder properties, String db, String collection, String query, 33 | String fields, 34 | boolean isPipeline, int docsToSample, MongoDbInputMeta step, 35 | VariableSpace vars, DiscoverFieldsCallback discoverFieldsCallback ) throws KettleException; 36 | } 37 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/trans/steps/mongodbinput/messages/messages_it_IT.properties: -------------------------------------------------------------------------------- 1 | #File generated by Hitachi Vantara Translator for package 'org.pentaho.di.trans.steps.mongodbinput' in locale 'it_IT' 2 | # 3 | # 4 | #Wed Jan 18 15:33:28 CET 2012 5 | MongoDbInputDialog.DbName.Label=Database 6 | MongoDbInputDialog.JsonField.Label=Nome del campo di output JSON 7 | MongoDbInput.ErrorAuthenticating.Exception=Non \u00E8 stato possibile autenticare l''utente fornito. Per favore guardare ai log di MongoDB per ulteriori informazioni. 8 | MongoDbInputDialog.PreviewSize.DialogMessage=Inserire il numero di righe per l''anteprima\: 9 | MongoDbInputDialog.AuthenticationPassword.Label=Password di autenticazione 10 | MongoDbInputMeta.Exception.UnableToSaveStepInfo=Impossibile salvare le informazioni del passo nel repository con id\= 11 | MongoDbInputDialog.JsonQuery.Label=Espressione Query (JSON) 12 | MongoDbInputDialog.PreviewSize.DialogTitle=Inserire la dimensione dell''anteprima 13 | MongoDbInputDialog.Collection.Label=Collezione 14 | MongoDbInputDialog.AuthenticationUser.Label=Utente di autenticazione 15 | MongoDbInputDialog.Shell.Title=Input MongoDB 16 | MongoDbInputMeta.Exception.UnableToLoadStepInfo=Impossibile caricare le informazioni del passo da XML\! 17 | MongoDbInputDialog.Hostname.Label=Nome host od indirizzo IP 18 | MongoDbInputDialog.Stepname.Label=Nome del passo 19 | MongoDbInputDialog.Port.Label=Porta 20 | MongoDbInput.ErrorConnectingToMongoDb.Exception=Errore di connessione a MongoDB con l''host {0}, porta {1}, database {2} e collezione {3}\: 21 | MongoDbInputMeta.Exception.UnexpectedErrorWhileReadingStepInfo=Impossibile leggere le informazioni del passo dal repository 22 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/trans/steps/mongodbinput/messages/messages_fr_FR.properties: -------------------------------------------------------------------------------- 1 | #File generated by Hitachi Vantara Translator for package 'org.pentaho.di.trans.steps.mongodbinput' in locale 'fr_FR' 2 | # 3 | # 4 | #Sat May 21 12:24:43 CEST 2011 5 | MongoDbInputDialog.DbName.Label=Base de donn\u00E9es 6 | MongoDbInputDialog.JsonField.Label=Champ en sortie JSON 7 | MongoDbInput.ErrorAuthenticating.Exception=Erreur lors de l''authentification avec le compte utilisateur\! Veuillez svp v\u00E9rifier la trace pour plus de d\u00E9tail. 8 | MongoDbInputDialog.PreviewSize.DialogMessage=Veuillez svp saisir le nombre de lignes max. \u00E0 retourner\: 9 | MongoDbInputDialog.AuthenticationPassword.Label=Mot de passe 10 | MongoDbInputMeta.Exception.UnableToSaveStepInfo=Erreur lors de la sauvegarde dans le r\u00E9f\u00E9rentiel, des informations de l''\u00E9tape identifi\u00E9e par l'id\= 11 | MongoDbInputDialog.Collection.Label=Collection 12 | MongoDbInputDialog.PreviewSize.DialogTitle=Limite pr\u00E9visualisation 13 | MongoDbInputMeta.Exception.UnableToLoadStepInfo=Erreur lors du chargement des informations de l''\u00E9tape depuis le fichier XML\! 14 | MongoDbInputDialog.Shell.Title=Extraction depuis une base Mongo 15 | MongoDbInputDialog.AuthenticationUser.Label=Compte utilisateur 16 | MongoDbInputDialog.Stepname.Label=Nom \u00E9tape 17 | MongoDbInputDialog.Hostname.Label=Nom ou adresse IP serveur 18 | MongoDbInputDialog.Port.Label=Port 19 | MongoDbInput.ErrorConnectingToMongoDb.Exception=Erreur lors de la connexion au serveur {0}, port {1}, base Mango {2} et collection [{3}\: 20 | MongoDbInputMeta.Exception.UnexpectedErrorWhileReadingStepInfo=Erreur lors de la lecture depuis le r\u00E9f\u00E9rentiel, des informations de l''\u00E9tape 21 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/mongo/wrapper/KettleMongoUtilLoggerTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper; 15 | 16 | import org.junit.Before; 17 | import org.junit.Test; 18 | import org.mockito.Mock; 19 | import org.mockito.MockitoAnnotations; 20 | import org.pentaho.di.core.logging.LogChannelInterface; 21 | 22 | import static org.mockito.Mockito.times; 23 | import static org.mockito.Mockito.verify; 24 | 25 | /** 26 | * Created by matt on 10/30/15. 27 | */ 28 | public class KettleMongoUtilLoggerTest { 29 | 30 | @Mock LogChannelInterface logChannelInterface; 31 | @Mock Exception exception; 32 | KettleMongoUtilLogger logger; 33 | 34 | @Before public void before() { 35 | MockitoAnnotations.initMocks( this ); 36 | logger = new KettleMongoUtilLogger( logChannelInterface ); 37 | } 38 | 39 | @Test public void testLoggingDelegates() throws Exception { 40 | logger.debug( "log" ); 41 | verify( logChannelInterface ).logDebug( "log" ); 42 | logger.info( "log" ); 43 | verify( logChannelInterface ).logBasic( "log" ); 44 | logger.warn( "log", exception ); 45 | logger.error( "log", exception ); 46 | // both warn and error are mapped to logError. 47 | verify( logChannelInterface, times( 2 ) ).logError( "log", exception ); 48 | logger.isDebugEnabled(); 49 | verify( logChannelInterface ).isDebug(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/trans/steps/mongodbinput/messages/messages_ja_JP.properties: -------------------------------------------------------------------------------- 1 | #File generated by Hitachi Vantara Translator for package 'org.pentaho.di.trans.steps.mongodbinput' in locale 'en_US' 2 | # 3 | # 4 | #Sat Apr 09 14:23:39 CEST 2011 5 | MongoDbInputDialog.DbName.Label=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9 6 | MongoDbInputDialog.JsonField.Label=JSON\u51fa\u529b\u30d5\u30a3\u30fc\u30eb\u30c9\u540d 7 | MongoDbInputDialog.PreviewSize.DialogMessage=\u30d7\u30ec\u30d3\u30e5\u30fc\u3059\u308b\u884c\u6570\u3092\u5165\u529b\u3057\u3066\u304f\u3060\u3055\u3044\: 8 | MongoDbInputMeta.Exception.UnableToSaveStepInfo=Unable to save in repository step informations with id\= 9 | MongoDbInputDialog.Collection.Label=\u30b3\u30ec\u30af\u30b7\u30e7\u30f3 10 | MongoDbInputDialog.PreviewSize.DialogTitle=\u30d7\u30ec\u30d3\u30e5\u30fc\u30b5\u30a4\u30ba\u3092\u5165\u529b\u3057\u3066\u304f\u3060\u3055\u3044 11 | MongoDbInputDialog.Shell.Title=MongoDB\u5165\u529b 12 | MongoDbInputMeta.Exception.UnableToLoadStepInfo=Unable to load step informations from XML\! 13 | MongoDbInputDialog.Hostname.Label=\u30db\u30b9\u30c8\u540d\u307e\u305f\u306fIP\u30a2\u30c9\u30ec\u30b9 14 | MongoDbInputDialog.Stepname.Label=\u30b9\u30c6\u30c3\u30d7\u540d 15 | MongoDbInputDialog.Port.Label=\u30dd\u30fc\u30c8 16 | MongoDbInput.ErrorConnectingToMongoDb.Exception=There was an error connecting to MongoDB with host {0}, port {1}, database {2} and collection {3}\: 17 | MongoDbInputMeta.Exception.UnexpectedErrorWhileReadingStepInfo=Unable to read step informations from repository 18 | MongoDbInputDialog.AuthenticationUser.Label=\u8a8d\u8a3c\u30e6\u30fc\u30b6\u30fc 19 | MongoDbInputDialog.AuthenticationPassword.Label=\u8a8d\u8a3c\u30d1\u30b9\u30ef\u30fc\u30c9 20 | MongoDbInput.ErrorAuthenticating.Exception=It was not possible to authenticate the supplied username. Please look at the MongoDB log for further information. -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/mongo/wrapper/KettleMongoUtilLogger.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper; 15 | 16 | import org.pentaho.di.core.logging.LogChannelInterface; 17 | import org.pentaho.mongo.MongoUtilLogger; 18 | 19 | /** 20 | * Created by bryan on 8/7/14. 21 | */ 22 | public class KettleMongoUtilLogger implements MongoUtilLogger { 23 | private final LogChannelInterface logChannelInterface; 24 | 25 | public KettleMongoUtilLogger( LogChannelInterface logChannelInterface ) { 26 | this.logChannelInterface = logChannelInterface; 27 | } 28 | 29 | @Override public void debug( String s ) { 30 | if ( logChannelInterface != null ) { 31 | logChannelInterface.logDebug( s ); 32 | } 33 | } 34 | 35 | @Override public void info( String s ) { 36 | if ( logChannelInterface != null ) { 37 | logChannelInterface.logBasic( s ); 38 | } 39 | } 40 | 41 | @Override public void warn( String s, Throwable throwable ) { 42 | if ( logChannelInterface != null ) { 43 | logChannelInterface.logError( s, throwable ); 44 | } 45 | } 46 | 47 | @Override public void error( String s, Throwable throwable ) { 48 | if ( logChannelInterface != null ) { 49 | logChannelInterface.logError( s, throwable ); 50 | } 51 | } 52 | 53 | @Override public boolean isDebugEnabled() { 54 | if ( logChannelInterface != null ) { 55 | return logChannelInterface.isDebug(); 56 | } else { 57 | return false; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/ui/trans/steps/mongodbinput/xul/mongodb_input_en_US_zh_TW.properties: -------------------------------------------------------------------------------- 1 | dialog-title=MongoDB \u67e5\u8a62 2 | simple-message=\u5c0d Kettle \u8f49\u63db\u6b65\u9a5f\u9032\u884c\u7c21\u6613\u7de8\u8f2f 3 | edit-query=\u8f38\u5165\u65b0\u7684 MongoDB \u67e5\u8a62\u5b57\u4e32\uff1a 4 | step-name=\u6b65\u9a5f\u540d\u7a31 5 | # Configure connection panel 6 | configure-connection=\u8a2d\u5b9a\u9023\u7dda 7 | options=\u8f38\u5165\u9078\u9805 8 | query=\u67e5\u8a62 9 | fields=\u6b04\u4f4d 10 | host-name=\u4e3b\u6a5f\u540d\u7a31\u6216 IP \u4f4d\u5740 11 | port=\u9023\u63a5\u57e0 12 | database=\u8cc7\u6599\u5eab 13 | collection=\u96c6\u5408 14 | auth-database=\u9a57\u8b49\u8cc7\u6599\u5eab 15 | auth-user=\u4f7f\u7528\u8005 16 | auth-password=\u5bc6\u78bc 17 | connection-timeout=\u9023\u7dda\u903e\u6642 18 | socket-timeout=\u901a\u8a0a\u7aef\u903e\u6642 19 | read-preference=\u8b80\u53d6\u504f\u597d\u8a2d\u5b9a 20 | use-all-replicas=\u4f7f\u7528\u6240\u6709\u8907\u672c\u96c6\u6210\u54e1 21 | use-kerberos=\u4f7f\u7528 Kerberos \u9032\u884c\u9a57\u8b49 22 | column-tag=\u6a19\u8a18\u96c6\u5408 23 | max-preview-rows=\u9810\u89bd\u8cc7\u6599\u5217\u4e0a\u9650 24 | preview=\u9810\u89bd 25 | get-dbs=\u53d6\u5f97 DB 26 | get-collections=\u53d6\u5f97\u96c6\u5408 27 | dialog-title=MongoDB \u8f38\u5165 28 | # Query panel 29 | query-expression=\u67e5\u8a62\u904b\u7b97\u5f0f (JSON) 30 | fields-expression=\u6b04\u4f4d\u904b\u7b97\u5f0f (JSON) 31 | is-agg-pipeline=\u67e5\u8a62\u662f\u5f59\u7e3d\u7ba1\u7dda 32 | # Fields panel 33 | column-name=\u540d\u7a31 34 | column-path=\u8def\u5f91 35 | column-type=\u985e\u578b 36 | column-indexed-values=\u5df2\u7de8\u88fd\u7d22\u5f15\u7684\u503c 37 | column-sample-array=\u7bc4\u4f8b\uff1a\u9663\u5217 min:max \u7d22\u5f15 38 | column-sample-occurences=\u7bc4\u4f8b\uff1a\u767c\u751f\u6b21\u6578/\u6587\u4ef6\u6578\u76ee 39 | column-sample-disparate-types=\u7bc4\u4f8b\uff1a\u76f8\u7570\u985e\u578b 40 | get-fields=\u53d6\u5f97\u6b04\u4f4d 41 | get-tags=\u53d6\u5f97\u6a19\u8a18 42 | join-tags=\u806f\u7d50\u6a19\u8a18 43 | test-tags=\u6e2c\u8a66\u6a19\u8a18\u96c6\u5408 44 | 45 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/lifecycle/KettleMongoDbPluginLifecycleListener.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.lifecycle; 15 | 16 | import org.pentaho.di.core.annotations.LifecyclePlugin; 17 | import org.pentaho.di.core.lifecycle.LifeEventHandler; 18 | import org.pentaho.di.core.lifecycle.LifecycleException; 19 | import org.pentaho.di.core.lifecycle.LifecycleListener; 20 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbConnectionAnalyzer; 21 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputExternalResourceConsumer; 22 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputStepAnalyzer; 23 | import org.pentaho.platform.engine.core.system.PentahoSystem; 24 | 25 | @LifecyclePlugin( id = "KettleMongoDbPlugin", name = "KettleMongoDbPlugin" ) 26 | public class KettleMongoDbPluginLifecycleListener implements LifecycleListener { 27 | 28 | @Override 29 | public void onStart( LifeEventHandler handler ) throws LifecycleException { 30 | MongoDbInputStepAnalyzer mongoDbInputStepAnalyzer = new MongoDbInputStepAnalyzer(); 31 | MongoDbInputExternalResourceConsumer mongoDbInputExternalResourceConsumer = new MongoDbInputExternalResourceConsumer(); 32 | MongoDbConnectionAnalyzer mongoDbConnectionAnalyzer = new MongoDbConnectionAnalyzer(); 33 | mongoDbInputStepAnalyzer.setConnectionAnalyzer( mongoDbConnectionAnalyzer ); 34 | mongoDbInputStepAnalyzer.setExternalResourceConsumer( mongoDbInputExternalResourceConsumer ); 35 | 36 | PentahoSystem.registerObject( mongoDbInputStepAnalyzer ); 37 | PentahoSystem.registerObject( mongoDbInputExternalResourceConsumer ); 38 | PentahoSystem.registerObject( mongoDbConnectionAnalyzer ); 39 | } 40 | 41 | @Override public void onExit( LifeEventHandler handler ) throws LifecycleException { 42 | // no-op 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputDiscoverFieldsHolderTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import com.google.common.collect.ImmutableMap; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import org.mockito.Mock; 20 | import org.mockito.MockitoAnnotations; 21 | 22 | import java.util.Collections; 23 | 24 | import static org.hamcrest.Matchers.equalTo; 25 | import static org.junit.Assert.assertThat; 26 | 27 | public class MongoDbInputDiscoverFieldsHolderTest { 28 | 29 | @Mock MongoDbInputDiscoverFields discoverFields; 30 | @Mock MongoDbInputDiscoverFields discoverFields2; 31 | @Mock MongoDbInputDiscoverFields discoverFields3b; 32 | @Mock MongoDbInputDiscoverFields discoverFields3; 33 | MongoDbInputDiscoverFieldsHolder holder = MongoDbInputDiscoverFieldsHolder.getInstance(); 34 | 35 | @Before public void before() { 36 | MockitoAnnotations.initMocks( this ); 37 | } 38 | 39 | @Test public void testImplAddedRemoved() throws Exception { 40 | holder.implAdded( discoverFields, ImmutableMap.of( "service.ranking", 1 ) ); 41 | holder.implAdded( discoverFields2, Collections.emptyMap() ); 42 | holder.implAdded( discoverFields3, ImmutableMap.of( "service.ranking", 5 ) ); 43 | holder.implAdded( discoverFields3b, ImmutableMap.of( "service.ranking", 5 ) ); // second impl at same rank 44 | 45 | assertThat( holder.getMongoDbInputDiscoverFields(), equalTo( discoverFields3 ) ); 46 | holder.implRemoved( discoverFields3, ImmutableMap.of( "service.ranking", 5 ) ); 47 | assertThat( holder.getMongoDbInputDiscoverFields(), equalTo( discoverFields3b ) ); 48 | holder.implRemoved( discoverFields3b, ImmutableMap.of( "service.ranking", 5 ) ); 49 | assertThat( holder.getMongoDbInputDiscoverFields(), equalTo( discoverFields ) ); 50 | holder.implRemoved( discoverFields, ImmutableMap.of( "service.ranking", 1 ) ); 51 | assertThat( holder.getMongoDbInputDiscoverFields(), equalTo( discoverFields2 ) ); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/README.md: -------------------------------------------------------------------------------- 1 | Pentaho MongoDB Plugin 2 | ======================= 3 | 4 | The Pentaho MongoDB Plugin Project provides support for MongoDB support within the Pentaho ecosystem. It is a plugin for the Pentaho Kettle engine which can be used within Pentaho Data Integration (Kettle), Pentaho Reporting, and the Pentaho BI Platform. 5 | 6 | 7 | How to build 8 | -------------- 9 | 10 | Pentaho MongoDB Plugin uses the maven framework. 11 | 12 | 13 | #### Pre-requisites for building the project: 14 | * Maven, version 3+ 15 | * Java JDK 1.8 16 | * This [settings.xml](https://raw.githubusercontent.com/pentaho/maven-parent-poms/master/maven-support-files/settings.xml) in your /.m2 directory 17 | 18 | #### Building it 19 | 20 | This is a maven project, and to build it use the following command 21 | 22 | ``` 23 | $ mvn clean install 24 | ``` 25 | Optionally you can specify -Drelease to trigger obfuscation and/or uglification (as needed) 26 | 27 | Optionally you can specify -Dmaven.test.skip=true to skip the tests (even though 28 | you shouldn't as you know) 29 | 30 | The build result will be a Pentaho package located in ```target```. 31 | 32 | #### Running the tests 33 | 34 | __Unit tests__ 35 | 36 | This will run all unit tests in the project (and sub-modules). To run integration tests as well, see Integration Tests below. 37 | 38 | ``` 39 | $ mvn test 40 | ``` 41 | 42 | If you want to remote debug a single java unit test (default port is 5005): 43 | 44 | ``` 45 | $ cd core 46 | $ mvn test -Dtest=<> -Dmaven.surefire.debug 47 | ``` 48 | 49 | __Integration tests__ 50 | 51 | In addition to the unit tests, there are integration tests that test cross-module operation. This will run the integration tests. 52 | 53 | ``` 54 | $ mvn verify -DrunITs 55 | ``` 56 | 57 | To run a single integration test: 58 | 59 | ``` 60 | $ mvn verify -DrunITs -Dit.test=<> 61 | ``` 62 | 63 | To run a single integration test in debug mode (for remote debugging in an IDE) on the default port of 5005: 64 | 65 | ``` 66 | $ mvn verify -DrunITs -Dit.test=<> -Dmaven.failsafe.debug 67 | ``` 68 | 69 | To skip test 70 | 71 | ``` 72 | $ mvn clean install -DskipTests 73 | ``` 74 | 75 | To get log as text file 76 | 77 | ``` 78 | $ mvn clean install test >log.txt 79 | ``` 80 | 81 | License 82 | ------- 83 | Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. 84 | 85 | __IntelliJ__ 86 | 87 | * Don't use IntelliJ's built-in maven. Make it use the same one you use from the commandline. 88 | * Project Preferences -> Build, Execution, Deployment -> Build Tools -> Maven ==> Maven home directory 89 | 90 | ```` 91 | 92 | -------------------------------------------------------------------------------- /pentaho-mongo.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/ui/trans/steps/mongodbinput/models/MongoTag.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.ui.trans.steps.mongodbinput.models; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | import org.pentaho.ui.xul.XulEventSourceAdapter; 20 | import org.pentaho.ui.xul.util.AbstractModelList; 21 | 22 | public class MongoTag extends XulEventSourceAdapter { 23 | 24 | private String m_tagName = ""; 25 | 26 | public MongoTag() { 27 | 28 | } 29 | 30 | public MongoTag( String name ) { 31 | m_tagName = name; 32 | } 33 | 34 | public String getTagName() { 35 | return this.m_tagName; 36 | } 37 | 38 | public void setTagName( String name ) { 39 | this.m_tagName = name; 40 | } 41 | 42 | public static void convertList( List tags, AbstractModelList docTags ) { 43 | 44 | if ( tags == null || tags.isEmpty() ) { 45 | return; 46 | } 47 | for ( String tag : tags ) { 48 | MongoTag docTag = new MongoTag(); 49 | 50 | if ( tag.startsWith( "{" ) ) { 51 | tag = tag.substring( 1 ); 52 | } 53 | 54 | if ( tag.endsWith( "}" ) ) { 55 | tag = tag.substring( 0, tag.length() - 1 ); 56 | } 57 | 58 | docTag.setTagName( tag ); 59 | docTags.add( docTag ); 60 | } 61 | } 62 | 63 | public static void trimList( List tags, AbstractModelList docTags ) { 64 | 65 | if ( tags == null || tags.isEmpty() ) { 66 | return; 67 | } 68 | 69 | for ( int i = tags.size() - 1; i >= 0; i-- ) { 70 | String tag = tags.get( i ); 71 | for ( MongoTag docTag : docTags.asList() ) { 72 | if ( tag.equalsIgnoreCase( docTag.getTagName() ) ) { 73 | tags.remove( tag ); 74 | break; 75 | } 76 | } 77 | } 78 | } 79 | 80 | public static List convertFromList( AbstractModelList docTags ) { 81 | List tags = new ArrayList(); 82 | 83 | if ( docTags == null || docTags.isEmpty() ) { 84 | return tags; 85 | } 86 | for ( MongoTag docTag : docTags ) { 87 | 88 | String tag = docTag.getTagName(); 89 | 90 | if ( !tag.startsWith( "{" ) ) { 91 | tag = "{" + tag; 92 | } 93 | 94 | if ( !tag.endsWith( "}" ) ) { 95 | tag += "}"; 96 | } 97 | 98 | tags.add( tag ); 99 | } 100 | return tags; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Pentaho Developer Edition 10.3 Copyright 2024 Hitachi Vantara, LLC; licensed under the 2 | Business Source License 1.1 (BSL). This project may include third party components that 3 | are individually licensed per the terms indicated by their respective copyright owners 4 | included in text file or in the source code. 5 | 6 | License text copyright (c) 2020 MariaDB Corporation Ab, All Rights Reserved. 7 | "Business Source License" is a trademark of MariaDB Corporation Ab. 8 | 9 | Parameters 10 | 11 | Licensor: Hitachi Vantara, LLC. 12 | Licensed Work: Pentaho Developer Edition 10.3. The Licensed Work is (c) 2024 13 | Hitachi Vantara, LLC. 14 | Additional Use Grant: None 15 | Change Date: Four years from the date the Licensed Work is published. 16 | Change License: Apache 2.0 17 | 18 | For information about alternative licensing arrangements for the Licensed Work, 19 | please contact support@pentaho.com. 20 | 21 | Notice 22 | 23 | Business Source License 1.1 24 | 25 | Terms 26 | 27 | The Licensor hereby grants you the right to copy, modify, create derivative 28 | works, redistribute, and make non-production use of the Licensed Work. The 29 | Licensor may make an Additional Use Grant, above, permitting limited production use. 30 | 31 | Effective on the Change Date, or the fourth anniversary of the first publicly 32 | available distribution of a specific version of the Licensed Work under this 33 | License, whichever comes first, the Licensor hereby grants you rights under 34 | the terms of the Change License, and the rights granted in the paragraph 35 | above terminate. 36 | 37 | If your use of the Licensed Work does not comply with the requirements 38 | currently in effect as described in this License, you must purchase a 39 | commercial license from the Licensor, its affiliated entities, or authorized 40 | resellers, or you must refrain from using the Licensed Work. 41 | 42 | All copies of the original and modified Licensed Work, and derivative works 43 | of the Licensed Work, are subject to this License. This License applies 44 | separately for each version of the Licensed Work and the Change Date may vary 45 | for each version of the Licensed Work released by Licensor. 46 | 47 | You must conspicuously display this License on each original or modified copy 48 | of the Licensed Work. If you receive the Licensed Work in original or 49 | modified form from a third party, the terms and conditions set forth in this 50 | License apply to your use of that work. 51 | 52 | Any use of the Licensed Work in violation of this License will automatically 53 | terminate your rights under this License for the current and all other 54 | versions of the Licensed Work. 55 | 56 | This License does not grant you any right in any trademark or logo of 57 | Licensor or its affiliates (provided that you may use a trademark or logo of 58 | Licensor as expressly required by this License). 59 | 60 | TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON 61 | AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, 62 | EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF 63 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND TITLE. -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/reporting/ui/datasources/mongodb/MongoDbDatasourceDialogTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.reporting.ui.datasources.mongodb; 15 | 16 | import org.junit.Test; 17 | import org.pentaho.di.core.logging.KettleLogStore; 18 | import org.pentaho.di.trans.TransMeta; 19 | import org.pentaho.di.trans.step.StepMeta; 20 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputMeta; 21 | import org.pentaho.di.ui.trans.steps.mongodbinput.MongoDbInputXulDialog; 22 | 23 | public class MongoDbDatasourceDialogTest { 24 | 25 | @Test public void dummyTest() { 26 | // Have to have at least one test method to run, otherwise JUnit packs a sad. 27 | // Main method contains the tests. 28 | } 29 | 30 | public static void main( String[] args ) { 31 | 32 | KettleLogStore.init(); 33 | 34 | MongoDbInputMeta meta = new MongoDbInputMeta(); 35 | meta.setJsonQuery( "{CITY:\"NYC\"}" ); 36 | meta.setAuthenticationPassword( "password" ); 37 | meta.setAuthenticationUser( "gmoran" ); 38 | meta.setCollection( "big" ); 39 | meta.setDbName( "data" ); 40 | meta.setHostnames( "" ); 41 | meta.setPort( "27017" ); 42 | meta.setReadPreference( "Secondary preferred" ); 43 | meta.setFieldsName( "{id:true}" ); 44 | meta.setQueryIsPipeline( true ); 45 | 46 | TransMeta trans = new TransMeta(); 47 | StepMeta stepMeta = new StepMeta( "mongo_source", meta ); 48 | trans.addStep( stepMeta ); 49 | 50 | MongoDbInputXulDialog dlg = new MongoDbInputXulDialog( null, meta, trans, "mongo_source" ); 51 | 52 | if ( dlg.open() != null ) { 53 | System.out.println( "Host name(s): ".concat( meta.getHostnames() != null ? meta.getHostnames() : "" ) ); 54 | System.out.println( "Port: ".concat( meta.getPort() != null ? meta.getPort() : "" ) ); 55 | System.out.println( "Database: ".concat( meta.getDbName() != null ? meta.getDbName() : "" ) ); 56 | System.out.println( "Collection: ".concat( meta.getCollection() != null ? meta.getCollection() : "" ) ); 57 | System.out.println( "User: ".concat( meta.getAuthenticationUser() != null ? meta.getAuthenticationUser() : "" ) ); 58 | System.out.println( 59 | "Password: ".concat( meta.getAuthenticationPassword() != null ? meta.getAuthenticationPassword() : "" ) ); 60 | System.out 61 | .println( "Connection Timeout: ".concat( meta.getConnectTimeout() != null ? meta.getConnectTimeout() : "" ) ); 62 | System.out.println( "Socket Timeout: ".concat( meta.getSocketTimeout() != null ? meta.getSocketTimeout() : "" ) ); 63 | System.out 64 | .println( "Read Preference: ".concat( meta.getReadPreference() != null ? meta.getReadPreference() : "" ) ); 65 | System.out.println( "JSON Query: ".concat( meta.getJsonQuery() != null ? meta.getJsonQuery() : "" ) ); 66 | System.out 67 | .println( "Is Agg Pipeline:".concat( meta.getQueryIsPipeline() ? "IS pipeline" : "IS NOT a pipeline" ) ); 68 | System.out.println( "Field Expression: ".concat( meta.getFieldsName() != null ? meta.getFieldsName() : "" ) ); 69 | } 70 | 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputDiscoverFieldsHolder.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.pentaho.mongo.wrapper.field.MongodbInputDiscoverFieldsImpl; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | /** 25 | * Created by bryan on 11/10/14. 26 | */ 27 | public class MongoDbInputDiscoverFieldsHolder { 28 | private static MongoDbInputDiscoverFieldsHolder INSTANCE = new MongoDbInputDiscoverFieldsHolder(); 29 | private final Map> mongoDbInputDiscoverFieldsMap; 30 | private MongoDbInputDiscoverFields mongoDbInputDiscoverFields = new MongodbInputDiscoverFieldsImpl(); 31 | 32 | private MongoDbInputDiscoverFieldsHolder() { 33 | mongoDbInputDiscoverFieldsMap = new HashMap>(); 34 | } 35 | 36 | public static MongoDbInputDiscoverFieldsHolder getInstance() { 37 | return INSTANCE; 38 | } 39 | 40 | public MongoDbInputDiscoverFields getMongoDbInputDiscoverFields() { 41 | return mongoDbInputDiscoverFields; 42 | } 43 | 44 | public void implAdded( MongoDbInputDiscoverFields mongoDbInputDiscoverFields, Map properties ) { 45 | synchronized ( mongoDbInputDiscoverFieldsMap ) { 46 | Integer ranking = (Integer) properties.get( "service.ranking" ); 47 | if ( ranking == null ) { 48 | ranking = 0; 49 | } 50 | List mongoDbInputDiscoverFieldsList = mongoDbInputDiscoverFieldsMap.get( ranking ); 51 | if ( mongoDbInputDiscoverFieldsList == null ) { 52 | mongoDbInputDiscoverFieldsList = new ArrayList(); 53 | mongoDbInputDiscoverFieldsMap.put( ranking, mongoDbInputDiscoverFieldsList ); 54 | } 55 | mongoDbInputDiscoverFieldsList.add( mongoDbInputDiscoverFields ); 56 | updateField(); 57 | } 58 | } 59 | 60 | public void implRemoved( MongoDbInputDiscoverFields mongoDbInputDiscoverFields, Map properties ) { 61 | synchronized ( mongoDbInputDiscoverFieldsMap ) { 62 | Integer ranking = (Integer) properties.get( "service.ranking" ); 63 | if ( ranking == null ) { 64 | ranking = 0; 65 | } 66 | List mongoDbInputDiscoverFieldsList = mongoDbInputDiscoverFieldsMap.get( ranking ); 67 | if ( mongoDbInputDiscoverFieldsList != null ) { 68 | mongoDbInputDiscoverFieldsList.remove( mongoDbInputDiscoverFields ); 69 | if ( mongoDbInputDiscoverFieldsList.size() == 0 ) { 70 | mongoDbInputDiscoverFieldsMap.remove( ranking ); 71 | } 72 | updateField(); 73 | } 74 | } 75 | } 76 | 77 | private void updateField() { 78 | List keys = new ArrayList( mongoDbInputDiscoverFieldsMap.keySet() ); 79 | if ( keys.size() == 0 ) { 80 | mongoDbInputDiscoverFields = null; 81 | } else { 82 | Collections.sort( keys ); 83 | mongoDbInputDiscoverFields = mongoDbInputDiscoverFieldsMap.get( keys.get( keys.size() - 1 ) ).get( 0 ); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/mongodb-input.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 8 | 10 | 12 | 17 | 18 | 19 | 20 | 21 | 22 | 24 | 26 | 31 | 33 | 35 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/BaseMongoDbStepTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import com.mongodb.Cursor; 17 | import com.mongodb.DBObject; 18 | import org.junit.After; 19 | import org.junit.Before; 20 | import org.mockito.ArgumentCaptor; 21 | import org.mockito.Captor; 22 | import org.mockito.Mock; 23 | import org.mockito.Mockito; 24 | import org.mockito.MockitoAnnotations; 25 | import org.pentaho.di.core.logging.KettleLogStore; 26 | import org.pentaho.di.core.logging.LogChannelInterface; 27 | import org.pentaho.di.core.logging.LogChannelInterfaceFactory; 28 | import org.pentaho.di.core.row.RowMeta; 29 | import org.pentaho.di.trans.Trans; 30 | import org.pentaho.di.trans.TransMeta; 31 | import org.pentaho.di.trans.step.BaseStep; 32 | import org.pentaho.di.trans.step.StepMeta; 33 | import org.pentaho.mongo.MongoDbException; 34 | import org.pentaho.mongo.MongoProperties; 35 | import org.pentaho.mongo.MongoUtilLogger; 36 | import org.pentaho.mongo.wrapper.MongoClientWrapper; 37 | import org.pentaho.mongo.wrapper.MongoWrapperClientFactory; 38 | import org.pentaho.mongo.wrapper.MongoWrapperUtil; 39 | import org.pentaho.mongo.wrapper.collection.MongoCollectionWrapper; 40 | 41 | import static org.mockito.Mockito.*; 42 | 43 | /** 44 | * Common mock setup for MongoDbOutputTest and MongoDbInput 45 | */ 46 | public class BaseMongoDbStepTest { 47 | 48 | @Mock protected StepMeta stepMeta; 49 | @Mock protected TransMeta transMeta; 50 | @Mock protected Trans trans; 51 | @Mock protected LogChannelInterface mockLog; 52 | @Mock protected MongoWrapperClientFactory mongoClientWrapperFactory; 53 | @Mock protected MongoClientWrapper mongoClientWrapper; 54 | @Mock protected MongoCollectionWrapper mongoCollectionWrapper; 55 | @Mock protected LogChannelInterfaceFactory logChannelFactory; 56 | @Mock protected Cursor cursor; 57 | @Captor protected ArgumentCaptor stringCaptor; 58 | @Captor protected ArgumentCaptor dbObjectCaptor; 59 | @Captor protected ArgumentCaptor throwableCaptor; 60 | 61 | protected RowMeta rowMeta = new RowMeta(); 62 | protected Object[] rowData; 63 | 64 | private MongoWrapperClientFactory cachedFactory; 65 | 66 | @Before public void before() throws MongoDbException { 67 | MockitoAnnotations.openMocks( BaseMongoDbStepTest.class ); 68 | cachedFactory = MongoWrapperUtil.getMongoWrapperClientFactory(); 69 | MongoWrapperUtil.setMongoWrapperClientFactory( mongoClientWrapperFactory ); 70 | when( mongoClientWrapperFactory 71 | .createMongoClientWrapper( Mockito.any(), Mockito.any() ) ) 72 | .thenReturn( mongoClientWrapper ); 73 | 74 | lenient().when( stepMeta.getName() ).thenReturn( "stepMetaName" ); 75 | lenient().when( transMeta.findStep( anyString() ) ).thenReturn( stepMeta ); 76 | lenient().when( logChannelFactory.create( any( BaseStep.class ), any( Trans.class ) ) ).thenReturn( mockLog ); 77 | KettleLogStore.setLogChannelInterfaceFactory( logChannelFactory ); 78 | } 79 | 80 | @After public void tearDown() { 81 | MongoWrapperUtil.setMongoWrapperClientFactory( cachedFactory ); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbConnectionAnalyzerTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.junit.Before; 17 | import org.junit.Test; 18 | import org.junit.runner.RunWith; 19 | import org.mockito.Mock; 20 | import org.mockito.junit.MockitoJUnitRunner; 21 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 22 | import org.pentaho.dictionary.DictionaryConst; 23 | import org.pentaho.metaverse.api.IAnalysisContext; 24 | import org.pentaho.metaverse.api.IComponentDescriptor; 25 | import org.pentaho.metaverse.api.IMetaverseBuilder; 26 | import org.pentaho.metaverse.api.IMetaverseNode; 27 | import org.pentaho.metaverse.api.IMetaverseObjectFactory; 28 | import org.pentaho.metaverse.api.INamespace; 29 | import org.pentaho.metaverse.api.MetaverseObjectFactory; 30 | 31 | import java.util.List; 32 | 33 | import static org.junit.Assert.assertEquals; 34 | import static org.junit.Assert.assertNotNull; 35 | import static org.mockito.Mockito.mock; 36 | import static org.mockito.Mockito.when; 37 | 38 | /** 39 | * User: RFellows Date: 3/6/15 40 | */ 41 | @RunWith( MockitoJUnitRunner.StrictStubs.class ) 42 | public class MongoDbConnectionAnalyzerTest { 43 | 44 | MongoDbConnectionAnalyzer analyzer; 45 | 46 | @Mock private IMetaverseBuilder mockBuilder; 47 | @Mock private MongoDbMeta mongoDbMeta; 48 | @Mock private IComponentDescriptor mockDescriptor; 49 | 50 | @Before 51 | public void setUp() throws Exception { 52 | IMetaverseObjectFactory factory = new MetaverseObjectFactory(); 53 | when( mockBuilder.getMetaverseObjectFactory() ).thenReturn( factory ); 54 | 55 | analyzer = new MongoDbConnectionAnalyzer(); 56 | analyzer.setMetaverseBuilder( mockBuilder ); 57 | 58 | when( mockDescriptor.getNamespace() ).thenReturn( mock( INamespace.class) ); 59 | when( mockDescriptor.getContext() ).thenReturn( mock( IAnalysisContext.class ) ); 60 | 61 | when( mongoDbMeta.getHostnames() ).thenReturn( "localhost" ); 62 | when( mongoDbMeta.getDbName() ).thenReturn( "db" ); 63 | when( mongoDbMeta.getAuthenticationUser() ).thenReturn( "user" ); 64 | when( mongoDbMeta.getPort() ).thenReturn( "12345" ); 65 | when( mongoDbMeta.isUseLegacyOptions() ).thenReturn( true ); 66 | } 67 | 68 | @Test 69 | public void testAnalyze() throws Exception { 70 | IMetaverseNode node = analyzer.analyze( mockDescriptor, mongoDbMeta ); 71 | assertNotNull( node ); 72 | assertEquals( "localhost", node.getProperty( MongoDbConnectionAnalyzer.HOST_NAMES ) ); 73 | assertEquals( "db", node.getProperty( MongoDbConnectionAnalyzer.DATABASE_NAME ) ); 74 | assertEquals( "user", node.getProperty( DictionaryConst.PROPERTY_USER_NAME ) ); 75 | assertEquals( "12345", node.getProperty( DictionaryConst.PROPERTY_PORT ) ); 76 | 77 | } 78 | 79 | @Test 80 | public void testGetUsedConnections() throws Exception { 81 | List dbMetaList = analyzer.getUsedConnections( mongoDbMeta ); 82 | assertEquals( 1, dbMetaList.size() ); 83 | 84 | // should just return the same MongoDbMeta object in list form as the only entry 85 | assertEquals( mongoDbMeta, dbMetaList.get( 0 ) ); 86 | } 87 | 88 | @Test 89 | public void testBuildComponentDescriptor() throws Exception { 90 | IComponentDescriptor dbDesc = analyzer.buildComponentDescriptor( mockDescriptor, mongoDbMeta ); 91 | assertNotNull( dbDesc ); 92 | assertEquals( "db", dbDesc.getName() ); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/ui/swing/preview/PreviewRowsSwingDialog.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.ui.swing.preview; 15 | 16 | import java.util.Iterator; 17 | import java.util.List; 18 | import java.util.ResourceBundle; 19 | 20 | import org.pentaho.di.trans.step.BaseStepMeta; 21 | import org.pentaho.di.ui.xul.common.preview.AbstractPreviewRowsXulDialog; 22 | import org.pentaho.ui.xul.XulException; 23 | import org.pentaho.ui.xul.XulSettingsManager; 24 | import org.pentaho.ui.xul.containers.XulTree; 25 | import org.pentaho.ui.xul.containers.XulTreeRow; 26 | import org.pentaho.ui.xul.swing.SwingBindingFactory; 27 | import org.pentaho.ui.xul.swing.SwingXulLoader; 28 | import org.pentaho.ui.xul.swing.SwingXulRunner; 29 | import org.pentaho.ui.xul.swing.tags.SwingTreeCell; 30 | import org.pentaho.ui.xul.swing.tags.SwingTreeCol; 31 | import org.pentaho.ui.xul.swing.tags.SwingTreeCols; 32 | 33 | public class PreviewRowsSwingDialog extends AbstractPreviewRowsXulDialog { 34 | 35 | @Override public void init() { 36 | super.init(); 37 | } 38 | 39 | public PreviewRowsSwingDialog( Object parent, BaseStepMeta meta, int previewRowCount ) { 40 | super( parent, meta, previewRowCount ); 41 | } 42 | 43 | @Override public XulSettingsManager getSettingsManager() { 44 | return null; 45 | } 46 | 47 | @Override public ResourceBundle getResourceBundle() { 48 | return null; 49 | } 50 | 51 | protected void initializeXul() throws XulException { 52 | 53 | initializeXul( new SwingXulLoader(), new SwingBindingFactory(), new SwingXulRunner(), parent ); 54 | 55 | } 56 | 57 | @Override public void onAccept() { 58 | close(); 59 | dispose(); 60 | 61 | } 62 | 63 | @Override public void onCancel() { 64 | close(); 65 | dispose(); 66 | 67 | } 68 | 69 | @Override protected Class getClassForMessages() { 70 | return this.getClass(); 71 | } 72 | 73 | @Override public void dispose() { 74 | 75 | } 76 | 77 | /** 78 | * TODO: replace this method with XUL bindings 79 | *

80 | * This is a bad bad method. We need a way to load the column definitions and 81 | * data through standard XUL bindings. 82 | * 83 | * @param data 84 | * @param columns 85 | */ 86 | protected void createPreviewRows( List data, List columns ) { 87 | 88 | // Adds table rows. 89 | Object[] theObj = null; 90 | XulTreeRow theRow = null; 91 | Object theValue = null; 92 | SwingTreeCell theCell = null; 93 | 94 | XulTree table = (XulTree) super.document.getElementById( "tableData" ); 95 | table.getRootChildren().removeAll(); 96 | Iterator theItr = data.iterator(); 97 | while ( theItr.hasNext() ) { 98 | theObj = theItr.next(); 99 | theRow = table.getRootChildren().addNewRow(); 100 | for ( int i = 0; i < theObj.length; i++ ) { 101 | theValue = theObj[i]; 102 | theCell = new SwingTreeCell( null ); 103 | theCell.setLabel( theValue == null ? "" : theValue.toString() ); 104 | theRow.addCell( theCell ); 105 | } 106 | } 107 | 108 | // Adds table columns. 109 | SwingTreeCol theColumn = null; 110 | 111 | SwingTreeCols theColumns = new SwingTreeCols( null, table, null, null ); 112 | for ( int i = 0; i < columns.size(); i++ ) { 113 | theColumn = new SwingTreeCol( null, null, null, null ); 114 | theColumn.setWidth( 100 ); 115 | theColumn.setLabel( columns.get( i ) ); 116 | theColumns.addColumn( theColumn ); 117 | } 118 | table.setColumns( theColumns ); 119 | table.update(); 120 | 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 19 | 20 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbConnectionAnalyzer.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 17 | import org.pentaho.dictionary.DictionaryConst; 18 | import org.pentaho.metaverse.api.IComponentDescriptor; 19 | import org.pentaho.metaverse.api.IConnectionAnalyzer; 20 | import org.pentaho.metaverse.api.ILogicalIdGenerator; 21 | import org.pentaho.metaverse.api.IMetaverseNode; 22 | import org.pentaho.metaverse.api.MetaverseAnalyzerException; 23 | import org.pentaho.metaverse.api.MetaverseComponentDescriptor; 24 | import org.pentaho.metaverse.api.MetaverseLogicalIdGenerator; 25 | import org.pentaho.metaverse.api.analyzer.kettle.BaseKettleMetaverseComponent; 26 | 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | 30 | /** 31 | * User: RFellows Date: 3/6/15 32 | */ 33 | public class MongoDbConnectionAnalyzer extends BaseKettleMetaverseComponent implements 34 | IConnectionAnalyzer { 35 | 36 | public static final String HOST_NAMES = "hostNames"; 37 | public static final String DATABASE_NAME = "databaseName"; 38 | public static final String USE_ALL_REPLICA_SET_MEMBERS = "useAllReplicaSetMembers"; 39 | public static final String USE_KERBEROS_AUTHENTICATION = "useKerberosAuthentication"; 40 | public static final String CONNECTION_TIMEOUT = "connectionTimeout"; 41 | public static final String SOCKET_TIMEOUT = "socketTimeout"; 42 | public static final String CONNECTION_STRING = "connectionString"; 43 | 44 | public static final ILogicalIdGenerator ID_GENERATOR = new MetaverseLogicalIdGenerator( 45 | HOST_NAMES, 46 | DATABASE_NAME, 47 | DictionaryConst.PROPERTY_PORT, 48 | DictionaryConst.PROPERTY_USER_NAME 49 | ); 50 | 51 | @Override 52 | public IMetaverseNode analyze( IComponentDescriptor descriptor, MongoDbMeta mongoDbMeta ) 53 | throws MetaverseAnalyzerException { 54 | 55 | IMetaverseNode datasourceNode = createNodeFromDescriptor( descriptor ); 56 | 57 | String database = mongoDbMeta.getDbName(); 58 | datasourceNode.setName( database ); 59 | datasourceNode.setProperty( DATABASE_NAME, database ); 60 | datasourceNode.setType( DictionaryConst.NODE_TYPE_MONGODB_CONNECTION ); 61 | datasourceNode.setLogicalIdGenerator( getLogicalIdGenerator() ); 62 | 63 | if ( mongoDbMeta.isUseLegacyOptions() ) { 64 | String port = mongoDbMeta.getPort(); 65 | String hostNames = mongoDbMeta.getHostnames(); 66 | String user = mongoDbMeta.getAuthenticationUser(); 67 | boolean useAllReplicaSetMembers = mongoDbMeta.getUseAllReplicaSetMembers(); 68 | boolean useKerberosAuthentication = mongoDbMeta.getUseKerberosAuthentication(); 69 | String connectTimeout = mongoDbMeta.getConnectTimeout(); 70 | String socketTimeout = mongoDbMeta.getSocketTimeout(); 71 | 72 | datasourceNode.setProperty( HOST_NAMES, hostNames ); 73 | datasourceNode.setProperty( DictionaryConst.PROPERTY_USER_NAME, user ); 74 | datasourceNode.setProperty( DictionaryConst.PROPERTY_PORT, port ); 75 | datasourceNode.setProperty( USE_ALL_REPLICA_SET_MEMBERS, useAllReplicaSetMembers ); 76 | datasourceNode.setProperty( USE_KERBEROS_AUTHENTICATION, useKerberosAuthentication ); 77 | datasourceNode.setProperty( CONNECTION_TIMEOUT, connectTimeout ); 78 | datasourceNode.setProperty( SOCKET_TIMEOUT, socketTimeout ); 79 | } else if ( mongoDbMeta.isUseConnectionString() ) { 80 | String connectionString = mongoDbMeta.getConnectionString(); 81 | datasourceNode.setProperty( CONNECTION_STRING, connectionString ); 82 | } 83 | return datasourceNode; 84 | } 85 | 86 | @Override 87 | public List getUsedConnections( MongoDbMeta meta ) { 88 | List metas = new ArrayList(); 89 | metas.add( meta ); 90 | return metas; 91 | } 92 | 93 | @Override 94 | public IComponentDescriptor buildComponentDescriptor( IComponentDescriptor parentDescriptor, 95 | MongoDbMeta connection ) { 96 | 97 | IComponentDescriptor dbDescriptor = new MetaverseComponentDescriptor( 98 | connection.getDbName(), 99 | DictionaryConst.NODE_TYPE_MONGODB_CONNECTION, 100 | parentDescriptor.getNamespace(), 101 | parentDescriptor.getContext() ); 102 | 103 | return dbDescriptor; 104 | } 105 | 106 | @Override 107 | protected ILogicalIdGenerator getLogicalIdGenerator() { 108 | return ID_GENERATOR; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | pentaho 7 | pentaho-mongo-plugin-parent 8 | 11.1.0.0-SNAPSHOT 9 | 10 | 11 | pentaho 12 | pentaho-mongodb-plugin 13 | 11.1.0.0-SNAPSHOT 14 | jar 15 | 16 | Pentaho Enterprise Edition Project: ${project.artifactId} 17 | a Pentaho closed source project 18 | http://www.pentaho.com 19 | 20 | 21 | 22 | Pentaho Proprietary License 23 | http://www.pentaho.com 24 | repo 25 | 26 | 27 | 28 | 29 | 30 | scm:git:git@github.com:pentaho/pentaho-mongodb-plugin.git 31 | 32 | 33 | 34 | 5.10.0 35 | 4.13.2 36 | 3.12.10 37 | 11.1.0.0-SNAPSHOT 38 | 39 | 40 | 41 | 42 | pentaho-kettle 43 | kettle-core 44 | ${project.version} 45 | provided 46 | 47 | 48 | pentaho-kettle 49 | kettle-engine 50 | ${project.version} 51 | provided 52 | 53 | 54 | pentaho-kettle 55 | kettle-engine 56 | ${project.version} 57 | tests 58 | test 59 | 60 | 61 | pentaho-kettle 62 | kettle-ui-swt 63 | ${project.version} 64 | provided 65 | 66 | 67 | pentaho 68 | pentaho-mongo-utils 69 | ${project.version} 70 | provided 71 | 72 | 73 | pentaho 74 | pentaho-metaverse-api 75 | ${project.version} 76 | provided 77 | 78 | 79 | pentaho 80 | pentaho-platform-core 81 | ${platform.version} 82 | provided 83 | 84 | 85 | org.easymock 86 | easymock 87 | 5.2.0 88 | test 89 | 90 | 91 | org.mockito 92 | mockito-core 93 | ${mockito.version} 94 | test 95 | 96 | 97 | * 98 | * 99 | 100 | 101 | 102 | 103 | junit 104 | junit 105 | ${junit.version} 106 | test 107 | 108 | 109 | org.hamcrest 110 | java-hamcrest 111 | 2.0.0.0 112 | test 113 | 114 | 115 | javax.servlet 116 | javax.servlet-api 117 | 3.0.1 118 | test 119 | 120 | 121 | xerces 122 | xercesImpl 123 | test 124 | 125 | 126 | pentaho-kettle 127 | kettle-core 128 | ${project.version} 129 | tests 130 | test 131 | 132 | 133 | net.bytebuddy 134 | byte-buddy-agent 135 | ${byte-buddy.version} 136 | test 137 | 138 | 139 | net.bytebuddy 140 | byte-buddy 141 | ${byte-buddy.version} 142 | test 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputMetaTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.junit.BeforeClass; 17 | import org.junit.Test; 18 | import org.pentaho.di.core.Const; 19 | import org.pentaho.di.core.encryption.Encr; 20 | import org.pentaho.di.core.encryption.TwoWayPasswordEncoderPluginType; 21 | import org.pentaho.di.core.exception.KettleException; 22 | import org.pentaho.di.core.plugins.PluginRegistry; 23 | import org.pentaho.di.core.util.EnvUtil; 24 | import org.pentaho.di.trans.steps.loadsave.LoadSaveTester; 25 | import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; 26 | import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidatorFactory; 27 | import org.pentaho.di.trans.steps.loadsave.validator.ListLoadSaveValidator; 28 | import org.pentaho.di.trans.steps.loadsave.validator.ObjectValidator; 29 | import org.pentaho.mongo.wrapper.field.MongoField; 30 | 31 | import java.util.Arrays; 32 | import java.util.HashMap; 33 | import java.util.List; 34 | import java.util.Map; 35 | import java.util.UUID; 36 | 37 | public class MongoDbInputMetaTest { 38 | @BeforeClass 39 | public static void beforeClass() throws KettleException { 40 | PluginRegistry.addPluginType( TwoWayPasswordEncoderPluginType.getInstance() ); 41 | PluginRegistry.init(); 42 | String passwordEncoderPluginID = Const.NVL( EnvUtil.getSystemProperty( Const.KETTLE_PASSWORD_ENCODER_PLUGIN ), "Kettle" ); 43 | Encr.init( passwordEncoderPluginID ); 44 | } 45 | 46 | @Test public void testRoundTrips() throws KettleException, NoSuchMethodException, SecurityException { 47 | Map getterMap = new HashMap(); 48 | getterMap.put( "hostname", "getHostnames" ); 49 | getterMap.put( "auth_user", "getAuthenticationUser" ); 50 | getterMap.put( "auth_password", "getAuthenticationPassword" ); 51 | getterMap.put( "auth_kerberos", "getUseKerberosAuthentication" ); 52 | getterMap.put( "use_all_replica_members", "getUseAllReplicaSetMembers" ); 53 | getterMap.put( "execute_for_each_row", "getExecuteForEachIncomingRow" ); 54 | getterMap.put( "mongo_fields", "getMongoFields" ); 55 | getterMap.put( "tag_sets", "getReadPrefTagSets" ); 56 | Map setterMap = new HashMap(); 57 | setterMap.put( "hostname", "setHostnames" ); 58 | setterMap.put( "auth_user", "setAuthenticationUser" ); 59 | setterMap.put( "auth_password", "setAuthenticationPassword" ); 60 | setterMap.put( "auth_kerberos", "setUseKerberosAuthentication" ); 61 | setterMap.put( "use_all_replica_members", "setUseAllReplicaSetMembers" ); 62 | setterMap.put( "execute_for_each_row", "setExecuteForEachIncomingRow" ); 63 | setterMap.put( "mongo_fields", "setMongoFields" ); 64 | setterMap.put( "tag_sets", "setReadPrefTagSets" ); 65 | 66 | Map> 67 | fieldLoadSaveValidatorAttributeMap = 68 | new HashMap>(); 69 | fieldLoadSaveValidatorAttributeMap.put( "tag_sets", new ListLoadSaveValidator( 70 | new FieldLoadSaveValidator() { 71 | 72 | @Override 73 | public String getTestObject() { 74 | return "{" + UUID.randomUUID() + "}"; 75 | } 76 | 77 | @Override 78 | public boolean validateTestObject( String testObject, Object actual ) { 79 | return testObject.equals( actual ); 80 | } 81 | } ) ); 82 | Map> 83 | fieldLoadSaveValidatorTypeMap = 84 | new HashMap>(); 85 | 86 | LoadSaveTester 87 | tester = 88 | new LoadSaveTester( MongoDbInputMeta.class, Arrays.asList( "hostname", "port", "db_name", 89 | "fields_name", "collection", "json_field_name", "json_query", "auth_user", "auth_password", 90 | "auth_kerberos", "connect_timeout", "socket_timeout", "read_preference", "output_json", 91 | "use_all_replica_members", "query_is_pipeline", "execute_for_each_row", "mongo_fields", "tag_sets" ), 92 | getterMap, setterMap, fieldLoadSaveValidatorAttributeMap, fieldLoadSaveValidatorTypeMap ); 93 | 94 | FieldLoadSaveValidatorFactory validatorFactory = tester.getFieldLoadSaveValidatorFactory(); 95 | 96 | validatorFactory.registerValidator( validatorFactory.getName( List.class, MongoField.class ), 97 | new ListLoadSaveValidator( new ObjectValidator( validatorFactory, MongoField.class, 98 | Arrays.asList( "m_fieldName", "m_fieldPath", "m_kettleType", "m_indexedVals" ) ) ) ); 99 | 100 | tester.testXmlRoundTrip(); 101 | tester.testRepoRoundTrip(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/MongoDB.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 12 | 14 | 19 | 21 | 23 | 26 | 27 | 28 | 30 | 32 | 35 | 41 | 42 | 43 | 44 | 46 | 48 | 51 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/ui/trans/steps/mongodbinput/models/MongoDocumentField.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.ui.trans.steps.mongodbinput.models; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Arrays; 18 | import java.util.List; 19 | 20 | import org.pentaho.di.core.Const; 21 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputData; 22 | import org.pentaho.mongo.wrapper.field.MongoField; 23 | import org.pentaho.ui.xul.XulEventSourceAdapter; 24 | import org.pentaho.ui.xul.util.AbstractModelList; 25 | 26 | public class MongoDocumentField extends XulEventSourceAdapter { 27 | 28 | private String m_fieldName; 29 | private String m_fieldPath; 30 | private String m_kettleType; 31 | private List m_indexedVals; 32 | private String m_arrayIndexInfo; 33 | private String m_occurenceFraction; 34 | private boolean m_disparateTypes; 35 | 36 | public String getFieldName() { 37 | return this.m_fieldName; 38 | } 39 | 40 | public void setFieldName( String name ) { 41 | this.m_fieldName = name; 42 | } 43 | 44 | public String getFieldPath() { 45 | return this.m_fieldPath; 46 | } 47 | 48 | public void setFieldPath( String path ) { 49 | this.m_fieldPath = path; 50 | } 51 | 52 | public String getKettleType() { 53 | return this.m_kettleType; 54 | } 55 | 56 | public void setKettleType( String type ) { 57 | this.m_kettleType = type; 58 | } 59 | 60 | public String getIndexedValues() { 61 | if ( !Const.isEmpty( m_indexedVals ) ) { 62 | return MongoDbInputData.indexedValsList( m_indexedVals ); 63 | } 64 | return ""; 65 | } 66 | 67 | public void setIndexedValues( String values ) { 68 | String[] vals = values.split( ",", -2 ); 69 | m_indexedVals = Arrays.asList( vals ); 70 | } 71 | 72 | public String getArrayIndexInfo() { 73 | return this.m_arrayIndexInfo; 74 | } 75 | 76 | public void setArrayIndexInfo( String info ) { 77 | this.m_arrayIndexInfo = info; 78 | } 79 | 80 | public String getOccurenceFraction() { 81 | return this.m_occurenceFraction; 82 | } 83 | 84 | public void setOccurenceFraction( String fraction ) { 85 | this.m_occurenceFraction = fraction; 86 | } 87 | 88 | public String getIsDisparateTypes() { 89 | return ( this.m_disparateTypes ? "Y" : "" ); 90 | } 91 | 92 | public void setIsDisparateTypes( String bool ) { 93 | this.m_disparateTypes = "Y".equals( bool ) ? true : false; 94 | } 95 | 96 | protected void convertFrom( MongoField field ) { 97 | this.m_arrayIndexInfo = field.m_arrayIndexInfo; 98 | this.m_disparateTypes = field.m_disparateTypes; 99 | this.m_fieldName = field.m_fieldName; 100 | this.m_fieldPath = field.m_fieldPath; 101 | this.m_indexedVals = field.m_indexedVals; 102 | this.m_kettleType = field.m_kettleType; 103 | this.m_occurenceFraction = field.m_occurenceFraction; 104 | } 105 | 106 | protected void convertTo( MongoField field ) { 107 | field.m_arrayIndexInfo = this.m_arrayIndexInfo; 108 | field.m_disparateTypes = this.m_disparateTypes; 109 | field.m_fieldName = this.m_fieldName; 110 | field.m_fieldPath = this.m_fieldPath; 111 | field.m_indexedVals = this.m_indexedVals; 112 | field.m_kettleType = this.m_kettleType; 113 | field.m_occurenceFraction = this.m_occurenceFraction; 114 | } 115 | 116 | public static void convertList( List fields, AbstractModelList docFields ) { 117 | 118 | if ( fields == null || fields.isEmpty() ) { 119 | return; 120 | } 121 | for ( MongoField field : fields ) { 122 | MongoDocumentField docField = new MongoDocumentField(); 123 | docField.convertFrom( field ); 124 | docFields.add( docField ); 125 | } 126 | } 127 | 128 | public static void trimList( List fields, AbstractModelList docFields ) { 129 | 130 | if ( fields == null || fields.isEmpty() ) { 131 | return; 132 | } 133 | for ( int i = fields.size() - 1; i >= 0; i-- ) { 134 | MongoField field = fields.get( i ); 135 | for ( MongoDocumentField docField : docFields.asList() ) { 136 | if ( ( field.m_fieldName.equalsIgnoreCase( docField.m_fieldName ) ) && ( field.m_fieldPath 137 | .equalsIgnoreCase( docField.m_fieldPath ) ) ) { 138 | fields.remove( field ); 139 | break; 140 | } 141 | } 142 | } 143 | } 144 | 145 | public static List convertFromList( AbstractModelList docFields ) { 146 | List fields = new ArrayList(); 147 | 148 | if ( docFields == null || docFields.isEmpty() ) { 149 | return fields; 150 | } 151 | for ( MongoDocumentField docField : docFields ) { 152 | MongoField field = new MongoField(); 153 | docField.convertTo( field ); 154 | fields.add( field ); 155 | } 156 | return fields; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbResourceInfoTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.junit.Before; 17 | import org.junit.BeforeClass; 18 | import org.junit.Test; 19 | import org.pentaho.di.core.encryption.Encr; 20 | import org.pentaho.di.core.encryption.TwoWayPasswordEncoderPluginType; 21 | import org.pentaho.di.core.plugins.PluginRegistry; 22 | import org.pentaho.di.core.variables.VariableSpace; 23 | import org.pentaho.di.core.variables.Variables; 24 | import org.pentaho.di.trans.TransMeta; 25 | import org.pentaho.di.trans.step.StepMeta; 26 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 27 | 28 | import static org.junit.Assert.assertEquals; 29 | import static org.junit.Assert.assertFalse; 30 | import static org.junit.Assert.assertNull; 31 | import static org.junit.Assert.assertTrue; 32 | import static org.mockito.Mockito.mock; 33 | import static org.mockito.Mockito.when; 34 | 35 | public class MongoDbResourceInfoTest { 36 | 37 | MongoDbResourceInfo info; 38 | 39 | MongoDbMeta meta; 40 | 41 | @BeforeClass 42 | public static void init() throws Exception { 43 | PluginRegistry.addPluginType( TwoWayPasswordEncoderPluginType.getInstance() ); 44 | PluginRegistry.init(); 45 | Encr.init( "Kettle" ); 46 | } 47 | 48 | @Before 49 | public void setUp() throws Exception { 50 | meta = mock( MongoDbMeta.class ); 51 | when( meta.getDbName() ).thenReturn( null ); 52 | when( meta.getPort() ).thenReturn( null ); 53 | when( meta.getHostnames() ).thenReturn( null ); 54 | when( meta.getAuthenticationUser() ).thenReturn( null ); 55 | when( meta.getAuthenticationPassword() ).thenReturn( null ); 56 | when( meta.getUseAllReplicaSetMembers() ).thenReturn( false ); 57 | when( meta.getUseKerberosAuthentication() ).thenReturn( false ); 58 | when( meta.getConnectTimeout() ).thenReturn( null ); 59 | when( meta.getSocketTimeout() ).thenReturn( null ); 60 | when( meta.getCollection() ).thenReturn( null ); 61 | StepMeta stepMeta = new StepMeta(); 62 | TransMeta transMeta = new TransMeta(); 63 | 64 | stepMeta.setParentTransMeta( transMeta ); 65 | VariableSpace variables = new Variables(); 66 | 67 | when( meta.getParentStepMeta() ).thenReturn( stepMeta ); 68 | info = new MongoDbResourceInfo( meta ); 69 | } 70 | 71 | @Test 72 | public void testStringConstructor() { 73 | info = new MongoDbResourceInfo( "localhost, remote.pentaho.com" , "1000", "myDb" ); 74 | assertEquals( "localhost, remote.pentaho.com", info.getHostNames() ); 75 | assertEquals( "1000", info.getPort() ); 76 | assertEquals( "myDb", info.getDatabase() ); 77 | } 78 | 79 | @Test 80 | public void testGetConnectTimeout() throws Exception { 81 | assertNull( info.getConnectTimeout() ); 82 | info.setConnectTimeout( "1000" ); 83 | assertEquals( "1000", info.getConnectTimeout() ); 84 | } 85 | 86 | @Test 87 | public void testGetDatabase() throws Exception { 88 | assertNull( info.getDatabase() ); 89 | info.setDatabase( "myDb" ); 90 | assertEquals( "myDb", info.getDatabase() ); 91 | } 92 | 93 | @Test 94 | public void testGetHostNames() throws Exception { 95 | assertNull( info.getHostNames() ); 96 | info.setHostNames( "localhost, remote.pentaho.com" ); 97 | assertEquals( "localhost, remote.pentaho.com", info.getHostNames() ); 98 | } 99 | 100 | @Test 101 | public void testGetPort() throws Exception { 102 | assertNull( info.getPort() ); 103 | info.setPort( "1000" ); 104 | assertEquals( "1000", info.getPort() ); 105 | } 106 | 107 | @Test 108 | public void testGetSocketTimeout() throws Exception { 109 | assertNull( info.getSocketTimeout() ); 110 | info.setSocketTimeout( "1000" ); 111 | assertEquals( "1000", info.getSocketTimeout() ); 112 | } 113 | 114 | @Test 115 | public void testIsUseAllReplicaSetMembers() throws Exception { 116 | assertFalse( info.isUseAllReplicaSetMembers() ); 117 | info.setUseAllReplicaSetMembers( true ); 118 | assertTrue( info.isUseAllReplicaSetMembers() ); 119 | } 120 | 121 | @Test 122 | public void testIsUseKerberosAuthentication() throws Exception { 123 | assertFalse( info.isUseKerberosAuthentication() ); 124 | info.setUseKerberosAuthentication( true ); 125 | assertTrue( info.isUseKerberosAuthentication() ); 126 | } 127 | 128 | @Test 129 | public void testGetUser() throws Exception { 130 | assertNull( info.getUser() ); 131 | info.setUser( "joe" ); 132 | assertEquals( "joe", info.getUser() ); 133 | } 134 | 135 | @Test 136 | public void testGetPassword() throws Exception { 137 | assertNull( info.getPassword() ); 138 | info.setPassword( "password" ); 139 | assertEquals( "password", info.getPassword() ); 140 | } 141 | 142 | @Test 143 | public void testGetEncryptedPassword() throws Exception { 144 | assertNull( info.getPassword() ); 145 | info.setPassword( "password" ); 146 | assertEquals( "Encrypted 2be98afc86aa7f2e4bb18bd63c99dbdde", info.getEncryptedPassword() ); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodboutput/MongoDbOutputMetaTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodboutput; 15 | 16 | import org.junit.BeforeClass; 17 | import org.junit.Test; 18 | import org.pentaho.di.core.Const; 19 | import org.pentaho.di.core.encryption.Encr; 20 | import org.pentaho.di.core.encryption.TwoWayPasswordEncoderPluginType; 21 | import org.pentaho.di.core.exception.KettleException; 22 | import org.pentaho.di.core.plugins.PluginRegistry; 23 | import org.pentaho.di.core.util.EnvUtil; 24 | import org.pentaho.di.repository.ObjectId; 25 | import org.pentaho.di.repository.Repository; 26 | import org.pentaho.di.trans.steps.loadsave.LoadSaveTester; 27 | import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidatorFactory; 28 | import org.pentaho.di.trans.steps.loadsave.validator.ListLoadSaveValidator; 29 | import org.pentaho.di.trans.steps.loadsave.validator.ObjectValidator; 30 | import org.pentaho.di.trans.steps.mongodboutput.MongoDbOutputMeta.MongoField; 31 | import org.pentaho.di.trans.steps.mongodboutput.MongoDbOutputMeta.MongoIndex; 32 | import org.pentaho.metastore.api.IMetaStore; 33 | 34 | import java.lang.reflect.Method; 35 | import java.util.Arrays; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | import java.util.Map; 39 | 40 | import static org.junit.Assert.assertTrue; 41 | import static org.junit.Assert.fail; 42 | 43 | public class MongoDbOutputMetaTest { 44 | @BeforeClass public static void beforeClass() throws KettleException { 45 | PluginRegistry.addPluginType( TwoWayPasswordEncoderPluginType.getInstance() ); 46 | PluginRegistry.init(); 47 | String 48 | passwordEncoderPluginID = 49 | Const.NVL( EnvUtil.getSystemProperty( Const.KETTLE_PASSWORD_ENCODER_PLUGIN ), "Kettle" ); 50 | Encr.init( passwordEncoderPluginID ); 51 | } 52 | 53 | @Test public void testRoundTrips() throws KettleException { 54 | List commonFields = 55 | Arrays.asList( "mongo_host", "mongo_port", "use_all_replica_members", "mongo_user", "mongo_password", 56 | "auth_kerberos", "mongo_db", "mongo_collection", "batch_insert_size", "connect_timeout", "socket_timeout", 57 | "read_preference", "write_concern", "w_timeout", "journaled_writes", "truncate", "update", "upsert", 58 | "multi", "modifier_update", "write_retries", "write_retry_delay", "mongo_fields", "mongo_indexes" ); 59 | Map getterMap = new HashMap(); 60 | getterMap.put( "mongo_host", "getHostnames" ); 61 | getterMap.put( "mongo_port", "getPort" ); 62 | getterMap.put( "use_all_replica_members", "getUseAllReplicaSetMembers" ); 63 | getterMap.put( "mongo_user", "getAuthenticationUser" ); 64 | getterMap.put( "mongo_password", "getAuthenticationPassword" ); 65 | getterMap.put( "auth_kerberos", "getUseKerberosAuthentication" ); 66 | getterMap.put( "mongo_db", "getDbName" ); 67 | getterMap.put( "mongo_collection", "getCollection" ); 68 | getterMap.put( "journaled_writes", "getJournal" ); 69 | 70 | Map setterMap = new HashMap(); 71 | setterMap.put( "mongo_host", "setHostnames" ); 72 | setterMap.put( "mongo_port", "setPort" ); 73 | setterMap.put( "use_all_replica_members", "setUseAllReplicaSetMembers" ); 74 | setterMap.put( "mongo_user", "setAuthenticationUser" ); 75 | setterMap.put( "mongo_password", "setAuthenticationPassword" ); 76 | setterMap.put( "auth_kerberos", "setUseKerberosAuthentication" ); 77 | setterMap.put( "mongo_db", "setDbName" ); 78 | setterMap.put( "mongo_collection", "setCollection" ); 79 | setterMap.put( "batch_insert_size", "setBatchInsertSize" ); 80 | setterMap.put( "journaled_writes", "setJournal" ); 81 | 82 | LoadSaveTester tester = new LoadSaveTester( MongoDbOutputMeta.class, commonFields, getterMap, setterMap ); 83 | 84 | FieldLoadSaveValidatorFactory validatorFactory = tester.getFieldLoadSaveValidatorFactory(); 85 | 86 | validatorFactory.registerValidator( validatorFactory.getName( List.class, MongoField.class ), 87 | new ListLoadSaveValidator( new ObjectValidator( validatorFactory, MongoField.class, 88 | Arrays.asList( "m_incomingFieldName", "m_mongoDocPath", "m_useIncomingFieldNameAsMongoFieldName", 89 | "m_updateMatchField", "m_modifierUpdateOperation", "m_modifierOperationApplyPolicy", "m_JSON", 90 | "insertNull" ) ) ) ); 91 | 92 | validatorFactory.registerValidator( validatorFactory.getName( List.class, MongoIndex.class ), 93 | new ListLoadSaveValidator( new ObjectValidator( validatorFactory, MongoIndex.class, 94 | Arrays.asList( "m_pathToFields", "m_drop", "m_unique", "m_sparse" ) ) ) ); 95 | 96 | tester.testXmlRoundTrip(); 97 | tester.testRepoRoundTrip(); 98 | } 99 | 100 | @Test public void testForPDI12155_NotDeprecatedSaveRepMethodImplemented() throws Exception { 101 | Class[] cArg = { Repository.class, IMetaStore.class, ObjectId.class, ObjectId.class }; 102 | try { 103 | MongoDbOutputMeta.class.getDeclaredMethod( "saveRep", cArg ); 104 | } catch ( NoSuchMethodException e ) { 105 | fail( "There is no such a method BUT should be: " + e.getMessage() ); 106 | } 107 | } 108 | 109 | @Test public void testForPDI12155_DeprecatedSaveRepMethodNotImplemented() throws Exception { 110 | Class[] cArg = { Repository.class, ObjectId.class, ObjectId.class }; 111 | try { 112 | Method declaredMethod = MongoDbOutputMeta.class.getDeclaredMethod( "saveRep", cArg ); 113 | fail( "There is a method BUT should not be: " + declaredMethod ); 114 | } catch ( NoSuchMethodException e ) { 115 | assertTrue( true ); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/mongo/wrapper/MongoWrapperUtil.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper; 15 | 16 | import com.google.common.annotations.VisibleForTesting; 17 | import org.pentaho.di.core.encryption.Encr; 18 | import org.pentaho.di.core.logging.LogChannelInterface; 19 | import org.pentaho.di.core.variables.VariableSpace; 20 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 21 | import org.pentaho.mongo.MongoDbException; 22 | import org.pentaho.mongo.MongoProp; 23 | import org.pentaho.mongo.MongoProperties; 24 | import org.pentaho.mongo.MongoUtilLogger; 25 | 26 | /** 27 | * Created by bryan on 8/7/14. 28 | */ 29 | public class MongoWrapperUtil { 30 | private static MongoWrapperClientFactory mongoWrapperClientFactory = new MongoWrapperClientFactory() { 31 | @Override public MongoClientWrapper createMongoClientWrapper( MongoProperties props, MongoUtilLogger log ) 32 | throws MongoDbException { 33 | return MongoClientWrapperFactory.createMongoClientWrapper( props, log ); 34 | } 35 | @Override public MongoClientWrapper createConnectionStringMongoClientWrapper( String connectionString, MongoUtilLogger log ) 36 | throws MongoDbException { 37 | return MongoClientWrapperFactory.createConnectionStringMongoClientWrapper( connectionString, log ); 38 | } 39 | }; 40 | 41 | public static void setMongoWrapperClientFactory( MongoWrapperClientFactory mongoWrapperClientFactory ) { 42 | MongoWrapperUtil.mongoWrapperClientFactory = mongoWrapperClientFactory; 43 | } 44 | 45 | @VisibleForTesting 46 | public static MongoWrapperClientFactory getMongoWrapperClientFactory() { 47 | return mongoWrapperClientFactory; 48 | } 49 | 50 | public static MongoClientWrapper createMongoClientWrapper( MongoDbMeta mongoDbMeta, VariableSpace vars, 51 | LogChannelInterface log ) throws MongoDbException { 52 | if ( mongoDbMeta.isUseConnectionString() ) { 53 | return mongoWrapperClientFactory.createConnectionStringMongoClientWrapper( 54 | Encr.decryptPasswordOptionallyEncrypted( vars.environmentSubstitute( mongoDbMeta.getConnectionString() ) ), new KettleMongoUtilLogger( log ) ); 55 | } else { 56 | MongoProperties.Builder propertiesBuilder = createPropertiesBuilder( mongoDbMeta, vars ); 57 | 58 | return mongoWrapperClientFactory 59 | .createMongoClientWrapper( propertiesBuilder.build(), new KettleMongoUtilLogger( log ) ); 60 | } 61 | } 62 | 63 | public static MongoProperties.Builder createPropertiesBuilder( MongoDbMeta mongoDbMeta, VariableSpace vars ) { 64 | MongoProperties.Builder propertiesBuilder = new MongoProperties.Builder(); 65 | 66 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.HOST, vars.environmentSubstitute( mongoDbMeta.getHostnames() ) ); 67 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.PORT, vars.environmentSubstitute( mongoDbMeta.getPort() ) ); 68 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.DBNAME, vars.environmentSubstitute( mongoDbMeta.getDbName() ) ); 69 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.connectTimeout, vars.environmentSubstitute( mongoDbMeta.getConnectTimeout() ) ); 70 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.socketTimeout, vars.environmentSubstitute( mongoDbMeta.getSocketTimeout() ) ); 71 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.readPreference, mongoDbMeta.getReadPreference() ); 72 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.writeConcern, mongoDbMeta.getWriteConcern() ); 73 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.wTimeout, mongoDbMeta.getWTimeout() ); 74 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.JOURNALED, Boolean.toString( mongoDbMeta.getJournal() ) ); 75 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.USE_ALL_REPLICA_SET_MEMBERS, 76 | Boolean.toString( mongoDbMeta.getUseAllReplicaSetMembers() ) ); 77 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.AUTH_DATABASE, vars.environmentSubstitute( mongoDbMeta.getAuthenticationDatabaseName() ) ); 78 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.USERNAME, vars.environmentSubstitute( mongoDbMeta.getAuthenticationUser() ) ); 79 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.PASSWORD, vars.environmentSubstitute( mongoDbMeta.getAuthenticationPassword() ) ); 80 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.AUTH_MECHA, mongoDbMeta.getAuthenticationMechanism() ); 81 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.USE_KERBEROS, 82 | Boolean.toString( mongoDbMeta.getUseKerberosAuthentication() ) ); 83 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.useSSL, 84 | Boolean.toString( mongoDbMeta.isUseSSLSocketFactory() ) ); 85 | if ( mongoDbMeta.getReadPrefTagSets() != null ) { 86 | StringBuilder tagSet = new StringBuilder(); 87 | for ( String tag : mongoDbMeta.getReadPrefTagSets() ) { 88 | tagSet.append( tag ); 89 | tagSet.append( "," ); 90 | } 91 | // Remove trailing comma 92 | if ( tagSet.length() > 0 ) { 93 | tagSet.setLength( tagSet.length() - 1 ); 94 | } 95 | setIfNotNullOrEmpty( propertiesBuilder, MongoProp.tagSet, tagSet.toString() ); 96 | } 97 | 98 | return propertiesBuilder; 99 | } 100 | 101 | public static MongoClientWrapper createMongoClientWrapper( MongoProperties.Builder properties, LogChannelInterface log ) 102 | throws MongoDbException { 103 | return mongoWrapperClientFactory 104 | .createMongoClientWrapper( properties.build(), new KettleMongoUtilLogger( log ) ); 105 | } 106 | 107 | private static void setIfNotNullOrEmpty( MongoProperties.Builder builder, MongoProp prop, String value ) { 108 | if ( value != null && value.trim().length() > 0 ) { 109 | boolean isPassword = MongoProp.PASSWORD.equals( prop ); 110 | if ( isPassword ) { 111 | value = Encr.decryptPasswordOptionallyEncrypted( value ); 112 | } 113 | builder.set( prop, value ); 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/resources/org/pentaho/di/ui/trans/steps/mongodbinput/xul/mongodb_input.xul: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 34 | 35 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 49 | 50 | 53 | 54 | 55 | 56 | 57 | 60 | 61 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 170 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/mongo/wrapper/MongoFieldTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper; 15 | 16 | import com.mongodb.BasicDBObject; 17 | import com.mongodb.util.JSON; 18 | import org.bson.types.Binary; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.mockito.Mock; 22 | import org.mockito.MockitoAnnotations; 23 | import org.mockito.invocation.InvocationOnMock; 24 | import org.mockito.stubbing.Answer; 25 | import org.pentaho.di.core.exception.KettleException; 26 | import org.pentaho.di.core.exception.KettlePluginException; 27 | import org.pentaho.di.core.plugins.PluginRegistry; 28 | import org.pentaho.di.core.row.value.ValueMetaPluginType; 29 | import org.pentaho.di.core.variables.VariableSpace; 30 | import org.pentaho.mongo.wrapper.field.MongoField; 31 | 32 | import java.math.BigDecimal; 33 | import java.util.Date; 34 | 35 | import static org.hamcrest.CoreMatchers.equalTo; 36 | import static org.hamcrest.CoreMatchers.instanceOf; 37 | import static org.junit.Assert.assertEquals; 38 | import static org.junit.Assert.assertFalse; 39 | import static org.junit.Assert.assertNull; 40 | import static org.junit.Assert.assertThat; 41 | import static org.junit.Assert.assertTrue; 42 | import static org.junit.Assert.fail; 43 | import static org.mockito.Mockito.any; 44 | import static org.mockito.Mockito.when; 45 | 46 | public class MongoFieldTest { 47 | 48 | @Mock VariableSpace space; 49 | private MongoField field; 50 | 51 | @Before 52 | public void before() throws KettlePluginException { 53 | MockitoAnnotations.initMocks( this ); 54 | when( space.environmentSubstitute( any( String.class ) ) ) 55 | .thenAnswer( new Answer() { 56 | @Override public String answer( InvocationOnMock invocationOnMock ) throws Throwable { 57 | return (String) invocationOnMock.getArguments()[0]; 58 | } 59 | } ); 60 | PluginRegistry.addPluginType( ValueMetaPluginType.getInstance() ); 61 | PluginRegistry.init(); 62 | } 63 | 64 | @Test 65 | public void testGetPath() throws Exception { 66 | MongoField mongoField = new MongoField(); 67 | 68 | mongoField.m_fieldPath = "$.parent[0].child[0]"; 69 | assertEquals( "parent.0.child.0", mongoField.getPath() ); 70 | 71 | mongoField.m_fieldPath = "$.field[*]"; 72 | assertEquals( "field", mongoField.getPath() ); 73 | 74 | mongoField.m_fieldPath = "$.parent.child"; 75 | assertEquals( "parent.child", mongoField.getPath() ); 76 | } 77 | 78 | //"Number", "String", "Date", "Boolean", "Integer", "BigNumber", "Serializable", 79 | // "Binary", "Timestamp", "Internet Address" 80 | @Test 81 | public void testDatatypes() throws KettleException { 82 | initField( "Number" ); 83 | assertThat( field.getKettleValue( 1.1 ), equalTo( (Object) 1.1 ) ); 84 | assertThat( field.getKettleValue( "1.1" ), equalTo( (Object) 1.1 ) ); 85 | assertThat( field.getKettleValue( 86 | new Binary( new byte[] { '1', '.', '1' } ) ), equalTo( (Object) 1.1 ) ); 87 | 88 | initField( "BigNumber" ); 89 | Date date = new Date(); 90 | assertThat( field.getKettleValue( date ), equalTo( (Object) BigDecimal.valueOf( date.getTime() ) ) ); 91 | assertThat( field.getKettleValue( 12341234 ), equalTo( (Object) BigDecimal.valueOf( 12341234 ) ) ); 92 | assertThat( field.getKettleValue( "12341234" ), equalTo( (Object) BigDecimal.valueOf( 12341234 ) ) ); 93 | 94 | initField( "Boolean" ); 95 | assertTrue( (Boolean) field.getKettleValue( 1 ) ); 96 | assertTrue( (Boolean) field.getKettleValue( "Y" ) ); 97 | assertFalse( (Boolean) field.getKettleValue( 0 ) ); 98 | assertTrue( (Boolean) field.getKettleValue( true ) ); 99 | 100 | initField( "Binary" ); 101 | byte[] data = new byte[] { 'a', 'b', 'c' }; 102 | assertThat( field.getKettleValue( new Binary( data ) ), equalTo( (Object) data ) ); 103 | assertThat( (byte[]) field.getKettleValue( data ), equalTo( data ) ); 104 | assertThat( field.getKettleValue( "abc" ), equalTo( (Object) data ) ); 105 | 106 | initField( "Date" ); 107 | assertThat( field.getKettleValue( date ), equalTo( (Object) date ) ); 108 | assertThat( field.getKettleValue( date.getTime() ), equalTo( (Object) date ) ); 109 | try { 110 | field.getKettleValue( "Not a date value" ); 111 | fail( "expected exception" ); 112 | } catch ( Exception e ) { 113 | assertThat( e, instanceOf( KettleException.class ) ); 114 | } 115 | 116 | initField( "Integer" ); 117 | assertThat( field.getKettleValue( 123 ), equalTo( (Object) 123l ) ); 118 | assertThat( field.getKettleValue( "123" ), equalTo( (Object) 123l ) ); 119 | assertThat( field.getKettleValue( 120 | new Binary( new byte[] { '1', '2', '3' } ) ), equalTo( (Object) 123l ) ); 121 | 122 | initField( "String" ); 123 | assertThat( field.getKettleValue( "foo" ), equalTo( (Object) "foo" ) ); 124 | assertThat( field.getKettleValue( 123 ), equalTo( (Object) "123" ) ); 125 | } 126 | 127 | @Test 128 | public void testConvertArrayIndicesToKettleValue() throws KettleException { 129 | BasicDBObject dbObj = (BasicDBObject) JSON.parse( "{ parent : { fieldName : ['valA', 'valB'] } } " ); 130 | 131 | initField( "fieldName", "$.parent.fieldName[0]", "String" ); 132 | assertThat( field.convertToKettleValue( dbObj ), equalTo( (Object) "valA" ) ); 133 | initField( "fieldName", "$.parent.fieldName[1]", "String" ); 134 | assertThat( field.convertToKettleValue( dbObj ), equalTo( (Object) "valB" ) ); 135 | } 136 | 137 | @Test 138 | public void testConvertUndefinedOrNullToKettleValue() throws KettleException { 139 | BasicDBObject dbObj = BasicDBObject.parse( "{ test1 : undefined, test2 : null } " ); 140 | initField( "fieldName", "$.test1", "String" ); 141 | //PDI-16090S 142 | assertNull( "Undefined should be interpreted as null ", field.convertToKettleValue( dbObj ) ); 143 | initField( "fieldName", "$.test2", "String" ); 144 | assertNull( field.convertToKettleValue( dbObj ) ); 145 | initField( "fieldName", "$.test3", "String" ); 146 | assertNull( field.convertToKettleValue( dbObj ) ); 147 | } 148 | 149 | private void initField( String type ) throws KettleException { 150 | initField( "fieldName", "$.parent.child.fieldName", type ); 151 | } 152 | 153 | private void initField( String name, String path, String type ) throws KettleException { 154 | field = new MongoField(); 155 | field.m_fieldName = name; 156 | field.m_fieldPath = path; 157 | field.m_kettleType = type; 158 | field.init( 0 ); 159 | field.reset( space ); 160 | 161 | } 162 | 163 | } 164 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputMetaInjectionTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.junit.After; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import org.pentaho.di.core.injection.BaseMetadataInjectionTest; 20 | import org.pentaho.di.core.logging.KettleLogStore; 21 | import org.pentaho.di.core.logging.LogChannelInterfaceFactory; 22 | import org.pentaho.di.trans.steps.mongodboutput.MongoDbOutputMetaInjectionTest; 23 | 24 | /** 25 | * MDI test for MongoDbInput. 26 | */ 27 | public class MongoDbInputMetaInjectionTest extends BaseMetadataInjectionTest { 28 | 29 | private LogChannelInterfaceFactory oldLogChannelInterfaceFactory; 30 | 31 | @Before 32 | public void setup() { 33 | oldLogChannelInterfaceFactory = KettleLogStore.getLogChannelInterfaceFactory(); 34 | MongoDbOutputMetaInjectionTest.setKettleLogFactoryWithMock(); 35 | setup( new MongoDbInputMeta() ); 36 | } 37 | 38 | @After 39 | public void tearDown() { 40 | KettleLogStore.setLogChannelInterfaceFactory( oldLogChannelInterfaceFactory ); 41 | } 42 | 43 | @Test 44 | public void test() throws Exception { 45 | check( "HOSTNAME", new StringGetter() { 46 | public String get() { 47 | return meta.getHostnames(); 48 | } 49 | } ); 50 | check( "JSON_FIELD", new StringGetter() { 51 | public String get() { 52 | return meta.getFieldsName(); 53 | } 54 | } ); 55 | check( "JSON_QUERY", new StringGetter() { 56 | public String get() { 57 | return meta.getJsonQuery(); 58 | } 59 | } ); 60 | check( "PORT", new StringGetter() { 61 | public String get() { 62 | return meta.getPort(); 63 | } 64 | } ); 65 | check( "DATABASE_NAME", new StringGetter() { 66 | public String get() { 67 | return meta.getDbName(); 68 | } 69 | } ); 70 | check( "COLLECTION", new StringGetter() { 71 | public String get() { 72 | return meta.getCollection(); 73 | } 74 | } ); 75 | check( "AUTH_DATABASE", new StringGetter() { 76 | public String get() { 77 | return meta.getAuthenticationDatabaseName(); 78 | } 79 | } ); 80 | check( "AUTH_USERNAME", new StringGetter() { 81 | public String get() { 82 | return meta.getAuthenticationUser(); 83 | } 84 | } ); 85 | check( "AUTH_PASSWORD", new StringGetter() { 86 | public String get() { 87 | return meta.getAuthenticationPassword(); 88 | } 89 | } ); 90 | check( "AUTH_MECHANISM", new StringGetter() { 91 | public String get() { 92 | return meta.getAuthenticationMechanism(); 93 | } 94 | } ); 95 | check( "AUTH_KERBEROS", new BooleanGetter() { 96 | public boolean get() { 97 | return meta.getUseKerberosAuthentication(); 98 | } 99 | } ); 100 | check( "TIMEOUT_CONNECTION", new StringGetter() { 101 | public String get() { 102 | return meta.getConnectTimeout(); 103 | } 104 | } ); 105 | check( "TIMEOUT_SOCKET", new StringGetter() { 106 | public String get() { 107 | return meta.getSocketTimeout(); 108 | } 109 | } ); 110 | check( "USE_SSL_SOCKET_FACTORY", new BooleanGetter() { 111 | public boolean get() { 112 | return meta.isUseSSLSocketFactory(); 113 | } 114 | } ); 115 | check( "USE_CONNECTION_STRING", new BooleanGetter() { 116 | @Override 117 | public boolean get() { 118 | return meta.isUseConnectionString(); 119 | } 120 | } ); 121 | check( "USE_LEGACY_OPTIONS", new BooleanGetter() { 122 | @Override 123 | public boolean get() { 124 | return meta.isUseLegacyOptions(); 125 | } 126 | } ); 127 | check( "CONNECTION_STRING", new StringGetter() { 128 | @Override 129 | public String get() { 130 | return meta.getConnectionString(); 131 | } 132 | } ); 133 | check( "READ_PREFERENCE", new StringGetter() { 134 | public String get() { 135 | return meta.getReadPreference(); 136 | } 137 | } ); 138 | check( "USE_ALL_REPLICA_SET_MEMBERS", new BooleanGetter() { 139 | public boolean get() { 140 | return meta.getUseAllReplicaSetMembers(); 141 | } 142 | } ); 143 | check( "TAG_SET", new StringGetter() { 144 | public String get() { 145 | return meta.getReadPrefTagSets().get( 0 ); 146 | } 147 | } ); 148 | check( "JSON_OUTPUT_FIELD", new StringGetter() { 149 | public String get() { 150 | return meta.getJsonFieldName(); 151 | } 152 | } ); 153 | check( "AGG_PIPELINE", new BooleanGetter() { 154 | public boolean get() { 155 | return meta.getQueryIsPipeline(); 156 | } 157 | } ); 158 | check( "ALLOWDISKUSE", new BooleanGetter() { 159 | public boolean get() { 160 | return meta.isAllowDiskUse(); 161 | } 162 | } ); 163 | check( "OUTPUT_JSON", new BooleanGetter() { 164 | public boolean get() { 165 | return meta.getOutputJson(); 166 | } 167 | } ); 168 | check( "EXECUTE_FOR_EACH_ROW", new BooleanGetter() { 169 | public boolean get() { 170 | return meta.getExecuteForEachIncomingRow(); 171 | } 172 | } ); 173 | check( "FIELD_NAME", new StringGetter() { 174 | public String get() { 175 | return meta.getMongoFields().get( 0 ).m_fieldName; 176 | } 177 | } ); 178 | check( "FIELD_PATH", new StringGetter() { 179 | public String get() { 180 | return meta.getMongoFields().get( 0 ).m_fieldPath; 181 | } 182 | } ); 183 | check( "FIELD_TYPE", new StringGetter() { 184 | public String get() { 185 | return meta.getMongoFields().get( 0 ).m_kettleType; 186 | } 187 | } ); 188 | check( "FIELD_INDEXED", new StringGetter() { 189 | public String get() { 190 | return meta.getMongoFields().get( 0 ).m_indexedVals.get( 0 ); 191 | } 192 | } ); 193 | check( "FIELD_ARRAY_INDEX", new StringGetter() { 194 | public String get() { 195 | return meta.getMongoFields().get( 0 ).m_arrayIndexInfo; 196 | } 197 | } ); 198 | check( "FIELD_PERCENTAGE", new IntGetter() { 199 | public int get() { 200 | return meta.getMongoFields().get( 0 ).m_percentageOfSample; 201 | } 202 | } ); 203 | check( "FIELD_DISPARATE_TYPES", new BooleanGetter() { 204 | public boolean get() { 205 | return meta.getMongoFields().get( 0 ).m_disparateTypes; 206 | } 207 | } ); 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputStepAnalyzer.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import org.pentaho.di.core.row.ValueMetaInterface; 17 | import org.pentaho.di.trans.TransMeta; 18 | import org.pentaho.di.trans.step.BaseStepMeta; 19 | import org.pentaho.di.trans.step.StepMeta; 20 | import org.pentaho.dictionary.DictionaryConst; 21 | import org.pentaho.metaverse.api.IAnalysisContext; 22 | import org.pentaho.metaverse.api.IMetaverseNode; 23 | import org.pentaho.metaverse.api.MetaverseAnalyzerException; 24 | import org.pentaho.metaverse.api.MetaverseComponentDescriptor; 25 | import org.pentaho.metaverse.api.StepField; 26 | import org.pentaho.metaverse.api.analyzer.kettle.step.ConnectionExternalResourceStepAnalyzer; 27 | import org.pentaho.metaverse.api.analyzer.kettle.step.IClonableStepAnalyzer; 28 | import org.pentaho.metaverse.api.model.IExternalResourceInfo; 29 | import org.pentaho.mongo.wrapper.field.MongoField; 30 | 31 | import java.util.HashSet; 32 | import java.util.List; 33 | import java.util.Set; 34 | 35 | /** 36 | * Analyzes MongoDbInput Steps for lineage related information 37 | */ 38 | public class MongoDbInputStepAnalyzer extends ConnectionExternalResourceStepAnalyzer { 39 | 40 | public static final String COLLECTION = "collection"; 41 | 42 | // Query property names 43 | public static final String AGG_PIPELINE = "isAggPipeline"; 44 | public static final String FIELDS_EXPRESSION = "fieldsExpression"; 45 | 46 | // Tag Set / Read Preference property names 47 | public static final String READ_PREF = "readPreference"; 48 | public static final String TAG_SETS = "tagSets"; 49 | 50 | // Field property names 51 | public static final String OUTPUT_JSON = "outputJson"; 52 | public static final String JSON_PATH = "jsonPath"; 53 | public static final String MINMAX_RANGE = "minMaxArrayRange"; 54 | public static final String OCCUR_RATIO = "occurRatio"; 55 | public static final String INDEXED_VALS = "indexedValues"; 56 | public static final String DISPARATE_TYPES = "disparateTypes"; 57 | 58 | @Override protected void customAnalyze( MongoDbInputMeta meta, IMetaverseNode node ) 59 | throws MetaverseAnalyzerException { 60 | getConnectionAnalyzer().setMetaverseBuilder( this.getMetaverseBuilder() ); 61 | super.customAnalyze( meta, node ); 62 | 63 | node.setProperty( OUTPUT_JSON, meta.getOutputJson() ); 64 | node.setProperty( COLLECTION, meta.getCollection() ); 65 | // If the output is the full JSON, we don't have to do any additional analysis 66 | if ( !meta.getOutputJson() ) { 67 | // add properties to the node - the query (and its characteristics) in particular 68 | node.setProperty( DictionaryConst.PROPERTY_QUERY, meta.getJsonQuery() ); 69 | node.setProperty( AGG_PIPELINE, meta.getQueryIsPipeline() ); 70 | node.setProperty( DictionaryConst.PROPERTY_EXECUTE_EACH_ROW, meta.getExecuteForEachIncomingRow() ); 71 | node.setProperty( FIELDS_EXPRESSION, meta.getFieldsName() ); 72 | 73 | // Add tag set and read preference information to the step node 74 | node.setProperty( READ_PREF, meta.getReadPreference() ); 75 | List tagSets = meta.getReadPrefTagSets(); 76 | if ( tagSets != null ) { 77 | // The string representation of the list is good enough for us. Each entry should already be in JSON format 78 | node.setProperty( TAG_SETS, tagSets.toString() ); 79 | } 80 | } 81 | } 82 | 83 | @Override 84 | protected IMetaverseNode createOutputFieldNode( IAnalysisContext context, ValueMetaInterface fieldMeta, 85 | String targetStepName, String nodeType ) { 86 | 87 | IMetaverseNode mongoFieldNode = super.createOutputFieldNode( context, fieldMeta, targetStepName, nodeType ); 88 | List mongoFields = baseStepMeta.getMongoFields(); 89 | if ( mongoFields != null ) { 90 | for ( MongoField mongoField : mongoFields ) { 91 | if ( fieldMeta.getName().equals( mongoField.getName() ) ) { 92 | mongoFieldNode.setProperty( JSON_PATH, mongoField.m_fieldPath ); 93 | mongoFieldNode.setProperty( MINMAX_RANGE, mongoField.m_arrayIndexInfo ); 94 | mongoFieldNode.setProperty( OCCUR_RATIO, mongoField.m_occurenceFraction ); 95 | mongoFieldNode.setProperty( INDEXED_VALS, mongoField.m_indexedVals ); 96 | mongoFieldNode.setProperty( DISPARATE_TYPES, mongoField.m_disparateTypes ); 97 | break; 98 | } 99 | } 100 | } 101 | return mongoFieldNode; 102 | } 103 | 104 | @Override protected Set getUsedFields( MongoDbInputMeta meta ) { 105 | return null; 106 | } 107 | 108 | @Override 109 | public Set> getSupportedSteps() { 110 | Set> supportedSteps = new HashSet<>(); 111 | supportedSteps.add( MongoDbInputMeta.class ); 112 | return supportedSteps; 113 | } 114 | 115 | @Override protected IMetaverseNode createTableNode( IExternalResourceInfo resource ) 116 | throws MetaverseAnalyzerException { 117 | MongoDbResourceInfo resourceInfo = (MongoDbResourceInfo) resource; 118 | 119 | // create a node for the collection 120 | MetaverseComponentDescriptor componentDescriptor = new MetaverseComponentDescriptor( 121 | resourceInfo.getCollection(), 122 | DictionaryConst.NODE_TYPE_MONGODB_COLLECTION, 123 | getConnectionNode(), 124 | getDescriptor().getContext() ); 125 | 126 | // set the namespace to be the id of the connection node. 127 | IMetaverseNode node = createNodeFromDescriptor( componentDescriptor ); 128 | node.setProperty( DictionaryConst.PROPERTY_NAMESPACE, componentDescriptor.getNamespace().getNamespaceId() ); 129 | node.setProperty( COLLECTION, resourceInfo.getCollection() ); 130 | node.setLogicalIdGenerator( DictionaryConst.LOGICAL_ID_GENERATOR_DEFAULT ); 131 | return node; 132 | } 133 | 134 | @Override public String getResourceInputNodeType() { 135 | return DictionaryConst.NODE_TYPE_DATA_COLUMN; 136 | } 137 | 138 | @Override public String getResourceOutputNodeType() { 139 | return null; 140 | } 141 | 142 | @Override public boolean isOutput() { 143 | return false; 144 | } 145 | 146 | @Override public boolean isInput() { 147 | return true; 148 | } 149 | 150 | @Override protected IClonableStepAnalyzer newInstance() { 151 | return new MongoDbInputStepAnalyzer(); 152 | } 153 | @Override public String toString() { 154 | return this.getClass().getName(); 155 | } 156 | 157 | ///////////// for unit testing 158 | protected void setBaseStepMeta( MongoDbInputMeta meta ) { 159 | baseStepMeta = meta; 160 | } 161 | protected void setRootNode( IMetaverseNode node ) { 162 | rootNode = node; 163 | } 164 | protected void setParentTransMeta( TransMeta tm ) { 165 | parentTransMeta = tm; 166 | } 167 | protected void setParentStepMeta( StepMeta sm ) { 168 | parentStepMeta = sm; 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbResourceInfo.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import com.fasterxml.jackson.annotation.JsonProperty; 17 | import org.apache.commons.lang.StringUtils; 18 | import org.pentaho.di.core.encryption.Encr; 19 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 20 | import org.pentaho.dictionary.DictionaryConst; 21 | import org.pentaho.metaverse.api.AnalysisContext; 22 | import org.pentaho.metaverse.api.IAnalysisContext; 23 | import org.pentaho.metaverse.api.model.BaseResourceInfo; 24 | import org.pentaho.metaverse.api.model.IExternalResourceInfo; 25 | 26 | /** 27 | * User: RFellows Date: 3/10/15 28 | */ 29 | public class MongoDbResourceInfo extends BaseResourceInfo implements IExternalResourceInfo { 30 | 31 | public static final String JSON_PROPERTY_PORT = "port"; 32 | public static final String JSON_PROPERTY_USERNAME = "username"; 33 | public static final String JSON_PROPERTY_PASSWORD = "password"; 34 | public static final String JSON_PROPERTY_DATABASE_NAME = "databaseName"; 35 | public static final String JSON_PROPERTY_CONNECTION_TIMEOUT = "connectionTimeout"; 36 | public static final String JSON_PROPERTY_HOST_NAMES = "hostNames"; 37 | public static final String JSON_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; 38 | public static final String JSON_PROPERTY_USE_ALL_REPLICA_SET_MEMBERS = "useAllReplicaSetMembers"; 39 | public static final String JSON_PROPERTY_USE_KERBEROS_AUTHENTICATION = "useKerberosAuthentication"; 40 | public static final String JSON_PROPERTY_COLLECTION = "collection"; 41 | 42 | private String database; 43 | private String port; 44 | private String hostNames; 45 | private String user; 46 | private String password; 47 | private boolean useAllReplicaSetMembers; 48 | private boolean useKerberosAuthentication; 49 | private String connectTimeout; 50 | private String socketTimeout; 51 | private String collection; 52 | 53 | public MongoDbResourceInfo( MongoDbMeta mongoDbMeta ) { 54 | this( mongoDbMeta, new AnalysisContext( DictionaryConst.CONTEXT_RUNTIME ) ); 55 | } 56 | 57 | public MongoDbResourceInfo( MongoDbMeta mongoDbMeta, IAnalysisContext context ) { 58 | setName( substituteIfNeeded( mongoDbMeta.getDbName(), mongoDbMeta, context ) ); 59 | setDatabase( substituteIfNeeded( mongoDbMeta.getDbName(), mongoDbMeta, context ) ); 60 | setPort( substituteIfNeeded( mongoDbMeta.getPort(), mongoDbMeta, context ) ); 61 | setHostNames( substituteIfNeeded( mongoDbMeta.getHostnames(), mongoDbMeta, context ) ); 62 | setUser( substituteIfNeeded( mongoDbMeta.getAuthenticationUser(), mongoDbMeta, context ) ); 63 | setPassword( substituteIfNeeded( mongoDbMeta.getAuthenticationPassword(), mongoDbMeta, context ) ); 64 | setUseAllReplicaSetMembers( mongoDbMeta.getUseAllReplicaSetMembers() ); 65 | setUseKerberosAuthentication( mongoDbMeta.getUseKerberosAuthentication() ); 66 | setConnectTimeout( substituteIfNeeded( mongoDbMeta.getConnectTimeout(), mongoDbMeta, context ) ); 67 | setSocketTimeout( substituteIfNeeded( mongoDbMeta.getSocketTimeout(), mongoDbMeta, context ) ); 68 | setCollection( substituteIfNeeded( mongoDbMeta.getCollection(), mongoDbMeta, context ) ); 69 | } 70 | 71 | public MongoDbResourceInfo( String hostNames, String port, String database ) { 72 | setHostNames( hostNames ); 73 | setPort( port ); 74 | setDatabase( database ); 75 | } 76 | 77 | private String substituteIfNeeded( String value, MongoDbMeta meta, IAnalysisContext context ) { 78 | String contextName = context != null ? context.getContextName() : ""; 79 | String ret = contextName.equals( DictionaryConst.CONTEXT_RUNTIME ) 80 | ? meta.getParentStepMeta().getParentTransMeta().environmentSubstitute( value ) : value; 81 | return ret; 82 | } 83 | 84 | @Override 85 | public String getType() { 86 | return "MongoDbResource"; 87 | } 88 | 89 | @JsonProperty( JSON_PROPERTY_CONNECTION_TIMEOUT ) 90 | public String getConnectTimeout() { 91 | return connectTimeout; 92 | } 93 | 94 | public void setConnectTimeout( String connectTimeout ) { 95 | this.connectTimeout = connectTimeout; 96 | } 97 | 98 | @JsonProperty( JSON_PROPERTY_DATABASE_NAME ) 99 | public String getDatabase() { 100 | return database; 101 | } 102 | 103 | public void setDatabase( String database ) { 104 | this.database = database; 105 | } 106 | 107 | @JsonProperty( JSON_PROPERTY_HOST_NAMES ) 108 | public String getHostNames() { 109 | return hostNames; 110 | } 111 | 112 | public void setHostNames( String hostNames ) { 113 | this.hostNames = hostNames; 114 | } 115 | 116 | @JsonProperty( JSON_PROPERTY_PORT ) 117 | public String getPort() { 118 | return port; 119 | } 120 | 121 | public void setPort( String port ) { 122 | this.port = port; 123 | } 124 | 125 | @JsonProperty( JSON_PROPERTY_SOCKET_TIMEOUT ) 126 | public String getSocketTimeout() { 127 | return socketTimeout; 128 | } 129 | 130 | public void setSocketTimeout( String socketTimeout ) { 131 | this.socketTimeout = socketTimeout; 132 | } 133 | 134 | @JsonProperty( JSON_PROPERTY_USE_ALL_REPLICA_SET_MEMBERS ) 135 | public boolean isUseAllReplicaSetMembers() { 136 | return useAllReplicaSetMembers; 137 | } 138 | 139 | public void setUseAllReplicaSetMembers( boolean useAllReplicaSetMembers ) { 140 | this.useAllReplicaSetMembers = useAllReplicaSetMembers; 141 | } 142 | 143 | @JsonProperty( JSON_PROPERTY_USE_KERBEROS_AUTHENTICATION ) 144 | public boolean isUseKerberosAuthentication() { 145 | return useKerberosAuthentication; 146 | } 147 | 148 | public void setUseKerberosAuthentication( boolean useKerberosAuthentication ) { 149 | this.useKerberosAuthentication = useKerberosAuthentication; 150 | } 151 | 152 | @JsonProperty( JSON_PROPERTY_USERNAME ) 153 | public String getUser() { 154 | return user; 155 | } 156 | 157 | public void setUser( String user ) { 158 | this.user = user; 159 | } 160 | 161 | public String getPassword() { 162 | return password; 163 | } 164 | 165 | @JsonProperty( JSON_PROPERTY_PASSWORD ) 166 | public void setPassword( String password ) { 167 | this.password = Encr.decryptPasswordOptionallyEncrypted( password ); 168 | } 169 | 170 | @JsonProperty( JSON_PROPERTY_PASSWORD ) 171 | protected String getEncryptedPassword() { 172 | if ( StringUtils.isEmpty( password ) ) { 173 | return StringUtils.EMPTY; 174 | } 175 | // Need "Encrypted prefix for decryptPasswordOptionallyEncrypted() to operate properly 176 | return Encr.PASSWORD_ENCRYPTED_PREFIX + Encr.encryptPassword( password ); 177 | } 178 | 179 | @JsonProperty( JSON_PROPERTY_COLLECTION ) 180 | public String getCollection() { 181 | return collection; 182 | } 183 | 184 | public void setCollection( String collection ) { 185 | this.collection = collection; 186 | } 187 | 188 | @Override 189 | public void cleanupSensitiveData() { 190 | password = null; 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/di/trans/steps/mongodboutput/MongoDbOutputMetaInjectionTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodboutput; 15 | 16 | import org.junit.After; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import org.pentaho.di.core.injection.BaseMetadataInjectionTest; 20 | import org.pentaho.di.core.logging.KettleLogStore; 21 | import org.pentaho.di.core.logging.LogChannelInterface; 22 | import org.pentaho.di.core.logging.LogChannelInterfaceFactory; 23 | 24 | import static org.mockito.Mockito.any; 25 | import static org.mockito.Mockito.mock; 26 | import static org.mockito.Mockito.when; 27 | 28 | /** 29 | * MDI test for MongoDbOutput. 30 | */ 31 | public class MongoDbOutputMetaInjectionTest extends BaseMetadataInjectionTest { 32 | private LogChannelInterfaceFactory oldLogChannelInterfaceFactory; 33 | 34 | @Before 35 | public void setup() throws IllegalAccessException { 36 | oldLogChannelInterfaceFactory = KettleLogStore.getLogChannelInterfaceFactory(); 37 | setKettleLogFactoryWithMock(); 38 | setup( new MongoDbOutputMeta() ); 39 | } 40 | 41 | public static void setKettleLogFactoryWithMock() { 42 | LogChannelInterfaceFactory logChannelInterfaceFactory = mock( LogChannelInterfaceFactory.class ); 43 | LogChannelInterface logChannelInterface = mock( LogChannelInterface.class ); 44 | when( logChannelInterfaceFactory.create( any() ) ).thenReturn( 45 | logChannelInterface ); 46 | KettleLogStore.setLogChannelInterfaceFactory( logChannelInterfaceFactory ); 47 | } 48 | 49 | @After 50 | public void tearDown() { 51 | KettleLogStore.setLogChannelInterfaceFactory( oldLogChannelInterfaceFactory ); 52 | } 53 | 54 | @Test 55 | public void test() throws Exception { 56 | check( "TRUNCATE", new BooleanGetter() { 57 | public boolean get() { 58 | return meta.m_truncate; 59 | } 60 | } ); 61 | check( "UPDATE", new BooleanGetter() { 62 | public boolean get() { 63 | return meta.m_update; 64 | } 65 | } ); 66 | check( "UPSERT", new BooleanGetter() { 67 | public boolean get() { 68 | return meta.m_upsert; 69 | } 70 | } ); 71 | check( "MULTI", new BooleanGetter() { 72 | public boolean get() { 73 | return meta.m_multi; 74 | } 75 | } ); 76 | check( "MODIFIER_UPDATE", new BooleanGetter() { 77 | public boolean get() { 78 | return meta.m_modifierUpdate; 79 | } 80 | } ); 81 | check( "BATCH_INSERT_SIZE", new StringGetter() { 82 | public String get() { 83 | return meta.m_batchInsertSize; 84 | } 85 | } ); 86 | check( "RETRY_NUMBER", new StringGetter() { 87 | public String get() { 88 | return meta.getWriteRetries(); 89 | } 90 | } ); 91 | check( "RETRY_DELAY", new StringGetter() { 92 | public String get() { 93 | return meta.getWriteRetryDelay(); 94 | } 95 | } ); 96 | check( "HOSTNAME", new StringGetter() { 97 | public String get() { 98 | return meta.getHostnames(); 99 | } 100 | } ); 101 | check( "PORT", new StringGetter() { 102 | public String get() { 103 | return meta.getPort(); 104 | } 105 | } ); 106 | check( "DATABASE_NAME", new StringGetter() { 107 | public String get() { 108 | return meta.getDbName(); 109 | } 110 | } ); 111 | check( "COLLECTION", new StringGetter() { 112 | public String get() { 113 | return meta.getCollection(); 114 | } 115 | } ); 116 | check( "AUTH_DATABASE", new StringGetter() { 117 | public String get() { 118 | return meta.getAuthenticationDatabaseName(); 119 | } 120 | } ); 121 | check( "AUTH_USERNAME", new StringGetter() { 122 | public String get() { 123 | return meta.getAuthenticationUser(); 124 | } 125 | } ); 126 | check( "AUTH_PASSWORD", new StringGetter() { 127 | public String get() { 128 | return meta.getAuthenticationPassword(); 129 | } 130 | } ); 131 | check( "AUTH_MECHANISM", new StringGetter() { 132 | public String get() { 133 | return meta.getAuthenticationMechanism(); 134 | } 135 | } ); 136 | check( "AUTH_KERBEROS", new BooleanGetter() { 137 | public boolean get() { 138 | return meta.getUseKerberosAuthentication(); 139 | } 140 | } ); 141 | check( "TIMEOUT_CONNECTION", new StringGetter() { 142 | public String get() { 143 | return meta.getConnectTimeout(); 144 | } 145 | } ); 146 | check( "TIMEOUT_SOCKET", new StringGetter() { 147 | public String get() { 148 | return meta.getSocketTimeout(); 149 | } 150 | } ); 151 | check( "USE_SSL_SOCKET_FACTORY", new BooleanGetter() { 152 | public boolean get() { 153 | return meta.isUseSSLSocketFactory(); 154 | } 155 | } ); 156 | check( "USE_CONNECTION_STRING", new BooleanGetter() { 157 | @Override 158 | public boolean get() { 159 | return meta.isUseConnectionString(); 160 | } 161 | } ); 162 | check( "USE_LEGACY_OPTIONS", new BooleanGetter() { 163 | @Override 164 | public boolean get() { 165 | return meta.isUseLegacyOptions(); 166 | } 167 | } ); 168 | check( "CONNECTION_STRING", new StringGetter() { 169 | @Override 170 | public String get() { 171 | return meta.getConnectionString(); 172 | } 173 | } ); 174 | check( "READ_PREFERENCE", new StringGetter() { 175 | public String get() { 176 | return meta.getReadPreference(); 177 | } 178 | } ); 179 | check( "USE_ALL_REPLICA_SET_MEMBERS", new BooleanGetter() { 180 | public boolean get() { 181 | return meta.getUseAllReplicaSetMembers(); 182 | } 183 | } ); 184 | check( "INCOMING_FIELD_NAME", new StringGetter() { 185 | public String get() { 186 | return meta.getMongoFields().get( 0 ).m_incomingFieldName; 187 | } 188 | } ); 189 | check( "MONGO_DOCUMENT_PATH", new StringGetter() { 190 | public String get() { 191 | return meta.getMongoFields().get( 0 ).m_mongoDocPath; 192 | } 193 | } ); 194 | check( "INCOMING_AS_MONGO", new BooleanGetter() { 195 | public boolean get() { 196 | return meta.getMongoFields().get( 0 ).m_useIncomingFieldNameAsMongoFieldName; 197 | } 198 | } ); 199 | check( "UPDATE_MATCH_FIELD", new BooleanGetter() { 200 | public boolean get() { 201 | return meta.getMongoFields().get( 0 ).m_updateMatchField; 202 | } 203 | } ); 204 | check( "MODIFIER_OPERATION", new StringGetter() { 205 | public String get() { 206 | return meta.getMongoFields().get( 0 ).m_modifierUpdateOperation; 207 | } 208 | } ); 209 | check( "MODIFIER_POLICY", new StringGetter() { 210 | public String get() { 211 | return meta.getMongoFields().get( 0 ).m_modifierOperationApplyPolicy; 212 | } 213 | } ); 214 | check( "INSERT_NULL", new BooleanGetter() { 215 | public boolean get() { 216 | return meta.getMongoFields().get( 0 ).insertNull; 217 | } 218 | } ); 219 | check( "JSON", new BooleanGetter() { 220 | public boolean get() { 221 | return meta.getMongoFields().get( 0 ).m_JSON; 222 | } 223 | } ); 224 | check( "INDEX_FIELD", new StringGetter() { 225 | public String get() { 226 | return meta.getMongoIndexes().get( 0 ).m_pathToFields; 227 | } 228 | } ); 229 | check( "DROP", new BooleanGetter() { 230 | public boolean get() { 231 | return meta.getMongoIndexes().get( 0 ).m_drop; 232 | } 233 | } ); 234 | check( "UNIQUE", new BooleanGetter() { 235 | public boolean get() { 236 | return meta.getMongoIndexes().get( 0 ).m_unique; 237 | } 238 | } ); 239 | check( "SPARSE", new BooleanGetter() { 240 | public boolean get() { 241 | return meta.getMongoIndexes().get( 0 ).m_sparse; 242 | } 243 | } ); 244 | check( "TAG_SET", new StringGetter() { 245 | public String get() { 246 | return meta.getReadPrefTagSets().get( 0 ); 247 | } 248 | } ); 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/mongo/wrapper/field/MongoArrayExpansion.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper.field; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | import org.pentaho.di.core.Const; 20 | import org.pentaho.di.core.exception.KettleException; 21 | import org.pentaho.di.core.row.RowDataUtil; 22 | import org.pentaho.di.core.row.RowMetaInterface; 23 | import org.pentaho.di.core.variables.VariableSpace; 24 | import org.pentaho.di.i18n.BaseMessages; 25 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputData; 26 | 27 | import com.mongodb.BasicDBList; 28 | import com.mongodb.BasicDBObject; 29 | 30 | public class MongoArrayExpansion { 31 | protected static Class PKG = MongoArrayExpansion.class; // for i18n purposes 32 | 33 | /** 34 | * The prefix of the full path that defines the expansion 35 | */ 36 | public String m_expansionPath; 37 | 38 | /** 39 | * Subfield objects that handle the processing of the path after the expansion prefix 40 | */ 41 | protected List m_subFields; 42 | 43 | private List m_pathParts; 44 | private List m_tempParts; 45 | 46 | public RowMetaInterface m_outputRowMeta; 47 | 48 | public MongoArrayExpansion( List subFields ) { 49 | m_subFields = subFields; 50 | } 51 | 52 | /** 53 | * Initialize this field by parsing the path etc. 54 | * 55 | * @throws KettleException if a problem occurs 56 | */ 57 | public void init() throws KettleException { 58 | if ( Const.isEmpty( m_expansionPath ) ) { 59 | throw new KettleException( BaseMessages.getString( PKG, "MongoDbInput.ErrorMessage.NoPathSet" ) ); //$NON-NLS-1$ 60 | } 61 | if ( m_pathParts != null ) { 62 | return; 63 | } 64 | 65 | String expansionPath = MongoDbInputData.cleansePath( m_expansionPath ); 66 | 67 | String[] temp = expansionPath.split( "\\." ); //$NON-NLS-1$ 68 | m_pathParts = new ArrayList(); 69 | for ( String part : temp ) { 70 | m_pathParts.add( part ); 71 | } 72 | 73 | if ( m_pathParts.get( 0 ).equals( "$" ) ) { //$NON-NLS-1$ 74 | m_pathParts.remove( 0 ); // root record indicator 75 | } else if ( m_pathParts.get( 0 ).startsWith( "$[" ) ) { //$NON-NLS-1$ 76 | 77 | // strip leading $ off of array 78 | String r = m_pathParts.get( 0 ).substring( 1, m_pathParts.get( 0 ).length() ); 79 | m_pathParts.set( 0, r ); 80 | } 81 | m_tempParts = new ArrayList(); 82 | 83 | // initialize the sub fields 84 | if ( m_subFields != null ) { 85 | for ( MongoField f : m_subFields ) { 86 | int outputIndex = m_outputRowMeta.indexOfValue( f.m_fieldName ); 87 | f.init( outputIndex ); 88 | } 89 | } 90 | } 91 | 92 | /** 93 | * Reset this field. Should be called prior to processing a new field value from the avro file 94 | * 95 | * @param space environment variables (values that environment variables resolve to cannot contain "."s) 96 | */ 97 | public void reset( VariableSpace space ) { 98 | m_tempParts.clear(); 99 | 100 | for ( String part : m_pathParts ) { 101 | m_tempParts.add( space.environmentSubstitute( part ) ); 102 | } 103 | 104 | // reset sub fields 105 | for ( MongoField f : m_subFields ) { 106 | f.reset( space ); 107 | } 108 | } 109 | 110 | protected Object[][] nullResult() { 111 | Object[][] result = new Object[1][m_outputRowMeta.size() + RowDataUtil.OVER_ALLOCATE_SIZE]; 112 | 113 | return result; 114 | } 115 | 116 | public Object[][] convertToKettleValue( BasicDBObject mongoObject, VariableSpace space ) throws KettleException { 117 | 118 | if ( mongoObject == null ) { 119 | return nullResult(); 120 | } 121 | 122 | if ( m_tempParts.size() == 0 ) { 123 | throw new KettleException( 124 | BaseMessages.getString( PKG, "MongoDbInput.ErrorMessage.MalformedPathRecord" ) ); //$NON-NLS-1$ 125 | } 126 | 127 | String part = m_tempParts.remove( 0 ); 128 | 129 | if ( part.charAt( 0 ) == '[' ) { 130 | // we're not expecting an array at this point - this document does not 131 | // contain our field(s) 132 | return nullResult(); 133 | } 134 | 135 | if ( part.indexOf( '[' ) > 0 ) { 136 | String arrayPart = part.substring( part.indexOf( '[' ) ); 137 | part = part.substring( 0, part.indexOf( '[' ) ); 138 | 139 | // put the array section back into location zero 140 | m_tempParts.add( 0, arrayPart ); 141 | } 142 | 143 | // part is a named field of this record 144 | Object fieldValue = mongoObject.get( part ); 145 | if ( fieldValue == null ) { 146 | return nullResult(); 147 | } 148 | 149 | if ( fieldValue instanceof BasicDBObject ) { 150 | return convertToKettleValue( ( (BasicDBObject) fieldValue ), space ); 151 | } 152 | 153 | if ( fieldValue instanceof BasicDBList ) { 154 | return convertToKettleValue( ( (BasicDBList) fieldValue ), space ); 155 | } 156 | 157 | // must mean we have a primitive here, but we're expecting to process more 158 | // path so this doesn't match us - return null 159 | return nullResult(); 160 | } 161 | 162 | public Object[][] convertToKettleValue( BasicDBList mongoList, VariableSpace space ) throws KettleException { 163 | 164 | if ( mongoList == null ) { 165 | return nullResult(); 166 | } 167 | 168 | if ( m_tempParts.size() == 0 ) { 169 | throw new KettleException( 170 | BaseMessages.getString( PKG, "MongoDbInput.ErrorMessage.MalformedPathArray" ) ); //$NON-NLS-1$ 171 | } 172 | 173 | String part = m_tempParts.remove( 0 ); 174 | if ( !( part.charAt( 0 ) == '[' ) ) { 175 | // we're expecting an array at this point - this document does not 176 | // contain our field 177 | return nullResult(); 178 | } 179 | 180 | String index = part.substring( 1, part.indexOf( ']' ) ); 181 | 182 | if ( part.indexOf( ']' ) < part.length() - 1 ) { 183 | // more dimensions to the array 184 | part = part.substring( part.indexOf( ']' ) + 1, part.length() ); 185 | m_tempParts.add( 0, part ); 186 | } 187 | 188 | if ( index.equals( "*" ) ) { //$NON-NLS-1$ 189 | // start the expansion - we delegate conversion to our subfields 190 | Object[][] result = new Object[mongoList.size()][m_outputRowMeta.size() + RowDataUtil.OVER_ALLOCATE_SIZE]; 191 | 192 | for ( int i = 0; i < mongoList.size(); i++ ) { 193 | Object element = mongoList.get( i ); 194 | 195 | for ( int j = 0; j < m_subFields.size(); j++ ) { 196 | MongoField sf = m_subFields.get( j ); 197 | sf.reset( space ); 198 | 199 | // what have we got? 200 | if ( element instanceof BasicDBObject ) { 201 | result[i][sf.m_outputIndex] = sf.convertToKettleValue( (BasicDBObject) element ); 202 | } else if ( element instanceof BasicDBList ) { 203 | result[i][sf.m_outputIndex] = sf.convertToKettleValue( (BasicDBList) element ); 204 | } else { 205 | // assume a primitive 206 | result[i][sf.m_outputIndex] = sf.getKettleValue( element ); 207 | } 208 | } 209 | } 210 | 211 | return result; 212 | } else { 213 | int arrayI = 0; 214 | try { 215 | arrayI = Integer.parseInt( index.trim() ); 216 | } catch ( NumberFormatException e ) { 217 | throw new KettleException( 218 | BaseMessages.getString( PKG, "MongoDbInput.ErrorMessage.UnableToParseArrayIndex", index ) ); //$NON-NLS-1$ 219 | } 220 | 221 | if ( arrayI >= mongoList.size() || arrayI < 0 ) { 222 | // index is out of bounds 223 | return nullResult(); 224 | } 225 | 226 | Object element = mongoList.get( arrayI ); 227 | 228 | if ( element == null ) { 229 | return nullResult(); 230 | } 231 | 232 | if ( element instanceof BasicDBObject ) { 233 | return convertToKettleValue( ( (BasicDBObject) element ), space ); 234 | } 235 | 236 | if ( element instanceof BasicDBList ) { 237 | return convertToKettleValue( ( (BasicDBList) element ), space ); 238 | } 239 | 240 | // must mean we have a primitive here, but we're expecting to process 241 | // more 242 | // path so this doesn't match us - return null 243 | return nullResult(); 244 | } 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodb/MongoDBHelper.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2025 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | ******************************************************************************/ 11 | package org.pentaho.di.trans.steps.mongodb; 12 | 13 | import com.mongodb.BasicDBObject; 14 | import com.mongodb.DBObject; 15 | import org.json.simple.JSONObject; 16 | import org.pentaho.di.core.exception.KettleException; 17 | import org.pentaho.di.i18n.BaseMessages; 18 | import org.pentaho.di.trans.TransMeta; 19 | import org.pentaho.di.trans.step.StepInterface; 20 | import org.pentaho.mongo.MongoDbException; 21 | import org.pentaho.mongo.NamedReadPreference; 22 | import org.pentaho.mongo.wrapper.MongoClientWrapper; 23 | import org.pentaho.mongo.wrapper.MongoWrapperUtil; 24 | import org.pentaho.reporting.libraries.base.util.StringUtils; 25 | 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | import java.util.Set; 29 | 30 | public class MongoDBHelper { 31 | 32 | private static final Class PKG = MongoDBHelper.class; 33 | public static final String ERROR_MESSAGE = "errorMessage"; 34 | private static final String ERROR_LABEL = "errorLabel"; 35 | private static final String MISSING_CONN_DETAILS = "MongoDb.ErrorMessage.MissingConnectionDetails"; 36 | private static final String MISSING_CONN_DETAILS_TITLE = "MongoDb.ErrorMessage.MissingConnectionDetails.Title"; 37 | private static final String UNABLE_TO_CONNECT = "MongoDb.ErrorMessage.UnableToConnect"; 38 | 39 | public JSONObject testConnectionAction( TransMeta transMeta, MongoDbMeta mongoDbMeta ) { 40 | JSONObject response = new JSONObject(); 41 | 42 | if ( StringUtils.isEmpty( mongoDbMeta.getConnectionString() ) ) { 43 | return errorResponse( response, 44 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 45 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, "Connection String" ) ); 46 | } 47 | 48 | try { 49 | MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, transMeta, transMeta.getLogChannel() ); 50 | try { 51 | wrapper.getDatabaseNames(); 52 | } finally { 53 | wrapper.dispose(); 54 | } 55 | } catch ( Exception ex ) { 56 | errorResponse( response, BaseMessages.getString( PKG, UNABLE_TO_CONNECT ), ex ); 57 | response.put( "isValidConnection", false ); 58 | return response; 59 | } 60 | 61 | response.put( "isValidConnection", true ); 62 | response.put( "title", BaseMessages.getString( PKG, "MongoDb.SuccessMessage.SuccessConnectionDetails.Title" ) ); 63 | response.put( "message", BaseMessages.getString( PKG, "MongoDb.SuccessMessage.SuccessConnectionDetails" ) ); 64 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 65 | return response; 66 | } 67 | 68 | public JSONObject getDBNamesAction( TransMeta transMeta, MongoDbMeta mongoDbMeta ) { 69 | JSONObject response = new JSONObject(); 70 | List dbNames; 71 | 72 | boolean isValidRequest = validateRequestForDbNamesAndCollections( response, mongoDbMeta ); 73 | if ( !isValidRequest ) { 74 | return response; 75 | } 76 | 77 | try { 78 | MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, transMeta, transMeta.getLogChannel() ); 79 | try { 80 | dbNames = wrapper.getDatabaseNames(); 81 | } finally { 82 | wrapper.dispose(); 83 | } 84 | } catch ( Exception ex ) { 85 | return errorResponse( response, BaseMessages.getString( PKG, UNABLE_TO_CONNECT ), ex ); 86 | } 87 | 88 | response.put( "dbNames", dbNames ); 89 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 90 | return response; 91 | } 92 | 93 | public JSONObject getCollectionNamesAction( TransMeta transMeta, MongoDbMeta mongoDbMeta ) { 94 | JSONObject response = new JSONObject(); 95 | Set collectionNames; 96 | 97 | boolean isValidRequest = validateRequestForDbNamesAndCollections( response, mongoDbMeta ); 98 | if ( !isValidRequest ) { 99 | return response; 100 | } 101 | 102 | if ( StringUtils.isEmpty( mongoDbMeta.getDbName() ) ) { 103 | errorResponse( response, 104 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 105 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, "database" ) ); 106 | return response; 107 | } 108 | 109 | try { 110 | MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, transMeta, transMeta.getLogChannel() ); 111 | try { 112 | collectionNames = wrapper.getCollectionsNames( transMeta.environmentSubstitute( mongoDbMeta.getDbName() ) ); 113 | } finally { 114 | wrapper.dispose(); 115 | } 116 | } catch ( Exception ex ) { 117 | return errorResponse( response, BaseMessages.getString( PKG, UNABLE_TO_CONNECT ), ex ); 118 | } 119 | 120 | response.put( "collectionNames", new ArrayList<>( collectionNames ) ); 121 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 122 | return response; 123 | } 124 | 125 | public JSONObject getPreferencesAction() { 126 | JSONObject response = new JSONObject(); 127 | List preferences = new ArrayList<>(); 128 | for ( NamedReadPreference preference : NamedReadPreference.values() ) { 129 | preferences.add( preference.getName() ); 130 | } 131 | 132 | response.put( "preferences", preferences ); 133 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 134 | return response; 135 | } 136 | 137 | public static JSONObject errorResponse( JSONObject response, String label, Exception ex ) { 138 | return errorResponse( response, label, ex.getCause() != null ? ex.getCause().getMessage() : ex.getMessage() ); 139 | } 140 | 141 | public static JSONObject errorResponse( JSONObject response, String label, String message ) { 142 | response.put( ERROR_LABEL, label ); 143 | response.put( ERROR_MESSAGE, message ); 144 | response.put( StepInterface.ACTION_STATUS, StepInterface.FAILURE_RESPONSE ); 145 | return response; 146 | } 147 | 148 | public static String validateRequestForFields( MongoDbMeta mongoDbMeta ) { 149 | String missingConDetails = ""; 150 | if ( !mongoDbMeta.isUseConnectionString() ) { 151 | if ( StringUtils.isEmpty( mongoDbMeta.getHostnames() ) ) { 152 | missingConDetails += " host name(s)"; 153 | } 154 | 155 | if ( StringUtils.isEmpty( mongoDbMeta.getDbName() ) ) { 156 | missingConDetails += " database"; 157 | } 158 | 159 | if ( StringUtils.isEmpty( mongoDbMeta.getCollection() ) ) { 160 | missingConDetails += " collection"; 161 | } 162 | } else { 163 | if ( StringUtils.isEmpty( mongoDbMeta.getConnectionString() ) ) { 164 | missingConDetails += " Connection string"; 165 | } 166 | } 167 | 168 | return missingConDetails; 169 | } 170 | 171 | private boolean validateRequestForDbNamesAndCollections( JSONObject response, MongoDbMeta mongoDbMeta ) { 172 | if ( mongoDbMeta != null ) { 173 | boolean isConnectionDetailsEmpty = StringUtils.isEmpty( mongoDbMeta.getHostnames() ) && StringUtils.isEmpty( mongoDbMeta.getConnectionString() ); 174 | if ( isConnectionDetailsEmpty ) { 175 | String errorString = mongoDbMeta.isUseConnectionString() ? "Connection String" : "Hostname"; 176 | errorResponse( response, 177 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 178 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, errorString ) ); 179 | return false; 180 | } 181 | } 182 | 183 | return true; 184 | } 185 | 186 | public static void setupDBObjects( List tagSets, String tagSet ) { 187 | String set = tagSet; 188 | if ( !tagSet.startsWith( "{" ) ) { 189 | set = "{" + tagSet; 190 | } 191 | 192 | if ( !tagSet.endsWith( "}" ) ) { 193 | set = set + "}"; 194 | } 195 | 196 | DBObject setO = BasicDBObject.parse( set ); 197 | tagSets.add( setO ); 198 | } 199 | 200 | public static List getReplicaMemberBasedOnTagSet( List tagSets, MongoClientWrapper wrapper ) throws KettleException { 201 | List satisfy; 202 | try { 203 | try { 204 | satisfy = wrapper.getReplicaSetMembersThatSatisfyTagSets( tagSets ); 205 | } catch ( MongoDbException e ) { 206 | throw new KettleException( e ); 207 | } 208 | } finally { 209 | try { 210 | wrapper.dispose(); 211 | } catch ( MongoDbException e ) { 212 | //Ignore 213 | } 214 | } 215 | return satisfy; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodboutput/MongoDbOutputHelper.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2025 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | package org.pentaho.di.trans.steps.mongodboutput; 13 | 14 | import org.json.simple.JSONArray; 15 | import org.json.simple.JSONObject; 16 | import org.pentaho.di.core.row.RowMetaInterface; 17 | import org.pentaho.di.core.row.ValueMetaInterface; 18 | import org.pentaho.di.core.util.Utils; 19 | import org.pentaho.di.i18n.BaseMessages; 20 | import org.pentaho.di.trans.TransMeta; 21 | import org.pentaho.di.trans.step.BaseStepHelper; 22 | import org.pentaho.di.trans.step.StepInterface; 23 | import org.pentaho.di.trans.steps.mongodb.MongoDBHelper; 24 | import org.pentaho.mongo.wrapper.MongoClientWrapper; 25 | import org.pentaho.mongo.wrapper.MongoWrapperUtil; 26 | import org.pentaho.reporting.libraries.base.util.StringUtils; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class MongoDbOutputHelper extends BaseStepHelper { 31 | 32 | private static final Class PKG = MongoDbOutputHelper.class; 33 | private final MongoDbOutputMeta mongoDbOutputMeta; 34 | private static final String TEST_CONNECTION = "testConnection"; 35 | private static final String GET_DB_NAMES = "getDBNames"; 36 | private static final String GET_COLLECTION_NAMES = "getCollectionNames"; 37 | private static final String GET_PREFERENCES = "getPreferences"; 38 | private static final String WRITE_CONCERNS = "writeConcerns"; 39 | private static final String GET_DOCUMENT_FIELDS = "getDocumentFields"; 40 | private static final String PREVIEW_DOCUMENT_STRUCTURE = "previewDocumentStructure"; 41 | private static final String SHOW_INDEXES = "showIndexes"; 42 | private static final String MISSING_CONN_DETAILS = "MongoDbOutputDialog.ErrorMessage.MissingConnectionDetails"; 43 | private static final String MISSING_CONN_DETAILS_TITLE = "MongoDbOutputDialog.ErrorMessage.MissingConnectionDetails.title"; 44 | 45 | public MongoDbOutputHelper( MongoDbOutputMeta mongoDbOutputMeta ) { 46 | this.mongoDbOutputMeta = mongoDbOutputMeta; 47 | } 48 | 49 | @Override 50 | protected JSONObject handleStepAction( String method, TransMeta transMeta, Map queryParams ) { 51 | JSONObject response = new JSONObject(); 52 | MongoDBHelper mongoDBHelper = new MongoDBHelper(); 53 | try { 54 | switch ( method ) { 55 | case TEST_CONNECTION: 56 | response = mongoDBHelper.testConnectionAction( transMeta, mongoDbOutputMeta ); 57 | break; 58 | case GET_DB_NAMES: 59 | response = mongoDBHelper.getDBNamesAction( transMeta, mongoDbOutputMeta ); 60 | break; 61 | case GET_COLLECTION_NAMES: 62 | response = mongoDBHelper.getCollectionNamesAction( transMeta, mongoDbOutputMeta ); 63 | break; 64 | case GET_PREFERENCES: 65 | response = mongoDBHelper.getPreferencesAction(); 66 | break; 67 | case WRITE_CONCERNS: 68 | response = getWriteConcernsAction( transMeta ); 69 | break; 70 | case GET_DOCUMENT_FIELDS: 71 | response = getDocumentFieldsAction( transMeta ); 72 | break; 73 | case PREVIEW_DOCUMENT_STRUCTURE: 74 | response = previewDocumentStructureAction( transMeta ); 75 | break; 76 | case SHOW_INDEXES: 77 | response = showIndexesAction( transMeta ); 78 | break; 79 | default: 80 | response.put( StepInterface.ACTION_STATUS, StepInterface.FAILURE_METHOD_NOT_RESPONSE ); 81 | break; 82 | } 83 | } catch ( Exception e ) { 84 | log.logError( e.getMessage() ); 85 | response.put( StepInterface.ACTION_STATUS, StepInterface.FAILURE_METHOD_NOT_RESPONSE ); 86 | } 87 | return response; 88 | } 89 | 90 | private JSONObject getWriteConcernsAction( TransMeta transMeta ) { 91 | JSONObject response = new JSONObject(); 92 | String hostname = transMeta.environmentSubstitute( mongoDbOutputMeta.getHostnames() ); 93 | 94 | if ( !Utils.isEmpty( hostname ) ) { 95 | try { 96 | MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( mongoDbOutputMeta, transMeta, transMeta.getLogChannel() ); 97 | List writeConcerns; 98 | try { 99 | writeConcerns = wrapper.getLastErrorModes(); 100 | response.put( WRITE_CONCERNS, writeConcerns ); 101 | } finally { 102 | wrapper.dispose(); 103 | } 104 | } catch ( Exception exception ) { 105 | MongoDBHelper.errorResponse( response, 106 | BaseMessages.getString( PKG, "MongoDbOutputDialog.ErrorMessage.UnableToConnect" ), 107 | exception ); 108 | return response; 109 | } 110 | } else { 111 | MongoDBHelper.errorResponse( response, 112 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 113 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, "host name(s)" ) ); 114 | return response; 115 | } 116 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 117 | return response; 118 | } 119 | 120 | private JSONObject getDocumentFieldsAction( TransMeta transMeta ) { 121 | JSONObject response = new JSONObject(); 122 | 123 | String missingConDetails = MongoDBHelper.validateRequestForFields( mongoDbOutputMeta ); 124 | if ( !StringUtils.isEmpty( missingConDetails ) ) { 125 | return MongoDBHelper.errorResponse( response, 126 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 127 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, missingConDetails ) ); 128 | } 129 | 130 | try { 131 | RowMetaInterface rowMetaInterface = transMeta.getPrevStepFields( mongoDbOutputMeta.getParentStepMeta().getName() ); 132 | response.put( "fields", setFieldResponse( rowMetaInterface.getValueMetaList() ) ); 133 | } catch ( Exception ex ) { 134 | MongoDBHelper.errorResponse( response, BaseMessages.getString( PKG, "System.Dialog.GetFieldsFailed.Message" ), ex ); 135 | return response; 136 | } 137 | 138 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 139 | return response; 140 | } 141 | 142 | private JSONObject previewDocumentStructureAction( TransMeta transMeta ) { 143 | JSONObject response = new JSONObject(); 144 | String missingConDetails = MongoDBHelper.validateRequestForFields( mongoDbOutputMeta ); 145 | if ( !StringUtils.isEmpty( missingConDetails ) ) { 146 | return MongoDBHelper.errorResponse( response, 147 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 148 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, missingConDetails ) ); 149 | } 150 | 151 | try { 152 | MongoDbOutputUtilHelper mongoDbOutputUtilHelper = new MongoDbOutputUtilHelper(); 153 | List mongoFields = mongoDbOutputMeta.getMongoFields(); 154 | Map displayDetails = mongoDbOutputUtilHelper.previewDocStructure( transMeta, mongoDbOutputMeta.getParentStepMeta().getName(), mongoFields, mongoDbOutputMeta.m_modifierUpdate ); 155 | response.put( "windowTitle", displayDetails.get( "windowTitle" ) ); 156 | response.put( "toDisplay", displayDetails.get( "toDisplay" ) ); 157 | } catch ( Exception ex ) { 158 | MongoDBHelper.errorResponse( response, BaseMessages.getString( PKG, "MongoDbOutputDialog.ErrorMessage.ProblemPreviewingDocStructure.Message" ), ex ); 159 | return response; 160 | } 161 | 162 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 163 | return response; 164 | } 165 | 166 | private JSONObject showIndexesAction( TransMeta transMeta ) { 167 | JSONObject response = new JSONObject(); 168 | String missingConDetails = MongoDBHelper.validateRequestForFields( mongoDbOutputMeta ); 169 | if ( !StringUtils.isEmpty( missingConDetails ) ) { 170 | return MongoDBHelper.errorResponse( response, 171 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS_TITLE ), 172 | BaseMessages.getString( PKG, MISSING_CONN_DETAILS, missingConDetails ) ); 173 | } 174 | 175 | String hostname = transMeta.environmentSubstitute( mongoDbOutputMeta.getHostnames() ); 176 | String dbName = transMeta.environmentSubstitute( mongoDbOutputMeta.getDbName() ); 177 | String collection = transMeta.environmentSubstitute( mongoDbOutputMeta.getCollection() ); 178 | if ( !Utils.isEmpty( hostname ) ) { 179 | try { 180 | MongoClientWrapper wrapper = MongoWrapperUtil.createMongoClientWrapper( mongoDbOutputMeta, transMeta, log ); 181 | StringBuilder result = new StringBuilder(); 182 | for ( String index : wrapper.getIndexInfo( dbName, collection ) ) { 183 | result.append( index ).append( "\n\n" ); 184 | } 185 | 186 | response.put( "indexes", result.toString() ); 187 | } catch ( Exception e ) { 188 | MongoDBHelper.errorResponse( response, BaseMessages.getString( PKG, "MongoDbOutputDialog.ErrorMessage.IndexPreview.Title" ), e ); 189 | return response; 190 | } 191 | } 192 | 193 | response.put( StepInterface.ACTION_STATUS, StepInterface.SUCCESS_RESPONSE ); 194 | return response; 195 | } 196 | 197 | private JSONArray setFieldResponse( List fields ) { 198 | JSONArray jsonArray = new JSONArray(); 199 | for ( ValueMetaInterface field : fields ) { 200 | JSONObject jsonObject = new JSONObject(); 201 | jsonObject.put( "name", field.getName() ); 202 | jsonArray.add( jsonObject ); 203 | } 204 | return jsonArray; 205 | } 206 | 207 | } 208 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/test/java/org/pentaho/mongo/wrapper/MongoWrapperUtilTest.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.mongo.wrapper; 15 | 16 | import org.junit.After; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import org.junit.runner.RunWith; 20 | import org.mockito.MockedStatic; 21 | import org.mockito.Mockito; 22 | import org.mockito.junit.MockitoJUnitRunner; 23 | import org.pentaho.di.core.Const; 24 | import org.pentaho.di.core.encryption.Encr; 25 | import org.pentaho.di.core.encryption.TwoWayPasswordEncoderPluginType; 26 | import org.pentaho.di.core.exception.KettleException; 27 | import org.pentaho.di.core.logging.LogChannelInterface; 28 | import org.pentaho.di.core.plugins.PluginRegistry; 29 | import org.pentaho.di.core.util.EnvUtil; 30 | import org.pentaho.di.core.variables.VariableSpace; 31 | import org.pentaho.di.core.variables.Variables; 32 | import org.pentaho.di.trans.steps.mongodb.MongoDbMeta; 33 | import org.pentaho.di.trans.steps.mongodbinput.MongoDbInputMeta; 34 | import org.pentaho.di.trans.steps.mongodboutput.MongoDbOutputMeta; 35 | import org.pentaho.mongo.MongoDbException; 36 | import org.pentaho.mongo.MongoProp; 37 | import org.pentaho.mongo.MongoProperties; 38 | import org.pentaho.mongo.MongoUtilLogger; 39 | 40 | import java.util.Arrays; 41 | 42 | import static org.junit.Assert.assertEquals; 43 | import static org.mockito.Mockito.any; 44 | import static org.mockito.Mockito.anyString; 45 | import static org.mockito.Mockito.doReturn; 46 | import static org.mockito.Mockito.mock; 47 | import static org.mockito.Mockito.mockStatic; 48 | import static org.mockito.Mockito.verify; 49 | import static org.mockito.Mockito.when; 50 | 51 | /** 52 | * Created by bryan on 8/22/14. 53 | */ 54 | @RunWith( MockitoJUnitRunner.StrictStubs.class ) 55 | public class MongoWrapperUtilTest { 56 | private static final String SOCKET_TIMEOUT = "mongoDbSocketTimeout"; 57 | private static final String CONNECTION_TIMEOUT = "mongoDbConnectionTimeout"; 58 | private static final String PASSWORD = "password"; 59 | 60 | private MongoWrapperClientFactory cachedFactory; 61 | private MongoWrapperClientFactory mockFactory; 62 | 63 | @Before public void setup() { 64 | cachedFactory = MongoWrapperUtil.getMongoWrapperClientFactory(); 65 | mockFactory = mock( MongoWrapperClientFactory.class ); 66 | } 67 | 68 | @After public void tearDown() { 69 | MongoWrapperUtil.setMongoWrapperClientFactory( cachedFactory ); 70 | } 71 | 72 | @Test public void testCreateCalledNoReadPrefs() throws MongoDbException { 73 | MongoDbMeta mongoDbMeta = mock( MongoDbMeta.class ); 74 | VariableSpace variableSpace = mock( VariableSpace.class ); 75 | LogChannelInterface logChannelInterface = mock( LogChannelInterface.class ); 76 | MongoWrapperUtil.setMongoWrapperClientFactory( mockFactory ); 77 | MongoClientWrapper wrapper = mock( MongoClientWrapper.class ); 78 | when( mockFactory.createMongoClientWrapper( any( MongoProperties.class ), any( KettleMongoUtilLogger.class ) ) ) 79 | .thenReturn( wrapper ); 80 | assertEquals( wrapper, 81 | MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, variableSpace, logChannelInterface ) ); 82 | } 83 | 84 | @Test public void testCreateCalledReadPrefs() throws MongoDbException { 85 | MongoDbMeta mongoDbMeta = mock( MongoDbMeta.class ); 86 | VariableSpace variableSpace = mock( VariableSpace.class ); 87 | LogChannelInterface logChannelInterface = mock( LogChannelInterface.class ); 88 | MongoWrapperUtil.setMongoWrapperClientFactory( mockFactory ); 89 | MongoClientWrapper wrapper = mock( MongoClientWrapper.class ); 90 | when( mongoDbMeta.getReadPrefTagSets() ).thenReturn( Arrays.asList( "test", "test2" ) ); 91 | when( mockFactory.createMongoClientWrapper( any( MongoProperties.class ), any( KettleMongoUtilLogger.class ) ) ) 92 | .thenReturn( wrapper ); 93 | assertEquals( wrapper, 94 | MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, variableSpace, logChannelInterface ) ); 95 | } 96 | 97 | @Test public void testCreatePropertiesBuilder() { 98 | MongoDbMeta input = new MongoDbInputMeta(); 99 | setSocetAndConnectionTimeouts( input, "${" + CONNECTION_TIMEOUT + "}", "${" + SOCKET_TIMEOUT + "}" ); 100 | MongoWrapperUtil.setMongoWrapperClientFactory( mockFactory ); 101 | MongoDbMeta output = new MongoDbOutputMeta(); 102 | setSocetAndConnectionTimeouts( output, "${" + CONNECTION_TIMEOUT + "}", "${" + SOCKET_TIMEOUT + "}" ); 103 | 104 | VariableSpace vars = new Variables(); 105 | vars.setVariable( CONNECTION_TIMEOUT, "200" ); 106 | vars.setVariable( SOCKET_TIMEOUT, "500" ); 107 | 108 | MongoProperties inProps = MongoWrapperUtil.createPropertiesBuilder( input, vars ).build(); 109 | MongoProperties outProps = MongoWrapperUtil.createPropertiesBuilder( output, vars ).build(); 110 | 111 | checkProps( inProps, "200", "500" ); 112 | checkProps( outProps, "200", "500" ); 113 | } 114 | 115 | @Test public void testPropertiesBuilderEncrPassword() throws KettleException { 116 | MongoWrapperUtil.setMongoWrapperClientFactory( mockFactory ); 117 | final String pass = "pass"; 118 | testPropertiesBuilderForPassword( true, pass ); 119 | testPropertiesBuilderForPassword( false, pass ); 120 | } 121 | 122 | @Test public void testCreateConnStringMongoClientWrapperUtil() throws MongoDbException { 123 | MongoWrapperUtil.setMongoWrapperClientFactory( mockFactory ); 124 | MongoDbMeta mongoDbMeta = mock( MongoDbMeta.class ); 125 | VariableSpace variableSpace = mock( VariableSpace.class ); 126 | LogChannelInterface logChannelInterface = mock( LogChannelInterface.class ); 127 | try ( MockedStatic mockedEncr = mockStatic( Encr.class ) ) { 128 | mockedEncr.when( () -> Encr.decryptPasswordOptionallyEncrypted( anyString() ) ).thenReturn( "SimpleString" ); 129 | when( mongoDbMeta.isUseConnectionString() ).thenReturn( true ); 130 | when( mongoDbMeta.getConnectionString() ).thenReturn( "SimpleString" ); 131 | MongoClientWrapper wrapper = mock( ConnectionStringMongoClientWrapper.class ); 132 | doReturn( wrapper ).when( mockFactory ).createConnectionStringMongoClientWrapper( Mockito.any(), Mockito.any() ); 133 | assertEquals( wrapper, 134 | MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, variableSpace, logChannelInterface ) ); 135 | verify( mongoDbMeta ).isUseConnectionString(); 136 | } 137 | } 138 | 139 | @Test public void testCreateConnStringMongoClientWrapperFactory() throws MongoDbException { 140 | MongoDbMeta mongoDbMeta = mock( MongoDbMeta.class ); 141 | VariableSpace variableSpace = mock( VariableSpace.class ); 142 | LogChannelInterface logChannelInterface = mock( LogChannelInterface.class ); 143 | doReturn( true ).when( mongoDbMeta ).isUseConnectionString(); 144 | doReturn( "SimpleString" ).when( mongoDbMeta ).getConnectionString(); 145 | try ( MockedStatic mockedEncr = mockStatic( Encr.class ); 146 | MockedStatic mockedWrapper = mockStatic( MongoClientWrapperFactory.class )) { 147 | MongoClientWrapper wrapper = mock( ConnectionStringMongoClientWrapper.class ); 148 | mockedEncr.when( () -> Encr.decryptPasswordOptionallyEncrypted( any() ) ).thenReturn( "SimpleString" ); 149 | mockedWrapper.when( () -> MongoClientWrapperFactory.createConnectionStringMongoClientWrapper( Mockito.any(), Mockito.any() ) ).thenReturn( wrapper ); 150 | 151 | assertEquals( wrapper, MongoWrapperUtil.createMongoClientWrapper( mongoDbMeta, variableSpace, logChannelInterface ) ); 152 | verify( mongoDbMeta ).isUseConnectionString(); 153 | } 154 | } 155 | 156 | private void testPropertiesBuilderForPassword( boolean isEncrypted, String password ) throws KettleException { 157 | MongoDbMeta input = new MongoDbInputMeta(); 158 | setPassword( input, "${" + PASSWORD + "}" ); 159 | 160 | MongoDbMeta output = new MongoDbOutputMeta(); 161 | setPassword( output, "${" + PASSWORD + "}" ); 162 | 163 | VariableSpace vars = new Variables(); 164 | 165 | initEncryptor(); 166 | 167 | String value; 168 | if ( isEncrypted ) { 169 | value = Encr.encryptPasswordIfNotUsingVariables( password ); 170 | } else { 171 | value = password; 172 | } 173 | vars.setVariable( PASSWORD, value ); 174 | 175 | MongoProperties inProps = MongoWrapperUtil.createPropertiesBuilder( input, vars ).build(); 176 | MongoProperties outProps = MongoWrapperUtil.createPropertiesBuilder( output, vars ).build(); 177 | 178 | checkPass( inProps, password ); 179 | checkPass( outProps, password ); 180 | } 181 | 182 | private void initEncryptor() throws KettleException { 183 | PluginRegistry.addPluginType( TwoWayPasswordEncoderPluginType.getInstance() ); 184 | PluginRegistry.init( true ); 185 | String passwordEncoderPluginID = Const.NVL( EnvUtil.getSystemProperty( Const.KETTLE_PASSWORD_ENCODER_PLUGIN ), "Kettle" ); 186 | Encr.init( passwordEncoderPluginID ); 187 | } 188 | 189 | private void setSocetAndConnectionTimeouts( MongoDbMeta meta, String connection, String session ) { 190 | meta.setConnectTimeout( connection ); 191 | meta.setSocketTimeout( session ); 192 | } 193 | 194 | private void setPassword( MongoDbMeta meta, String password ) { 195 | meta.setAuthenticationPassword( password ); 196 | } 197 | 198 | private void checkProps( MongoProperties props, String cTimeout, String sTimeout ) { 199 | assertEquals( cTimeout, props.get( MongoProp.connectTimeout ) ); 200 | assertEquals( sTimeout, props.get( MongoProp.socketTimeout ) ); 201 | } 202 | 203 | private void checkPass( MongoProperties props, String password ) { 204 | assertEquals( password, props.get( MongoProp.PASSWORD ) ); 205 | } 206 | } -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodboutput/MongoDbOutputUtilHelper.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2025 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | ******************************************************************************/ 11 | 12 | package org.pentaho.di.trans.steps.mongodboutput; 13 | 14 | import com.mongodb.DBObject; 15 | import org.pentaho.di.core.exception.KettleException; 16 | import org.pentaho.di.core.row.RowMeta; 17 | import org.pentaho.di.core.row.RowMetaInterface; 18 | import org.pentaho.di.core.row.ValueMetaInterface; 19 | import org.pentaho.di.core.row.value.ValueMetaFactory; 20 | import org.pentaho.di.core.util.Utils; 21 | import org.pentaho.di.core.variables.VariableSpace; 22 | import org.pentaho.di.core.variables.Variables; 23 | import org.pentaho.di.i18n.BaseMessages; 24 | import org.pentaho.di.trans.TransMeta; 25 | import org.pentaho.mongo.MongoDbException; 26 | 27 | import java.util.HashMap; 28 | import java.util.List; 29 | import java.util.Map; 30 | 31 | public class MongoDbOutputUtilHelper { 32 | 33 | private static final Class PKG = MongoDbOutputUtilHelper.class; 34 | private enum Element { 35 | OPEN_BRACE, CLOSE_BRACE, OPEN_BRACKET, CLOSE_BRACKET, COMMA 36 | } 37 | 38 | public Map previewDocStructure( TransMeta transMeta, String stepName, List mongoFields, boolean updateSelection ) throws KettleException, MongoDbException { 39 | Map displayDetails = new HashMap<>(); 40 | String windowTitle = getString( "MongoDbOutputDialog.PreviewDocStructure.Title" ); 41 | String toDisplay = ""; 42 | 43 | // Try and get metadata on incoming fields 44 | RowMetaInterface actualR = null; 45 | RowMetaInterface r; 46 | boolean gotGenuineRowMeta = false; 47 | try { 48 | actualR = transMeta.getPrevStepFields( stepName ); 49 | gotGenuineRowMeta = true; 50 | } catch ( KettleException e ) { 51 | // don't complain if we can't 52 | } 53 | 54 | r = new RowMeta(); 55 | 56 | Object[] dummyRow = new Object[mongoFields.size()]; 57 | int i = 0; 58 | // Initialize Variable space to allow for environment substitution during doc preview. 59 | VariableSpace vs = new Variables(); 60 | vs.initializeVariablesFrom( transMeta ); 61 | boolean hasTopLevelJSONDocInsert = MongoDbOutputData.scanForInsertTopLevelJSONDoc( mongoFields ); 62 | 63 | for ( MongoDbOutputMeta.MongoField field : mongoFields ) { 64 | field.init( vs ); 65 | // set up dummy row meta 66 | ValueMetaInterface vm = ValueMetaFactory.createValueMeta( ValueMetaInterface.TYPE_STRING ); 67 | vm.setName( field.environUpdatedFieldName ); 68 | r.addValueMeta( vm ); 69 | 70 | String val = ""; //$NON-NLS-1$ 71 | if ( gotGenuineRowMeta && actualR.indexOfValue( field.environUpdatedFieldName ) >= 0 ) { 72 | val = getMongoDataTypeDocument( field, actualR ); 73 | } else { 74 | val = ""; //$NON-NLS-1$ 75 | } 76 | 77 | dummyRow[i++] = val; 78 | } 79 | 80 | MongoDbOutputData.MongoTopLevel topLevelStruct = MongoDbOutputData.checkTopLevelConsistency( mongoFields, vs ); 81 | for ( MongoDbOutputMeta.MongoField m : mongoFields ) { 82 | m.m_modifierOperationApplyPolicy = "Insert&Update"; //$NON-NLS-1$ 83 | } 84 | 85 | if ( !updateSelection ) { 86 | DBObject result = MongoDbOutputData.kettleRowToMongo( mongoFields, r, dummyRow, topLevelStruct, hasTopLevelJSONDocInsert ); 87 | if ( result != null ) { 88 | toDisplay = prettyPrintDocStructure( result.toString() ); 89 | } 90 | } else { 91 | DBObject query = MongoDbOutputData.getQueryObject( mongoFields, r, dummyRow, vs, topLevelStruct ); 92 | DBObject modifier = new MongoDbOutputData().getModifierUpdateObject( mongoFields, r, dummyRow, vs, topLevelStruct ); 93 | toDisplay = getDisplayStringForUpdate( query, modifier ); 94 | windowTitle = getString( "MongoDbOutputDialog.PreviewModifierUpdate.Title" ); //$NON-NLS-1$ 95 | } 96 | 97 | displayDetails.put( "windowTitle", windowTitle ); 98 | displayDetails.put( "toDisplay", toDisplay ); 99 | return displayDetails; 100 | } 101 | 102 | /** 103 | * Format JSON document structure for printing to the preview dialog 104 | * 105 | * @param toFormat the document to format 106 | * @return a String containing the formatted document structure 107 | */ 108 | public String prettyPrintDocStructure( String toFormat ) { 109 | StringBuilder result = new StringBuilder(); 110 | int indent = 0; 111 | String source = formatSource( toFormat ); 112 | Element next = Element.OPEN_BRACE; 113 | 114 | while ( !source.isEmpty() ) { 115 | source = source.trim(); 116 | String toIndent = ""; //$NON-NLS-1$ 117 | 118 | Map setUpValues = setMinIndexAndNextElementAndTargetChar( source, next ); 119 | int minIndex = (Integer) setUpValues.get( "minIndex" ); 120 | char targetChar = (Character) setUpValues.get( "targetChar" ); 121 | next = (Element) setUpValues.get( "next" ); 122 | 123 | if ( minIndex == 0 ) { 124 | if ( next == Element.CLOSE_BRACE || next == Element.CLOSE_BRACKET ) { 125 | indent -= 2; 126 | } 127 | 128 | pad( result, indent ); 129 | 130 | String comma = ""; //$NON-NLS-1$ 131 | int offset = 1; 132 | if ( source.length() >= 2 && source.charAt( 1 ) == ',' ) { 133 | comma = ","; //$NON-NLS-1$ 134 | offset = 2; 135 | } 136 | result.append( targetChar ).append( comma ).append( "\n" ); //$NON-NLS-1$ 137 | source = source.substring( offset ); 138 | } else { 139 | pad( result, indent ); 140 | Map sourceAndToIndent = getSourceAndToIndentForNonMinIndex( next, source, minIndex ); 141 | toIndent = sourceAndToIndent.get( "toIndent" ); 142 | source = sourceAndToIndent.get( "source" ); 143 | result.append( toIndent.trim() ).append( "\n" ); //$NON-NLS-1$ 144 | } 145 | 146 | if ( next == Element.OPEN_BRACE || next == Element.OPEN_BRACKET ) { 147 | indent += 2; 148 | } 149 | } 150 | 151 | return result.toString(); 152 | } 153 | 154 | private String formatSource( String input ) { 155 | if ( input == null || input.isEmpty() ) { 156 | return input; 157 | } 158 | 159 | String[] parts = input.split( "," ); 160 | for ( int i = 0; i < parts.length; i++ ) { 161 | parts[i] = parts[i].trim(); 162 | } 163 | return String.join( ",", parts ); 164 | } 165 | 166 | private Map getSourceAndToIndentForNonMinIndex( Element next, String source, int minIndex ) { 167 | Map result = new HashMap<>(); 168 | String toIndent; 169 | if ( next == Element.CLOSE_BRACE || next == Element.CLOSE_BRACKET ) { 170 | toIndent = source.substring( 0, minIndex ); 171 | source = source.substring( minIndex ); 172 | } else { 173 | toIndent = source.substring( 0, minIndex + 1 ); 174 | source = source.substring( minIndex + 1 ); 175 | } 176 | 177 | result.put( "toIndent", toIndent ); 178 | result.put( "source", source ); 179 | return result; 180 | } 181 | 182 | private Map setMinIndexAndNextElementAndTargetChar( String source, Element next ) { 183 | Map result = new HashMap<>(); 184 | int minIndex = Integer.MAX_VALUE; 185 | char targetChar = '{'; 186 | if ( source.indexOf( '{' ) > -1 ) { 187 | next = Element.OPEN_BRACE; 188 | minIndex = source.indexOf( '{' ); 189 | } 190 | 191 | if ( source.indexOf( '}' ) > -1 && source.indexOf( '}' ) < minIndex ) { 192 | next = Element.CLOSE_BRACE; 193 | minIndex = source.indexOf( '}' ); 194 | targetChar = '}'; 195 | } 196 | 197 | if ( source.indexOf( '[' ) > -1 && source.indexOf( '[' ) < minIndex ) { 198 | next = Element.OPEN_BRACKET; 199 | minIndex = source.indexOf( '[' ); 200 | targetChar = '['; 201 | } 202 | 203 | if ( source.indexOf( ']' ) > -1 && source.indexOf( ']' ) < minIndex ) { 204 | next = Element.CLOSE_BRACKET; 205 | minIndex = source.indexOf( ']' ); 206 | targetChar = ']'; 207 | } 208 | 209 | if ( source.indexOf( ',' ) > -1 && source.indexOf( ',' ) < minIndex ) { 210 | next = Element.COMMA; 211 | minIndex = source.indexOf( ',' ); 212 | targetChar = ','; 213 | } 214 | 215 | result.put( "minIndex", minIndex ); 216 | result.put( "next", next ); 217 | result.put( "targetChar", targetChar ); 218 | return result; 219 | } 220 | 221 | private String getDisplayStringForUpdate( DBObject query, DBObject modifier ) { 222 | return getString( "MongoDbOutputDialog.PreviewModifierUpdate.Heading1" ) //$NON-NLS-1$ 223 | + ":\n\n" //$NON-NLS-1$ 224 | + ( query != null ? prettyPrintDocStructure( query.toString() ) : "" ) 225 | + getString( "MongoDbOutputDialog.PreviewModifierUpdate.Heading2" ) //$NON-NLS-1$ 226 | + ":\n\n" //$NON-NLS-1$ 227 | + ( modifier != null ? prettyPrintDocStructure( modifier.toString() ) : "" ); 228 | } 229 | 230 | private String getMongoDataTypeDocument( MongoDbOutputMeta.MongoField field, RowMetaInterface actualR ) { 231 | int index = actualR.indexOfValue( field.environUpdatedFieldName ); 232 | String val; 233 | switch ( actualR.getValueMeta( index ).getType() ) { 234 | case ValueMetaInterface.TYPE_STRING: 235 | if ( field.m_JSON ) { 236 | if ( !field.m_useIncomingFieldNameAsMongoFieldName && Utils.isEmpty( field.environUpdateMongoDocPath ) ) { 237 | // we will actually have to parse some kind of JSON doc 238 | // here in the case where the matching doc/doc to be inserted is 239 | // a full top-level incoming JSON doc 240 | val = "{\"IncomingJSONDoc\" : \"\"}"; //$NON-NLS-1$ 241 | } else { 242 | val = ""; //$NON-NLS-1$ 243 | // turn this off for the purpose of doc structure 244 | // visualization so that we don't screw up for the 245 | // lack of a real JSON doc to parse :-) 246 | field.m_JSON = false; 247 | } 248 | } else { 249 | val = ""; //$NON-NLS-1$ 250 | } 251 | break; 252 | case ValueMetaInterface.TYPE_INTEGER: 253 | val = ""; //$NON-NLS-1$ 254 | break; 255 | case ValueMetaInterface.TYPE_NUMBER: 256 | val = ""; //$NON-NLS-1$ 257 | break; 258 | case ValueMetaInterface.TYPE_BOOLEAN: 259 | val = ""; //$NON-NLS-1$ 260 | break; 261 | case ValueMetaInterface.TYPE_DATE: 262 | val = ""; //$NON-NLS-1$ 263 | break; 264 | case ValueMetaInterface.TYPE_BINARY: 265 | val = ""; //$NON-NLS-1$ 266 | break; 267 | default: 268 | val = ""; //$NON-NLS-1$ 269 | } 270 | 271 | return val; 272 | } 273 | 274 | private void pad( StringBuilder toPad, int numBlanks ) { 275 | toPad.append( " ".repeat( Math.max( 0, numBlanks ) ) ); 276 | } 277 | 278 | private String getString( String key ) { 279 | return BaseMessages.getString( PKG, key ); 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /pentaho-mongodb-plugin/src/main/java/org/pentaho/di/trans/steps/mongodbinput/MongoDbInputData.java: -------------------------------------------------------------------------------- 1 | /*! ****************************************************************************** 2 | * 3 | * Pentaho 4 | * 5 | * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com 6 | * 7 | * Use of this software is governed by the Business Source License included 8 | * in the LICENSE.TXT file. 9 | * 10 | * Change Date: 2029-07-20 11 | ******************************************************************************/ 12 | 13 | 14 | package org.pentaho.di.trans.steps.mongodbinput; 15 | 16 | import com.mongodb.BasicDBList; 17 | import com.mongodb.BasicDBObject; 18 | import com.mongodb.DBObject; 19 | import org.pentaho.di.core.exception.KettleException; 20 | import org.pentaho.di.core.row.RowDataUtil; 21 | import org.pentaho.di.core.row.RowMetaInterface; 22 | import org.pentaho.di.core.variables.VariableSpace; 23 | import org.pentaho.di.i18n.BaseMessages; 24 | import org.pentaho.di.trans.step.BaseStepData; 25 | import org.pentaho.di.trans.step.StepDataInterface; 26 | import org.pentaho.mongo.wrapper.MongoClientWrapper; 27 | import org.pentaho.mongo.wrapper.collection.MongoCollectionWrapper; 28 | import org.pentaho.mongo.wrapper.cursor.MongoCursorWrapper; 29 | import org.pentaho.mongo.wrapper.field.MongoArrayExpansion; 30 | import org.pentaho.mongo.wrapper.field.MongoField; 31 | 32 | import java.util.ArrayList; 33 | import java.util.Iterator; 34 | import java.util.List; 35 | 36 | /** 37 | * @author Matt 38 | * @author Mark Hall 39 | * @since 24-jan-2005 40 | */ 41 | public class MongoDbInputData extends BaseStepData implements StepDataInterface { 42 | 43 | public static final int MONGO_DEFAULT_PORT = 27017; 44 | public RowMetaInterface outputRowMeta; 45 | public MongoClientWrapper clientWrapper; 46 | public MongoCollectionWrapper collection; 47 | 48 | /** 49 | * cursor for a standard query 50 | */ 51 | public MongoCursorWrapper cursor; 52 | 53 | /** 54 | * results of an aggregation pipeline 55 | */ 56 | Iterator m_pipelineResult; 57 | 58 | private List m_userFields; 59 | private MongoArrayExpansion m_expansionHandler; 60 | private static MongoDbInputDiscoverFieldsHolder mongoDbInputDiscoverFieldsHolder = 61 | MongoDbInputDiscoverFieldsHolder.getInstance(); 62 | 63 | public static MongoDbInputDiscoverFieldsHolder getMongoDbInputDiscoverFieldsHolder() { 64 | return mongoDbInputDiscoverFieldsHolder; 65 | } 66 | 67 | protected static void setMongoDbInputDiscoverFieldsHolder( MongoDbInputDiscoverFieldsHolder holder ) { 68 | mongoDbInputDiscoverFieldsHolder = holder; 69 | } 70 | 71 | protected static MongoArrayExpansion checkFieldPaths( List normalFields, RowMetaInterface outputRowMeta ) 72 | throws KettleException { 73 | 74 | // here we check whether there are any full array expansions 75 | // specified in the paths (via [*]). If so, we want to make sure 76 | // that only one is present across all paths. E.g. we can handle 77 | // multiple fields like $.person[*].first, $.person[*].last etc. 78 | // but not $.person[*].first, $.person[*].address[*].street. 79 | 80 | String expansion = null; 81 | List normalList = new ArrayList(); 82 | List expansionList = new ArrayList(); 83 | 84 | for ( MongoField f : normalFields ) { 85 | String path = f.m_fieldPath; 86 | 87 | if ( path != null && path.lastIndexOf( "[*]" ) >= 0 ) { //$NON-NLS-1$ 88 | 89 | if ( path.indexOf( "[*]" ) != path.lastIndexOf( "[*]" ) ) { //$NON-NLS-1$ //$NON-NLS-2$ 90 | throw new KettleException( BaseMessages 91 | .getString( MongoDbInputMeta.PKG, "MongoInput.ErrorMessage.PathContainsMultipleExpansions", 92 | path ) ); //$NON-NLS-1$ 93 | } 94 | 95 | String pathPart = path.substring( 0, path.lastIndexOf( "[*]" ) + 3 ); //$NON-NLS-1$ 96 | 97 | if ( expansion == null ) { 98 | expansion = pathPart; 99 | } else { 100 | if ( !expansion.equals( pathPart ) ) { 101 | throw new KettleException( BaseMessages.getString( MongoDbInputMeta.PKG, 102 | "MongoDbInput.ErrorMessage.MutipleDifferentExpansions" ) ); //$NON-NLS-1$ 103 | } 104 | } 105 | 106 | expansionList.add( f ); 107 | } else { 108 | normalList.add( f ); 109 | } 110 | } 111 | 112 | normalFields.clear(); 113 | for ( MongoField f : normalList ) { 114 | normalFields.add( f ); 115 | } 116 | 117 | if ( expansionList.size() > 0 ) { 118 | 119 | List subFields = new ArrayList(); 120 | 121 | for ( MongoField ef : expansionList ) { 122 | MongoField subField = new MongoField(); 123 | subField.m_fieldName = ef.m_fieldName; 124 | String path = ef.m_fieldPath; 125 | if ( path.charAt( path.length() - 2 ) == '*' ) { 126 | path = "dummy"; // pulling a primitive out of the array (path //$NON-NLS-1$ 127 | // doesn't matter) 128 | } else { 129 | path = path.substring( path.lastIndexOf( "[*]" ) + 3, path.length() ); //$NON-NLS-1$ 130 | path = "$" + path; //$NON-NLS-1$ 131 | } 132 | 133 | subField.m_fieldPath = path; 134 | subField.m_indexedVals = ef.m_indexedVals; 135 | subField.m_kettleType = ef.m_kettleType; 136 | 137 | subFields.add( subField ); 138 | } 139 | 140 | MongoArrayExpansion exp = new MongoArrayExpansion( subFields ); 141 | exp.m_expansionPath = expansion; 142 | exp.m_outputRowMeta = outputRowMeta; 143 | 144 | return exp; 145 | } 146 | 147 | return null; 148 | } 149 | 150 | public MongoDbInputData() { 151 | super(); 152 | } 153 | 154 | /** 155 | * Initialize all the paths by locating the index for their field name in the outgoing row structure. 156 | * 157 | * @throws KettleException 158 | */ 159 | public void init() throws KettleException { 160 | if ( m_userFields != null ) { 161 | 162 | // set up array expansion/unwinding (if necessary) 163 | m_expansionHandler = checkFieldPaths( m_userFields, outputRowMeta ); 164 | 165 | for ( MongoField f : m_userFields ) { 166 | int outputIndex = outputRowMeta.indexOfValue( f.m_fieldName ); 167 | f.init( outputIndex ); 168 | } 169 | 170 | if ( m_expansionHandler != null ) { 171 | m_expansionHandler.init(); 172 | } 173 | } 174 | } 175 | 176 | /** 177 | * Convert a mongo document to outgoing row field values with respect to the user-specified paths. May return more 178 | * than one Kettle row if an array is being expanded/unwound 179 | * 180 | * @param mongoObj the mongo document 181 | * @param space variables to use 182 | * @return populated Kettle row(s) 183 | * @throws KettleException if a problem occurs 184 | */ 185 | public Object[][] mongoDocumentToKettle( DBObject mongoObj, VariableSpace space ) throws KettleException { 186 | 187 | Object[][] result = null; 188 | 189 | if ( m_expansionHandler != null ) { 190 | m_expansionHandler.reset( space ); 191 | 192 | if ( mongoObj instanceof BasicDBObject ) { 193 | result = m_expansionHandler.convertToKettleValue( (BasicDBObject) mongoObj, space ); 194 | } else { 195 | result = m_expansionHandler.convertToKettleValue( (BasicDBList) mongoObj, space ); 196 | } 197 | } else { 198 | result = new Object[1][]; 199 | } 200 | 201 | // get the normal (non expansion-related fields) 202 | Object[] normalData = RowDataUtil.allocateRowData( outputRowMeta.size() ); 203 | Object value; 204 | for ( MongoField f : m_userFields ) { 205 | value = null; 206 | f.reset( space ); 207 | 208 | if ( mongoObj instanceof BasicDBObject ) { 209 | value = f.convertToKettleValue( (BasicDBObject) mongoObj ); 210 | } else if ( mongoObj instanceof BasicDBList ) { 211 | value = f.convertToKettleValue( (BasicDBList) mongoObj ); 212 | } 213 | 214 | normalData[f.m_outputIndex] = value; 215 | } 216 | 217 | // copy normal fields over to each expansion row (if necessary) 218 | if ( m_expansionHandler == null ) { 219 | result[0] = normalData; 220 | } else { 221 | for ( int i = 0; i < result.length; i++ ) { 222 | Object[] row = result[i]; 223 | for ( MongoField f : m_userFields ) { 224 | row[f.m_outputIndex] = normalData[f.m_outputIndex]; 225 | } 226 | } 227 | } 228 | 229 | return result; 230 | } 231 | 232 | /** 233 | * Cleanses a string path by ensuring that any variables names present in the path do not contain "."s (replaces any 234 | * dots with underscores). 235 | * 236 | * @param path the path to cleanse 237 | * @return the cleansed path 238 | */ 239 | public static String cleansePath( String path ) { 240 | // look for variables and convert any "." to "_" 241 | 242 | int index = path.indexOf( "${" ); //$NON-NLS-1$ 243 | 244 | int endIndex = 0; 245 | String tempStr = path; 246 | while ( index >= 0 ) { 247 | index += 2; 248 | endIndex += tempStr.indexOf( "}" ); //$NON-NLS-1$ 249 | if ( endIndex > 0 && endIndex > index + 1 ) { 250 | String key = path.substring( index, endIndex ); 251 | 252 | String cleanKey = key.replace( '.', '_' ); 253 | path = path.replace( key, cleanKey ); 254 | } else { 255 | break; 256 | } 257 | 258 | if ( endIndex + 1 < path.length() ) { 259 | tempStr = path.substring( endIndex + 1, path.length() ); 260 | } else { 261 | break; 262 | } 263 | 264 | index = tempStr.indexOf( "${" ); //$NON-NLS-1$ 265 | 266 | if ( index > 0 ) { 267 | index += endIndex; 268 | } 269 | } 270 | 271 | return path; 272 | } 273 | 274 | /** 275 | * Set user-defined paths for extracting field values from Mongo documents 276 | * 277 | * @param fields the field path specifications 278 | */ 279 | public void setMongoFields( List fields ) { 280 | // copy this list 281 | m_userFields = new ArrayList(); 282 | if ( fields != null ) { 283 | for ( MongoField f : fields ) { 284 | m_userFields.add( f.copy() ); 285 | } 286 | } 287 | } 288 | 289 | /** 290 | * Helper function that takes a list of indexed values and returns them as a String in comma-separated form. 291 | * 292 | * @param indexedVals a list of indexed values 293 | * @return the list a String in comma-separated form 294 | */ 295 | public static String indexedValsList( List indexedVals ) { 296 | StringBuffer temp = new StringBuffer(); 297 | 298 | for ( int i = 0; i < indexedVals.size(); i++ ) { 299 | temp.append( indexedVals.get( i ) ); 300 | if ( i < indexedVals.size() - 1 ) { 301 | temp.append( "," ); //$NON-NLS-1$ 302 | } 303 | } 304 | 305 | return temp.toString(); 306 | } 307 | 308 | /** 309 | * Helper function that takes a comma-separated list in a String and returns a list. 310 | * 311 | * @param indexedVals the String containing the lsit 312 | * @return a List containing the values 313 | */ 314 | public static List indexedValsList( String indexedVals ) { 315 | 316 | String[] parts = indexedVals.split( "," ); //$NON-NLS-1$ 317 | List list = new ArrayList(); 318 | for ( String s : parts ) { 319 | list.add( s.trim() ); 320 | } 321 | 322 | return list; 323 | } 324 | } 325 | --------------------------------------------------------------------------------