├── target ├── .gitignore ├── maven-status │ └── maven-compiler-plugin │ │ ├── testCompile │ │ └── default-testCompile │ │ │ ├── inputFiles.lst │ │ │ └── createdFiles.lst │ │ └── compile │ │ └── default-compile │ │ ├── createdFiles.lst │ │ └── inputFiles.lst ├── .DS_Store ├── maven-archiver │ └── pom.properties └── classes │ ├── com │ └── force │ │ └── crma │ │ └── olympus │ │ ├── FileUtils.class │ │ ├── HeaderJSON.class │ │ ├── StringUtils.class │ │ ├── CreateDataset.class │ │ ├── DataPartJSON.class │ │ ├── DatasetCreator.class │ │ ├── RESTHttpClient.class │ │ ├── AddDataPartThread.class │ │ ├── DatasetCreatorParams.class │ │ └── DatasetCreatorException.class │ ├── META-INF │ ├── MANIFEST.MF │ └── maven │ │ └── com.force.crma.olympus │ │ └── dataset-creator │ │ ├── pom.properties │ │ └── pom.xml │ └── Complaints_schema.json ├── .DS_Store ├── .gitattributes ├── src ├── .DS_Store ├── main │ ├── .DS_Store │ ├── java │ │ ├── .DS_Store │ │ └── com │ │ │ ├── .DS_Store │ │ │ └── force │ │ │ ├── .DS_Store │ │ │ └── crma │ │ │ └── olympus │ │ │ ├── DatasetCreatorParams.java │ │ │ ├── StringUtils.java │ │ │ ├── DatasetCreatorException.java │ │ │ ├── DataPartJSON.java │ │ │ ├── AddDataPartThread.java │ │ │ ├── HeaderJSON.java │ │ │ ├── DatasetCreator.java │ │ │ ├── FileUtils.java │ │ │ ├── RESTHttpClient.java │ │ │ └── CreateDataset.java │ └── resources │ │ ├── .DS_Store │ │ └── Complaints_schema.json └── test │ └── .DS_Store ├── .settings ├── org.eclipse.m2e.core.prefs └── org.eclipse.jdt.core.prefs ├── CONTRIBUTING-ARCHIVED.md ├── .gitignore ├── SECURITY.md ├── .project ├── logging.properties ├── .classpath ├── LICENSE.md ├── pom.xml └── README.md /target/.gitignore: -------------------------------------------------------------------------------- 1 | /classes/ 2 | /test-classes/ 3 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/.DS_Store -------------------------------------------------------------------------------- /target/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/.DS_Store -------------------------------------------------------------------------------- /src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/main/.DS_Store -------------------------------------------------------------------------------- /src/test/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/test/.DS_Store -------------------------------------------------------------------------------- /src/main/java/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/main/java/.DS_Store -------------------------------------------------------------------------------- /src/main/java/com/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/main/java/com/.DS_Store -------------------------------------------------------------------------------- /src/main/resources/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/main/resources/.DS_Store -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /src/main/java/com/force/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/src/main/java/com/force/.DS_Store -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 3 | -------------------------------------------------------------------------------- /target/maven-archiver/pom.properties: -------------------------------------------------------------------------------- 1 | #Created by Apache Maven 3.6.1 2 | groupId=com.force.crma.olympus 3 | artifactId=dataset-creator 4 | version=0.54.1 5 | -------------------------------------------------------------------------------- /CONTRIBUTING-ARCHIVED.md: -------------------------------------------------------------------------------- 1 | # ARCHIVED 2 | 3 | This project is `Archived` and is no longer actively maintained; 4 | We are not accepting contributions or Pull Requests. 5 | 6 | -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/FileUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/FileUtils.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/HeaderJSON.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/HeaderJSON.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/StringUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/StringUtils.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/CreateDataset.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/CreateDataset.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/DataPartJSON.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/DataPartJSON.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/DatasetCreator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/DatasetCreator.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/RESTHttpClient.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/RESTHttpClient.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/AddDataPartThread.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/AddDataPartThread.class -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.class 3 | target/classes/META-INF/maven/com.force.crma.olympus/dataset-creator/pom.properties 4 | target/classes/META-INF/maven/com.force.crma.olympus/dataset-creator/pom.xml 5 | -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/DatasetCreatorParams.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/DatasetCreatorParams.class -------------------------------------------------------------------------------- /target/classes/com/force/crma/olympus/DatasetCreatorException.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forcedotcom/CRMA-dataset-creator/HEAD/target/classes/com/force/crma/olympus/DatasetCreatorException.class -------------------------------------------------------------------------------- /target/classes/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Built-By: terence.wilson 3 | Build-Jdk: 11.0.4 4 | Main-Class: com.force.crma.olympus.CreateDataset 5 | Created-By: Maven Integration for Eclipse 6 | 7 | -------------------------------------------------------------------------------- /target/classes/META-INF/maven/com.force.crma.olympus/dataset-creator/pom.properties: -------------------------------------------------------------------------------- 1 | #Generated by Maven Integration for Eclipse 2 | #Tue Sep 13 19:39:12 MDT 2022 3 | m2e.projectLocation=/Users/terence.wilson/eclipse-workspace/dataset-creator 4 | m2e.projectName=dataset-creator 5 | groupId=com.force.crma.olympus 6 | artifactId=dataset-creator 7 | version=0.54.1 8 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com) 4 | as soon as it is discovered. This library limits its runtime dependencies in 5 | order to reduce the total cost of ownership as much as can be, but all consumers 6 | should remain vigilant and have their security stakeholders review all third-party 7 | products (3PP) like this one and their dependencies. -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst: -------------------------------------------------------------------------------- 1 | com/force/crma/olympus/StringUtils.class 2 | com/force/crma/olympus/RESTHttpClient.class 3 | com/force/crma/olympus/FileUtils.class 4 | com/force/crma/olympus/HeaderJSON.class 5 | com/force/crma/olympus/DatasetCreatorException.class 6 | com/force/crma/olympus/AddDataPartThread.class 7 | com/force/crma/olympus/DatasetCreator.class 8 | com/force/crma/olympus/DataPartJSON.class 9 | com/force/crma/olympus/DatasetCreatorParams.class 10 | com/force/crma/olympus/CreateDataset.class 11 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | dataset-creator 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/DatasetCreatorParams.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | public class DatasetCreatorParams { 4 | 5 | 6 | String datasetAlias = null; 7 | String datasetLabel = null; 8 | String app =null; 9 | String username = null; 10 | String password = null; 11 | String cSecret = null; 12 | String cKey = null; 13 | String endpoint = null; 14 | String inputFile = null; 15 | String schemaFile = null; 16 | String operation = null; 17 | String mode=null; 18 | String proxyHost =null; 19 | int proxyPort =0; 20 | String proxyUsername =null; 21 | String proxyPassword=null; 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /logging.properties: -------------------------------------------------------------------------------- 1 | 2 | # Logs to file and console 3 | handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler 4 | # Global logging levels, 7 levels 5 | .level= FINE 6 | 7 | # Log file output in user's home directory, %h 8 | java.util.logging.FileHandler.pattern = %h/java%u.log 9 | java.util.logging.FileHandler.limit = 50000 10 | java.util.logging.FileHandler.count = 1 11 | #java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter 12 | java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter 13 | 14 | java.util.logging.ConsoleHandler.level = INFO 15 | java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter 16 | 17 | java.util.logging.SimpleFormatter.format=[%1$tc] %4$s: %5$s %n 18 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/StringUtils.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | public class StringUtils { 7 | 8 | public StringUtils() { 9 | // TODO Auto-generated constructor stub 10 | } 11 | 12 | protected static boolean isBlank(String aString) { 13 | return aString == null || aString.isEmpty(); 14 | 15 | } 16 | 17 | protected static boolean arrayContainsValue(String[] aArray, String aString) { 18 | if(!isEmptyArray(aArray)) { 19 | List list =Arrays.asList(aArray); 20 | return list.contains(aString); 21 | }else 22 | return false; 23 | 24 | } 25 | 26 | protected static boolean isEmptyArray(String[] aArray) { 27 | return aArray==null||aArray.length==0; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/DatasetCreatorException.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | public class DatasetCreatorException extends Exception { 4 | 5 | /** 6 | * 7 | */ 8 | private static final long serialVersionUID = 1L; 9 | 10 | public DatasetCreatorException() { 11 | 12 | } 13 | 14 | public DatasetCreatorException(String message) { 15 | super(message); 16 | 17 | } 18 | 19 | public DatasetCreatorException(Throwable cause) { 20 | super(cause); 21 | 22 | } 23 | 24 | public DatasetCreatorException(String message, Throwable cause) { 25 | super(message, cause); 26 | 27 | } 28 | 29 | public DatasetCreatorException(String message, Throwable cause, boolean enableSuppression, 30 | boolean writableStackTrace) { 31 | super(message, cause, enableSuppression, writableStackTrace); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst: -------------------------------------------------------------------------------- 1 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/DataPartJSON.java 2 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/HeaderJSON.java 3 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/StringUtils.java 4 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/RESTHttpClient.java 5 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/DatasetCreator.java 6 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/DatasetCreatorException.java 7 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/DatasetCreatorParams.java 8 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/FileUtils.java 9 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/AddDataPartThread.java 10 | /Users/terence.wilson/eclipse-workspace/dataset-creator/src/main/java/com/force/crma/olympus/CreateDataset.java 11 | -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/DataPartJSON.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | 4 | /* 5 | * Simple JAVA object representing REST JSON of the datapart object InsightsExternalDataPart 6 | * to support REST insert calls; 7 | * 8 | */ 9 | public class DataPartJSON { 10 | 11 | 12 | protected static final String EXT_DATA_OBJECT_PARTS="InsightsExternalDataPart"; 13 | protected static final String FIELD_DATA_FILE ="\"DataFile\""; 14 | protected static final String FIELD_INSIGHTS_EXT_DATA_ID="\"InsightsExternalDataId\""; 15 | protected static final String FIELD_PART_NUMBER="\"PartNumber\""; 16 | 17 | 18 | private static String headerId=null; 19 | private String partNumber=null; 20 | private String dataPart=null; 21 | 22 | public DataPartJSON(String aHeaderId) { 23 | if (headerId ==null) headerId=aHeaderId; 24 | } 25 | 26 | public void setPartNumber(String aPartNumber) { 27 | partNumber=aPartNumber; 28 | } 29 | 30 | public void setDataPart(String aDataPart) { 31 | dataPart=aDataPart; 32 | } 33 | public String toString() { 34 | StringBuilder requestBody =new StringBuilder(); 35 | requestBody.append("{").append(HeaderJSON.LF); 36 | requestBody.append(FIELD_INSIGHTS_EXT_DATA_ID).append(" : \"").append(headerId).append("\",").append(HeaderJSON.LF); 37 | requestBody.append(FIELD_PART_NUMBER).append(" : \"").append(partNumber).append("\",").append(HeaderJSON.LF); 38 | requestBody.append(FIELD_DATA_FILE).append(" : \"").append(dataPart).append("\"").append(HeaderJSON.LF); 39 | requestBody.append("}").append(HeaderJSON.LF); 40 | return requestBody.toString(); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # Salesforce.com Analytics Cloud DatasetCreator End User License Agreement 2 | 3 | Except as described below, Salesforce.com Analytics Cloud DatasetCreator is Copyright (c) 2022, salesforce.com, inc. All rights reserved. 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.force.crma.olympus 4 | dataset-creator 5 | 0.56.0 6 | dataset-creator 7 | CRMA Dataset Creator 8 | 9 | 56.0 10 | 11 11 | UTF-8 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | org.apache.maven.plugins 20 | maven-assembly-plugin 21 | 3.2.0 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-compiler-plugin 26 | 3.8.1 27 | 28 | 11 29 | 11 30 | 31 | 32 | 33 | 34 | 35 | maven-jar-plugin 36 | 3.0.2 37 | 38 | 39 | 40 | true 41 | 42 | 43 | com.force.crma.olympus.CreateDataset 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /target/classes/META-INF/maven/com.force.crma.olympus/dataset-creator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.force.crma.olympus 4 | dataset-creator 5 | 0.54.1 6 | dataset-creator 7 | TCRM Dataset Creator 8 | 9 | 54.0 10 | 11 11 | UTF-8 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | org.apache.maven.plugins 20 | maven-assembly-plugin 21 | 3.2.0 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-compiler-plugin 26 | 3.8.1 27 | 28 | 11 29 | 11 30 | 31 | 32 | 33 | 34 | 35 | maven-jar-plugin 36 | 3.0.2 37 | 38 | 39 | 40 | true 41 | 42 | 43 | com.force.crma.olympus.CreateDataset 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/AddDataPartThread.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | import java.io.File; 4 | import java.util.Map; 5 | import java.util.concurrent.BlockingQueue; 6 | import java.util.concurrent.atomic.AtomicBoolean; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import java.util.logging.Level; 9 | import java.util.logging.Logger; 10 | 11 | public class AddDataPartThread implements Runnable { 12 | 13 | private final BlockingQueue> queue; 14 | private final int threadNo; 15 | private final static Logger logger = Logger.getLogger(AddDataPartThread.class.getName()); 16 | private volatile AtomicInteger dataPartCount; 17 | 18 | private volatile AtomicBoolean done = new AtomicBoolean(false); 19 | 20 | public AddDataPartThread(BlockingQueue> aFileParts, int aThreadNo, AtomicInteger aCnt) { 21 | queue= aFileParts; 22 | threadNo=aThreadNo+1; 23 | dataPartCount=aCnt; 24 | logger.info("Creating AddDataPartThread : "+threadNo); 25 | } 26 | 27 | @Override 28 | public void run() { 29 | 30 | 31 | try { 32 | RESTHttpClient restClient = new RESTHttpClient(); 33 | logger.fine("Before queue take on thread : " + threadNo); 34 | Map filePartRow = queue.take(); 35 | 36 | File filePart = null; 37 | Integer partNo = 0; 38 | for (Integer key : filePartRow.keySet()) { 39 | filePart = filePartRow.get(key); 40 | partNo = key; 41 | 42 | } 43 | logger.fine("After queue take on thread : " + threadNo + " & Part No " + partNo); 44 | while (filePartRow != null && !filePartRow.isEmpty()) { 45 | restClient.createDataPart(filePart, partNo); 46 | dataPartCount.incrementAndGet(); 47 | if (!restClient.isDataPartSuccess()) { 48 | throw new DatasetCreatorException("REST insert of Data part failed, part number: " + partNo); 49 | } 50 | logger.info("Adding data part number : " + partNo + " of rows added: " + dataPartCount.get()); 51 | filePartRow = queue.take(); 52 | for (Integer key : filePartRow.keySet()) { 53 | filePart = filePartRow.get(key); 54 | partNo = key; 55 | logger.info("Thread : " + threadNo + " File Name :" + filePart.getName()); 56 | 57 | } 58 | if (filePart != null && FileUtils.END_THREAD.equalsIgnoreCase(filePart.getName())) { 59 | logger.info("Checking queue and break: " + partNo); 60 | break; 61 | } 62 | 63 | 64 | } 65 | done.set(true); 66 | logger.info("End :" + Thread.currentThread().getName()); 67 | 68 | } catch (Exception e) { 69 | logger.log(Level.SEVERE, "Error :" + e.getMessage()); 70 | Thread.currentThread().interrupt(); 71 | DatasetCreator.dataPartErrors.incrementAndGet(); 72 | done.set(true); 73 | 74 | 75 | } 76 | } 77 | 78 | protected boolean isDone() { 79 | return done.get(); 80 | } 81 | 82 | protected void initAtomicInteger() { 83 | 84 | } 85 | 86 | 87 | 88 | } 89 | 90 | 91 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/HeaderJSON.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | /* 4 | * Simple JAVA object representing REST JSON of the header object InsightsExternalData 5 | * for to support both REST insert and update Calls 6 | * 7 | */ 8 | 9 | public class HeaderJSON { 10 | 11 | protected static final String EXT_DATA_OBJECT="InsightsExternalData"; 12 | protected static final String FIELD_FORMAT="\"Format\""; 13 | protected static final String FIELD_OPERATION="\"Operation\""; 14 | protected static final String FIELD_MODE="\"Mode\""; 15 | protected static final String FIELD_EDGEMART_LABEL="\"EdgemartLabel\""; 16 | protected static final String FIELD_EDGEMART_ALIAS="\"EdgemartAlias\""; 17 | protected static final String FIELD_EDGEMART_CONTAINER="\"EdgemartContainer\""; 18 | protected static final String FIELD_METADATA_JSON="\"MetadataJson\""; 19 | protected static final String FIELD_ACTION="\"Action\""; 20 | protected static final String FIELD_DATA_FILE ="DataFile"; 21 | protected static final String[] OPERATIONS_VALUE= {"Append","Overwrite","Upsert"}; 22 | protected static final String[] MODE_VALUE = {"Incremental"}; 23 | protected static final char LF = '\n'; 24 | 25 | 26 | 27 | 28 | private String operation="Overwrite"; 29 | private String mode=null; 30 | private String datasetLabel=null; 31 | private String datasetAlias=null; 32 | private String metajson =null; 33 | private String action="None"; 34 | private String app=null; 35 | 36 | 37 | 38 | 39 | 40 | public HeaderJSON(String aOperation,String aMode, String aDatasetLabel,String aDatasetAlias, String aApp) { 41 | if(!StringUtils.isBlank(aOperation)) operation =aOperation; 42 | mode=aMode; 43 | datasetAlias=aDatasetAlias; 44 | datasetLabel=aDatasetLabel; 45 | app=aApp; 46 | 47 | }; 48 | 49 | public HeaderJSON() { 50 | 51 | } 52 | 53 | protected void setMetaJSON(String aMetajson) { 54 | metajson =aMetajson; 55 | } 56 | 57 | 58 | protected void setAction(String aAction) { 59 | action =aAction; 60 | } 61 | 62 | 63 | public String toString() { 64 | StringBuilder requestBody =new StringBuilder(); 65 | requestBody.append("{").append(LF); 66 | requestBody.append(FIELD_FORMAT).append(" : \"CSV\",").append(LF); 67 | requestBody.append(FIELD_EDGEMART_LABEL).append(" : \"").append(datasetLabel).append("\",").append(LF); 68 | requestBody.append(FIELD_EDGEMART_ALIAS).append(" : \"").append(datasetAlias).append("\",").append(LF); 69 | if(!StringUtils.isBlank(app)) { 70 | requestBody.append(FIELD_EDGEMART_CONTAINER).append(" : \"").append(app).append("\",").append(LF); 71 | } 72 | requestBody.append(FIELD_OPERATION).append(" : \"").append(operation).append("\",").append(LF); 73 | if(!StringUtils.isBlank(mode)) { 74 | requestBody.append(FIELD_MODE).append(" : \"").append(mode).append("\",").append(LF); 75 | } 76 | requestBody.append(FIELD_METADATA_JSON).append(" : \"").append(metajson).append("\",").append(LF); 77 | requestBody.append(FIELD_ACTION).append(" : \"").append(action).append("\"").append(LF); 78 | requestBody.append("}").append(LF); 79 | 80 | return requestBody.toString(); 81 | } 82 | 83 | 84 | public String toPatchString() { 85 | StringBuilder requestBody =new StringBuilder(); 86 | requestBody.append("{").append(LF); 87 | requestBody.append(FIELD_ACTION).append(" : \"").append(action).append("\"").append(LF); 88 | requestBody.append("}").append(LF); 89 | return requestBody.toString(); 90 | 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # DatasetCreator 4 | 5 | DatasetCreator is lightweight RESTFul client implementation of the CRM Analytics External Data API. This tool is free to use, but it is not officially supported by Salesforce. It has been deliberately developed with no 3rd party jars with the goal of being a lean, reliable and scalable solution. 6 | This is a community project that have not been officially tested or documented. Please do not contact Salesforce for support when using this application. 7 | 8 | 9 | ## Log4j2 Issues (CVE-2021-44228 and CVE-2021-45046) 10 | 11 | DatasetCreator uses no third party jars and is therefore free from Log4j issues 12 | 13 | 14 | ## Downloading DatasetCreator 15 | 16 | Download the latest version from [releases](https://github.com/forcedotcom/CRMA-Dataset-Creator/releases) and follow the examples below: 17 | 18 | ## Running DatasetCreator 19 | 20 | ## Prerequisite 21 | 22 | Download and install Java JDK (not JRE) from Zulu Open JDK 23 | 24 | * [Zulu Open JDK](https://www.azul.com/downloads/zulu-community/?&architecture=x86-64-bit&package=jdk) 25 | 26 | After installation is complete. Different versions of DatasetUtils require different versions of JDK, the latest release API 54 requires JDK 11. Open a console and check that the java version is correct for your DatasetCreator version by running the following command: 27 | 28 | 29 | ``java -version`` 30 | 31 | 32 | **Windows**: 33 | 34 | Download exectuable jar. 35 | 36 | **Mac**: 37 | 38 | Download executable jar. 39 | 40 | 41 | ### Command Line 42 | 43 | Open a terminal and type in the following command and follow the prompts on the console: 44 | 45 | 46 | java -jar dataset-creator-0.54.jar --inputFile .csv|.zip --schemaFile .json --operation Overwrite|Append|Upsert --u --p --dataset --label --app --endpoint --cSecret --cKey 47 | 48 | 49 | --u : Salesforce.com login 50 | 51 | --p : Salesforce.com password,if omitted you will be prompted 52 | 53 | --endpoint: The Salesforce soap api endpoint (test/prod) Default: https://login.salesforce.com 54 | 55 | --dataset : the dataset alias. required 56 | 57 | --label : (Optional) the dataset label. 58 | 59 | --app : (Optional) the app/folder name for the dataset. 60 | 61 | --operation : the operation for load (Overwrite/Upsert/Append/Delete). 62 | 63 | --inputFile : the input csv or gzip file. 64 | 65 | --schemaFile : the dataset meta json file. 66 | 67 | --cKey : the client key. 68 | 69 | --cSecret : the client secret. 70 | 71 | --mode : optional to use with beta feature of true append to dataset (only value Incremental is available as of this date). Mode is only applied with the beta feature enabled on the org and with operation Append. 72 | 73 | 74 | 75 | ## Usage Example 1: Upload a local csv to a dataset in production 76 | 77 | java -jar dataset-creator-0.54.jar --inputFile Complaints.csv --schemaFile Complaints_schema.json --operation Overwrite --u twilson@olympus.crma.com --p AppassWd --dataset Complaints --label Complaints --app SharedApp --endpoint https://login.salesforce.com --cSecret 844383111F0F755E420B23E1EA0B4AEDB --cKey 3MVG9F0F755E420B23E1EA9BC10B4AEDBmV9T7ZMnfw4C 78 | 79 | ## Usage Example 2: Append a local csv to a dataset 80 | 81 | java -jar dataset-creator-0.54.jar --inputFile Complaints.csv --schemaFile Complaints_schema.json --operation Append --u twilson@olympus.crma.com --p AppassWd --dataset Complaints --label Complaints --app SharedApp --endpoint https://login.salesforce.com --cSecret 844383111F0F755E420B23E1EA0B4AEDB --cKey 3MVG9F0F755E420B23E1EA9BC10B4AEDBmV9T7ZMnfw4C 82 | 83 | ## Usage Example 3: Upload a local csv to a dataset in sandbox 84 | 85 | java -jar dataset-creator-0.54.jar --inputFile Complaints.csv --schemaFile Complaints_schema.json --operation Overwrite --u twilson@olympus.crma.com --p AppassWd --dataset Complaints --label Complaints --app SharedApp --endpoint https://test.salesforce.com --cSecret 844383111F0F755E420B23E1EA0B4AEDB --cKey 3MVG9F0F755E420B23E1EA9BC10B4AEDBmV9T7ZMnfw4C 86 | 87 | ## Usage Example 4: Append a local csv to a dataset in incremental mode (Beta) 88 | 89 | java -jar dataset-creator-0.54.jar --inputFile Complaints.csv --schemaFile Complaints_schema.json --operation Append --mode Incremental --u twilson@olympus.crma.com --p AppassWd --dataset Complaints --label Complaints --app SharedApp --endpoint https://login.salesforce.com --cSecret 844383111F0F755E420B23E1EA0B4AEDB --cKey 3MVG9F0F755E420B23E1EA9BC10B4AEDBmV9T7ZMnfw4C 90 | 91 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/DatasetCreator.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | import java.io.File; 4 | import java.util.HashMap; 5 | import java.util.LinkedList; 6 | import java.util.Map; 7 | import java.util.concurrent.BlockingQueue; 8 | import java.util.concurrent.LinkedBlockingQueue; 9 | import java.util.concurrent.TimeUnit; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import java.util.logging.Logger ; 12 | 13 | /** 14 | * Class containing code to upload csv files to Salesforce CRMA to create a Dataset. 15 | * The class uses the standard Salesforce REST API in conjunction with the External Data API to create the dataset. 16 | * It supports Overwrite/Append/Upsert Operations and requires a valid data csv file and schema file 17 | * in accordance with CRMA external data upload specifications. 18 | * The file format is csv or gzip with csv or zip extension respectively. 19 | * The encoding of the file must be UTF-8. 20 | * The process attempts to upload the file once, no retries are attempted. 21 | * Current API limit for upload is 40Gb per file and 50Gb total in 24 hours. 22 | * 23 | * @author T J Wilson, 06/12/2022 24 | * 25 | */ 26 | 27 | public class DatasetCreator { 28 | 29 | 30 | //datapart size limits 31 | protected static final int MAX_BYTES_PER_CHUNK=33554432; // 32 Mb per datapart 32 | private static int ADD_DATAPART_THREADS =3; //loading threads count 33 | private static Map fileParts =null; 34 | //logger 35 | protected static AtomicInteger dataPartErrors =new AtomicInteger(0); 36 | 37 | private final static Logger logger = Logger.getLogger(DatasetCreator.class.getName()); 38 | 39 | 40 | 41 | 42 | /** 43 | * Main method to Upload the datafile to CMRA platform and create a Dataset.Using the Salesforce REST API and oauth 44 | * it creates the Data header with the meta information that describes the dataset configuration. It compresses 45 | * the data file if necessary and chunks it into 32Mb parts and writes those file parts to disk. Using a blocking 46 | * queue of 10 and upload thread count of 3 it controls the sequencing of loading each data part file into the 47 | * Data part object. 48 | * @param aMetadataJson the schema file containing field definitions 49 | * @param aDataFile the comma separated value data file containing records, must be zip or csv extension 50 | * @param aDatasetAlias, the alias name of the Dataset to be created. 51 | * @param aDatasetLabel, the dataset label 52 | * @param aUserName, the login to be used to upload in the target Org 53 | * @param aPassword, the password for the upload login 54 | * @param aCSecret, the consumer secret. 55 | * @param aCKey, the consumer key. 56 | * @param aEndpoint, the REST API endpoint url. 57 | * @param aOperation, the load operation ; Overwrite, Append or Upsert 58 | * @throws Exception 59 | */ 60 | 61 | public static void uploadData(File aMetadataJson, File aDataFile,String aDatasetAlias,String aDatasetLabel, 62 | String aUserName, String aPassword, String aCSecret, String aCKey, 63 | String aEndpoint,String aOperation, String aApp, String aMode) throws Exception{ 64 | 65 | 66 | RESTHttpClient.setOauthTokens(aUserName, aPassword, aCSecret, aCKey,(aEndpoint)); 67 | 68 | RESTHttpClient.createHeader(aOperation,aMode, aDatasetLabel,aDatasetAlias, aMetadataJson, aApp); 69 | try { 70 | addDataParts(aDataFile); 71 | RESTHttpClient.updateHeader(); 72 | }catch (Exception e) { 73 | throw new DatasetCreatorException("Unable to process dataparts "+e.getMessage()); 74 | }finally { 75 | FileUtils.deleteQuietly(fileParts); 76 | } 77 | } 78 | 79 | 80 | 81 | 82 | 83 | private static void addDataParts(File aDataFile) throws DatasetCreatorException { 84 | logger.info("Adding data parts"); 85 | try { 86 | 87 | File fileToChunk = null; 88 | //if file extension is zip it is already in gzipped format 89 | if (FileUtils.FILE_EXT_ZIP.equals(FileUtils.getFileExt(aDataFile))) { 90 | fileToChunk = aDataFile; 91 | 92 | } else { 93 | // Try to compress file, if unable use original file 94 | File gzipFile = new File(FileUtils.getBaseName(aDataFile)+".zip"); 95 | if (FileUtils.gzipFile(aDataFile, gzipFile)) { 96 | fileToChunk = gzipFile; 97 | } else { 98 | fileToChunk = aDataFile; 99 | } 100 | } 101 | 102 | fileParts = FileUtils.chunkBinary(fileToChunk); // Split the file 103 | //created queue bounded to 10 104 | BlockingQueue> queue = new LinkedBlockingQueue>(10); 105 | LinkedList loadThreads = new LinkedList(); 106 | if(fileParts.size()<=ADD_DATAPART_THREADS) 107 | ADD_DATAPART_THREADS = 1; 108 | AtomicInteger cnt = new AtomicInteger(); 109 | for (int i = 0; i < ADD_DATAPART_THREADS; i++) { 110 | AddDataPartThread addDataPart = new AddDataPartThread(queue,i, cnt); 111 | Thread th = new Thread(addDataPart,"AddDataPartsThread-"+i); 112 | th.setDaemon(true); 113 | th.start(); 114 | loadThreads.add(addDataPart); 115 | 116 | } 117 | for (Map.Entry entry : fileParts.entrySet()) { 118 | Map row =new HashMap(); 119 | logger.info("Adding data part number to queue: "+entry.getKey()); 120 | row.put(entry.getKey(), entry.getValue()); 121 | queue.offer(row,240,TimeUnit.SECONDS); 122 | 123 | } 124 | logger.info("Completed queue entries "); 125 | Integer endPart =4000; 126 | for(int i=0;i endRow =new HashMap(); 129 | File endFile= new File(FileUtils.END_THREAD); 130 | logger.info("Adding data part number to queue: End File "+endPart); 131 | endRow.put(endPart, endFile); 132 | queue.put(endRow); 133 | } 134 | if(dataPartErrors.get()>0) { 135 | throw new DatasetCreatorException("There were "+dataPartErrors.get()+" errors with data part entry"); 136 | } 137 | for(int i=0;i fileParts) { 57 | for (Map.Entry entry : fileParts.entrySet()) { 58 | cleanExistingFile(entry.getValue()); 59 | } 60 | 61 | } 62 | 63 | 64 | 65 | //Gzips single file 66 | protected static boolean gzipFile(File aDataFile, File aGZIPFile) { 67 | logger.log(Level.INFO,"In gzipFile"); 68 | FileInputStream fis =null; 69 | FileOutputStream fos=null; 70 | GZIPOutputStream gzipos=null; 71 | boolean gzipOK=false; 72 | try { 73 | fis = new FileInputStream(aDataFile); 74 | fos = new FileOutputStream(aGZIPFile); 75 | gzipos = new GZIPOutputStream(fos); 76 | byte[] buffer = new byte[1024]; 77 | int len; 78 | while((len=fis.read(buffer)) != -1){ 79 | gzipos.write(buffer, 0, len); 80 | } 81 | gzipOK= true; 82 | 83 | } catch (IOException e) { 84 | logger.log(Level.WARNING,"Unable to gzip file ",e); 85 | //do not throw 86 | }finally{ 87 | if(gzipos!=null)try{gzipos.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 88 | if(fis!=null)try{fis.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 89 | if(fos!=null)try{fos.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 90 | 91 | } 92 | return gzipOK; 93 | 94 | } 95 | 96 | protected File unzip(File aGZIPFile) throws IOException { 97 | logger.log(Level.INFO,"In unzip"); 98 | FileInputStream fis =null; 99 | FileOutputStream fos=null; 100 | GZIPInputStream gzipis=null; 101 | File csvFilePart=new File(aGZIPFile.getName()+"."+FILE_EXT_CSV); 102 | try { 103 | fis = new FileInputStream(aGZIPFile); 104 | fos = new FileOutputStream(csvFilePart); 105 | gzipis = new GZIPInputStream(fis); 106 | byte[] buffer = new byte[1024]; 107 | int len; 108 | while((len=gzipis.read(buffer)) > 0){ 109 | fos.write(buffer, 0, len); 110 | } 111 | 112 | } catch (IOException e) { 113 | logger.log(Level.WARNING,"Unable to unzip file ",e); 114 | throw e; 115 | }finally{ 116 | if(gzipis!=null)try{gzipis.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 117 | if(fis!=null)try{fis.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 118 | if(fos!=null)try{fos.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 119 | 120 | } 121 | return csvFilePart; 122 | 123 | } 124 | 125 | 126 | 127 | 128 | //detect if compressed file is GZip 129 | protected static boolean detectGZipped(File aDataFile) { 130 | int magic = 0; 131 | try { 132 | RandomAccessFile raf = new RandomAccessFile(aDataFile, "r"); 133 | magic = raf.read() & 0xff | ((raf.read() << 8) & 0xff00); 134 | raf.close(); 135 | } catch (Throwable e) { 136 | logger.log(Level.WARNING,"Unable to detect if file is GZipped " +e.getMessage()); 137 | } 138 | return magic == GZIPInputStream.GZIP_MAGIC; 139 | } 140 | 141 | 142 | 143 | protected static Map chunkBinary(File aDataFile) throws DatasetCreatorException{ 144 | 145 | ConcurrentHashMap filePartMap = new ConcurrentHashMap(); 146 | //split file into chunks 147 | logger.info("In chunkBinary"); 148 | String fileExt="."+getFileExt(aDataFile); 149 | String fileBaseName= getBaseName(aDataFile); 150 | int maxBytesToChunk=DatasetCreator.MAX_BYTES_PER_CHUNK; 151 | 152 | byte[] buffer =new byte[maxBytesToChunk]; 153 | FileOutputStream tmpOut = null; 154 | FileInputStream fis=null; 155 | BufferedInputStream bis=null; 156 | try{ 157 | fis=new FileInputStream(aDataFile); 158 | bis=new BufferedInputStream(fis); 159 | int bytesRead=0; 160 | int filePartNo=1; 161 | while((bytesRead=bis.read(buffer))>0){ 162 | logger.log(Level.INFO,"Creating file part number "+filePartNo); 163 | File filePart=new File(fileBaseName+".part."+filePartNo +fileExt); 164 | 165 | cleanExistingFile(filePart); 166 | tmpOut=new FileOutputStream(filePart); 167 | tmpOut.write(buffer,0,bytesRead); 168 | tmpOut.close(); 169 | tmpOut=null; 170 | filePartMap.put(filePartNo, filePart); 171 | filePartNo++; 172 | } 173 | 174 | }catch(IOException ioe){ 175 | throw new DatasetCreatorException("Unable to create data file part "+ ioe.getMessage(), ioe.getCause()); 176 | }finally { 177 | if(bis!=null)try{fis.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 178 | if(fis!=null)try{fis.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 179 | if(tmpOut!=null)try{tmpOut.close();}catch(IOException ioe){ logger.log(Level.WARNING,ioe.getMessage());} 180 | } 181 | 182 | return filePartMap; 183 | 184 | } 185 | 186 | protected byte[] readFile(File aDataFile) throws IOException { 187 | Path path = Paths.get(aDataFile.getPath()); 188 | return Files.readAllBytes(path); 189 | 190 | } 191 | 192 | protected byte[] getBase64ByteArray(String aFileString) { 193 | return Base64.getEncoder().encode(aFileString.getBytes()); 194 | } 195 | 196 | protected String getBase64String(byte[] aFileBytes) { 197 | return Base64.getEncoder().encodeToString(aFileBytes); 198 | } 199 | 200 | protected static String getBaseName(File aFile) { 201 | String fileName =aFile.getName(); 202 | String bN = null; 203 | if (fileName.contains(".")) { 204 | int i = fileName.lastIndexOf('.'); 205 | bN = i > 0 ? fileName.substring(0,fileName.length() - i) : ""; 206 | } 207 | return bN; 208 | } 209 | 210 | 211 | protected static String getFileExt(File aFile){ 212 | String fileName =aFile.getName(); 213 | String fe = null; 214 | if (fileName.contains(".")) { 215 | int i = fileName.lastIndexOf('.'); 216 | fe = i > 0 ? fileName.substring(i + 1) : ""; 217 | } 218 | return fe; 219 | 220 | } 221 | 222 | } 223 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/RESTHttpClient.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | import java.io.File; 4 | import java.net.URI; 5 | import java.net.URLEncoder; 6 | import java.net.http.HttpClient; 7 | import java.net.http.HttpRequest; 8 | import java.net.http.HttpRequest.BodyPublisher; 9 | import java.net.http.HttpRequest.BodyPublishers; 10 | import java.net.http.HttpResponse; 11 | import java.net.http.HttpResponse.BodyHandlers; 12 | import java.nio.charset.StandardCharsets; 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | import java.util.logging.Logger; 16 | 17 | 18 | /** 19 | * Class that communicates with CRMA External Data API via the REST API and parses 20 | * response JSON. Specifically it gets the oauth token, inserts the meta data into the header 21 | * object and provides the method to insert BLOBs into the data part object. 22 | * In addition it updates the header with the process command. 23 | * 24 | * @author T J Wilson, 06/12/2022 25 | * 26 | */ 27 | 28 | 29 | 30 | public class RESTHttpClient { 31 | 32 | 33 | public static final String OAUTH_2_URL ="https://login.salesforce.com/services/oauth2/token"; 34 | public static final String OAUTH_2_URL_STEM ="/services/oauth2/token"; 35 | public static final String END_POINT="/services/data/"; 36 | public static final String API_VERSION="v56.0"; 37 | public static final String SOBJECTS="/sobjects/"; 38 | public static final String SOQL_URL_STEM ="/query/?q=SELECT+FolderId+FROM+InsightsApplication+WHERE+DeveloperName+='"; 39 | 40 | private static String[] oauthTokens =new String[2]; 41 | private static String headerId=null; 42 | private boolean dataPartSuccess=false; 43 | 44 | public RESTHttpClient() { 45 | 46 | } 47 | private final static Logger logger = Logger.getLogger(RESTHttpClient.class.getName()); 48 | 49 | 50 | public static void setOauthTokens(String aUserName, String aPassword, 51 | String aCSecret, String aCKey, String aEndpoint) throws Exception{ 52 | String oauth2Url=OAUTH_2_URL; 53 | if(!StringUtils.isBlank(aEndpoint)) { 54 | oauth2Url=aEndpoint +OAUTH_2_URL_STEM; 55 | } 56 | 57 | 58 | Map data = new HashMap<>(); 59 | data.put("grant_type", "password"); 60 | data.put("client_id", aCKey); 61 | data.put("client_secret", aCSecret); 62 | data.put("username", aUserName); 63 | data.put("password", aPassword); 64 | 65 | HttpRequest request =HttpRequest.newBuilder() 66 | .uri(new URI(oauth2Url)) 67 | .POST(setFormData(data)) 68 | .header("Content-Type", "application/x-www-form-urlencoded") 69 | .build(); 70 | 71 | HttpClient client = HttpClient.newHttpClient(); 72 | 73 | HttpResponse resp=client.send(request,BodyHandlers.ofString()); 74 | // logger.fine("Query response : "+resp.body()); 75 | if(!parseAccessIsSuccess(resp.body())) { 76 | throw new DatasetCreatorException(resp.body()); 77 | } 78 | 79 | oauthTokens[0] = parseAccessToken(resp.body()); 80 | oauthTokens[1] =parseIntanceUrl(resp.body()); 81 | 82 | } 83 | 84 | 85 | 86 | public static String[] getOauthTokens() { 87 | return oauthTokens; 88 | } 89 | 90 | public static BodyPublisher setFormData(Map data) { 91 | var builder = new StringBuilder(); 92 | for (Map.Entry entry : data.entrySet()) { 93 | if (builder.length() > 0) { 94 | builder.append("&"); 95 | } 96 | builder 97 | .append(URLEncoder.encode(entry.getKey().toString(), StandardCharsets.UTF_8)); 98 | builder.append("="); 99 | builder 100 | .append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); 101 | } 102 | return BodyPublishers.ofString(builder.toString()); 103 | } 104 | 105 | // Simple method to parse Access Token from salesforce response 106 | 107 | 108 | 109 | public static String parseAccessToken(String body) { 110 | int posStart= body.indexOf(":"); 111 | body=body.substring(posStart+1); 112 | int posEnd =body.indexOf(","); 113 | body=body.substring(1,posEnd-1); 114 | return body; 115 | 116 | } 117 | public static String parseIntanceUrl(String body){ 118 | int posStart= body.indexOf("instance_url"); 119 | body=body.substring(posStart+14); 120 | int posEnd =body.indexOf(","); 121 | body=body.substring(1,posEnd-1); 122 | return body; 123 | 124 | } 125 | 126 | 127 | public static String parseHeaderId(String body){ 128 | int posStart= body.indexOf("id"); 129 | body=body.substring(posStart+4); 130 | int posEnd =body.indexOf(","); 131 | body=body.substring(1,posEnd-1); 132 | return body; 133 | 134 | } 135 | 136 | public static String parseFolderId(String body){ 137 | int posStart= body.indexOf("FolderId"); 138 | if(posStart>-1) { 139 | body=body.substring(posStart+10); 140 | int posEnd =body.indexOf("}"); 141 | body=body.substring(1,posEnd-1); 142 | }else { 143 | body=null; 144 | } 145 | 146 | return body; 147 | 148 | } 149 | 150 | public static boolean parseResponseIsSuccess(String aBody) { 151 | int posStart= aBody.indexOf("success"); 152 | if(posStart>-1) { 153 | String body=aBody.substring(posStart+9); 154 | int posEnd =body.indexOf(","); 155 | body=body.substring(0,posEnd); 156 | if (!StringUtils.isBlank(body)&&body.equalsIgnoreCase("true")){ 157 | return true; 158 | }else 159 | return false; 160 | }else return false; 161 | 162 | } 163 | 164 | public static boolean parseAccessIsSuccess(String aBody) { 165 | int posStart= aBody.indexOf("error"); 166 | if(posStart>-1) { 167 | return false; 168 | }else return true; 169 | 170 | } 171 | 172 | 173 | protected boolean isDataPartSuccess() { 174 | return dataPartSuccess; 175 | } 176 | 177 | 178 | 179 | public static void createHeader(String aOperation,String aMode, String aDatasetLabel, 180 | String aDatasetAlias,File aMetadataJson, String aApp ) throws Exception { 181 | 182 | String appId=null; 183 | if (aApp!=null) { 184 | appId=getAppId(aApp); 185 | } 186 | 187 | logger.info("App Folder Id : "+appId); 188 | 189 | HeaderJSON header= new HeaderJSON(aOperation,aMode,aDatasetLabel,aDatasetAlias, appId); 190 | FileUtils fileUtils = new FileUtils(); 191 | header.setMetaJSON(fileUtils.getBase64String(fileUtils.readFile(aMetadataJson))); 192 | 193 | HttpRequest request =HttpRequest.newBuilder() 194 | .uri(new URI(getOauthTokens()[1]+END_POINT+API_VERSION+SOBJECTS+HeaderJSON.EXT_DATA_OBJECT+"/")) 195 | .header("Content-Type", "application/json") 196 | .header("Authorization", "Bearer "+getOauthTokens()[0]) 197 | .POST(BodyPublishers.ofString(header.toString())) 198 | .build(); 199 | 200 | HttpClient client = HttpClient.newHttpClient(); 201 | 202 | HttpResponse resp=client.send(request,BodyHandlers.ofString()); 203 | if(!parseResponseIsSuccess(resp.body())) { 204 | throw new DatasetCreatorException("Unable to create header : " + resp.body()); 205 | } 206 | headerId=parseHeaderId(resp.body()); 207 | 208 | 209 | } 210 | 211 | 212 | public void createDataPart(File aDataPart, int aPartNumber) throws Exception { 213 | 214 | 215 | DataPartJSON dataPart= new DataPartJSON(headerId) ; 216 | FileUtils fileUtils = new FileUtils(); 217 | dataPart.setDataPart(fileUtils.getBase64String(fileUtils.readFile(aDataPart))); 218 | dataPart.setPartNumber(String.valueOf(aPartNumber)); 219 | 220 | HttpRequest request =HttpRequest.newBuilder() 221 | .uri(new URI(getOauthTokens()[1]+END_POINT+API_VERSION+SOBJECTS+DataPartJSON.EXT_DATA_OBJECT_PARTS+"/")) 222 | .header("Content-Type", "application/json") 223 | .header("Authorization", "Bearer "+getOauthTokens()[0]) 224 | .POST(BodyPublishers.ofString(dataPart.toString())) 225 | .build(); 226 | 227 | 228 | HttpClient client = HttpClient.newHttpClient(); 229 | 230 | HttpResponse resp=client.send(request,BodyHandlers.ofString()); 231 | logger.fine("File Part:"+aPartNumber +" Response :"+resp.body()); 232 | if(!parseResponseIsSuccess(resp.body())) { 233 | // this will be changed with new data part error handling code 234 | // obvious two ways that are contradictory her 235 | throw new DatasetCreatorException(resp.body()); 236 | }else { 237 | dataPartSuccess=true; 238 | } 239 | 240 | } 241 | 242 | public static void updateHeader() throws Exception{ 243 | HeaderJSON header= new HeaderJSON(); 244 | logger.fine("Setting action to Process"); 245 | header.setAction("Process"); 246 | 247 | HttpRequest request =HttpRequest.newBuilder() 248 | .uri(new URI(getOauthTokens()[1]+END_POINT+API_VERSION+SOBJECTS+ 249 | HeaderJSON.EXT_DATA_OBJECT+"/" +headerId)) 250 | .header("Content-Type", "application/json") 251 | .header("Authorization", "Bearer "+getOauthTokens()[0]) 252 | .method("PATCH",BodyPublishers.ofString(header.toPatchString())) 253 | .build(); 254 | 255 | HttpClient client = HttpClient.newHttpClient(); 256 | HttpResponse resp=client.send(request,BodyHandlers.ofString()); 257 | logger.info("Header action set to process "); 258 | 259 | 260 | } 261 | 262 | public static String getAppId(String aApp ) throws Exception { 263 | 264 | //SOQL query 265 | 266 | logger.fine("App Id Query : "+ END_POINT+API_VERSION+SOQL_URL_STEM +aApp+"'" ); 267 | 268 | HttpRequest request =HttpRequest.newBuilder() 269 | .uri(new URI(getOauthTokens()[1]+END_POINT+API_VERSION+SOQL_URL_STEM +aApp+"'")) 270 | .header("Content-Type", "application/json") 271 | .header("Authorization", "Bearer "+getOauthTokens()[0]) 272 | .GET() 273 | .build(); 274 | 275 | HttpClient client = HttpClient.newHttpClient(); 276 | 277 | HttpResponse resp=client.send(request,BodyHandlers.ofString()); 278 | logger.fine("Query response : "+resp.body()); 279 | 280 | return parseFolderId(resp.body()); 281 | 282 | 283 | 284 | 285 | } 286 | 287 | 288 | 289 | 290 | } 291 | -------------------------------------------------------------------------------- /src/main/java/com/force/crma/olympus/CreateDataset.java: -------------------------------------------------------------------------------- 1 | package com.force.crma.olympus; 2 | 3 | import java.io.File; 4 | 5 | 6 | public class CreateDataset { 7 | 8 | 9 | 10 | public static void main(String[] args) 11 | { 12 | DatasetCreatorParams params = new DatasetCreatorParams(); 13 | if (args.length >= 1) { 14 | 15 | for (int i = 1; i < args.length; i += 2) { 16 | // System.out.println("Curent Argument " + args[(i - 1)]); 17 | // System.out.println("Curent Value " + args[i]); 18 | if ((args[(i - 1)].equalsIgnoreCase("--help")) || (args[(i - 1)].equalsIgnoreCase("-help")) || (args[(i - 1)].equalsIgnoreCase("help"))) 19 | { 20 | printUsage(); 21 | } 22 | else if (args[(i - 1)].equalsIgnoreCase("--u")) 23 | { 24 | params.username = args[i]; 25 | } 26 | else if (args[(i - 1)].equalsIgnoreCase("--p")) 27 | { 28 | params.password = args[i]; 29 | } 30 | else if (args[(i - 1)].equalsIgnoreCase("--cSecret")) 31 | { 32 | params.cSecret = args[i]; 33 | } 34 | else if (args[(i - 1)].equalsIgnoreCase("--cKey")) 35 | { 36 | params.cKey = args[i]; 37 | } 38 | else if (args[(i - 1)].equalsIgnoreCase("--schemaFile")) 39 | { 40 | String tmp = args[i]; 41 | if (tmp != null) 42 | { 43 | File tempFile = new File(tmp); 44 | if (tempFile.exists()) 45 | { 46 | params.schemaFile = tempFile.toString(); 47 | } 48 | else 49 | { 50 | System.out.println("File {" + args[i] + "} does not exist"); 51 | System.exit(-1); 52 | } 53 | } 54 | } 55 | else if (args[(i - 1)].equalsIgnoreCase("--inputFile")) 56 | { 57 | String tmp = args[i]; 58 | if (tmp != null) 59 | { 60 | File tempFile = new File(tmp); 61 | if (tempFile.exists()) 62 | { 63 | params.inputFile = tempFile.toString(); 64 | } 65 | else 66 | { 67 | System.out.println("File {" + args[i] + "} does not exist"); 68 | System.exit(-1); 69 | } 70 | } 71 | } 72 | 73 | else if (args[(i - 1)].equalsIgnoreCase("--dataset")) 74 | { 75 | params.datasetAlias = args[i]; 76 | } 77 | else if (args[(i - 1)].equalsIgnoreCase("--label")) 78 | { 79 | params.datasetLabel = args[i]; 80 | } 81 | else if (args[(i - 1)].equalsIgnoreCase("--app")) 82 | { 83 | params.app = args[i]; 84 | } 85 | 86 | else if (args[(i - 1)].equalsIgnoreCase("--endpoint")) 87 | { 88 | params.endpoint = args[i]; 89 | } 90 | //proxy parameters not used but left in for future use 91 | else if (args[(i - 1)].equalsIgnoreCase("--proxyHost")) 92 | { 93 | params.proxyHost = args[i]; 94 | } 95 | else if (args[(i - 1)].equalsIgnoreCase("--proxyPort")) 96 | { 97 | params.proxyPort = Integer.valueOf(args[i]); 98 | } 99 | else if (args[(i - 1)].equalsIgnoreCase("--proxyUserName")) 100 | { 101 | params.proxyUsername = args[i]; 102 | } 103 | else if (args[(i - 1)].equalsIgnoreCase("--proxyPassword")) 104 | { 105 | params.proxyPassword = args[i]; 106 | } 107 | else if (args[(i - 1)].equalsIgnoreCase("--operation")) 108 | { 109 | if (args[i] != null) { 110 | if (args[i].equalsIgnoreCase("Overwrite")) 111 | { 112 | params.operation = args[i]; 113 | } 114 | else if (args[i].equalsIgnoreCase("Upsert")) 115 | { 116 | params.operation = args[i]; 117 | } 118 | else if (args[i].equalsIgnoreCase("Append")) 119 | { 120 | params.operation = args[i]; 121 | } 122 | else if (args[i].equalsIgnoreCase("Delete")) 123 | { 124 | params.operation = args[i]; 125 | } 126 | else 127 | { 128 | System.out.println("Invalid Operation {" + args[i] + "} Must be Overwrite or Upsert or Append or Delete"); 129 | System.exit(-1); 130 | } 131 | } 132 | } 133 | else if (args[(i - 1)].equalsIgnoreCase("--mode")) 134 | { 135 | if (args[i] != null) { 136 | if (args[i].equalsIgnoreCase("Incremental")) 137 | { 138 | params.mode = args[i]; 139 | } 140 | } 141 | } 142 | 143 | } 144 | } 145 | // params.schemaFile = (params.inputFile.substring(0, params.inputFile.length() - 4) + "_schema.json"); 146 | // System.out.println("Schema file " + params.schemaFile); 147 | 148 | long startTime = System.currentTimeMillis(); 149 | System.out.println("Start Time in ms : " + startTime); 150 | File schemaFile = new File(params.schemaFile); 151 | File dataFile = new File (params.inputFile); 152 | try 153 | { 154 | checkArguments(schemaFile, dataFile, params.datasetAlias, params.username, 155 | params.password,params.cSecret,params.cKey, params.operation,params.mode); 156 | 157 | DatasetCreator.uploadData(schemaFile, dataFile, params.datasetAlias,params.datasetLabel, 158 | params.username, params.password,params.cSecret,params.cKey, 159 | params.endpoint, params.operation, params.app, params.mode ); 160 | } 161 | catch (Exception e) 162 | { 163 | e.printStackTrace(); 164 | } 165 | long endTime = System.currentTimeMillis(); 166 | System.out.println("End Time in ms : " + endTime); 167 | } 168 | 169 | 170 | //Simple validation method for the arguments 171 | private static void checkArguments(File aMetadataJson, File aDataFile,String aDatasetAlias,String aUserName, String aPassword, 172 | String aCSecret, String aCKey,String aOperation, String aMode) throws DatasetCreatorException{ 173 | //Validate JSON file 174 | if(aMetadataJson==null){ 175 | throw new DatasetCreatorException("The JSON schema file cannot be null"); 176 | } 177 | if(aMetadataJson.length()==0){ 178 | throw new DatasetCreatorException("The JSON schema file cannot be empty"); 179 | } 180 | //Validate security credentials 181 | if(StringUtils.isBlank(aUserName)||StringUtils.isBlank(aPassword)){ 182 | throw new DatasetCreatorException("The username or password cannot be null or empty"); 183 | } 184 | if(StringUtils.isBlank(aCSecret)||StringUtils.isBlank(aCKey)){ 185 | throw new DatasetCreatorException("The client secret or key cannot be null or empty"); 186 | } 187 | //Validate DatasetAlias credentials 188 | if(StringUtils.isBlank(aDatasetAlias)){ 189 | throw new DatasetCreatorException("The Dataset Alias name cannot be null or empty"); 190 | } 191 | if(!StringUtils.isBlank(aOperation)&&!StringUtils.arrayContainsValue(HeaderJSON.OPERATIONS_VALUE,aOperation)){ 192 | throw new DatasetCreatorException("The operation "+aOperation +" is not valid, it must be one of : Append, Overwrite or Upsert"); 193 | } 194 | if(!StringUtils.isBlank(aMode)&&!StringUtils.arrayContainsValue(HeaderJSON.MODE_VALUE,aMode)){ 195 | throw new DatasetCreatorException("The mode "+aMode +" is not valid, it must be one of : Incremental "); 196 | } 197 | //Validate data file file 198 | if(aDataFile==null){ 199 | throw new DatasetCreatorException("The data file cannot be null"); 200 | } 201 | if(aDataFile.length()==0){ 202 | throw new DatasetCreatorException("The data file cannot be empty"); 203 | } 204 | //check if csv or zip extension 205 | String fileExt=FileUtils.getFileExt(aDataFile); 206 | if(!(FileUtils.FILE_EXT_CSV.equals(fileExt)||FileUtils.FILE_EXT_ZIP.equals(fileExt))){ 207 | throw new DatasetCreatorException("The data file must have .csv or .zip extension"); 208 | } 209 | //if zip check that the format is gzip 210 | if(FileUtils.FILE_EXT_ZIP.equals(fileExt)&&!FileUtils.detectGZipped(aDataFile)){ 211 | throw new DatasetCreatorException("The zip file must be GZip format"); 212 | } 213 | 214 | } 215 | 216 | public static void printUsage() 217 | { 218 | System.out.println("\n*******************************************************************************"); 219 | System.out.println("NOT COMPLETE YET Usage:"); 220 | System.out.print("java -jar DatasetCreator.jar --u userName --p password --operation operatiom --endpoint endPoint "); 221 | System.out.println("--dataset datasetAlias --label datasetLabel --inputFile inputFile --schemaFile metajsonfile "); 222 | System.out.println("--cKey custKey --cSecret custSecret --inputFile inputFile --schemaFile metajsonfile "); 223 | System.out.println("--p : required Salesforce.com password"); 224 | System.out.println("--endpoint: (Optional) The salesforce REST api endpoint (test/prod)"); 225 | System.out.println(" : Default: https://login.salesforce.com/"); 226 | System.out.println("--dataset : the dataset alias. required"); 227 | System.out.println("--label : the dataset label"); 228 | System.out.println("--inputFile :the input csv file. required "); 229 | System.out.println("--schemaFile :the meta json file. required "); 230 | System.out.println("--cKey :client key, required"); 231 | System.out.println("--cSecret :client secret, required"); 232 | System.out.println("--operation :the load operation, Overwrite, Append, Upsert or Delete. required "); 233 | System.out.println("*******************************************************************************\n"); 234 | System.out.println("Usage Example 1: Upload a csv to a dataset"); 235 | System.out.println("java -jar DatasetCreator.jar --operation Overwrite --schemaFile complaints_schema.json --inputFile complaints.csv --dataset Complaints"); 236 | System.out.println("--label Complaints --u yname@acme.com --p pa33w@rd --endpoint https://login.salesforce.com --cSecret 844383BE0F0FVV --cKey 3MVG9FMtW0XJDLd2hVV"); 237 | System.out.println(""); 238 | } 239 | 240 | 241 | } 242 | -------------------------------------------------------------------------------- /target/classes/Complaints_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fileFormat" : { 3 | "charsetName" : "UTF-8", 4 | "fieldsDelimitedBy" : ",", 5 | "fieldsEnclosedBy" : "\"", 6 | "linesTerminatedBy" : "\n", 7 | "numberOfLinesToIgnore" : 1 8 | }, 9 | "objects" : [ { 10 | "name" : "ComplaintsSamples_csv", 11 | "fullyQualifiedName" : "ComplaintsSamples_csv", 12 | "connector" : "CSV", 13 | "label" : "ComplaintsSamples.csv", 14 | "description" : "ComplaintsSamples_csv", 15 | "rowLevelSecurityFilter" : null, 16 | "fields" : [ { 17 | "name" : "Lab_country", 18 | "fullyQualifiedName" : "Lab_country", 19 | "label" : "Lab_country", 20 | "description" : null, 21 | "type" : "Text", 22 | "precision" : 255, 23 | "scale" : 0, 24 | "decimalSeparator" : ".", 25 | "defaultValue" : null, 26 | "format" : null, 27 | "isSystemField" : false, 28 | "isUniqueId" : false, 29 | "isMultiValue" : false, 30 | "multiValueSeparator" : null, 31 | "fiscalMonthOffset" : 0, 32 | "firstDayOfWeek" : -1, 33 | "isYearEndFiscalYear" : true, 34 | "canTruncateValue" : true, 35 | "isSkipped" : false 36 | }, { 37 | "name" : "Lab_city", 38 | "fullyQualifiedName" : "Lab_city", 39 | "label" : "Lab_city", 40 | "description" : null, 41 | "type" : "Text", 42 | "precision" : 255, 43 | "scale" : 0, 44 | "decimalSeparator" : ".", 45 | "defaultValue" : null, 46 | "format" : null, 47 | "isSystemField" : false, 48 | "isUniqueId" : false, 49 | "isMultiValue" : false, 50 | "multiValueSeparator" : null, 51 | "fiscalMonthOffset" : 0, 52 | "firstDayOfWeek" : -1, 53 | "isYearEndFiscalYear" : true, 54 | "canTruncateValue" : true, 55 | "isSkipped" : false 56 | }, { 57 | "name" : "Lab_name", 58 | "fullyQualifiedName" : "Lab_name", 59 | "label" : "Lab_name", 60 | "description" : null, 61 | "type" : "Text", 62 | "precision" : 255, 63 | "scale" : 0, 64 | "decimalSeparator" : ".", 65 | "defaultValue" : null, 66 | "format" : null, 67 | "isSystemField" : false, 68 | "isUniqueId" : false, 69 | "isMultiValue" : false, 70 | "multiValueSeparator" : null, 71 | "fiscalMonthOffset" : 0, 72 | "firstDayOfWeek" : -1, 73 | "isYearEndFiscalYear" : true, 74 | "canTruncateValue" : true, 75 | "isSkipped" : false 76 | }, { 77 | "name" : "System_Name", 78 | "fullyQualifiedName" : "System_Name", 79 | "label" : "System_Name", 80 | "description" : null, 81 | "type" : "Text", 82 | "precision" : 255, 83 | "scale" : 0, 84 | "decimalSeparator" : ".", 85 | "defaultValue" : null, 86 | "format" : null, 87 | "isSystemField" : false, 88 | "isUniqueId" : false, 89 | "isMultiValue" : false, 90 | "multiValueSeparator" : null, 91 | "fiscalMonthOffset" : 0, 92 | "firstDayOfWeek" : -1, 93 | "isYearEndFiscalYear" : true, 94 | "canTruncateValue" : true, 95 | "isSkipped" : false 96 | }, { 97 | "name" : "Institution", 98 | "fullyQualifiedName" : "Institution", 99 | "label" : "Institution", 100 | "description" : null, 101 | "type" : "Text", 102 | "precision" : 255, 103 | "scale" : 0, 104 | "decimalSeparator" : ".", 105 | "defaultValue" : null, 106 | "format" : null, 107 | "isSystemField" : false, 108 | "isUniqueId" : false, 109 | "isMultiValue" : false, 110 | "multiValueSeparator" : null, 111 | "fiscalMonthOffset" : 0, 112 | "firstDayOfWeek" : -1, 113 | "isYearEndFiscalYear" : true, 114 | "canTruncateValue" : true, 115 | "isSkipped" : false 116 | }, { 117 | "name" : "System_serial_number", 118 | "fullyQualifiedName" : "System_serial_number", 119 | "label" : "System_serial_number", 120 | "description" : null, 121 | "type" : "Text", 122 | "precision" : 255, 123 | "scale" : 0, 124 | "decimalSeparator" : ".", 125 | "defaultValue" : null, 126 | "format" : null, 127 | "isSystemField" : false, 128 | "isUniqueId" : false, 129 | "isMultiValue" : false, 130 | "multiValueSeparator" : null, 131 | "fiscalMonthOffset" : 0, 132 | "firstDayOfWeek" : -1, 133 | "isYearEndFiscalYear" : true, 134 | "canTruncateValue" : true, 135 | "isSkipped" : false 136 | }, { 137 | "name" : "System_model", 138 | "fullyQualifiedName" : "System_model", 139 | "label" : "System_model", 140 | "description" : null, 141 | "type" : "Text", 142 | "precision" : 255, 143 | "scale" : 0, 144 | "decimalSeparator" : ".", 145 | "defaultValue" : null, 146 | "format" : null, 147 | "isSystemField" : false, 148 | "isUniqueId" : false, 149 | "isMultiValue" : false, 150 | "multiValueSeparator" : null, 151 | "fiscalMonthOffset" : 0, 152 | "firstDayOfWeek" : -1, 153 | "isYearEndFiscalYear" : true, 154 | "canTruncateValue" : true, 155 | "isSkipped" : false 156 | }, { 157 | "name" : "Lab_region", 158 | "fullyQualifiedName" : "Lab_region", 159 | "label" : "Lab_region", 160 | "description" : null, 161 | "type" : "Text", 162 | "precision" : 255, 163 | "scale" : 0, 164 | "decimalSeparator" : ".", 165 | "defaultValue" : null, 166 | "format" : null, 167 | "isSystemField" : false, 168 | "isUniqueId" : false, 169 | "isMultiValue" : false, 170 | "multiValueSeparator" : null, 171 | "fiscalMonthOffset" : 0, 172 | "firstDayOfWeek" : -1, 173 | "isYearEndFiscalYear" : true, 174 | "canTruncateValue" : true, 175 | "isSkipped" : false 176 | }, { 177 | "name" : "Assay_group", 178 | "fullyQualifiedName" : "Assay_group", 179 | "label" : "Assay_group", 180 | "description" : null, 181 | "type" : "Text", 182 | "precision" : 255, 183 | "scale" : 0, 184 | "decimalSeparator" : ".", 185 | "defaultValue" : null, 186 | "format" : null, 187 | "isSystemField" : false, 188 | "isUniqueId" : false, 189 | "isMultiValue" : false, 190 | "multiValueSeparator" : null, 191 | "fiscalMonthOffset" : 0, 192 | "firstDayOfWeek" : -1, 193 | "isYearEndFiscalYear" : true, 194 | "canTruncateValue" : true, 195 | "isSkipped" : false 196 | }, { 197 | "name" : "Assay_name", 198 | "fullyQualifiedName" : "Assay_name", 199 | "label" : "Assay_name", 200 | "description" : null, 201 | "type" : "Text", 202 | "precision" : 255, 203 | "scale" : 0, 204 | "decimalSeparator" : ".", 205 | "defaultValue" : null, 206 | "format" : null, 207 | "isSystemField" : false, 208 | "isUniqueId" : false, 209 | "isMultiValue" : false, 210 | "multiValueSeparator" : null, 211 | "fiscalMonthOffset" : 0, 212 | "firstDayOfWeek" : -1, 213 | "isYearEndFiscalYear" : true, 214 | "canTruncateValue" : true, 215 | "isSkipped" : false 216 | }, { 217 | "name" : "Assay_version", 218 | "fullyQualifiedName" : "Assay_version", 219 | "label" : "Assay_version", 220 | "description" : null, 221 | "type" : "Text", 222 | "precision" : 255, 223 | "scale" : 0, 224 | "decimalSeparator" : ".", 225 | "defaultValue" : null, 226 | "format" : null, 227 | "isSystemField" : false, 228 | "isUniqueId" : false, 229 | "isMultiValue" : false, 230 | "multiValueSeparator" : null, 231 | "fiscalMonthOffset" : 0, 232 | "firstDayOfWeek" : -1, 233 | "isYearEndFiscalYear" : true, 234 | "canTruncateValue" : true, 235 | "isSkipped" : false 236 | }, { 237 | "name" : "Cartridge_S_N", 238 | "fullyQualifiedName" : "Cartridge_S_N", 239 | "label" : "Cartridge_S_N", 240 | "description" : null, 241 | "type" : "Text", 242 | "precision" : 255, 243 | "scale" : 0, 244 | "decimalSeparator" : ".", 245 | "defaultValue" : null, 246 | "format" : null, 247 | "isSystemField" : false, 248 | "isUniqueId" : false, 249 | "isMultiValue" : false, 250 | "multiValueSeparator" : null, 251 | "fiscalMonthOffset" : 0, 252 | "firstDayOfWeek" : -1, 253 | "isYearEndFiscalYear" : true, 254 | "canTruncateValue" : true, 255 | "isSkipped" : false 256 | }, { 257 | "name" : "Reagent_lot_ID", 258 | "fullyQualifiedName" : "Reagent_lot_ID", 259 | "label" : "Reagent_lot_ID", 260 | "description" : null, 261 | "type" : "Text", 262 | "precision" : 255, 263 | "scale" : 0, 264 | "decimalSeparator" : ".", 265 | "defaultValue" : null, 266 | "format" : null, 267 | "isSystemField" : false, 268 | "isUniqueId" : false, 269 | "isMultiValue" : false, 270 | "multiValueSeparator" : null, 271 | "fiscalMonthOffset" : 0, 272 | "firstDayOfWeek" : -1, 273 | "isYearEndFiscalYear" : true, 274 | "canTruncateValue" : true, 275 | "isSkipped" : false 276 | }, { 277 | "name" : "Main_error_code", 278 | "fullyQualifiedName" : "Main_error_code", 279 | "label" : "Main_error_code", 280 | "description" : null, 281 | "type" : "Text", 282 | "precision" : 255, 283 | "scale" : 0, 284 | "decimalSeparator" : ".", 285 | "defaultValue" : null, 286 | "format" : null, 287 | "isSystemField" : false, 288 | "isUniqueId" : false, 289 | "isMultiValue" : false, 290 | "multiValueSeparator" : null, 291 | "fiscalMonthOffset" : 0, 292 | "firstDayOfWeek" : -1, 293 | "isYearEndFiscalYear" : true, 294 | "canTruncateValue" : true, 295 | "isSkipped" : false 296 | }, { 297 | "name" : "Software_version", 298 | "fullyQualifiedName" : "Software_version", 299 | "label" : "Software_version", 300 | "description" : null, 301 | "type" : "Text", 302 | "precision" : 255, 303 | "scale" : 0, 304 | "decimalSeparator" : ".", 305 | "defaultValue" : null, 306 | "format" : null, 307 | "isSystemField" : false, 308 | "isUniqueId" : false, 309 | "isMultiValue" : false, 310 | "multiValueSeparator" : null, 311 | "fiscalMonthOffset" : 0, 312 | "firstDayOfWeek" : -1, 313 | "isYearEndFiscalYear" : true, 314 | "canTruncateValue" : true, 315 | "isSkipped" : false 316 | }, { 317 | "name" : "Sample_type", 318 | "fullyQualifiedName" : "Sample_type", 319 | "label" : "Sample_type", 320 | "description" : null, 321 | "type" : "Text", 322 | "precision" : 255, 323 | "scale" : 0, 324 | "decimalSeparator" : ".", 325 | "defaultValue" : null, 326 | "format" : null, 327 | "isSystemField" : false, 328 | "isUniqueId" : false, 329 | "isMultiValue" : false, 330 | "multiValueSeparator" : null, 331 | "fiscalMonthOffset" : 0, 332 | "firstDayOfWeek" : -1, 333 | "isYearEndFiscalYear" : true, 334 | "canTruncateValue" : true, 335 | "isSkipped" : false 336 | }, { 337 | "name" : "Module_name", 338 | "fullyQualifiedName" : "Module_name", 339 | "label" : "Module_name", 340 | "description" : null, 341 | "type" : "Text", 342 | "precision" : 255, 343 | "scale" : 0, 344 | "decimalSeparator" : ".", 345 | "defaultValue" : null, 346 | "format" : null, 347 | "isSystemField" : false, 348 | "isUniqueId" : false, 349 | "isMultiValue" : false, 350 | "multiValueSeparator" : null, 351 | "fiscalMonthOffset" : 0, 352 | "firstDayOfWeek" : -1, 353 | "isYearEndFiscalYear" : true, 354 | "canTruncateValue" : true, 355 | "isSkipped" : false 356 | }, { 357 | "name" : "Module_S_N", 358 | "fullyQualifiedName" : "Module_S_N", 359 | "label" : "Module_S_N", 360 | "description" : null, 361 | "type" : "Text", 362 | "precision" : 255, 363 | "scale" : 0, 364 | "decimalSeparator" : ".", 365 | "defaultValue" : null, 366 | "format" : null, 367 | "isSystemField" : false, 368 | "isUniqueId" : false, 369 | "isMultiValue" : false, 370 | "multiValueSeparator" : null, 371 | "fiscalMonthOffset" : 0, 372 | "firstDayOfWeek" : -1, 373 | "isYearEndFiscalYear" : true, 374 | "canTruncateValue" : true, 375 | "isSkipped" : false 376 | }, { 377 | "name" : "Initiation_date", 378 | "fullyQualifiedName" : "Initiation_date", 379 | "label" : "Initiation_date", 380 | "description" : null, 381 | "type" : "Date", 382 | "precision" : 0, 383 | "scale" : 0, 384 | "decimalSeparator" : ".", 385 | "defaultValue" : null, 386 | "format" : "MM/dd/yyyy", 387 | "isSystemField" : false, 388 | "isUniqueId" : false, 389 | "isMultiValue" : false, 390 | "multiValueSeparator" : null, 391 | "fiscalMonthOffset" : 0, 392 | "firstDayOfWeek" : -1, 393 | "isYearEndFiscalYear" : true, 394 | "canTruncateValue" : true, 395 | "isSkipped" : false 396 | }, { 397 | "name" : "Status", 398 | "fullyQualifiedName" : "Status", 399 | "label" : "Status", 400 | "description" : null, 401 | "type" : "Text", 402 | "precision" : 255, 403 | "scale" : 0, 404 | "decimalSeparator" : ".", 405 | "defaultValue" : null, 406 | "format" : null, 407 | "isSystemField" : false, 408 | "isUniqueId" : false, 409 | "isMultiValue" : false, 410 | "multiValueSeparator" : null, 411 | "fiscalMonthOffset" : 0, 412 | "firstDayOfWeek" : -1, 413 | "isYearEndFiscalYear" : true, 414 | "canTruncateValue" : true, 415 | "isSkipped" : false 416 | }, { 417 | "name" : "Closed_date", 418 | "fullyQualifiedName" : "Closed_date", 419 | "label" : "Closed_date", 420 | "description" : null, 421 | "type" : "Text", 422 | "precision" : 255, 423 | "scale" : 0, 424 | "decimalSeparator" : ".", 425 | "defaultValue" : null, 426 | "format" : null, 427 | "isSystemField" : false, 428 | "isUniqueId" : false, 429 | "isMultiValue" : false, 430 | "multiValueSeparator" : null, 431 | "fiscalMonthOffset" : 0, 432 | "firstDayOfWeek" : -1, 433 | "isYearEndFiscalYear" : true, 434 | "canTruncateValue" : true, 435 | "isSkipped" : false 436 | }, { 437 | "name" : "Phase", 438 | "fullyQualifiedName" : "Phase", 439 | "label" : "Phase", 440 | "description" : null, 441 | "type" : "Text", 442 | "precision" : 255, 443 | "scale" : 0, 444 | "decimalSeparator" : ".", 445 | "defaultValue" : null, 446 | "format" : null, 447 | "isSystemField" : false, 448 | "isUniqueId" : false, 449 | "isMultiValue" : false, 450 | "multiValueSeparator" : null, 451 | "fiscalMonthOffset" : 0, 452 | "firstDayOfWeek" : -1, 453 | "isYearEndFiscalYear" : true, 454 | "canTruncateValue" : true, 455 | "isSkipped" : false 456 | }, { 457 | "name" : "Case_Intake_Source", 458 | "fullyQualifiedName" : "Case_Intake_Source", 459 | "label" : "Case_Intake_Source", 460 | "description" : null, 461 | "type" : "Text", 462 | "precision" : 255, 463 | "scale" : 0, 464 | "decimalSeparator" : ".", 465 | "defaultValue" : null, 466 | "format" : null, 467 | "isSystemField" : false, 468 | "isUniqueId" : false, 469 | "isMultiValue" : false, 470 | "multiValueSeparator" : null, 471 | "fiscalMonthOffset" : 0, 472 | "firstDayOfWeek" : -1, 473 | "isYearEndFiscalYear" : true, 474 | "canTruncateValue" : true, 475 | "isSkipped" : false 476 | }, { 477 | "name" : "Domain_Pref_Locale", 478 | "fullyQualifiedName" : "Domain_Pref_Locale", 479 | "label" : "Domain_Pref_Locale", 480 | "description" : null, 481 | "type" : "Text", 482 | "precision" : 255, 483 | "scale" : 0, 484 | "decimalSeparator" : ".", 485 | "defaultValue" : null, 486 | "format" : null, 487 | "isSystemField" : false, 488 | "isUniqueId" : false, 489 | "isMultiValue" : false, 490 | "multiValueSeparator" : null, 491 | "fiscalMonthOffset" : 0, 492 | "firstDayOfWeek" : -1, 493 | "isYearEndFiscalYear" : true, 494 | "canTruncateValue" : true, 495 | "isSkipped" : false 496 | }, { 497 | "name" : "Market", 498 | "fullyQualifiedName" : "Market", 499 | "label" : "Market", 500 | "description" : null, 501 | "type" : "Text", 502 | "precision" : 255, 503 | "scale" : 0, 504 | "decimalSeparator" : ".", 505 | "defaultValue" : null, 506 | "format" : null, 507 | "isSystemField" : false, 508 | "isUniqueId" : false, 509 | "isMultiValue" : false, 510 | "multiValueSeparator" : null, 511 | "fiscalMonthOffset" : 0, 512 | "firstDayOfWeek" : -1, 513 | "isYearEndFiscalYear" : true, 514 | "canTruncateValue" : true, 515 | "isSkipped" : false 516 | }, { 517 | "name" : "Name", 518 | "fullyQualifiedName" : "Name", 519 | "label" : "Name", 520 | "description" : null, 521 | "type" : "Text", 522 | "precision" : 255, 523 | "scale" : 0, 524 | "decimalSeparator" : ".", 525 | "defaultValue" : null, 526 | "format" : null, 527 | "isSystemField" : false, 528 | "isUniqueId" : false, 529 | "isMultiValue" : false, 530 | "multiValueSeparator" : null, 531 | "fiscalMonthOffset" : 0, 532 | "firstDayOfWeek" : -1, 533 | "isYearEndFiscalYear" : true, 534 | "canTruncateValue" : true, 535 | "isSkipped" : false 536 | }, { 537 | "name" : "Current_Owner", 538 | "fullyQualifiedName" : "Current_Owner", 539 | "label" : "Current_Owner", 540 | "description" : null, 541 | "type" : "Text", 542 | "precision" : 255, 543 | "scale" : 0, 544 | "decimalSeparator" : ".", 545 | "defaultValue" : null, 546 | "format" : null, 547 | "isSystemField" : false, 548 | "isUniqueId" : false, 549 | "isMultiValue" : false, 550 | "multiValueSeparator" : null, 551 | "fiscalMonthOffset" : 0, 552 | "firstDayOfWeek" : -1, 553 | "isYearEndFiscalYear" : true, 554 | "canTruncateValue" : true, 555 | "isSkipped" : false 556 | }, { 557 | "name" : "Comments", 558 | "fullyQualifiedName" : "Comments", 559 | "label" : "Comments", 560 | "description" : null, 561 | "type" : "Text", 562 | "precision" : 1024, 563 | "scale" : 0, 564 | "decimalSeparator" : ".", 565 | "defaultValue" : null, 566 | "format" : null, 567 | "isSystemField" : false, 568 | "isUniqueId" : false, 569 | "isMultiValue" : false, 570 | "multiValueSeparator" : null, 571 | "fiscalMonthOffset" : 0, 572 | "firstDayOfWeek" : -1, 573 | "isYearEndFiscalYear" : true, 574 | "canTruncateValue" : true, 575 | "isSkipped" : false 576 | }, { 577 | "name" : "Verified_Reported_Failure", 578 | "fullyQualifiedName" : "Verified_Reported_Failure", 579 | "label" : "Verified_Reported_Failure", 580 | "description" : null, 581 | "type" : "Text", 582 | "precision" : 255, 583 | "scale" : 0, 584 | "decimalSeparator" : ".", 585 | "defaultValue" : null, 586 | "format" : null, 587 | "isSystemField" : false, 588 | "isUniqueId" : false, 589 | "isMultiValue" : false, 590 | "multiValueSeparator" : null, 591 | "fiscalMonthOffset" : 0, 592 | "firstDayOfWeek" : -1, 593 | "isYearEndFiscalYear" : true, 594 | "canTruncateValue" : true, 595 | "isSkipped" : false 596 | }, { 597 | "name" : "Contact_Phone", 598 | "fullyQualifiedName" : "Contact_Phone", 599 | "label" : "Contact_Phone", 600 | "description" : null, 601 | "type" : "Text", 602 | "precision" : 255, 603 | "scale" : 0, 604 | "decimalSeparator" : ".", 605 | "defaultValue" : null, 606 | "format" : null, 607 | "isSystemField" : false, 608 | "isUniqueId" : false, 609 | "isMultiValue" : false, 610 | "multiValueSeparator" : null, 611 | "fiscalMonthOffset" : 0, 612 | "firstDayOfWeek" : -1, 613 | "isYearEndFiscalYear" : true, 614 | "canTruncateValue" : true, 615 | "isSkipped" : false 616 | }, { 617 | "name" : "Category", 618 | "fullyQualifiedName" : "Category", 619 | "label" : "Category", 620 | "description" : null, 621 | "type" : "Text", 622 | "precision" : 255, 623 | "scale" : 0, 624 | "decimalSeparator" : ".", 625 | "defaultValue" : null, 626 | "format" : null, 627 | "isSystemField" : false, 628 | "isUniqueId" : false, 629 | "isMultiValue" : false, 630 | "multiValueSeparator" : null, 631 | "fiscalMonthOffset" : 0, 632 | "firstDayOfWeek" : -1, 633 | "isYearEndFiscalYear" : true, 634 | "canTruncateValue" : true, 635 | "isSkipped" : false 636 | } ] 637 | } ] 638 | } 639 | -------------------------------------------------------------------------------- /src/main/resources/Complaints_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fileFormat" : { 3 | "charsetName" : "UTF-8", 4 | "fieldsDelimitedBy" : ",", 5 | "fieldsEnclosedBy" : "\"", 6 | "linesTerminatedBy" : "\n", 7 | "numberOfLinesToIgnore" : 1 8 | }, 9 | "objects" : [ { 10 | "name" : "ComplaintsSamples_csv", 11 | "fullyQualifiedName" : "ComplaintsSamples_csv", 12 | "connector" : "CSV", 13 | "label" : "ComplaintsSamples.csv", 14 | "description" : "ComplaintsSamples_csv", 15 | "rowLevelSecurityFilter" : null, 16 | "fields" : [ { 17 | "name" : "Lab_country", 18 | "fullyQualifiedName" : "Lab_country", 19 | "label" : "Lab_country", 20 | "description" : null, 21 | "type" : "Text", 22 | "precision" : 255, 23 | "scale" : 0, 24 | "decimalSeparator" : ".", 25 | "defaultValue" : null, 26 | "format" : null, 27 | "isSystemField" : false, 28 | "isUniqueId" : false, 29 | "isMultiValue" : false, 30 | "multiValueSeparator" : null, 31 | "fiscalMonthOffset" : 0, 32 | "firstDayOfWeek" : -1, 33 | "isYearEndFiscalYear" : true, 34 | "canTruncateValue" : true, 35 | "isSkipped" : false 36 | }, { 37 | "name" : "Lab_city", 38 | "fullyQualifiedName" : "Lab_city", 39 | "label" : "Lab_city", 40 | "description" : null, 41 | "type" : "Text", 42 | "precision" : 255, 43 | "scale" : 0, 44 | "decimalSeparator" : ".", 45 | "defaultValue" : null, 46 | "format" : null, 47 | "isSystemField" : false, 48 | "isUniqueId" : false, 49 | "isMultiValue" : false, 50 | "multiValueSeparator" : null, 51 | "fiscalMonthOffset" : 0, 52 | "firstDayOfWeek" : -1, 53 | "isYearEndFiscalYear" : true, 54 | "canTruncateValue" : true, 55 | "isSkipped" : false 56 | }, { 57 | "name" : "Lab_name", 58 | "fullyQualifiedName" : "Lab_name", 59 | "label" : "Lab_name", 60 | "description" : null, 61 | "type" : "Text", 62 | "precision" : 255, 63 | "scale" : 0, 64 | "decimalSeparator" : ".", 65 | "defaultValue" : null, 66 | "format" : null, 67 | "isSystemField" : false, 68 | "isUniqueId" : false, 69 | "isMultiValue" : false, 70 | "multiValueSeparator" : null, 71 | "fiscalMonthOffset" : 0, 72 | "firstDayOfWeek" : -1, 73 | "isYearEndFiscalYear" : true, 74 | "canTruncateValue" : true, 75 | "isSkipped" : false 76 | }, { 77 | "name" : "System_Name", 78 | "fullyQualifiedName" : "System_Name", 79 | "label" : "System_Name", 80 | "description" : null, 81 | "type" : "Text", 82 | "precision" : 255, 83 | "scale" : 0, 84 | "decimalSeparator" : ".", 85 | "defaultValue" : null, 86 | "format" : null, 87 | "isSystemField" : false, 88 | "isUniqueId" : false, 89 | "isMultiValue" : false, 90 | "multiValueSeparator" : null, 91 | "fiscalMonthOffset" : 0, 92 | "firstDayOfWeek" : -1, 93 | "isYearEndFiscalYear" : true, 94 | "canTruncateValue" : true, 95 | "isSkipped" : false 96 | }, { 97 | "name" : "Institution", 98 | "fullyQualifiedName" : "Institution", 99 | "label" : "Institution", 100 | "description" : null, 101 | "type" : "Text", 102 | "precision" : 255, 103 | "scale" : 0, 104 | "decimalSeparator" : ".", 105 | "defaultValue" : null, 106 | "format" : null, 107 | "isSystemField" : false, 108 | "isUniqueId" : false, 109 | "isMultiValue" : false, 110 | "multiValueSeparator" : null, 111 | "fiscalMonthOffset" : 0, 112 | "firstDayOfWeek" : -1, 113 | "isYearEndFiscalYear" : true, 114 | "canTruncateValue" : true, 115 | "isSkipped" : false 116 | }, { 117 | "name" : "System_serial_number", 118 | "fullyQualifiedName" : "System_serial_number", 119 | "label" : "System_serial_number", 120 | "description" : null, 121 | "type" : "Text", 122 | "precision" : 255, 123 | "scale" : 0, 124 | "decimalSeparator" : ".", 125 | "defaultValue" : null, 126 | "format" : null, 127 | "isSystemField" : false, 128 | "isUniqueId" : false, 129 | "isMultiValue" : false, 130 | "multiValueSeparator" : null, 131 | "fiscalMonthOffset" : 0, 132 | "firstDayOfWeek" : -1, 133 | "isYearEndFiscalYear" : true, 134 | "canTruncateValue" : true, 135 | "isSkipped" : false 136 | }, { 137 | "name" : "System_model", 138 | "fullyQualifiedName" : "System_model", 139 | "label" : "System_model", 140 | "description" : null, 141 | "type" : "Text", 142 | "precision" : 255, 143 | "scale" : 0, 144 | "decimalSeparator" : ".", 145 | "defaultValue" : null, 146 | "format" : null, 147 | "isSystemField" : false, 148 | "isUniqueId" : false, 149 | "isMultiValue" : false, 150 | "multiValueSeparator" : null, 151 | "fiscalMonthOffset" : 0, 152 | "firstDayOfWeek" : -1, 153 | "isYearEndFiscalYear" : true, 154 | "canTruncateValue" : true, 155 | "isSkipped" : false 156 | }, { 157 | "name" : "Lab_region", 158 | "fullyQualifiedName" : "Lab_region", 159 | "label" : "Lab_region", 160 | "description" : null, 161 | "type" : "Text", 162 | "precision" : 255, 163 | "scale" : 0, 164 | "decimalSeparator" : ".", 165 | "defaultValue" : null, 166 | "format" : null, 167 | "isSystemField" : false, 168 | "isUniqueId" : false, 169 | "isMultiValue" : false, 170 | "multiValueSeparator" : null, 171 | "fiscalMonthOffset" : 0, 172 | "firstDayOfWeek" : -1, 173 | "isYearEndFiscalYear" : true, 174 | "canTruncateValue" : true, 175 | "isSkipped" : false 176 | }, { 177 | "name" : "Assay_group", 178 | "fullyQualifiedName" : "Assay_group", 179 | "label" : "Assay_group", 180 | "description" : null, 181 | "type" : "Text", 182 | "precision" : 255, 183 | "scale" : 0, 184 | "decimalSeparator" : ".", 185 | "defaultValue" : null, 186 | "format" : null, 187 | "isSystemField" : false, 188 | "isUniqueId" : false, 189 | "isMultiValue" : false, 190 | "multiValueSeparator" : null, 191 | "fiscalMonthOffset" : 0, 192 | "firstDayOfWeek" : -1, 193 | "isYearEndFiscalYear" : true, 194 | "canTruncateValue" : true, 195 | "isSkipped" : false 196 | }, { 197 | "name" : "Assay_name", 198 | "fullyQualifiedName" : "Assay_name", 199 | "label" : "Assay_name", 200 | "description" : null, 201 | "type" : "Text", 202 | "precision" : 255, 203 | "scale" : 0, 204 | "decimalSeparator" : ".", 205 | "defaultValue" : null, 206 | "format" : null, 207 | "isSystemField" : false, 208 | "isUniqueId" : false, 209 | "isMultiValue" : false, 210 | "multiValueSeparator" : null, 211 | "fiscalMonthOffset" : 0, 212 | "firstDayOfWeek" : -1, 213 | "isYearEndFiscalYear" : true, 214 | "canTruncateValue" : true, 215 | "isSkipped" : false 216 | }, { 217 | "name" : "Assay_version", 218 | "fullyQualifiedName" : "Assay_version", 219 | "label" : "Assay_version", 220 | "description" : null, 221 | "type" : "Text", 222 | "precision" : 255, 223 | "scale" : 0, 224 | "decimalSeparator" : ".", 225 | "defaultValue" : null, 226 | "format" : null, 227 | "isSystemField" : false, 228 | "isUniqueId" : false, 229 | "isMultiValue" : false, 230 | "multiValueSeparator" : null, 231 | "fiscalMonthOffset" : 0, 232 | "firstDayOfWeek" : -1, 233 | "isYearEndFiscalYear" : true, 234 | "canTruncateValue" : true, 235 | "isSkipped" : false 236 | }, { 237 | "name" : "Cartridge_S_N", 238 | "fullyQualifiedName" : "Cartridge_S_N", 239 | "label" : "Cartridge_S_N", 240 | "description" : null, 241 | "type" : "Text", 242 | "precision" : 255, 243 | "scale" : 0, 244 | "decimalSeparator" : ".", 245 | "defaultValue" : null, 246 | "format" : null, 247 | "isSystemField" : false, 248 | "isUniqueId" : false, 249 | "isMultiValue" : false, 250 | "multiValueSeparator" : null, 251 | "fiscalMonthOffset" : 0, 252 | "firstDayOfWeek" : -1, 253 | "isYearEndFiscalYear" : true, 254 | "canTruncateValue" : true, 255 | "isSkipped" : false 256 | }, { 257 | "name" : "Reagent_lot_ID", 258 | "fullyQualifiedName" : "Reagent_lot_ID", 259 | "label" : "Reagent_lot_ID", 260 | "description" : null, 261 | "type" : "Text", 262 | "precision" : 255, 263 | "scale" : 0, 264 | "decimalSeparator" : ".", 265 | "defaultValue" : null, 266 | "format" : null, 267 | "isSystemField" : false, 268 | "isUniqueId" : false, 269 | "isMultiValue" : false, 270 | "multiValueSeparator" : null, 271 | "fiscalMonthOffset" : 0, 272 | "firstDayOfWeek" : -1, 273 | "isYearEndFiscalYear" : true, 274 | "canTruncateValue" : true, 275 | "isSkipped" : false 276 | }, { 277 | "name" : "Main_error_code", 278 | "fullyQualifiedName" : "Main_error_code", 279 | "label" : "Main_error_code", 280 | "description" : null, 281 | "type" : "Text", 282 | "precision" : 255, 283 | "scale" : 0, 284 | "decimalSeparator" : ".", 285 | "defaultValue" : null, 286 | "format" : null, 287 | "isSystemField" : false, 288 | "isUniqueId" : false, 289 | "isMultiValue" : false, 290 | "multiValueSeparator" : null, 291 | "fiscalMonthOffset" : 0, 292 | "firstDayOfWeek" : -1, 293 | "isYearEndFiscalYear" : true, 294 | "canTruncateValue" : true, 295 | "isSkipped" : false 296 | }, { 297 | "name" : "Software_version", 298 | "fullyQualifiedName" : "Software_version", 299 | "label" : "Software_version", 300 | "description" : null, 301 | "type" : "Text", 302 | "precision" : 255, 303 | "scale" : 0, 304 | "decimalSeparator" : ".", 305 | "defaultValue" : null, 306 | "format" : null, 307 | "isSystemField" : false, 308 | "isUniqueId" : false, 309 | "isMultiValue" : false, 310 | "multiValueSeparator" : null, 311 | "fiscalMonthOffset" : 0, 312 | "firstDayOfWeek" : -1, 313 | "isYearEndFiscalYear" : true, 314 | "canTruncateValue" : true, 315 | "isSkipped" : false 316 | }, { 317 | "name" : "Sample_type", 318 | "fullyQualifiedName" : "Sample_type", 319 | "label" : "Sample_type", 320 | "description" : null, 321 | "type" : "Text", 322 | "precision" : 255, 323 | "scale" : 0, 324 | "decimalSeparator" : ".", 325 | "defaultValue" : null, 326 | "format" : null, 327 | "isSystemField" : false, 328 | "isUniqueId" : false, 329 | "isMultiValue" : false, 330 | "multiValueSeparator" : null, 331 | "fiscalMonthOffset" : 0, 332 | "firstDayOfWeek" : -1, 333 | "isYearEndFiscalYear" : true, 334 | "canTruncateValue" : true, 335 | "isSkipped" : false 336 | }, { 337 | "name" : "Module_name", 338 | "fullyQualifiedName" : "Module_name", 339 | "label" : "Module_name", 340 | "description" : null, 341 | "type" : "Text", 342 | "precision" : 255, 343 | "scale" : 0, 344 | "decimalSeparator" : ".", 345 | "defaultValue" : null, 346 | "format" : null, 347 | "isSystemField" : false, 348 | "isUniqueId" : false, 349 | "isMultiValue" : false, 350 | "multiValueSeparator" : null, 351 | "fiscalMonthOffset" : 0, 352 | "firstDayOfWeek" : -1, 353 | "isYearEndFiscalYear" : true, 354 | "canTruncateValue" : true, 355 | "isSkipped" : false 356 | }, { 357 | "name" : "Module_S_N", 358 | "fullyQualifiedName" : "Module_S_N", 359 | "label" : "Module_S_N", 360 | "description" : null, 361 | "type" : "Text", 362 | "precision" : 255, 363 | "scale" : 0, 364 | "decimalSeparator" : ".", 365 | "defaultValue" : null, 366 | "format" : null, 367 | "isSystemField" : false, 368 | "isUniqueId" : false, 369 | "isMultiValue" : false, 370 | "multiValueSeparator" : null, 371 | "fiscalMonthOffset" : 0, 372 | "firstDayOfWeek" : -1, 373 | "isYearEndFiscalYear" : true, 374 | "canTruncateValue" : true, 375 | "isSkipped" : false 376 | }, { 377 | "name" : "Initiation_date", 378 | "fullyQualifiedName" : "Initiation_date", 379 | "label" : "Initiation_date", 380 | "description" : null, 381 | "type" : "Date", 382 | "precision" : 0, 383 | "scale" : 0, 384 | "decimalSeparator" : ".", 385 | "defaultValue" : null, 386 | "format" : "MM/dd/yyyy", 387 | "isSystemField" : false, 388 | "isUniqueId" : false, 389 | "isMultiValue" : false, 390 | "multiValueSeparator" : null, 391 | "fiscalMonthOffset" : 0, 392 | "firstDayOfWeek" : -1, 393 | "isYearEndFiscalYear" : true, 394 | "canTruncateValue" : true, 395 | "isSkipped" : false 396 | }, { 397 | "name" : "Status", 398 | "fullyQualifiedName" : "Status", 399 | "label" : "Status", 400 | "description" : null, 401 | "type" : "Text", 402 | "precision" : 255, 403 | "scale" : 0, 404 | "decimalSeparator" : ".", 405 | "defaultValue" : null, 406 | "format" : null, 407 | "isSystemField" : false, 408 | "isUniqueId" : false, 409 | "isMultiValue" : false, 410 | "multiValueSeparator" : null, 411 | "fiscalMonthOffset" : 0, 412 | "firstDayOfWeek" : -1, 413 | "isYearEndFiscalYear" : true, 414 | "canTruncateValue" : true, 415 | "isSkipped" : false 416 | }, { 417 | "name" : "Closed_date", 418 | "fullyQualifiedName" : "Closed_date", 419 | "label" : "Closed_date", 420 | "description" : null, 421 | "type" : "Text", 422 | "precision" : 255, 423 | "scale" : 0, 424 | "decimalSeparator" : ".", 425 | "defaultValue" : null, 426 | "format" : null, 427 | "isSystemField" : false, 428 | "isUniqueId" : false, 429 | "isMultiValue" : false, 430 | "multiValueSeparator" : null, 431 | "fiscalMonthOffset" : 0, 432 | "firstDayOfWeek" : -1, 433 | "isYearEndFiscalYear" : true, 434 | "canTruncateValue" : true, 435 | "isSkipped" : false 436 | }, { 437 | "name" : "Phase", 438 | "fullyQualifiedName" : "Phase", 439 | "label" : "Phase", 440 | "description" : null, 441 | "type" : "Text", 442 | "precision" : 255, 443 | "scale" : 0, 444 | "decimalSeparator" : ".", 445 | "defaultValue" : null, 446 | "format" : null, 447 | "isSystemField" : false, 448 | "isUniqueId" : false, 449 | "isMultiValue" : false, 450 | "multiValueSeparator" : null, 451 | "fiscalMonthOffset" : 0, 452 | "firstDayOfWeek" : -1, 453 | "isYearEndFiscalYear" : true, 454 | "canTruncateValue" : true, 455 | "isSkipped" : false 456 | }, { 457 | "name" : "Case_Intake_Source", 458 | "fullyQualifiedName" : "Case_Intake_Source", 459 | "label" : "Case_Intake_Source", 460 | "description" : null, 461 | "type" : "Text", 462 | "precision" : 255, 463 | "scale" : 0, 464 | "decimalSeparator" : ".", 465 | "defaultValue" : null, 466 | "format" : null, 467 | "isSystemField" : false, 468 | "isUniqueId" : false, 469 | "isMultiValue" : false, 470 | "multiValueSeparator" : null, 471 | "fiscalMonthOffset" : 0, 472 | "firstDayOfWeek" : -1, 473 | "isYearEndFiscalYear" : true, 474 | "canTruncateValue" : true, 475 | "isSkipped" : false 476 | }, { 477 | "name" : "Domain_Pref_Locale", 478 | "fullyQualifiedName" : "Domain_Pref_Locale", 479 | "label" : "Domain_Pref_Locale", 480 | "description" : null, 481 | "type" : "Text", 482 | "precision" : 255, 483 | "scale" : 0, 484 | "decimalSeparator" : ".", 485 | "defaultValue" : null, 486 | "format" : null, 487 | "isSystemField" : false, 488 | "isUniqueId" : false, 489 | "isMultiValue" : false, 490 | "multiValueSeparator" : null, 491 | "fiscalMonthOffset" : 0, 492 | "firstDayOfWeek" : -1, 493 | "isYearEndFiscalYear" : true, 494 | "canTruncateValue" : true, 495 | "isSkipped" : false 496 | }, { 497 | "name" : "Market", 498 | "fullyQualifiedName" : "Market", 499 | "label" : "Market", 500 | "description" : null, 501 | "type" : "Text", 502 | "precision" : 255, 503 | "scale" : 0, 504 | "decimalSeparator" : ".", 505 | "defaultValue" : null, 506 | "format" : null, 507 | "isSystemField" : false, 508 | "isUniqueId" : false, 509 | "isMultiValue" : false, 510 | "multiValueSeparator" : null, 511 | "fiscalMonthOffset" : 0, 512 | "firstDayOfWeek" : -1, 513 | "isYearEndFiscalYear" : true, 514 | "canTruncateValue" : true, 515 | "isSkipped" : false 516 | }, { 517 | "name" : "Name", 518 | "fullyQualifiedName" : "Name", 519 | "label" : "Name", 520 | "description" : null, 521 | "type" : "Text", 522 | "precision" : 255, 523 | "scale" : 0, 524 | "decimalSeparator" : ".", 525 | "defaultValue" : null, 526 | "format" : null, 527 | "isSystemField" : false, 528 | "isUniqueId" : false, 529 | "isMultiValue" : false, 530 | "multiValueSeparator" : null, 531 | "fiscalMonthOffset" : 0, 532 | "firstDayOfWeek" : -1, 533 | "isYearEndFiscalYear" : true, 534 | "canTruncateValue" : true, 535 | "isSkipped" : false 536 | }, { 537 | "name" : "Current_Owner", 538 | "fullyQualifiedName" : "Current_Owner", 539 | "label" : "Current_Owner", 540 | "description" : null, 541 | "type" : "Text", 542 | "precision" : 255, 543 | "scale" : 0, 544 | "decimalSeparator" : ".", 545 | "defaultValue" : null, 546 | "format" : null, 547 | "isSystemField" : false, 548 | "isUniqueId" : false, 549 | "isMultiValue" : false, 550 | "multiValueSeparator" : null, 551 | "fiscalMonthOffset" : 0, 552 | "firstDayOfWeek" : -1, 553 | "isYearEndFiscalYear" : true, 554 | "canTruncateValue" : true, 555 | "isSkipped" : false 556 | }, { 557 | "name" : "Comments", 558 | "fullyQualifiedName" : "Comments", 559 | "label" : "Comments", 560 | "description" : null, 561 | "type" : "Text", 562 | "precision" : 1024, 563 | "scale" : 0, 564 | "decimalSeparator" : ".", 565 | "defaultValue" : null, 566 | "format" : null, 567 | "isSystemField" : false, 568 | "isUniqueId" : false, 569 | "isMultiValue" : false, 570 | "multiValueSeparator" : null, 571 | "fiscalMonthOffset" : 0, 572 | "firstDayOfWeek" : -1, 573 | "isYearEndFiscalYear" : true, 574 | "canTruncateValue" : true, 575 | "isSkipped" : false 576 | }, { 577 | "name" : "Verified_Reported_Failure", 578 | "fullyQualifiedName" : "Verified_Reported_Failure", 579 | "label" : "Verified_Reported_Failure", 580 | "description" : null, 581 | "type" : "Text", 582 | "precision" : 255, 583 | "scale" : 0, 584 | "decimalSeparator" : ".", 585 | "defaultValue" : null, 586 | "format" : null, 587 | "isSystemField" : false, 588 | "isUniqueId" : false, 589 | "isMultiValue" : false, 590 | "multiValueSeparator" : null, 591 | "fiscalMonthOffset" : 0, 592 | "firstDayOfWeek" : -1, 593 | "isYearEndFiscalYear" : true, 594 | "canTruncateValue" : true, 595 | "isSkipped" : false 596 | }, { 597 | "name" : "Contact_Phone", 598 | "fullyQualifiedName" : "Contact_Phone", 599 | "label" : "Contact_Phone", 600 | "description" : null, 601 | "type" : "Text", 602 | "precision" : 255, 603 | "scale" : 0, 604 | "decimalSeparator" : ".", 605 | "defaultValue" : null, 606 | "format" : null, 607 | "isSystemField" : false, 608 | "isUniqueId" : false, 609 | "isMultiValue" : false, 610 | "multiValueSeparator" : null, 611 | "fiscalMonthOffset" : 0, 612 | "firstDayOfWeek" : -1, 613 | "isYearEndFiscalYear" : true, 614 | "canTruncateValue" : true, 615 | "isSkipped" : false 616 | }, { 617 | "name" : "Category", 618 | "fullyQualifiedName" : "Category", 619 | "label" : "Category", 620 | "description" : null, 621 | "type" : "Text", 622 | "precision" : 255, 623 | "scale" : 0, 624 | "decimalSeparator" : ".", 625 | "defaultValue" : null, 626 | "format" : null, 627 | "isSystemField" : false, 628 | "isUniqueId" : false, 629 | "isMultiValue" : false, 630 | "multiValueSeparator" : null, 631 | "fiscalMonthOffset" : 0, 632 | "firstDayOfWeek" : -1, 633 | "isYearEndFiscalYear" : true, 634 | "canTruncateValue" : true, 635 | "isSkipped" : false 636 | } ] 637 | } ] 638 | } 639 | --------------------------------------------------------------------------------