├── src ├── test │ ├── resources │ │ ├── YAML │ │ │ ├── LoaderConfigTest │ │ │ │ ├── incident-sync.yaml │ │ │ │ └── multi-table-load.yaml │ │ │ ├── InsertTest │ │ │ │ ├── load_truncate.yaml │ │ │ │ └── load_twice.yaml │ │ │ ├── YamlParseErrorTest │ │ │ │ ├── bad_option_drop.yaml │ │ │ │ ├── load_no_table.yaml │ │ │ │ ├── baddate1.yaml │ │ │ │ ├── badsync1.yaml │ │ │ │ ├── badinteger.yaml │ │ │ │ └── baddate2.yaml │ │ │ ├── YamlParseValidTest │ │ │ │ ├── incident-sync.yaml │ │ │ │ ├── incident_load.yaml │ │ │ │ ├── incident_refresh.yaml │ │ │ │ ├── incident_load_created_range.yaml │ │ │ │ ├── incident_load_created_date.yaml │ │ │ │ └── multi-table-load1.yaml │ │ │ ├── BigTableLoaderTest │ │ │ │ └── incident-load.yaml │ │ │ ├── ColumnsTest │ │ │ │ ├── load-incident-truncate.yaml │ │ │ │ ├── incident-exclude-columns.yaml │ │ │ │ └── incident-include-columns.yaml │ │ │ ├── TimestampTest │ │ │ │ └── incident-load.yaml │ │ │ ├── RowCountExceptionTest │ │ │ │ ├── too-many-rows.yaml │ │ │ │ └── too-few-rows.yaml │ │ │ └── TableLoaderTest │ │ │ │ ├── load_kb_knowledge.yaml │ │ │ │ ├── load_sys_template.yaml │ │ │ │ └── load_incident_drop.yaml │ │ ├── snlogo1.png │ │ ├── Exceptions │ │ │ └── primary_key_violation.yaml │ │ ├── profile_test.properties │ │ ├── log4j2-test.xml │ │ └── junit.properties │ └── java │ │ └── sndml │ │ ├── util │ │ ├── AllTests.java │ │ ├── ParametersTest.java │ │ └── MetricsTest.java │ │ ├── loader │ │ ├── TestingException.java │ │ ├── AllTests.java │ │ ├── EvaluateTest.java │ │ ├── TestFolder.java │ │ ├── TestingProfile.java │ │ ├── PropertySetTest.java │ │ ├── YamlParseErrorTest.java │ │ ├── YamlParseValidTest.java │ │ ├── TimestampHashTest.java │ │ ├── DateTimeFactoryTest.java │ │ ├── TableLoaderTest.java │ │ ├── RowCountExceptionTest.java │ │ ├── ColumnsTest.java │ │ ├── RefreshTest1.java │ │ ├── RefreshTest2.java │ │ └── LoaderConfigTest.java │ │ └── servicenow │ │ ├── AllTests.java │ │ ├── SampleTest.java │ │ ├── SessionIDTest.java │ │ ├── InstanceTest.java │ │ ├── FieldNamesTest.java │ │ ├── GetKeysTest.java │ │ ├── GetRecordTest.java │ │ ├── SessionVerificationTest.java │ │ ├── SetFieldsTest.java │ │ ├── TableSchemaTest.java │ │ ├── GetRecordsTest.java │ │ └── TableWSDLTest.java ├── main │ ├── java │ │ └── sndml │ │ │ ├── servicenow │ │ │ ├── HttpMethod.java │ │ │ ├── InsertResponse.java │ │ │ ├── NoContentException.java │ │ │ ├── InvalidFieldNameException.java │ │ │ ├── JsonResponseError.java │ │ │ ├── InvalidTableNameException.java │ │ │ ├── JsonResponseException.java │ │ │ ├── InstanceUnavailableException.java │ │ │ ├── XmlParseException.java │ │ │ ├── QueryLimitReachedException.java │ │ │ ├── WSDLException.java │ │ │ ├── InsufficientRightsException.java │ │ │ ├── TooFewRowsException.java │ │ │ ├── TooManyRowsException.java │ │ │ ├── RowCountMismatchException.java │ │ │ ├── ServiceNowError.java │ │ │ ├── Domain.java │ │ │ ├── NoSuchRecordException.java │ │ │ ├── TableStats.java │ │ │ ├── RecordIterator.java │ │ │ ├── SchemaReader.java │ │ │ ├── NullWriter.java │ │ │ ├── TableTimestampReader.java │ │ │ ├── RecordListAccumulator.java │ │ │ ├── package-info.java │ │ │ ├── RecordWriter.java │ │ │ ├── SoapResponseException.java │ │ │ ├── RestPetitTableReader.java │ │ │ ├── ServiceNowException.java │ │ │ ├── SchemaCache.java │ │ │ ├── ServiceNowRequest.java │ │ │ ├── XmlFormatter.java │ │ │ ├── RecordKey.java │ │ │ ├── JsonRecord.java │ │ │ ├── RecordList.java │ │ │ ├── Instance.java │ │ │ ├── FieldDefinition.java │ │ │ ├── SoapKeySetTableReader.java │ │ │ ├── RecordKeySet.java │ │ │ └── XmlRecord.java │ │ │ ├── agent │ │ │ ├── package-info.java │ │ │ ├── WorkerList.java │ │ │ ├── AppJobStatus.java │ │ │ ├── AgentURLException.java │ │ │ ├── JobCancelledException.java │ │ │ ├── WorkerEntry.java │ │ │ ├── AgentHandlerException.java │ │ │ ├── AppStatusQueue.java │ │ │ ├── ServletPathParser.java │ │ │ ├── JobActionRequest.java │ │ │ ├── HeartbeatTask.java │ │ │ ├── ShutdownHook.java │ │ │ ├── AppStatusPayload.java │ │ │ ├── AppJobConfig.java │ │ │ ├── ScannerJobRunner.java │ │ │ ├── AppSchemaReader.java │ │ │ ├── SingleThreadScanner.java │ │ │ ├── GetRunListRequest.java │ │ │ ├── AgentMain.java │ │ │ ├── AppConfigFactory.java │ │ │ └── MultiThreadScanner.java │ │ │ ├── util │ │ │ ├── package-info.java │ │ │ ├── PartitionInterval.java │ │ │ ├── MissingPropertyException.java │ │ │ ├── LogManager.java │ │ │ ├── InvalidDateTimeException.java │ │ │ ├── ResourceException.java │ │ │ ├── DateTimeSerializer.java │ │ │ ├── NullProgressLogger.java │ │ │ ├── FieldNames.java │ │ │ ├── ProgressLogger.java │ │ │ ├── DatePartition.java │ │ │ ├── Parameters.java │ │ │ ├── PropertiesEditor.java │ │ │ ├── FieldValues.java │ │ │ ├── DatePartitionSet.java │ │ │ └── DateTimeRange.java │ │ │ └── loader │ │ │ ├── CommandOptionsException.java │ │ │ ├── TimestampHash.java │ │ │ ├── package-info.java │ │ │ ├── ConfigParseException.java │ │ │ ├── ReaderSession.java │ │ │ ├── TestJobRunner.java │ │ │ ├── DatabaseFieldDefinition.java │ │ │ ├── DatabaseDeleteStatement.java │ │ │ ├── SimpleTableLoader.java │ │ │ ├── DatabaseUpdateWriter.java │ │ │ ├── DatabaseInsertStatement.java │ │ │ ├── JobFactory.java │ │ │ ├── NameMap.java │ │ │ ├── DatabaseUpdateStatement.java │ │ │ ├── Action.java │ │ │ ├── DatabaseDeleteWriter.java │ │ │ ├── DatabaseInsertWriter.java │ │ │ ├── YamlFile.java │ │ │ ├── CompositeProgressLogger.java │ │ │ ├── YamlLoaderConfig.java │ │ │ └── DatabaseTableWriter.java │ ├── resources │ │ ├── log4j2-quiet.xml │ │ ├── log4j2-console.xml │ │ ├── log4j2.xml │ │ ├── log4j2-file.xml │ │ ├── log4j2-daemon.xml │ │ ├── log4j2-server.xml │ │ └── log4j2-debug.xml │ └── webapp │ │ └── WEB-INF │ │ └── web.xml └── assembly │ └── release.xml ├── docs ├── images │ ├── 2025-01-28-job-run-parts.png │ ├── 2025-01-30-table-advanced.png │ ├── 2025-01-28-http-server-via-mid.png │ └── 2025-01-30-schedule-with-3-jobs.png ├── _config.yml ├── _includes │ └── head-custom.html └── _layouts │ └── index.html ├── bin ├── sndml-table ├── sndml-cleanup ├── sndml-test ├── sndml-lastlog ├── sndml-fg ├── sndml-debug ├── sndml-export ├── sndml-generate ├── sndml-kill ├── sndml-scan ├── sndml-bg ├── sndml-jobrun ├── sndml-single ├── sndml-server-fg ├── sndml-daemon-fg ├── start-server.sh ├── sndml-daemon-bg └── sndml-server-bg ├── LICENSE ├── README.md └── .github └── workflows └── jekyll-gh-pages.yml /src/test/resources/YAML/LoaderConfigTest/incident-sync.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, action: sync} -------------------------------------------------------------------------------- /src/test/resources/YAML/InsertTest/load_truncate.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: change_request, truncate: true} -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/bad_option_drop.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, drop: true} -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/load_no_table.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {action: load, truncate: true} -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/incident-sync.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, action: sync} -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/incident_load.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, truncate: true} -------------------------------------------------------------------------------- /src/test/resources/snlogo1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gflewis/sndml3/HEAD/src/test/resources/snlogo1.png -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/baddate1.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, created: [2017-01-01, yellow]} 3 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/badsync1.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, action: sync, since: 2018-01-01} -------------------------------------------------------------------------------- /docs/images/2025-01-28-job-run-parts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gflewis/sndml3/HEAD/docs/images/2025-01-28-job-run-parts.png -------------------------------------------------------------------------------- /docs/images/2025-01-30-table-advanced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gflewis/sndml3/HEAD/docs/images/2025-01-30-table-advanced.png -------------------------------------------------------------------------------- /src/test/resources/YAML/BigTableLoaderTest/incident-load.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - name: incident 3 | action: load 4 | partition: month -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/badinteger.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, target: incident_2017, minrows: green} 3 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/incident_refresh.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, action: refresh, since: 2021-01-01} -------------------------------------------------------------------------------- /bin/sndml-table: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source sndml-setup $1 3 | env | grep SNDML | sort 4 | java -ea -jar $SNDML_JAR -p $SNDML_PROFILE -t $2 5 | -------------------------------------------------------------------------------- /src/test/resources/YAML/ColumnsTest/load-incident-truncate.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, truncate: true, created: 2020-01-01} 3 | -------------------------------------------------------------------------------- /docs/images/2025-01-28-http-server-via-mid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gflewis/sndml3/HEAD/docs/images/2025-01-28-http-server-via-mid.png -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/HttpMethod.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | public enum HttpMethod { DELETE, GET, PATCH, POST, PUT } 4 | -------------------------------------------------------------------------------- /src/test/resources/YAML/TimestampTest/incident-load.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - source: incident 3 | target: incident 4 | columns: number state 5 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/incident_load_created_range.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, action: load, created: 2019-01-01} -------------------------------------------------------------------------------- /docs/images/2025-01-30-schedule-with-3-jobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gflewis/sndml3/HEAD/docs/images/2025-01-30-schedule-with-3-jobs.png -------------------------------------------------------------------------------- /bin/sndml-cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # delete log files older than 10 days 3 | source sndml-setup $1 4 | find $SNDML_CONFIG/log -mtime +10 -type f -delete 5 | -------------------------------------------------------------------------------- /bin/sndml-test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run JUnit Tests 4 | # 5 | cd $SNDML_HOME 6 | mvn surefire:test -P test.api 7 | mvn surefire:test -P test.datamart 8 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseErrorTest/baddate2.yaml: -------------------------------------------------------------------------------- 1 | # test badly formatted date 2 | tables: 3 | - {name: incident, created: [1/1/2017, 2/28/2017]} 4 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/incident_load_created_date.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, action: load, created: [2019-01-01, 2020-01-01]} -------------------------------------------------------------------------------- /src/test/resources/YAML/ColumnsTest/incident-exclude-columns.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {source: incident, truncate: true, created: 2020-01-01, exclude: short_description} -------------------------------------------------------------------------------- /src/test/resources/YAML/ColumnsTest/incident-include-columns.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident, truncate: true, created: 2020-01-01, columns: sys_id number state} -------------------------------------------------------------------------------- /src/test/resources/YAML/RowCountExceptionTest/too-many-rows.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident_load, source: incident, truncate: true, action: insert, maxrows: 5} -------------------------------------------------------------------------------- /src/test/resources/YAML/RowCountExceptionTest/too-few-rows.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: user_load, source: sys_user, action: insert, filter: "name=bozo_the_clown", minrows: 1} -------------------------------------------------------------------------------- /src/main/java/sndml/agent/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Classes used to interact with a scoped application installed in the ServiceNow instance. 3 | */ 4 | package sndml.agent; -------------------------------------------------------------------------------- /src/main/java/sndml/util/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Classes which are not specific to the {@link sndml.loader} or {@link sndml.servicenow} package. 3 | */ 4 | package sndml.util; -------------------------------------------------------------------------------- /src/main/java/sndml/util/PartitionInterval.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | public enum PartitionInterval { 4 | YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, FIVE_MINUTE, MINUTE; 5 | } 6 | -------------------------------------------------------------------------------- /bin/sndml-lastlog: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # print the name of the most recent log file 3 | source sndml-setup $1 4 | lastlog=`ls -t $SNDML_CONFIG/log | sed 1q` 5 | echo $SNDML_CONFIG/log/$lastlog 6 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/InsertResponse.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | public interface InsertResponse { 4 | 5 | public RecordKey getKey(); 6 | public String getNumber(); 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/WorkerList.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.util.ArrayList; 4 | 5 | @SuppressWarnings("serial") 6 | public class WorkerList extends ArrayList { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/test/resources/YAML/TableLoaderTest/load_kb_knowledge.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: kb_knowledge_drop, source: kb_knowledge, action: droptable} 3 | - {name: kb_knowledge_load, source: kb_knowledge, action: insert} -------------------------------------------------------------------------------- /src/test/resources/YAML/TableLoaderTest/load_sys_template.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: sys_template_drop, source: sys_template, action: droptable} 3 | - {name: sys_template_load, source: sys_template, action: insert} -------------------------------------------------------------------------------- /src/test/resources/YAML/TableLoaderTest/load_incident_drop.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - {name: incident_drop, source: incident, action: droptable} 3 | - {name: incident_load, source: incident, action: insert, created: 2019-01-01} -------------------------------------------------------------------------------- /src/main/java/sndml/util/MissingPropertyException.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | @SuppressWarnings("serial") 4 | public class MissingPropertyException extends ResourceException { 5 | 6 | public MissingPropertyException(String message) { 7 | super(message); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /bin/sndml-fg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run a load job in foreground with output to the console. 4 | # This command is intended to be run from an interactive terminal session. 5 | # 6 | source sndml-setup $1 $2 7 | env | grep SNDML | sort 8 | java -ea -jar $SNDML_JAR -p $SNDML_PROFILE -y $SNDML_YAML 9 | -------------------------------------------------------------------------------- /bin/sndml-debug: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source sndml-setup $@ 3 | sndml_set_log $2 4 | env | grep SNDML | sort 5 | java -ea -Dlog4j2.configurationFile=log4j2-debug.xml -Dsndml.logFolder=$SNDML_CONFIG/log -Dsndml.logPrefix=$2 -jar $SNDML_JAR -p $SNDML_PROFILE -y $SNDML_YAML 6 | echo logfile=`sndml-lastlog $1` 7 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/CommandOptionsException.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | @SuppressWarnings("serial") 4 | public class CommandOptionsException extends IllegalArgumentException { 5 | 6 | public CommandOptionsException(String message) { 7 | super(message); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/NoContentException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class NoContentException extends ServiceNowException { 5 | 6 | public NoContentException(ServiceNowRequest request) { 7 | super(request); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/InvalidFieldNameException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class InvalidFieldNameException extends ServiceNowException { 5 | 6 | public InvalidFieldNameException(String message) { 7 | super(message); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/JsonResponseError.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | public class JsonResponseError extends ServiceNowError { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public JsonResponseError(String message) { 8 | super(message); 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/InvalidTableNameException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class InvalidTableNameException extends ServiceNowException { 5 | 6 | public InvalidTableNameException(String name) { 7 | super("InvalidTableName: " + name); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/JsonResponseException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | public class JsonResponseException extends ServiceNowException { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public JsonResponseException(JsonRequest request) { 8 | super(request); 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/test/resources/Exceptions/primary_key_violation.yaml: -------------------------------------------------------------------------------- 1 | # this file is not used 2 | tables: 3 | - {name: incident_drop, source: incident, action: droptable} 4 | # first step should be successful 5 | - {name: incident1, source: incident, truncate: true} 6 | # second step should fail 7 | - {name: incident2, source: incident, action: insert} -------------------------------------------------------------------------------- /src/test/resources/YAML/InsertTest/load_twice.yaml: -------------------------------------------------------------------------------- 1 | metrics: /tmp/load_twice.metrics 2 | tables: 3 | - {name: drop, action: droptable, target: change_request} 4 | # first step should be successful 5 | - {name: load1, source: change_request, truncate: true} 6 | # second step should fail 7 | - {name: load2, source: change_request, action: insert} -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/InstanceUnavailableException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | public class InstanceUnavailableException extends ServiceNowException { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public InstanceUnavailableException(ServiceNowRequest request) { 8 | super(request); 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/XmlParseException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.net.URI; 4 | 5 | class XmlParseException extends SoapResponseException { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | public XmlParseException(URI uri, Exception cause) { 10 | super(uri, cause); 11 | } 12 | 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/QueryLimitReachedException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.net.URI; 4 | 5 | public class QueryLimitReachedException extends ServiceNowException { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | public QueryLimitReachedException(URI uri) { 10 | super(uri); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /bin/sndml-export: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Generate SQL Create Table statement 4 | # 5 | # Usage: 6 | # sndml-export 7 | # 8 | source sndml-setup $1 9 | tablename=$2 10 | log4j=$SNDML_HOME/src/main/resources/log4j2-quiet.xml 11 | java -ea -cp $SNDML_JAR -Dlog4j.configurationFile=$log4j sndml.servicenow.FileWriter -p $SNDML_PROFILE -t $tablename 12 | -------------------------------------------------------------------------------- /bin/sndml-generate: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Generate SQL Create Table statement 4 | # 5 | # Usage: 6 | # sndml-generate 7 | # 8 | source sndml-setup $1 9 | tablename=$2 10 | log4j=$SNDML_HOME/src/main/resources/log4j2-quiet.xml 11 | java -ea -cp $SNDML_JAR -Dlog4j.configurationFile=$log4j sndml.datamart.Generator -t $tablename -p $SNDML_PROFILE 12 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/WSDLException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | 5 | @SuppressWarnings("serial") 6 | class WSDLException extends IOException { 7 | 8 | public WSDLException(String message) { 9 | super(message); 10 | } 11 | 12 | public WSDLException(Throwable cause) { 13 | super(cause); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/test/resources/profile_test.properties: -------------------------------------------------------------------------------- 1 | # This file is used to test the ConnectionProfile class. 2 | # Values in backticks are passed to Runtime.exec() for evaluation. 3 | servicenow.instance=dev00000 4 | servicenow.username=admin 5 | servicenow.password=`echo orange` 6 | datamart.url=jdbc:mysql://localhost/sndm 7 | datamart.username=admin 8 | datamart.password=`echo yellow` -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppJobStatus.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import com.fasterxml.jackson.annotation.JsonFormat; 4 | 5 | @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_VALUES) 6 | public enum AppJobStatus { 7 | DRAFT, 8 | SCHEDULED, 9 | READY, 10 | QUEUED, 11 | PREPARE, 12 | RUNNING, 13 | COMPLETE, 14 | RENAME, 15 | CANCELLED, 16 | FAILED 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/TimestampHash.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.util.Hashtable; 4 | 5 | import sndml.servicenow.*; 6 | import sndml.util.DateTime; 7 | 8 | @SuppressWarnings("serial") 9 | public class TimestampHash extends Hashtable { 10 | 11 | RecordKeySet getKeys() { 12 | return new RecordKeySet(this.keySet()); 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /bin/sndml-kill: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source sndml-setup $1 3 | SNDML_PIDFILE=`awk -F= '/\.pidfile=/{print $2}' <$SNDML_PROFILE` 4 | if [[ ! -f $SNDML_PIDFILE ]] 5 | then 6 | echo $SNDML_PIDFILE not found 7 | exit -1 8 | fi 9 | SNDML_PID=`cat $SNDML_PIDFILE` 10 | ps -p $SNDML_PID 11 | status=$? 12 | if [[ $status -ne 0 ]] 13 | then 14 | echo Process not found 15 | exit -1 16 | fi 17 | kill $SNDML_PID 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/LogManager.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | public class LogManager { 4 | 5 | @SuppressWarnings("rawtypes") 6 | static public org.slf4j.Logger logger(Class cls) { 7 | return org.slf4j.LoggerFactory.getLogger(cls); 8 | } 9 | 10 | static public org.slf4j.Logger logger(String name) { 11 | return org.slf4j.LoggerFactory.getLogger(name); 12 | } 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/test/java/sndml/util/AllTests.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | 7 | @RunWith(Suite.class) 8 | @SuiteClasses({ 9 | DateTimeTest.class, 10 | MetricsTest.class, 11 | ParametersTest.class, 12 | DatePartitionsTest.class }) 13 | 14 | public class AllTests { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | title: 2 | description: 3 | lsi: false 4 | safe: true 5 | source: docs 6 | incremental: false 7 | highlighter: rouge 8 | github: 9 | is_project_page: false 10 | gist: 11 | noscript: false 12 | kramdown: 13 | math_engine: mathjax 14 | syntax_highlighter: rouge 15 | theme: jekyll-theme-cayman 16 | remote_theme: pages-themes/cayman@v0.2.0 17 | plugins: 18 | - jekyll-remote-theme 19 | -------------------------------------------------------------------------------- /bin/sndml-scan: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the DataPump Daemon 4 | # 5 | source sndml-setup $1 6 | env | grep SNDML | sort 7 | options="" 8 | if [ "$SNDML_DEBUG" = "TRUE" ]; then 9 | options="$options -Dlog4j2.configurationFile=log4j2-debug.xml" 10 | options="$options -Dsndml.logFolder=$SNDML_CONFIG/log -Dsndml.logPrefix=scan" 11 | fi 12 | java -ea $options -jar $SNDML_JAR --profile=$SNDML_PROFILE --scan 13 | -------------------------------------------------------------------------------- /src/test/resources/YAML/LoaderConfigTest/multi-table-load.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - name: sys_user 3 | - name: sys_user_group 4 | - {name: sys_user_grmember, truncate: true} 5 | - {name: core_company, truncate: false} 6 | - {name: rm_story, created: [2017-01-01]} 7 | - {name: incident, target: incident_2017, created: [2017-01-01, start], minrows: 2} 8 | - {name: problem, since: 2017-03-30} 9 | - {name: cmdb_ci_service, since: today} -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/InsufficientRightsException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class InsufficientRightsException extends ServiceNowException { 5 | 6 | public InsufficientRightsException(String message) { 7 | super(message); 8 | } 9 | 10 | public InsufficientRightsException(ServiceNowRequest request) { 11 | super(request); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/TooFewRowsException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class TooFewRowsException extends ServiceNowException { 5 | 6 | public TooFewRowsException(Table table, int minimum, int actual) { 7 | super(String.format( 8 | "Error loading from %s: processed %d rows; minimum is %d", 9 | table.getName(), actual, minimum)); 10 | 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/TooManyRowsException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class TooManyRowsException extends ServiceNowException { 5 | 6 | public TooManyRowsException(Table table, int maximum, int current) { 7 | super(String.format( 8 | "Error processing %s: processed %d rows; maximum is %d", 9 | table.getName(), current, maximum)); 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/test/resources/YAML/YamlParseValidTest/multi-table-load1.yaml: -------------------------------------------------------------------------------- 1 | tables: 2 | - name: sys_user 3 | - name: sys_user_group 4 | - {name: sys_user_grmember, truncate: true} 5 | - {name: core_company, truncate: false} 6 | - {name: rm_story, created: [2017-01-01]} 7 | - {name: incident, target: incident_2017, created: [2017-01-01, start], minrows: 2} 8 | - {name: problem, since: 2017-03-30} 9 | - {name: cmdb_ci_service, since: today} 10 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RowCountMismatchException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @Deprecated 4 | public class RowCountMismatchException extends ServiceNowException { 5 | 6 | private static final long serialVersionUID = 1L; 7 | 8 | public RowCountMismatchException(Table table, int expected, int retrieved) { 9 | super(String.format("%s expected=%d retrieved=%d", table.getName(), expected, retrieved)); 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /bin/sndml-bg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run a load job with all output directed to a file in the log directory. 4 | # This command is intended to be used in cron jobs or other backgound contexts. 5 | # 6 | source sndml-setup $1 $2 7 | env | grep SNDML | sort >$SNDML_LOG 8 | cd $SNDML_CONFIG 9 | java -ea -Dlog4j2.configurationFile=log4j2-file.xml -Dsndml.logFolder=$SNDML_CONFIG/log -Dsndml.logPrefix=$2 -jar $SNDML_JAR -p $SNDML_PROFILE -y $SNDML_YAML 10 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

The main controlling code of the ServiceNow Datamart Loader (SNDML).

3 | * 4 | *

The Datamart Loader reads data and metadata from ServiceNow 5 | * to create and load tables in a JDBC database. 6 | * This package contains all classes used to interact with the databases. 7 | * Classes which interact with ServiceNow are in the @{link sndml.servicenow} package.

8 | */ 9 | package sndml.loader; -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/ServiceNowError.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | /* 4 | * A serious problem that probably indicates a bug in the implementation. 5 | */ 6 | public class ServiceNowError extends RuntimeException { 7 | 8 | private static final long serialVersionUID = 1L; 9 | 10 | public ServiceNowError(Throwable e) { 11 | super(e); 12 | } 13 | 14 | public ServiceNowError(String message) { 15 | super(message); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AgentURLException.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.net.HttpURLConnection; 4 | import java.net.URI; 5 | 6 | @SuppressWarnings("serial") 7 | public class AgentURLException extends AgentHandlerException { 8 | 9 | public AgentURLException(URI uri) { 10 | super(uri, HttpURLConnection.HTTP_BAD_REQUEST); 11 | } 12 | 13 | public AgentURLException(String path) { 14 | super(path, HttpURLConnection.HTTP_BAD_REQUEST); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/InvalidDateTimeException.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | 4 | @SuppressWarnings("serial") 5 | public class InvalidDateTimeException extends IllegalArgumentException { 6 | 7 | InvalidDateTimeException(String value) { 8 | super(String.format("Invalid date time \"%s\"", value)); 9 | } 10 | 11 | InvalidDateTimeException(String tablename, String fieldname, String value) { 12 | super(tablename + "." + fieldname + "=" + value); 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /bin/sndml-jobrun: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run a job from x_108443_sndml_action_run 4 | # $1 is config directory 5 | # $2 is sys_id of the job 6 | # 7 | source sndml-setup $1 8 | env | grep SNDML | sort 9 | options="" 10 | if [ "$SNDML_DEBUG" = "TRUE" ]; then 11 | options="$options -Dlog4j2.configurationFile=log4j2-debug.xml" 12 | options="$options -Dsndml.logFolder=$SNDML_CONFIG/log -Dsndml.logPrefix=jobrun" 13 | fi 14 | java -ea $options -jar $SNDML_JAR -p $SNDML_PROFILE --jobrun $2 -------------------------------------------------------------------------------- /src/main/java/sndml/loader/ConfigParseException.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | @SuppressWarnings("serial") 4 | public class ConfigParseException extends IllegalArgumentException { 5 | 6 | public ConfigParseException(String message) { 7 | super(message); 8 | } 9 | 10 | public ConfigParseException(Throwable cause) { 11 | super(cause); 12 | } 13 | 14 | public ConfigParseException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | 18 | 19 | } 20 | -------------------------------------------------------------------------------- /bin/sndml-single: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run a job from x_108443_sndml_action_run 4 | # $1 is config directory 5 | # $2 is table name 6 | # $3 is sys_id of the record 7 | # 8 | source sndml-setup $1 9 | env | grep SNDML | sort 10 | options="" 11 | if [ "$SNDML_DEBUG" = "TRUE" ]; then 12 | options="$options -Dlog4j2.configurationFile=log4j2-debug.xml" 13 | options="$options -Dsndml.logFolder=$SNDML_CONFIG/log" 14 | fi 15 | java -ea $options -jar $SNDML_JAR -p $SNDML_PROFILE --table $2 --sys_id $3 16 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/Domain.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | /** 4 | *

Wrapper for a sys_domain. 5 | * This could be a single value or a comma separated list.

6 | *

If Domain Separation is not in use then the domain will be null.

7 | * 8 | */ 9 | public class Domain { 10 | 11 | final String value; 12 | 13 | public Domain(String value) { 14 | this.value = value; 15 | } 16 | 17 | public String toString() { 18 | return value; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/NoSuchRecordException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | @SuppressWarnings("serial") 4 | public class NoSuchRecordException extends ServiceNowException { 5 | 6 | public NoSuchRecordException(ServiceNowRequest request, String message) { 7 | super(request, message); 8 | } 9 | 10 | public NoSuchRecordException(ServiceNowRequest request) { 11 | super(request); 12 | } 13 | 14 | public NoSuchRecordException(String message) { 15 | super(message); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/sndml/util/ParametersTest.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | 7 | public class ParametersTest { 8 | 9 | 10 | @Test 11 | public void testCreate() { 12 | Parameters p1 = new Parameters(); 13 | p1.add("animal", "giraffe"); 14 | assertEquals("giraffe", p1.get("animal")); 15 | Parameters p2 = new Parameters(p1); 16 | p1.add("animal", "lion"); 17 | assertEquals("lion", p1.get("animal")); 18 | assertEquals("giraffe", p2.get("animal")); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/ReaderSession.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import sndml.servicenow.Session; 4 | import sndml.util.PropertySet; 5 | import sndml.util.ResourceException; 6 | 7 | /** 8 | * {@link Session} that is used to read data records from the instance, 9 | * and is NOT used to communicate with a scoped app. 10 | * 11 | */ 12 | public class ReaderSession extends Session { 13 | 14 | public ReaderSession(PropertySet propset) throws ResourceException { 15 | super(propset); 16 | verifySession(propset); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/TestingException.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | /** 4 | * An exception thrown by the unit testing framework. 5 | */ 6 | public class TestingException extends RuntimeException { 7 | 8 | private static final long serialVersionUID = 1L; 9 | 10 | public TestingException(Throwable cause) { 11 | super(cause); 12 | } 13 | 14 | public TestingException(String message) { 15 | super(message); 16 | } 17 | 18 | public TestingException(String message, Throwable cause) { 19 | super(message, cause); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/JobCancelledException.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import sndml.servicenow.RecordKey; 4 | import sndml.servicenow.ServiceNowException; 5 | 6 | /** 7 | * This exception is thrown when an attempt to update the ServiceNow scoped app 8 | * receives a 410 error indicating that the Job Run has been cancelled. 9 | */ 10 | @SuppressWarnings("serial") 11 | public class JobCancelledException extends ServiceNowException { 12 | 13 | public JobCancelledException(RecordKey jobKey) { 14 | super("Job cancellation detected for " + jobKey.toString()); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/TableStats.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import sndml.util.DateTimeRange; 4 | 5 | public class TableStats { 6 | 7 | public int count; 8 | public DateTimeRange created; 9 | 10 | public int getCount() { 11 | return count; 12 | } 13 | 14 | public TableStats setCount(int count) { 15 | this.count = count; 16 | return this; 17 | } 18 | 19 | public TableStats setCreated(DateTimeRange range) { 20 | this.created = range; 21 | return this; 22 | } 23 | 24 | public DateTimeRange getCreated() { 25 | return this.created; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordIterator.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.util.*; 4 | 5 | /** 6 | * An iterator used to loop through a {@link RecordList}. 7 | */ 8 | public class RecordIterator implements Iterator { 9 | 10 | private ListIterator iter; 11 | 12 | RecordIterator(RecordList list) { 13 | iter = list.listIterator(); 14 | } 15 | 16 | public boolean hasNext() { 17 | return iter.hasNext(); 18 | } 19 | 20 | public TableRecord next() { 21 | return iter.next(); 22 | } 23 | 24 | public void remove() { 25 | iter.remove(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-quiet.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | %d{HH:mm:ss} %level [%thread] %logger{1} %X{job} %marker: %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | > 17 | 18 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/SchemaReader.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | *

An abstract class used to read table definitions from the ServiceNow instance. 7 | * There are two implementations:

8 | *
    9 | *
  • {@link sndml.servicenow.TableSchemaReader}
    • 10 | *
  • {@link sndml.agent.AppSchemaReader}
    • 11 | *
12 | */ 13 | public interface SchemaReader { 14 | 15 | TableSchema getSchema(String tablename) throws IOException, InterruptedException; 16 | 17 | TableSchema getSchema(Table table) throws IOException, InterruptedException; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/NullWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.util.Metrics; 7 | import sndml.util.ProgressLogger; 8 | 9 | /** 10 | * {@link RecordWriter} which discards all input. 11 | * 12 | */ 13 | public class NullWriter extends RecordWriter { 14 | 15 | public NullWriter() { 16 | super(); 17 | } 18 | 19 | @Override 20 | public void processRecords( 21 | RecordList recs, Metrics writerMetrics, ProgressLogger progressLogger) 22 | throws IOException, SQLException { 23 | writerMetrics.addSkipped(recs.size()); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-console.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | %d{HH:mm:ss} %level %logger{1} [%thread] %X{job} %marker: %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | > 17 | 18 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | %d{HH:mm:ss} %level %logger{1} [%thread] %X{job} %marker: %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | > 17 | 18 | -------------------------------------------------------------------------------- /docs/_includes/head-custom.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head-custom-google-analytics.html %} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/AllTests.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | 7 | @RunWith(Suite.class) 8 | @SuiteClasses({ 9 | EvaluateTest.class, 10 | YamlParseValidTest.class, 11 | YamlParseErrorTest.class, 12 | LoaderConfigTest.class, 13 | DateTimeFactoryTest.class, 14 | CreateTableTest.class, 15 | InsertTest.class, 16 | RefreshTest1.class, 17 | RefreshTest2.class, 18 | RowCountExceptionTest.class, 19 | PruneTest.class, 20 | TimestampTest.class, 21 | TableLoaderTest.class, 22 | }) 23 | 24 | public class AllTests { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/ResourceException.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | /** 4 | * An exception thrown while trying to obtain a resource such as a database connection 5 | * or a ServiceNow session. These exceptions are generally not recoverable 6 | * and will cause the Server to abend. 7 | */ 8 | @SuppressWarnings("serial") 9 | public class ResourceException extends RuntimeException { 10 | 11 | public ResourceException(String message) { 12 | super(message); 13 | } 14 | 15 | public ResourceException(Throwable e) { 16 | super(e); 17 | } 18 | 19 | public ResourceException(String message, Throwable e) { 20 | super(message, e); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/sndml/util/MetricsTest.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import org.junit.*; 4 | import static org.junit.Assert.*; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | public class MetricsTest { 10 | 11 | Logger logger = LoggerFactory.getLogger(this.getClass()); 12 | 13 | @Test 14 | public void testClone() throws CloneNotSupportedException { 15 | Metrics m1 = new Metrics("clone-test"); 16 | m1.addInserted(17); 17 | assertEquals(17, m1.getInserted()); 18 | assertEquals(0, m1.getUpdated()); 19 | Metrics m2 = m1.clone(); 20 | assertEquals(17, m2.getInserted()); 21 | assertEquals(0, m2.getUpdated()); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/WorkerEntry.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.util.concurrent.Future; 4 | 5 | import sndml.servicenow.RecordKey; 6 | import sndml.util.Metrics; 7 | 8 | class WorkerEntry { 9 | 10 | final AppJobConfig config; 11 | final AppJobRunner runner; 12 | final String number; 13 | final RecordKey key; 14 | final Future future; 15 | 16 | WorkerEntry(AppJobRunner runner, Future future) { 17 | this.runner = runner; 18 | this.config = runner.config; 19 | assert config != null; 20 | this.number = config.getNumber(); 21 | assert number != null; 22 | this.key = config.getRunKey(); 23 | this.future = future; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/TableTimestampReader.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import sndml.util.FieldNames; 4 | 5 | /** 6 | * Class to read values of sys_id, sys_created_on and sys_updated_on 7 | * from a ServiceNow table. 8 | * 9 | */ 10 | public class TableTimestampReader extends RestTableReader { 11 | 12 | final FieldNames fieldNames = new FieldNames("sys_id,sys_created_on,sys_updated_on"); 13 | 14 | public TableTimestampReader(Table table) { 15 | super(table); 16 | super.setFields(fieldNames); 17 | super.setPageSize(10000); 18 | } 19 | 20 | public TableTimestampReader setFields(FieldNames names) { 21 | throw new UnsupportedOperationException(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | webapp4 4 | 5 | index.html 6 | index.jsp 7 | index.htm 8 | default.html 9 | default.jsp 10 | default.htm 11 | 12 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/TestJobRunner.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.util.ResourceException; 7 | 8 | /** 9 | * Used for JUnit Tests 10 | */ 11 | 12 | public class TestJobRunner extends JobRunner { 13 | 14 | public TestJobRunner(ConnectionProfile profile, JobConfig config) 15 | throws ResourceException, SQLException { 16 | super(new Resources(profile), config); 17 | } 18 | 19 | public TestJobRunner(ConnectionProfile profile, YamlFile file) 20 | throws IOException, ConfigParseException, ResourceException, SQLException { 21 | super(new Resources(profile), file.getJobConfig(profile)); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/AllTests.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | 7 | import sndml.util.DateTimeTest; 8 | import sndml.util.ParametersTest; 9 | 10 | @RunWith(Suite.class) 11 | @SuiteClasses({ 12 | DateTimeTest.class, 13 | ParametersTest.class, 14 | FieldNamesTest.class, 15 | InstanceTest.class, 16 | SessionIDTest.class, 17 | SessionVerificationTest.class, 18 | TableWSDLTest.class, 19 | TableSchemaTest.class, 20 | GetKeysTest.class, 21 | RestTableReaderTest.class, 22 | SetFieldsTest.class, 23 | CRUDTest.class, 24 | }) 25 | 26 | public class AllTests { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /bin/sndml-server-fg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the DataPump Http Server 4 | # 5 | source sndml-setup $1 6 | cd $SNDML_CONFIG 7 | 8 | # select log4j config file from the jar (src/main/resources) 9 | export SNDML_LOG4J=log4j2-console.xml 10 | 11 | # the next 2 variables are referenced in the selected log4j config file 12 | export SNDML_AGENT=`awk -F= '/\.agent=/{print $2}' <$SNDML_PROFILE` 13 | export SNDML_LOG_DIR=$SNDML_CONFIG/log 14 | OPT_LOG="-Dlog4j2.configurationFile=$SNDML_LOG4J -Dsndml.logFolder=$SNDML_LOG_DIR -Dsndml.logPrefix=$SNDML_AGENT" 15 | 16 | # print exported variables 17 | env | grep SNDML | sort 18 | 19 | CMD="java -ea $OPT_LOG -jar $SNDML_JAR --profile=$SNDML_PROFILE --server" 20 | 21 | echo Running $CMD 22 | $CMD -------------------------------------------------------------------------------- /bin/sndml-daemon-fg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the DataPump Scanner Daemon 4 | # 5 | source sndml-setup $1 6 | cd $SNDML_CONFIG 7 | 8 | # select log4j config file from the jar (src/main/resources) 9 | export SNDML_LOG4J=log4j2-console.xml 10 | 11 | # the next 2 variables are referenced in the selected log4j config file 12 | export SNDML_AGENT=`awk -F= '/\.agent=/{print $2}' <$SNDML_PROFILE` 13 | export SNDML_LOG_DIR=$SNDML_CONFIG/log 14 | OPT_LOG="-Dlog4j2.configurationFile=$SNDML_LOG4J -Dsndml.logFolder=$SNDML_LOG_DIR -Dsndml.logPrefix=$SNDML_AGENT" 15 | 16 | # print exported variables 17 | env | grep SNDML | sort 18 | 19 | CMD="java -ea $OPT_LOG -jar $SNDML_JAR --profile=$SNDML_PROFILE --daemon" 20 | 21 | echo Running $CMD 22 | $CMD -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/SampleTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.After; 6 | import org.junit.AfterClass; 7 | import org.junit.Before; 8 | import org.junit.BeforeClass; 9 | import org.junit.Test; 10 | 11 | public class SampleTest { 12 | 13 | @BeforeClass 14 | public static void setUpBeforeClass() throws Exception { 15 | } 16 | 17 | @AfterClass 18 | public static void tearDownAfterClass() throws Exception { 19 | } 20 | 21 | @Before 22 | public void setUp() throws Exception { 23 | } 24 | 25 | @After 26 | public void tearDown() throws Exception { 27 | } 28 | 29 | @Test 30 | public void test() { 31 | fail("Not yet implemented"); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AgentHandlerException.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.net.URI; 4 | 5 | @SuppressWarnings("serial") 6 | public class AgentHandlerException extends Exception { 7 | 8 | int returnCode; 9 | 10 | public AgentHandlerException(Throwable cause, int returnCode) { 11 | super(cause); 12 | this.returnCode = returnCode; 13 | } 14 | 15 | public AgentHandlerException(String path, int returnCode) { 16 | super("Path=" + path); 17 | this.returnCode = returnCode; 18 | } 19 | 20 | public AgentHandlerException(URI uri, int returnCode) { 21 | super("URI=" + uri.toString()); 22 | this.returnCode = returnCode; 23 | } 24 | 25 | int getReturnCode() { 26 | return this.returnCode; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/DateTimeSerializer.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import java.io.IOException; 4 | 5 | import com.fasterxml.jackson.core.JsonGenerator; 6 | import com.fasterxml.jackson.databind.SerializerProvider; 7 | import com.fasterxml.jackson.databind.ser.std.*; 8 | 9 | 10 | @SuppressWarnings("serial") 11 | public class DateTimeSerializer extends StdSerializer { 12 | 13 | public DateTimeSerializer() { 14 | this(null); 15 | } 16 | 17 | public DateTimeSerializer(Class t) { 18 | super(t); 19 | } 20 | 21 | @Override 22 | public void serialize(DateTime value, JsonGenerator gen, SerializerProvider provider) 23 | throws IOException { 24 | gen.writeString(value.toString()); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseFieldDefinition.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | /** 4 | * Contains the JDBC/SQL type and the Glide type for a single field. 5 | */ 6 | public class DatabaseFieldDefinition { 7 | 8 | final String name; 9 | final int sqltype; 10 | final int size; 11 | final String glidename; 12 | 13 | DatabaseFieldDefinition(String name, int sqltype, int size, String glidename) { 14 | this.name = name; 15 | this.sqltype = sqltype; 16 | this.size = size; 17 | this.glidename = glidename; 18 | } 19 | 20 | public String getName() { return this.name; } 21 | public int getType() { return this.sqltype; } 22 | public int getSize() { return this.size; } 23 | public String getGlideName() { return this.glidename; } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppStatusQueue.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.util.concurrent.LinkedBlockingQueue; 4 | import org.slf4j.Logger; 5 | 6 | import sndml.util.Log; 7 | 8 | public class AppStatusQueue extends LinkedBlockingQueue { 9 | 10 | private static final long serialVersionUID = 1L; 11 | 12 | /** 13 | * Wait for this queue to become empty. 14 | * Designed to be used by publisher at end of job. 15 | */ 16 | void flush(Logger logger) { 17 | if (peek() != null) { 18 | logger.info(Log.FINISH, "Flushing status queue"); 19 | while (peek() != null) { 20 | try { 21 | Thread.sleep(200); 22 | } catch (InterruptedException e) { 23 | // ignore interrupt 24 | } 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/ServletPathParser.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import sndml.servicenow.RecordKey; 4 | 5 | public class ServletPathParser { 6 | 7 | final String action; 8 | final String parm; 9 | final RecordKey key; 10 | 11 | public ServletPathParser(String path) throws AgentURLException { 12 | String[] parts = path.split("/"); 13 | if (parts.length < 2) throw new AgentURLException(path); 14 | this.action = parts.length > 1 ? parts[1] : null; 15 | this.parm = parts.length > 2 ? parts[2] : null; 16 | if (action != "startjobrun") throw new AgentURLException(path); 17 | key = new RecordKey(parm); 18 | if (!key.isGUID()) throw new AgentURLException(path); 19 | } 20 | 21 | public RecordKey getSysID() { 22 | return this.key; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordListAccumulator.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import sndml.util.Metrics; 4 | import sndml.util.ProgressLogger; 5 | 6 | /** 7 | * Simple class to collect a bunch of records in a list. 8 | */ 9 | public class RecordListAccumulator extends RecordWriter { 10 | 11 | RecordList allRecords; 12 | 13 | public RecordListAccumulator(TableReader reader) { 14 | this(reader.table); 15 | } 16 | 17 | public RecordListAccumulator(Table table) { 18 | super(); 19 | allRecords = new RecordList(table); 20 | } 21 | 22 | public void processRecords(RecordList recs, Metrics metrics, ProgressLogger progressLogger) { 23 | allRecords.addAll(recs); 24 | metrics.addInserted(recs.size()); 25 | } 26 | 27 | public RecordList getRecords() { 28 | return allRecords; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/JobActionRequest.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | 6 | import com.fasterxml.jackson.core.exc.StreamReadException; 7 | import com.fasterxml.jackson.databind.DatabindException; 8 | import com.fasterxml.jackson.databind.ObjectMapper; 9 | 10 | @Deprecated 11 | public class JobActionRequest { 12 | 13 | enum JobActionType { 14 | START, 15 | CANCEL 16 | }; 17 | 18 | public String instance; 19 | public String agent; 20 | public String sys_id; 21 | public JobActionType action; 22 | 23 | static final ObjectMapper mapper = new ObjectMapper(); 24 | 25 | static public JobActionRequest load(InputStream input) 26 | throws StreamReadException, DatabindException, IOException { 27 | return mapper.readValue(input, JobActionRequest.class); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseDeleteStatement.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.sql.SQLException; 4 | import java.util.HashMap; 5 | 6 | import sndml.servicenow.*; 7 | 8 | public class DatabaseDeleteStatement extends DatabaseStatement { 9 | 10 | public DatabaseDeleteStatement(DatabaseWrapper db, String sqlTableName) throws SQLException { 11 | super(db, "delete", sqlTableName, null); 12 | } 13 | 14 | @Override 15 | String buildStatement() throws SQLException { 16 | HashMap map = new HashMap(); 17 | map.put("keyvalue", "?"); 18 | return generator.getTemplate(templateName, sqlTableName, map); 19 | } 20 | 21 | public boolean deleteRecord(RecordKey key) throws SQLException { 22 | this.bindField(1, key.toString()); 23 | int count = stmt.executeUpdate(); 24 | return (count > 0); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/NullProgressLogger.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | /** 4 | * {@link ProgressLogger} that discards metrics. 5 | */ 6 | public class NullProgressLogger extends ProgressLogger { 7 | 8 | public NullProgressLogger() { 9 | super(null, null); 10 | } 11 | 12 | public NullProgressLogger(Metrics metrics, DatePartition datePart) { 13 | super(metrics, datePart); 14 | } 15 | 16 | @Override 17 | public NullProgressLogger newPartLogger(Metrics newMetrics, DatePartition newPart) { 18 | return new NullProgressLogger(null, newPart); 19 | } 20 | 21 | @Override 22 | public void logPrepare() { 23 | } 24 | 25 | @Override 26 | public void logStart() { 27 | return; 28 | } 29 | 30 | @Override 31 | public void logProgress() { 32 | return; 33 | } 34 | 35 | @Override 36 | public void logComplete() { 37 | return; 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Classes used to access a ServiceNow instance. 3 | * Most significantly, this package contains classes which know how to 4 | * efficiently and reliably read and process 5 | * large amounts of data from ServiceNow. 6 | *

7 | *

This package implements the Java "visitor" pattern using a pair of abstract classes: 8 | * {@link TableReader} and {@link RecordWriter}. 9 | * Classes extended from {@link TableReader} are implented in this package, 10 | * whereas classes extended from {@link RecordWriter} are implemented in the {@link sndml.loader} package. 11 | * The most common {@link TableReader} is a {@link RestTableReader}. 12 | * This package also contains some older classes (e.g. {@link SoapKeySetTableReader}) 13 | * which implement the SOAP API, but are no longer used. 14 | *

15 | */ 16 | package sndml.servicenow; -------------------------------------------------------------------------------- /src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | %d{HH:mm:ss} %level %marker %logger{1} %X{job}[%X{table}]: %msg%n 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | > 23 | 24 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.agent.JobCancelledException; 7 | import sndml.util.Metrics; 8 | import sndml.util.ProgressLogger; 9 | 10 | /** 11 | * A class which knows how to process records retrieved from ServiceNow. 12 | * 13 | * Although this is normally instantiated as a {@link sndml.loader.DatabaseTableWriter}. 14 | * there are several other subclasses. 15 | * 16 | */ 17 | public abstract class RecordWriter { 18 | 19 | public RecordWriter() { 20 | } 21 | 22 | public abstract void processRecords( 23 | RecordList recs, Metrics metrics, ProgressLogger progressLogger) 24 | throws JobCancelledException, IOException, SQLException; 25 | 26 | public RecordWriter open(Metrics metrics) throws IOException, SQLException { 27 | metrics.start(); 28 | return this; 29 | } 30 | 31 | public void close(Metrics metrics) { 32 | metrics.finish(); 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-file.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | 12 | %d{MM-dd HH:mm:ss} %level %logger{1} [%thread] %marker %X{job}: %msg%n 13 | 14 | 15 | 16 | 17 | 18 | %d{HH:mm:ss} %level %logger{1} [%thread] %X{job} %marker: %msg%n 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/SessionIDTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import org.junit.Test; 5 | 6 | import sndml.loader.TestManager; 7 | 8 | public class SessionIDTest { 9 | 10 | @Test 11 | public void testSession() throws Exception { 12 | Session session = TestManager.getDefaultProfile().newReaderSession(); 13 | TableAPI location = session.table("cmn_location").api(); 14 | location.getRecord("name", TestManager.getProperty("location1"));; 15 | String session1 = session.getSessionID(); 16 | System.out.println("JSESSIONID=" + session1); 17 | location.getRecord("name", TestManager.getProperty("location2")); 18 | String session2 = session.getSessionID(); 19 | System.out.println("JSESSIONID=" + session2); 20 | location.getRecord("name", TestManager.getProperty("location3")); 21 | String session3 = session.getSessionID(); 22 | System.out.println("JSESSIONID=" + session3); 23 | assertEquals(session1, session2); 24 | assertEquals(session2, session3); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/EvaluateTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.io.File; 6 | import sndml.util.PropertiesEditor; 7 | import org.junit.Test; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /* 12 | * Test that values enclosed in backtics are evaluated 13 | */ 14 | public class EvaluateTest { 15 | 16 | static private Logger logger = LoggerFactory.getLogger(EvaluateTest.class); 17 | 18 | @Test 19 | public void testExecute() throws Exception { 20 | String result = PropertiesEditor.evaluate("echo Lorem ipsum dolor"); 21 | logger.debug("result=" + result); 22 | assertEquals("Lorem ipsum dolor", result); 23 | } 24 | 25 | @Test 26 | public void testSampleFile() throws Exception { 27 | File file = new File("src/test/resources/profile_test.properties"); 28 | ConnectionProfile profile = new ConnectionProfile(file); 29 | assertEquals("orange", profile.reader.getProperty("password")); 30 | assertEquals("yellow", profile.database.getProperty("password")); 31 | 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-daemon.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 13 | 14 | %d{MM-dd HH:mm:ss} %level %logger{1} [%thread] %marker %X{job}: %msg%n 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-server.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 13 | 14 | %d{MM-dd HH:mm:ss} %level %logger{1} [%thread] %marker %X{job}: %msg%n 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/FieldNames.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.Set; 6 | 7 | @SuppressWarnings("serial") 8 | public class FieldNames extends ArrayList { 9 | 10 | public FieldNames() { 11 | super(); 12 | } 13 | 14 | public FieldNames(int size) { 15 | super(size); 16 | } 17 | 18 | public FieldNames(String str) { 19 | super(); 20 | addAll(Arrays.asList(str.split("[,\\s]+"))); 21 | } 22 | 23 | public FieldNames(Set names) { 24 | super(names); 25 | } 26 | 27 | public FieldNames addKey() { 28 | if (!this.contains("sys_id")) { 29 | this.add(0, "sys_id"); 30 | } 31 | return this; 32 | } 33 | 34 | public String toString() { 35 | StringBuffer result = new StringBuffer(); 36 | String delim = ""; 37 | for (String name : this) { 38 | result.append(delim).append(name); 39 | delim = ","; 40 | } 41 | return result.toString(); 42 | } 43 | 44 | @Override 45 | public String[] toArray() { 46 | String[] result = new String[this.size()]; 47 | result = this.toArray(result); 48 | return result; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-2021 Giles Lewis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/TestFolder.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | 6 | /** 7 | * A folder in src/test/resources which contains files for JUnit tests. 8 | * 9 | */ 10 | @SuppressWarnings("serial") 11 | public class TestFolder extends File { 12 | 13 | FilenameFilter yamlFilter = (dir, name) -> name.endsWith(".yaml"); 14 | 15 | @SuppressWarnings("rawtypes") 16 | public TestFolder(Class myclass) { 17 | this(myclass.getSimpleName()); 18 | } 19 | 20 | public TestFolder(String name) { 21 | super("src/test/resources/YAML", name); 22 | } 23 | 24 | /** 25 | * Get a file from this folder 26 | */ 27 | public File getFile(String filename) { 28 | return new File(this, filename); 29 | } 30 | 31 | public YamlFile getYaml(String name) { 32 | return new YamlFile(new File(this, name + ".yaml")); 33 | } 34 | 35 | public YamlFile[] yamlFiles() { 36 | File[] files = listFiles(yamlFilter); 37 | YamlFile[] result = new YamlFile[files.length]; 38 | for (int i = 0; i < files.length; ++i) result[i] = new YamlFile(files[i]); 39 | return result; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/InstanceTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.net.URI; 6 | import java.net.URISyntaxException; 7 | 8 | import org.junit.Test; 9 | 10 | import sndml.util.Parameters; 11 | 12 | public class InstanceTest { 13 | 14 | @Test 15 | public void testInstance() throws URISyntaxException { 16 | Instance instance = new Instance("dev12345"); 17 | URI uri1 = instance.getURI("incident.do"); 18 | assertEquals("https://dev12345.service-now.com/incident.do", uri1.toString()); 19 | Parameters parms2 = new Parameters(); 20 | parms2.add("sysparm_query", "nameINJohn Doe,Mary Smith"); 21 | URI uri2 = instance.getURI("incident.do", parms2); 22 | assertEquals("https://dev12345.service-now.com/incident.do?sysparm_query=nameINJohn+Doe%2CMary+Smith", uri2.toString()); 23 | } 24 | 25 | @Test(expected = AssertionError.class) 26 | public void testBadInstance1() throws Exception { 27 | new Instance("dev 1234"); 28 | } 29 | 30 | @Test(expected = AssertionError.class) 31 | public void testBadInstance2() throws Exception { 32 | new Instance(""); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/SoapResponseException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | 6 | import org.jdom2.Element; 7 | 8 | import sndml.util.Log; 9 | 10 | /** 11 | * Exception thrown when there is an undetermined problem with a SOAP response. 12 | */ 13 | public class SoapResponseException extends IOException { 14 | 15 | private static final long serialVersionUID = 1L; 16 | 17 | SoapResponseException(URI uri, Exception cause) { 18 | super(uri.toString(), cause); 19 | } 20 | 21 | SoapResponseException(URI uri) { 22 | super(uri.toString()); 23 | } 24 | 25 | SoapResponseException(Table table, String method, String message, Element responseElement) { 26 | super(Log.joinLines( 27 | String.format("table=%s method=%s %s", table.getName(), method, message), 28 | XmlFormatter.format(responseElement))); 29 | } 30 | 31 | SoapResponseException(String tablename, String message) { 32 | super("table=" + tablename + " " + message); 33 | } 34 | 35 | SoapResponseException(Table table, Exception cause, String response) { 36 | super(response + "\ntable=" + table.getName(), cause); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/assembly/release.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | 10 | zip 11 | 12 | 13 | 14 | ${project.basedir} 15 | 16 | 17 | LICENSE* 18 | 19 | 20 | 21 | ${project.basedir}/UpdateSets 22 | 23 | 24 | *${app.version}*.xml 25 | 26 | 27 | 28 | ${project.build.directory} 29 | 30 | 31 | *${project.version}*.jar 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/TestingProfile.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.file.Path; 6 | import java.nio.file.Paths; 7 | 8 | public class TestingProfile extends ConnectionProfile { 9 | 10 | private String name; 11 | 12 | public TestingProfile(String profileName) throws IOException { 13 | super(getProfilePath(profileName)); 14 | name = profileName; 15 | } 16 | 17 | /** 18 | * Get a file named "sndml_profile" or ".sndml_profile" 19 | * from the directory configs/profileName/ 20 | * 21 | * Note that the profiles directory is NOT not stored in github 22 | * as it is likely to contain passwords. 23 | */ 24 | private static File getProfilePath(String profileName) { 25 | File file; 26 | Path directory = Paths.get("configs", profileName); 27 | file = directory.resolve("sndml_profile").toFile(); 28 | if (file.exists()) return file; 29 | file = directory.resolve(".sndml_profile").toFile(); 30 | return file; 31 | } 32 | 33 | public String getName() { 34 | return this.name; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return this.name; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RestPetitTableReader.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | /** 4 | *

A {@link RestTableReader} which attempts to read all qualifying records in a single 5 | * Web Services call. 6 | *

7 | *

A normal {@link RestTableReader} will continue fetching pages until an empty page 8 | * is returned, or the number of records reaches the expected number based on the stats. 9 | * A {@link RestPetitTableReader} skips the stats and it stops reading as soon as the 10 | * number of records returned is smaller than the page size. However, this class could 11 | * miss some records if ACLs are in place. This class should only be used if the page size 12 | * is much larger than the maximum number of records that are expected. 13 | *

14 | * 15 | */ 16 | public class RestPetitTableReader extends RestTableReader { 17 | 18 | public RestPetitTableReader(Table table) { 19 | super(table); 20 | this.statsEnabled = false; 21 | this.orderBy = OrderBy.NONE; 22 | } 23 | 24 | @Override 25 | protected boolean isFinished(int pageRows, int totalRows) { 26 | if (pageRows == 0 || pageRows < this.pageSize) return true; 27 | if (statsEnabled && totalRows >= getExpected()) return true; 28 | return false; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/FieldNamesTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | 7 | import sndml.util.FieldNames; 8 | 9 | public class FieldNamesTest { 10 | 11 | @Test 12 | public void testCommas() { 13 | FieldNames f1 = new FieldNames("alpha,beta,gamma"); 14 | assertTrue(f1.contains("beta")); 15 | assertTrue(f1.contains("alpha")); 16 | assertFalse(f1.contains("delta")); 17 | } 18 | 19 | @Test 20 | public void testSpaces() { 21 | FieldNames f1 = new FieldNames("alpha beta gamma"); 22 | assertTrue(f1.contains("beta")); 23 | assertTrue(f1.contains("alpha")); 24 | assertFalse(f1.contains("delta")); 25 | } 26 | 27 | @Test 28 | public void testMixture() { 29 | FieldNames f1 = new FieldNames("alpha, beta, gamma"); 30 | assertTrue(f1.contains("beta")); 31 | assertTrue(f1.contains("alpha")); 32 | assertFalse(f1.contains("delta")); 33 | } 34 | 35 | @Test 36 | public void testMultiLine() { 37 | String s1 = "alpha\n beta\ngamma"; 38 | FieldNames f1 = new FieldNames(s1); 39 | assertEquals(3, f1.size()); 40 | assertTrue(f1.contains("beta")); 41 | assertTrue(f1.contains("alpha")); 42 | assertFalse(f1.contains("delta")); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /bin/start-server.sh: -------------------------------------------------------------------------------- 1 | !/bin/bash 2 | # start SNDML as an HTTP server background process 3 | # file names are relative to the current working directory 4 | 5 | # verify that these 3 variables are correct 6 | export SNDML_PROFILE=sndml.profile 7 | export SNDML_JAR=sndml.jar 8 | export SNDML_LOG_DIR=log 9 | 10 | # extract these 3 variables from the connection profile 11 | export SNDML_AGENT=`awk -F= '/\.agent=/{print $2}' <$SNDML_PROFILE` 12 | export SNDML_PORT=`awk -F= '/\.port=/{print $2}' <$SNDML_PROFILE` 13 | export SNDML_PIDFILE=`awk -F= '/\.pidfile=/{print $2}' <$SNDML_PROFILE` 14 | 15 | OPT_LOG="-Dlog4j2.configurationFile=log4j2-daemon.xml -Dsndml.logFolder=$SNDML_LOG_DIR -Dsndml.logPrefix=$SNDML_AGENT" 16 | CMD="java -ea $OPT_LOG -jar $SNDML_JAR --profile=$SNDML_PROFILE --server" 17 | 18 | # print exported variables 19 | env | grep SNDML | sort 20 | 21 | # delete the pidfile 22 | if [ -f $SNDML_PIDFILE ]; then 23 | rm $SNDML_PIDFILE 24 | fi 25 | if [ -f nohup.out ]; then 26 | rm nohup.out 27 | fi 28 | 29 | echo starting $CMD 30 | nohup $CMD & 31 | 32 | # wait for pidfile 33 | sleep 3 34 | if [ -f $SNDML_PIDFILE ]; then 35 | PID=`cat $SNDML_PIDFILE` 36 | echo Server is running on PID $PID 37 | exit 0 38 | else 39 | cat nohup.out 40 | exit 1 41 | fi 42 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/PropertySetTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.io.File; 6 | 7 | import org.junit.After; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | public class PropertySetTest { 14 | 15 | static private Logger logger = LoggerFactory.getLogger(PropertySetTest.class); 16 | 17 | @Before 18 | public void setUp() throws Exception { 19 | } 20 | 21 | @After 22 | public void tearDown() throws Exception { 23 | } 24 | 25 | @Test 26 | public void testSampleFile() throws Exception { 27 | File file = new File("src/test/resources/profile_test.properties"); 28 | ConnectionProfile profile = new ConnectionProfile(file); 29 | int size = profile.reader.size(); 30 | logger.debug("size=" + size); 31 | assertEquals("dev00000", profile.getProperty("reader.instance")); 32 | assertEquals("dev00000", profile.reader.getProperty("instance")); 33 | assertEquals("admin", profile.database.getProperty("username")); 34 | assertEquals("200", profile.getProperty("reader.pagesize")); 35 | assertEquals("200", profile.reader.getProperty("pagesize")); 36 | assertEquals("200", profile.app.getProperty("pagesize")); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/ProgressLogger.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import sndml.agent.JobCancelledException; 4 | 5 | public abstract class ProgressLogger { 6 | 7 | protected final DatePartition datePart; 8 | protected final Metrics metrics; 9 | 10 | public ProgressLogger(Metrics metrics, DatePartition datePart) { 11 | this.datePart = datePart; 12 | this.metrics = metrics; 13 | } 14 | 15 | public DatePartition getPart() { 16 | return datePart; 17 | } 18 | 19 | public boolean hasPart() { 20 | return datePart != null; 21 | } 22 | 23 | public Metrics getMetrics() { 24 | return metrics; 25 | } 26 | 27 | public abstract ProgressLogger newPartLogger(Metrics newMetrics, DatePartition newPart); 28 | 29 | /** 30 | * We are starting the initialization process, which includes 31 | * calculating the number of expected records that will be processed. 32 | */ 33 | public abstract void logPrepare(); 34 | 35 | /** 36 | * We are starting the actual processing of records. 37 | * @throws JobCancelledException 38 | */ 39 | public abstract void logStart() throws JobCancelledException; 40 | 41 | public abstract void logProgress() throws JobCancelledException; 42 | 43 | /** 44 | * We have completed all processing of record. 45 | */ 46 | public abstract void logComplete(); 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/ServiceNowException.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import org.apache.http.StatusLine; 6 | 7 | import sndml.util.Log; 8 | 9 | @SuppressWarnings("serial") 10 | public class ServiceNowException extends IOException { 11 | 12 | private ServiceNowRequest request; 13 | 14 | public ServiceNowException(URI uri) { 15 | super(uri.toString()); 16 | } 17 | 18 | public ServiceNowException(URI uri, String requestText) { 19 | super(Log.joinLines(uri.toString(), requestText)); 20 | } 21 | 22 | public ServiceNowException(String message) { 23 | super(message); 24 | } 25 | 26 | public ServiceNowException(ServiceNowRequest request) { 27 | super(request.dump()); 28 | this.request = request; 29 | } 30 | 31 | public ServiceNowException(ServiceNowRequest request, String message) { 32 | super(message); 33 | this.request = request; 34 | } 35 | 36 | public ServiceNowException(ServiceNowRequest request, StatusLine status, String message) { 37 | super(message); 38 | this.request = request; 39 | } 40 | 41 | public int getStatusCode() { 42 | assert request != null: "No request in throw"; 43 | return request.getStatusCode(); 44 | } 45 | 46 | public URI getURL() { 47 | return request.getURI(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/test/resources/junit.properties: -------------------------------------------------------------------------------- 1 | # 2 | # These properties are used for testing the servicenow.api classes 3 | # They should work with an OOB PDI 4 | # 5 | # These next two must match (i.e. same record) 6 | junit.some_incident_number=INC0000002 7 | junit.some_incident_sys_id=9d385017c611228701d22104cc95c371 8 | 9 | # These next two must match 10 | junit.some_group_name=Database 11 | junit.some_group_manager=Don Goodliffe 12 | 13 | junit.location1=San Diego 14 | junit.location2=Frankfurt 15 | junit.location3=Chicago 16 | 17 | # These next three must match 18 | junit.some_department_id=0023 19 | junit.some_department_name=Customer Support 20 | junit.some_department_head=Rob Woodbyrne 21 | 22 | junit.some_ci=106c5c13c61122750194a1e96cfde951 23 | 24 | # Is this used for anything? 25 | junit.sample_jpg_file=src/test/resources/snlogo1.png 26 | 27 | # where can we put temporary files? 28 | junit.temp=/tmp 29 | 30 | # This is the name of the configuration directory for servicenow.api tests 31 | # The actual profile file path is: configs//.sndm_profile 32 | # where is the name of the profile 33 | junit.api.default_profile=junit 34 | 35 | # This is a list of configurations used for servicenow.datamart tests 36 | #junit.datamart.profile_list=awsmy,awspg,awsora,awsms 37 | junit.datamart.profile_list=junit 38 | 39 | # servicenow.sys_email.limit=10 40 | -------------------------------------------------------------------------------- /bin/sndml-daemon-bg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the DataPump Scanner Daemon as a background process 4 | # 5 | source sndml-setup $1 6 | cd $SNDML_CONFIG 7 | 8 | # select log4j config file from the jar (src/main/resources) 9 | export SNDML_LOG4J=log4j2-daemon.xml 10 | 11 | # the next 2 variables are referenced in the selected log4j config file 12 | export SNDML_AGENT=`awk -F= '/\.agent=/{print $2}' <$SNDML_PROFILE` 13 | export SNDML_PIDFILE=`awk -F= '/\.pidfile=/{print $2}' <$SNDML_PROFILE` 14 | export SNDML_LOG_DIR=$SNDML_CONFIG/log 15 | OPT_LOG="-Dlog4j2.configurationFile=$SNDML_LOG4J -Dsndml.logFolder=$SNDML_LOG_DIR -Dsndml.logPrefix=$SNDML_AGENT" 16 | 17 | # print exported variables 18 | env | grep SNDML | sort 19 | CMD="java -ea $OPT_LOG -jar $SNDML_JAR --profile=$SNDML_PROFILE --daemon" 20 | 21 | # delete the pidfile and nohup.out 22 | if [ -f $SNDML_PIDFILE ]; then 23 | rm $SNDML_PIDFILE 24 | fi 25 | if [ -f nohup.out ]; then 26 | rm nohup.out 27 | fi 28 | 29 | echo Starting $CMD 30 | nohup $CMD & 31 | 32 | # wait 20 seconds for pidfile 33 | count=0 34 | until [[ -f $SNDML_PIDFILE || count -ge 20 ]]; do 35 | let "count+=1" 36 | sleep 1 37 | done 38 | if [[ -f $SNDML_PIDFILE ]]; then 39 | PID=`cat $SNDML_PIDFILE` 40 | echo Server is running on PID $PID 41 | exit 0 42 | else 43 | echo $SNDML_PIDFILE NOT FOUND 44 | cat nohup.out 45 | exit 1 46 | fi 47 | -------------------------------------------------------------------------------- /bin/sndml-server-bg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Run the DataPump Scanner HTTP Server as a background process 4 | # 5 | source sndml-setup $1 6 | cd $SNDML_CONFIG 7 | 8 | # select log4j config file from the jar (src/main/resources) 9 | export SNDML_LOG4J=log4j2-daemon.xml 10 | 11 | # the next 2 variables are referenced in the selected log4j config file 12 | export SNDML_AGENT=`awk -F= '/\.agent=/{print $2}' <$SNDML_PROFILE` 13 | export SNDML_PIDFILE=`awk -F= '/\.pidfile=/{print $2}' <$SNDML_PROFILE` 14 | export SNDML_LOG_DIR=$SNDML_CONFIG/log 15 | OPT_LOG="-Dlog4j2.configurationFile=$SNDML_LOG4J -Dsndml.logFolder=$SNDML_LOG_DIR -Dsndml.logPrefix=$SNDML_AGENT" 16 | 17 | # print exported variables 18 | env | grep SNDML | sort 19 | CMD="java -ea $OPT_LOG -jar $SNDML_JAR --profile=$SNDML_PROFILE --server" 20 | 21 | # delete the pidfile and nohup.out 22 | if [ -f $SNDML_PIDFILE ]; then 23 | rm $SNDML_PIDFILE 24 | fi 25 | if [ -f nohup.out ]; then 26 | rm nohup.out 27 | fi 28 | 29 | echo Starting $CMD 30 | nohup $CMD & 31 | 32 | # wait 20 seconds for pidfile 33 | count=0 34 | until [[ -f $SNDML_PIDFILE || count -ge 20 ]]; do 35 | let "count+=1" 36 | sleep 1 37 | done 38 | if [[ -f $SNDML_PIDFILE ]]; then 39 | PID=`cat $SNDML_PIDFILE` 40 | echo Server is running on PID $PID 41 | exit 0 42 | else 43 | echo $SNDML_PIDFILE NOT FOUND 44 | cat nohup.out 45 | exit 1 46 | fi 47 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/SchemaCache.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.util.concurrent.ConcurrentHashMap; 5 | 6 | import sndml.util.Log; 7 | 8 | /** 9 | * A class that maintains a cache of {@link TableSchema} objects that have been read 10 | * from the instance, and reuses them if they are required a second time. 11 | * 12 | */ 13 | public class SchemaCache { 14 | 15 | private final SchemaReader schemaReader; 16 | 17 | private final ConcurrentHashMap schemaCache = 18 | new ConcurrentHashMap(); 19 | 20 | public SchemaCache(SchemaReader r) { 21 | this.schemaReader = r; 22 | } 23 | 24 | /** 25 | * Generate {@link TableSchema} or retrieve from cache. 26 | */ 27 | public TableSchema getSchema(String tablename) throws IOException, InterruptedException { 28 | if (schemaCache.containsKey(tablename)) 29 | return schemaCache.get(tablename); 30 | assert schemaReader != null : SchemaCache.class.getSimpleName() + " not initialized"; 31 | String saveJob = Log.getJobContext(); 32 | Log.setJobContext(tablename + ".schema"); 33 | TableSchema schema = schemaReader.getSchema(tablename); 34 | if (schema.isEmpty()) throw new InvalidTableNameException(tablename); 35 | schemaCache.put(tablename, schema); 36 | Log.setJobContext(saveJob); 37 | return schema; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/HeartbeatTask.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import java.util.TimerTask; 6 | 7 | import org.slf4j.Logger; 8 | 9 | import sndml.loader.Resources; 10 | import sndml.servicenow.JsonRequest; 11 | import sndml.util.Log; 12 | 13 | public class HeartbeatTask extends TimerTask { 14 | 15 | final AppSession appSession; 16 | final String agentName; 17 | final URI uri; 18 | 19 | // TODO If consecutive heartbeatFailures exceeds threshold then abort the program 20 | // use property server.heartbeat_failure_limit 21 | int consecutiveFailures = 0; 22 | 23 | final static Logger logger = Log.getLogger(HeartbeatTask.class); 24 | 25 | public HeartbeatTask(Resources resources) { 26 | this.appSession = resources.getAppSession(); 27 | this.agentName = resources.getAgentName(); 28 | this.uri = appSession.uriGetAgent(); 29 | } 30 | 31 | @Override 32 | public void run() { 33 | Log.setGlobalContext(); 34 | JsonRequest request = new JsonRequest(appSession, uri); 35 | try { 36 | request.execute(); 37 | consecutiveFailures = 0; 38 | logger.info(Log.PROCESS, "heartbeat sent"); 39 | } catch (IOException e) { 40 | consecutiveFailures += 1; 41 | logger.error(Log.REQUEST, e.getMessage(), e); 42 | logger.warn(Log.ERROR, String.format("%d heartbeat failure", consecutiveFailures)); 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/GetKeysTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import sndml.loader.TestManager; 6 | import sndml.loader.TestingProfile; 7 | import sndml.util.Log; 8 | import sndml.util.Metrics; 9 | import sndml.util.NullProgressLogger; 10 | 11 | import static org.junit.Assert.*; 12 | 13 | import org.junit.Test; 14 | 15 | public class GetKeysTest { 16 | 17 | final TestingProfile profile; 18 | final Logger logger = TestManager.getLogger(this.getClass()); 19 | 20 | public GetKeysTest() { 21 | this.profile = TestManager.getDefaultProfile(); 22 | } 23 | 24 | @Test 25 | public void test() throws Exception { 26 | Session session = profile.newReaderSession(); 27 | Table table = session.table("sys_user"); 28 | logger.info(Log.TEST, "Calling getStats"); 29 | TableStats stats = table.rest().getStats(null, false); 30 | Integer numRecs = stats.getCount(); 31 | assertTrue(numRecs > 200); 32 | // assertTrue(numRecs > 20000); 33 | KeySetTableReader reader = new KeySetTableReader(table); 34 | logger.info(Log.TEST, "Calling initialize"); 35 | Metrics metrics = new Metrics(null, null); 36 | reader.prepare(new NullWriter(), metrics, new NullProgressLogger()); 37 | Integer numKeys = reader.getExpected(); 38 | logger.info(Log.TEST, String.format("Count=%d Keys=%d", numRecs, numKeys)); 39 | assertEquals(numRecs, numKeys); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/SimpleTableLoader.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import sndml.servicenow.EncodedQuery; 4 | import sndml.servicenow.RecordKey; 5 | import sndml.servicenow.Table; 6 | import sndml.util.DateTime; 7 | import sndml.util.Log; 8 | import sndml.util.ResourceException; 9 | 10 | public class SimpleTableLoader extends JobRunner implements Runnable { 11 | 12 | public SimpleTableLoader(Resources resources, Table table, EncodedQuery filter) { 13 | super(resources, jobConfig(resources.getProfile(), table, filter)); 14 | } 15 | 16 | public SimpleTableLoader(Resources resources, Table table, RecordKey docKey) { 17 | super(resources, jobConfig(resources.getProfile(), table, docKey)); 18 | } 19 | 20 | private static JobConfig jobConfig(ConnectionProfile profile, Table table, EncodedQuery query) { 21 | ConfigFactory configFactory = new ConfigFactory(DateTime.now()); 22 | return configFactory.tableLoader(profile, table, query); 23 | } 24 | 25 | private static JobConfig jobConfig(ConnectionProfile profile, Table table, RecordKey docKey) { 26 | ConfigFactory configFactory = new ConfigFactory(DateTime.now()); 27 | return configFactory.singleRecordSync(profile, table, docKey); 28 | } 29 | 30 | @Override 31 | public void run() { 32 | try { 33 | super.call(); 34 | } catch (Exception e) { 35 | logger.error(Log.INIT, e.getMessage(), e); 36 | throw new ResourceException(e); 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/GetRecordTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | import java.io.IOException; 5 | import org.junit.Ignore; 6 | import org.junit.Test; 7 | 8 | import sndml.loader.TestManager; 9 | 10 | public class GetRecordTest { 11 | 12 | Session session; 13 | 14 | GetRecordTest() { 15 | session = TestManager.getDefaultProfile().newReaderSession(); 16 | } 17 | 18 | @Ignore @Test 19 | public void testRecordByKey() throws IOException { 20 | String sys_id = TestManager.getProperty("some_incident_sys_id"); 21 | RecordKey key = new RecordKey(sys_id); 22 | Table inc = session.table("incident"); 23 | TableRecord rec = inc.getRecord(key); 24 | assertNotNull(rec); 25 | } 26 | 27 | @Test 28 | public void testGetRecordByNumber() throws IOException { 29 | Table inc = session.table("incident"); 30 | String number = TestManager.getProperty("some_incident_number"); 31 | TableRecord rec1 = inc.api().getRecord("number", number); 32 | RecordKey key = rec1.getKey(); 33 | assertEquals(32, key.toString().length()); 34 | TableRecord rec2 = inc.getRecord(key); 35 | assertEquals(number, rec2.getValue("number")); 36 | } 37 | 38 | @Ignore @Test 39 | public void testGetNullRecord() throws IOException { 40 | RecordKey key = new RecordKey("00000000000000000000000000000000"); 41 | Table inc = session.table("incident"); 42 | TableRecord rec = inc.getRecord(key); 43 | assertNull(rec); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/YamlParseErrorTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.io.IOException; 6 | 7 | import org.junit.After; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.slf4j.Logger; 11 | 12 | import sndml.util.Log; 13 | 14 | public class YamlParseErrorTest { 15 | 16 | final Logger logger = TestManager.getLogger(this.getClass()); 17 | TestingProfile profile = TestManager.getDefaultProfile(); 18 | 19 | @Before 20 | public void setUp() throws Exception { 21 | } 22 | 23 | @After 24 | public void tearDown() throws Exception { 25 | } 26 | 27 | @Test 28 | public void testFail() throws IOException { 29 | TestFolder folder = new TestFolder(this.getClass().getSimpleName()); 30 | ConfigFactory factory = new ConfigFactory(); 31 | for (YamlFile file : folder.yamlFiles()) { 32 | String text = TestManager.readFully(file); 33 | logger.info(Log.TEST, "Testing " + file.getPath() + "\n" + text); 34 | Throwable err = null; 35 | try { 36 | @SuppressWarnings("unused") 37 | YamlLoaderConfig config = factory.loaderConfig(profile, file); 38 | fail("No exception for " + file.getPath()); 39 | } catch (ConfigParseException e) { 40 | err = e; 41 | logger.info(Log.TEST, e.getMessage()); 42 | } 43 | assertNotNull(err); 44 | logger.warn(Log.TEST, err.getMessage()); 45 | assertTrue(err instanceof ConfigParseException); 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseUpdateWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.servicenow.RecordKey; 7 | import sndml.servicenow.TableRecord; 8 | import sndml.util.Log; 9 | import sndml.util.Metrics; 10 | import sndml.servicenow.Table; 11 | 12 | public class DatabaseUpdateWriter extends DatabaseTableWriter { 13 | 14 | protected DatabaseInsertStatement insertStmt; 15 | protected DatabaseUpdateStatement updateStmt; 16 | 17 | public DatabaseUpdateWriter(DatabaseWrapper db, Table table, String sqlTableName, String writerName) 18 | throws IOException, SQLException { 19 | super(db, table, sqlTableName, writerName); 20 | } 21 | 22 | @Override 23 | public DatabaseUpdateWriter open(Metrics writerMetrics) throws SQLException, IOException { 24 | super.open(writerMetrics); 25 | insertStmt = new DatabaseInsertStatement(this.db, this.sqlTableName, columns); 26 | updateStmt = new DatabaseUpdateStatement(this.db, this.sqlTableName, columns); 27 | return this; 28 | } 29 | 30 | @Override 31 | void writeRecord(TableRecord rec, Metrics writerMetrics) throws SQLException { 32 | RecordKey key = rec.getKey(); 33 | logger.debug(Log.PROCESS, "Update " + key); 34 | if (updateStmt.update(rec)) { 35 | writerMetrics.incrementUpdated(); 36 | } else { 37 | logger.debug(Log.PROCESS, "Insert " + key); 38 | insertStmt.insert(rec); 39 | writerMetrics.incrementInserted(); 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseInsertStatement.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.sql.SQLException; 4 | import java.util.HashMap; 5 | 6 | import sndml.servicenow.TableRecord; 7 | 8 | public class DatabaseInsertStatement extends DatabaseStatement { 9 | 10 | public DatabaseInsertStatement(DatabaseWrapper db, String sqlTableName, ColumnDefinitions columns) 11 | throws SQLException { 12 | super(db, "insert", sqlTableName, columns); 13 | } 14 | 15 | String buildStatement() throws SQLException { 16 | final String fieldSeparator = ",\n"; 17 | StringBuilder fieldnames = new StringBuilder(); 18 | StringBuilder fieldvalues = new StringBuilder(); 19 | for (int i = 0; i < columns.size(); ++i) { 20 | if (i > 0) { 21 | fieldnames.append(fieldSeparator); 22 | fieldvalues.append(","); 23 | } 24 | fieldnames.append(generator.sqlQuote(columns.get(i).getName())); 25 | fieldvalues.append("?"); 26 | } 27 | HashMap map = new HashMap(); 28 | map.put("fieldnames", fieldnames.toString()); 29 | map.put("fieldvalues", fieldvalues.toString()); 30 | return generator.getTemplate(templateName, sqlTableName, map); 31 | } 32 | 33 | public void insert(TableRecord rec) throws SQLException { 34 | setRecord(rec); 35 | int n = columns.size(); 36 | for (int i = 0; i < n; ++i) { 37 | bindField(i + 1, i); 38 | } 39 | int count = stmt.executeUpdate(); 40 | if (count > 1) throw new AssertionError("insert count=" + count); 41 | } 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/JobFactory.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.File; 4 | import java.io.FileReader; 5 | import java.io.IOException; 6 | import java.io.Reader; 7 | import java.io.StringReader; 8 | 9 | import sndml.servicenow.Session; 10 | import sndml.util.DateTime;; 11 | 12 | public class JobFactory { 13 | 14 | final Resources resources; 15 | final ConnectionProfile profile; 16 | final Session session; 17 | final DatabaseWrapper database; 18 | final DateCalculator dateCalculator; 19 | ConfigFactory configFactory = new ConfigFactory(); 20 | 21 | public JobFactory(Resources resources, DateTime start) { 22 | this.resources = resources; 23 | this.profile = resources.getProfile(); 24 | this.session = resources.getReaderSession(); 25 | this.database = resources.getDatabaseWrapper(); 26 | this.dateCalculator = new DateCalculator(start); 27 | } 28 | 29 | public JobRunner yamlJob(Reader yamlReader) 30 | throws ConfigParseException, IOException { 31 | JobConfig config = configFactory.yamlJob(profile, yamlReader); 32 | config.initialize(profile, dateCalculator); 33 | config.validate(); 34 | return new JobRunner(resources, config); 35 | } 36 | 37 | public JobRunner yamlJob(File yamlFile) 38 | throws ConfigParseException, IOException { 39 | return yamlJob(new FileReader(yamlFile)); 40 | } 41 | 42 | public JobRunner yamlJob(String yamlText) 43 | throws ConfigParseException, IOException { 44 | return yamlJob(new StringReader(yamlText)); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | The **ServiceNow Data Mart Loader (SNDML)** is a Java application which exports data from ServiceNow 2 | into an SQL database such as MySQL, PostgreSQL, Oracle or Microsoft SQL Server. 3 | SNDML uses the ServiceNow REST API to extract data from ServiceNow. 4 | It uses JDBC to load target tables. It creates tables in the target database based on ServiceNow meta-data. 5 | It supports a variety of load and synchronization operations. 6 | 7 | **ServiceNow DataPump** is a scoped ServiceNow app which is installed in the ServiceNow instance and is used to configure and manage SNDML jobs. 8 | For an introduction to the DataPump app, please refer to 9 | - [Getting Started with ServiceNow DataPump](https://gflewis.github.io/sndml3/) (GitHub Pages) 10 | - [Exporting ServiceNow Data using DataPump 3.5](https://www.youtube.com/watch?v=r3TOvHVKeDQ) (YouTube Video) 11 | 12 | For an overview of the changes in **Relase 3.5** please refer to 13 | - https://github.com/gflewis/sndml3/wiki/Release-3.5 14 | 15 | If you are using YAML to configure the application, please refer to these wiki pages 16 | - https://github.com/gflewis/sndml3/wiki/Home 17 | - https://github.com/gflewis/sndml3/wiki/Getting-Started 18 | - https://github.com/gflewis/sndml3/wiki/YAML-Configuration 19 | - https://github.com/gflewis/sndml3/wiki/Options 20 | 21 | This program is freely distributed software. You are welcome to redistribute and/or modify it. 22 | This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY, explicit or implied. 23 | -------------------------------------------------------------------------------- /src/main/resources/log4j2-debug.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | %d{MM-dd HH:mm:ss} %level %marker [%thread] %logger{1} %X{job}[%X{table}] %msg%n 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss} %level %marker %logger{1} %X{job} %msg%n 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | > 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/NameMap.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.*; 4 | import java.util.*; 5 | import org.jdom2.*; 6 | 7 | /** 8 | * Contains a mapping between the Glide field name and the SQL field name 9 | * as loaded from the fieldnames element of the sqltemplates.xml file. 10 | * @author Giles Lewis 11 | * 12 | */ 13 | public class NameMap { 14 | 15 | Map toGlide = new HashMap(); 16 | Map toSql = new HashMap(); 17 | 18 | public NameMap(Element fieldnames) { 19 | loadFromXML(fieldnames); 20 | } 21 | 22 | public String getGlideName(String sqlName) { 23 | return toGlide.get(sqlName); 24 | } 25 | 26 | public String getSqlName(String glideName) { 27 | return toSql.get(glideName); 28 | } 29 | 30 | public void printMap(PrintStream out) { 31 | Set keys = toSql.keySet(); 32 | Iterator iter = keys.iterator(); 33 | while (iter.hasNext()) { 34 | String key = iter.next(); 35 | String val = toSql.get(key); 36 | out.println(key + "=" + val); 37 | } 38 | } 39 | 40 | public void loadFromXML(Element fieldnames) { 41 | List children = fieldnames.getChildren("namemap"); 42 | ListIterator iter = children.listIterator(); 43 | while (iter.hasNext()) { 44 | Element namemap = iter.next(); 45 | String glidename = namemap.getAttribute("glidename").getValue(); 46 | String sqlname = namemap.getTextTrim(); 47 | this.toSql.put(glidename, sqlname); 48 | this.toGlide.put(sqlname, glidename); 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/YamlParseValidTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.io.FileNotFoundException; 6 | 7 | import org.junit.After; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.slf4j.Logger; 11 | 12 | import sndml.util.Log; 13 | 14 | public class YamlParseValidTest { 15 | 16 | final Logger logger = TestManager.getLogger(this.getClass()); 17 | final TestingProfile profile = TestManager.getDefaultProfile(); 18 | 19 | @Before 20 | public void setUp() throws Exception { 21 | } 22 | 23 | @After 24 | public void tearDown() throws Exception { 25 | } 26 | 27 | @Test 28 | public void testPass() throws Exception { 29 | TestFolder folder = new TestFolder(this.getClass().getSimpleName()); 30 | ConfigFactory factory = new ConfigFactory(); 31 | for (YamlFile file : folder.yamlFiles()) { 32 | logger.info(Log.TEST, "Testing " + file.getPath() + 33 | "\n" + TestManager.readFully(file).trim()); 34 | try { 35 | YamlLoaderConfig config = factory.loaderConfig(profile, file); 36 | String json = factory.jsonMapper.writeValueAsString(config); 37 | logger.info(Log.TEST, json); 38 | } catch (FileNotFoundException e) { 39 | logger.warn(Log.TEST, e.getMessage()); 40 | e.printStackTrace(); 41 | fail("File not found: " + file.getPath()); 42 | return; 43 | } catch (ConfigParseException e) { 44 | logger.warn(Log.TEST, e.getMessage()); 45 | e.printStackTrace(); 46 | fail(e.getMessage()); 47 | return; 48 | } 49 | } 50 | } 51 | 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/TimestampHashTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import sndml.servicenow.*; 4 | import sndml.util.DateTime; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | import org.junit.After; 9 | import org.junit.AfterClass; 10 | import org.junit.Before; 11 | import org.junit.BeforeClass; 12 | import org.junit.Test; 13 | 14 | public class TimestampHashTest { 15 | 16 | @BeforeClass 17 | public static void setUpBeforeClass() throws Exception { 18 | } 19 | 20 | @AfterClass 21 | public static void tearDownAfterClass() throws Exception { 22 | } 23 | 24 | @Before 25 | public void setUp() throws Exception { 26 | } 27 | 28 | @After 29 | public void tearDown() throws Exception { 30 | } 31 | 32 | @Test 33 | public void testTimestampHash() { 34 | RecordKey key1 = new RecordKey("4715ab62a9fe1981018c3efb96143495"); 35 | RecordKey key2 = new RecordKey("4715ab62a9fe1981018c3efb96143495"); 36 | assertEquals(key1, key2); 37 | RecordKey key3 = new RecordKey("d71da88ac0a801670061eabfe4b28f77"); 38 | assertNotEquals(key1, key3); 39 | DateTime d1 = new DateTime("2015-11-02 20:49:08"); 40 | DateTime d2 = new DateTime("2016-03-24 17:47:36"); 41 | 42 | TimestampHash h = new TimestampHash(); 43 | assertEquals(0, h.size()); 44 | h.put(key1, d1); 45 | assertEquals(1, h.size()); 46 | assertNotNull(h.get(key1)); 47 | assertTrue(h.containsKey(key1)); 48 | assertTrue(h.containsKey(key2)); 49 | assertNotNull(h.get(key2)); 50 | h.put(key3, d2); 51 | assertEquals(h.get(key2), d1); 52 | assertNotNull(h.get(key3)); 53 | assertNotEquals(h.get(key1), h.get(key3)); 54 | } 55 | 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/ShutdownHook.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.Future; 5 | 6 | import org.slf4j.Logger; 7 | 8 | import sndml.loader.Resources; 9 | import sndml.util.Log; 10 | import sndml.util.Metrics; 11 | 12 | public class ShutdownHook extends Thread { 13 | 14 | final WorkerPool workerPool; 15 | final AppStatusLogger statusLogger; 16 | final Resources resources; 17 | 18 | final Logger logger = Log.getLogger(ShutdownHook.class); 19 | 20 | ShutdownHook(Resources resources) { 21 | this.resources = resources; 22 | this.workerPool = resources.getWorkerPool(); 23 | this.statusLogger = new AppStatusLogger(resources.getAppSession()); 24 | this.setName(this.getClass().getSimpleName()); 25 | } 26 | 27 | void cancelAll() { 28 | int count = 0; 29 | List list = workerPool.activeTasks(); 30 | for (WorkerEntry entry : list) { 31 | Future future = entry.future; 32 | if (!future.isCancelled() && !future.isDone()) { 33 | logger.info(Log.FINISH, String.format("Cancel %s", entry.number)); 34 | future.cancel(true); 35 | statusLogger.cancelJob(entry.key, this.getClass().getSimpleName()); 36 | count += 1; 37 | } 38 | } 39 | if (count > 0) 40 | logger.info(Log.FINISH, String.format("%d job(s) cancelled", count)); 41 | else 42 | logger.info(Log.FINISH, "No active jobs"); 43 | } 44 | 45 | @Override 46 | public void run() { 47 | Log.setGlobalContext(); 48 | logger.info(Log.FINISH, "ShutdownHook invoked"); 49 | cancelAll(); 50 | logger.info(Log.FINISH, "ShutdownHook complete"); 51 | Log.shutdown(); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /.github/workflows/jekyll-gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | # push: 7 | # branches: ["master"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | # Build job 26 | build: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | - name: Setup Pages 32 | uses: actions/configure-pages@v5 33 | - name: Build with Jekyll 34 | uses: actions/jekyll-build-pages@v1 35 | with: 36 | source: ./docs 37 | destination: ./_site 38 | - name: Upload artifact 39 | uses: actions/upload-pages-artifact@v3 40 | 41 | # Deployment job 42 | deploy: 43 | environment: 44 | name: github-pages 45 | url: ${{ steps.deployment.outputs.page_url }} 46 | runs-on: ubuntu-latest 47 | needs: build 48 | steps: 49 | - name: Deploy to GitHub Pages 50 | id: deployment 51 | uses: actions/deploy-pages@v4 52 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/SessionVerificationTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.junit.*; 4 | import org.slf4j.Logger; 5 | 6 | import sndml.loader.Resources; 7 | import sndml.loader.TestManager; 8 | 9 | import static org.junit.Assert.*; 10 | 11 | public class SessionVerificationTest { 12 | 13 | Logger logger = TestManager.getLogger(this.getClass()); 14 | 15 | @Test 16 | public void testValidate() throws Exception { 17 | Resources resources = new Resources(TestManager.getDefaultProfile()); 18 | Session session = resources.getReaderSession(); 19 | session.verifyUser(true); 20 | Table user = session.table("sys_user"); 21 | TableWSDL wsdl = user.getWSDL(); 22 | int wsdlCount = wsdl.getReadFieldNames().size(); 23 | logger.info("wsdl fields=" + wsdlCount); 24 | TableSchema schema = resources.getSchemaReader().getSchema(user); 25 | int schemaCount = schema.getFieldNames().size(); 26 | logger.info("schema fields=" + schemaCount); 27 | session.verifyUser(true); 28 | assertEquals(wsdlCount, schemaCount); 29 | } 30 | 31 | // @Test 32 | // public void testAutoVerify() throws Exception { 33 | // Properties props = new Properties(); 34 | // props.setProperty("servicenow.instance", "dev00000"); 35 | // props.setProperty("servicenow.username", "admin"); 36 | // props.setProperty("servicenow.password", "secret"); 37 | // PropertySet propset = new PropertySet(props, "servicenow"); 38 | // Session session1 = new Session(propset); 39 | // assertNotNull(session1); 40 | // props.setProperty("servicenow.verify_session", "true"); 41 | // Session session2 = new Session(propset); 42 | // assertNull(session2); 43 | // } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseUpdateStatement.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.sql.SQLException; 4 | import java.util.HashMap; 5 | 6 | import sndml.servicenow.TableRecord; 7 | 8 | public class DatabaseUpdateStatement extends DatabaseStatement { 9 | 10 | public DatabaseUpdateStatement(DatabaseWrapper db, String sqlTableName, ColumnDefinitions columns) 11 | throws SQLException { 12 | super(db, "update", sqlTableName, columns); 13 | } 14 | 15 | String buildStatement() throws SQLException { 16 | final String fieldSeparator = ",\n"; 17 | StringBuilder fieldmap = new StringBuilder(); 18 | for (int i = 1; i < columns.size(); ++i) { 19 | if (i > 1) fieldmap.append(fieldSeparator); 20 | fieldmap.append(generator.sqlQuote(columns.get(i).getName())); 21 | fieldmap.append("=?"); 22 | } 23 | HashMap map = new HashMap(); 24 | map.put("fieldmap", fieldmap.toString()); 25 | map.put("keyvalue", "?"); 26 | return generator.getTemplate(templateName, sqlTableName, map); 27 | } 28 | 29 | public boolean update(TableRecord rec) throws SQLException { 30 | setRecord(rec); 31 | // Checked when columns is instantiated 32 | // assert columns.get(0).getName().toLowerCase().equals("sys_id"); 33 | int n = columns.size(); 34 | // Skip column 0 which is the sys_id 35 | for (int i = 1; i < n; ++i) { 36 | bindField(i, i); 37 | } 38 | // Bind sys_id to the last position 39 | // bindField(n, columns.get(0), "sys_id", rec.getKey().toString()); 40 | bindField(n, 0); 41 | int count = stmt.executeUpdate(); 42 | if (count > 1) throw new AssertionError("update count=" + count); 43 | return (count > 0); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/SetFieldsTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import sndml.loader.TestManager; 10 | import sndml.util.FieldNames; 11 | import sndml.util.Log; 12 | import sndml.util.Metrics; 13 | import sndml.util.NullProgressLogger; 14 | 15 | public class SetFieldsTest { 16 | 17 | Logger logger = LoggerFactory.getLogger(this.getClass()); 18 | 19 | @Test 20 | public void testSetColumns() throws Exception { 21 | Session session = TestManager.getDefaultProfile().newReaderSession(); 22 | Table incident = session.table("change_request"); 23 | RecordListAccumulator accumulator = new RecordListAccumulator(incident); 24 | TableReader reader = incident.rest().getDefaultReader(); 25 | Metrics metrics = new Metrics(this.getClass().getSimpleName()); 26 | reader.setFields(new FieldNames("number,state,short_description")); 27 | reader.prepare(accumulator, metrics, new NullProgressLogger()); 28 | reader.call(); 29 | int rows = accumulator.getRecords().size(); 30 | assertTrue(rows > 0); 31 | for (TableRecord rec : accumulator.getRecords()) { 32 | assertNotNull(rec.getValue("sys_id")); 33 | assertNotNull(rec.getValue("sys_created_on")); 34 | assertNotNull(rec.getValue("sys_updated_on")); 35 | assertNotNull(rec.getValue("number")); 36 | assertNotNull(rec.getValue("state")); 37 | // assertNotNull(rec.getValue("short_description")); 38 | assertNull(rec.getValue("close_notes")); 39 | assertNull(rec.getValue("assignment_group")); 40 | } 41 | logger.info(Log.TEST, rows + " rows processed"); 42 | 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppStatusPayload.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import sndml.util.DatePartition; 4 | import sndml.util.Metrics; 5 | 6 | public class AppStatusPayload { 7 | 8 | public enum Type { 9 | STATUS, 10 | PROGRESS, 11 | HEARTBEAT 12 | } 13 | 14 | final Type type; 15 | final AppJobRunner runner; 16 | final AppJobStatus status; 17 | final DatePartition datePart; 18 | final Metrics metrics; 19 | 20 | public AppStatusPayload( 21 | Type type, 22 | AppJobRunner runner, 23 | AppJobStatus status, 24 | DatePartition datePart, 25 | Metrics metrics) 26 | { 27 | this.type = type; 28 | this.runner = runner; 29 | this.status = status; 30 | this.datePart = datePart; 31 | this.metrics = metrics; 32 | } 33 | 34 | public AppStatusPayload newHeartBeat() { 35 | return new AppStatusPayload( 36 | Type.HEARTBEAT, null, null, null, null); 37 | } 38 | 39 | public AppStatusPayload newStatusPayload( 40 | AppJobRunner runner, 41 | AppJobStatus status) { 42 | return new AppStatusPayload( 43 | Type.STATUS, runner, status, null, null); 44 | } 45 | 46 | public AppStatusPayload newProgressStatusPayload( 47 | AppJobRunner runner, 48 | AppJobStatus status, 49 | Metrics metrics) { 50 | return new AppStatusPayload(Type.PROGRESS, runner, status, null, metrics); 51 | } 52 | 53 | public AppStatusPayload newPartitionProgressPayload( 54 | AppJobRunner runner, 55 | AppJobStatus status, 56 | DatePartition datePart, 57 | Metrics metrics) { 58 | return new AppStatusPayload(Type.PROGRESS, runner, status, datePart, metrics); 59 | } 60 | 61 | public void process() { 62 | // TODO Implement AppStatusPayload.process() 63 | throw new IllegalStateException(); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppJobConfig.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import com.fasterxml.jackson.core.JsonProcessingException; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.fasterxml.jackson.databind.node.ObjectNode; 8 | 9 | import sndml.loader.ConfigParseException; 10 | import sndml.loader.JobConfig; 11 | import sndml.servicenow.RecordKey; 12 | import sndml.util.Log; 13 | 14 | public class AppJobConfig extends JobConfig { 15 | 16 | // TODO Rename sys_id to runkey. sys_id is ambiguous. 17 | public RecordKey runKey; 18 | public String number; 19 | 20 | static final Logger logger = Log.getLogger(AppJobConfig.class); 21 | 22 | public AppJobConfig() { 23 | super(); 24 | } 25 | 26 | @Override 27 | synchronized protected void updateCoreFields() { 28 | super.updateCoreFields(); 29 | if (jobName == null && number != null) jobName = number; 30 | } 31 | 32 | @Override 33 | synchronized public String toString() { 34 | ObjectMapper mapper = new ObjectMapper(); 35 | ObjectNode node = mapper.createObjectNode(); 36 | if (runKey != null) node.put("sys_id", runKey.toString()); 37 | if (number != null) node.put("number", getNumber()); 38 | addFieldsToObject(node); 39 | String yaml; 40 | try { 41 | yaml = mapper.writeValueAsString(node); 42 | } catch (JsonProcessingException e) { 43 | throw new ConfigParseException(e); 44 | } 45 | return yaml; 46 | } 47 | 48 | public RecordKey getRunKey() { 49 | assert this.runKey != null; 50 | return this.runKey; 51 | } 52 | 53 | public String getNumber() { 54 | assert this.number != null; 55 | return this.number; 56 | } 57 | 58 | @Override 59 | public String getName() { 60 | return getNumber(); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/ScannerJobRunner.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.loader.Resources; 7 | import sndml.util.Log; 8 | import sndml.util.Metrics; 9 | 10 | /** 11 | * An AppJobRunner which performs a re-scan after a job is complete 12 | */ 13 | public class ScannerJobRunner extends AppJobRunner { 14 | 15 | final AgentScanner scanner; // my parent 16 | final boolean onExceptionContinue; 17 | 18 | 19 | public ScannerJobRunner(AgentScanner scanner, Resources resources, AppJobConfig config) { 20 | super(resources, config); 21 | this.scanner = scanner; 22 | onExceptionContinue = Boolean.parseBoolean(profile.getProperty("daemon.continue")); 23 | } 24 | 25 | @Override 26 | public Metrics call() throws JobCancelledException { 27 | String myname = this.getClass().getSimpleName() + ".call"; 28 | logger.debug(Log.INIT, myname + " begin"); 29 | assert profile != null; 30 | assert config.getNumber() != null; 31 | setThreadName(); 32 | try { 33 | super.call(); 34 | } catch (JobCancelledException e) { 35 | logger.error(Log.ERROR, e.getMessage()); 36 | statusLogger.cancelJob(runKey, e); 37 | } catch (SQLException | IOException e) { 38 | Log.setJobContext(this.getName()); 39 | logger.error(Log.ERROR, myname + ": " + e.getClass().getName(), e); 40 | statusLogger.logError(runKey, e); 41 | if (!onExceptionContinue) AgentDaemon.abort(); 42 | } catch (Error e) { 43 | logger.error(Log.ERROR, myname + ": " + e.getClass().getName(), e); 44 | logger.error(Log.ERROR, "Critical error detected. Halting JVM."); 45 | Runtime.getRuntime().halt(-1); 46 | } 47 | logger.debug(Log.FINISH, myname + " end"); 48 | return jobMetrics; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/Action.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.util.EnumSet; 4 | 5 | import com.fasterxml.jackson.annotation.JsonFormat; 6 | 7 | @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_VALUES) 8 | public enum Action { 9 | /** 10 | * Insert new records and generate a warning if a primary key violation occurs. 11 | */ 12 | INSERT, 13 | 14 | /** 15 | * Insert new records and update existing records. 16 | */ 17 | UPDATE, 18 | 19 | /** 20 | * Compare all timestampls and insert, update or delete records accordingly. 21 | */ 22 | SYNC, 23 | 24 | /** 25 | * Delete records based on sys_audit_delete. 26 | */ 27 | PRUNE, 28 | 29 | /** 30 | * Execute an SQL command. 31 | */ 32 | EXECUTE, 33 | 34 | /** 35 | * Synchronize a single record 36 | */ 37 | ROWSYNC, 38 | 39 | /** 40 | * Create a table. 41 | */ 42 | CREATE, 43 | 44 | /** 45 | * Drop a table. Used for JUnit tests. 46 | */ 47 | DROPTABLE, 48 | 49 | /** 50 | * Alias for INSERT. 51 | */ 52 | LOAD, 53 | 54 | /** 55 | * Alias for UPDATE. 56 | */ 57 | REFRESH; 58 | 59 | public static EnumSet INSERT_UPDATE = 60 | EnumSet.of(INSERT, UPDATE, LOAD, REFRESH); 61 | 62 | public static EnumSet INSERT_UPDATE_SYNC = 63 | EnumSet.of(INSERT, UPDATE, SYNC, LOAD, REFRESH); 64 | 65 | public static EnumSet INSERT_UPDATE_PRUNE = 66 | EnumSet.of(INSERT, UPDATE, PRUNE, LOAD, REFRESH); 67 | 68 | public static EnumSet ANY_TABLE_ACTION = 69 | EnumSet.of(INSERT, UPDATE, SYNC, PRUNE, CREATE, DROPTABLE, LOAD, REFRESH, ROWSYNC); 70 | 71 | public static EnumSet EXECUTE_ONLY = 72 | EnumSet.of(EXECUTE); 73 | 74 | public static EnumSet ROWSYNC_ONLY = 75 | EnumSet.of(ROWSYNC); 76 | 77 | } -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppSchemaReader.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import com.fasterxml.jackson.databind.node.ArrayNode; 8 | import com.fasterxml.jackson.databind.node.ObjectNode; 9 | 10 | //import sndml.loader.ConnectionProfile; 11 | import sndml.servicenow.*; 12 | 13 | public class AppSchemaReader implements SchemaReader { 14 | 15 | private final AppSession appSession; 16 | // private final ConnectionProfile profile; 17 | 18 | public AppSchemaReader(AppSession session) { 19 | // this.profile = AgentDaemon.getConnectionProfile(); 20 | this.appSession = session; 21 | } 22 | 23 | @Override 24 | public TableSchema getSchema(Table table) throws IOException { 25 | return getSchema(table.getName()); 26 | } 27 | 28 | @Override 29 | public TableSchema getSchema(String tablename) throws IOException { 30 | assert tablename != null; 31 | Table table = appSession.table(tablename); 32 | TableSchema schema = new TableSchema(table); 33 | URI apiTableSchema = appSession.uriGetTableSchema(tablename); 34 | JsonRequest request = new JsonRequest(appSession, apiTableSchema); 35 | ObjectNode response = request.execute(); 36 | JsonNode result = response.get("result"); 37 | assert result.isArray(); 38 | ArrayNode elements = (ArrayNode) result; 39 | for (JsonNode element : elements) { 40 | String name = element.get("name").asText(); 41 | String type = element.get("type").asText(); 42 | int len = element.get("len").asInt(); 43 | String ref = element.get("ref").asText(); 44 | assert name.length() > 0; 45 | assert type.length() > 0; 46 | assert len > 0; 47 | schema.addField(name, type, len, ref); 48 | } 49 | assert schema.numFields() > 0; 50 | return schema; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/DatePartition.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | /** 4 | * One entry of a {@link DatePartitionSet}. 5 | * Both start and end must be on an {@link PartitionInterval} boundary. 6 | * 7 | */ 8 | public class DatePartition extends DateTimeRange { 9 | 10 | protected final PartitionInterval interval; 11 | 12 | public DatePartition(PartitionInterval interval, DateTime start, DateTime end) { 13 | super(start, end); 14 | this.interval = interval; 15 | assert interval != null; 16 | assert start != null; 17 | assert end != null; 18 | assert start.compareTo(end) < 0; 19 | assert start.equals(start.truncate(interval)) : String.format("DatePart.start=%s", start); 20 | assert end.equals(end.truncate(interval)) : String.format("DatePart end=%s", end); 21 | assert start.incrementBy(interval).equals(end) : 22 | String.format("DatePart start=%s end=%s", start, end); 23 | } 24 | 25 | public String getName() { 26 | return getName(interval, start); 27 | } 28 | 29 | public DateTimeRange getRange() { 30 | return new DateTimeRange(this.start, this.end); 31 | } 32 | 33 | static public String getName(PartitionInterval interval, DateTime start) { 34 | String prefix = (interval.equals(PartitionInterval.FIVE_MINUTE) || interval.equals(PartitionInterval.MINUTE)) ? 35 | "" : interval.toString().substring(0,1); 36 | switch (interval) { 37 | case YEAR: 38 | return prefix + "-" + start.toString().substring(0, 4); 39 | case QUARTER: 40 | case MONTH: 41 | return prefix + "-" + start.toString().substring(0, 7); 42 | case WEEK: 43 | case DAY: 44 | return prefix + start.toString().substring(0, 10); 45 | default: 46 | return prefix + start.toFullString().replaceAll(" ", "T"); 47 | } 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return getName(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/TableSchemaTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import sndml.loader.TestManager; 10 | 11 | public class TableSchemaTest { 12 | 13 | Logger logger = LoggerFactory.getLogger(this.getClass()); 14 | 15 | public TableSchemaTest() { 16 | TestManager.setDefaultProfile(this.getClass()); 17 | } 18 | 19 | @Test 20 | public void testReport() throws Exception { 21 | Session session = TestManager.getProfile().newReaderSession(); 22 | TableSchemaReader factory = new TableSchemaReader(session); 23 | TableSchema schema = factory.getSchema("incident"); 24 | assertFalse(schema.isEmpty()); 25 | assertTrue(schema.contains("sys_id")); 26 | assertTrue(schema.contains("short_description")); 27 | assertTrue(schema.contains("assignment_group")); 28 | assertTrue(schema.contains("parent_incident")); 29 | assertFalse(schema.contains("georgia")); 30 | schema.report(System.out); 31 | } 32 | 33 | @Test 34 | public void testSysTemplate() throws Exception { 35 | Session session = TestManager.getProfile().newReaderSession(); 36 | TableSchemaReader factory = new TableSchemaReader(session); 37 | TableSchema schema = factory.getSchema("sys_template"); 38 | assertFalse(schema.isEmpty()); 39 | assertTrue(schema.contains("table")); 40 | assertTrue(schema.contains("show_on_template_bar")); 41 | } 42 | 43 | @Test(expected = InvalidTableNameException.class) 44 | public void testBadTable() throws Exception { 45 | Session session = TestManager.getProfile().newReaderSession(); 46 | TableSchemaReader factory = new TableSchemaReader(session); 47 | TableSchema schema = factory.getSchema("incidentxx"); 48 | assertTrue(schema.isEmpty()); 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/DateTimeFactoryTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | import java.util.Properties; 5 | import org.junit.Test; 6 | 7 | import sndml.util.DateTime; 8 | 9 | public class DateTimeFactoryTest { 10 | 11 | DateTime myStart = new DateTime("2017-06-15 10:15:00"); 12 | 13 | Properties getProperties() { 14 | Properties props = new Properties(); 15 | props.setProperty("start", "2017-06-14 17:35:35"); 16 | props.setProperty("incident.start", "2017-06-14 17:35:37"); 17 | props.setProperty("incident.finish", "2017-06-14 18:20:20"); 18 | return props; 19 | } 20 | 21 | DateCalculator getFactory() { 22 | return new DateCalculator(myStart, getProperties()); 23 | } 24 | 25 | @Test 26 | public void testDateTimeFactory() throws Exception { 27 | DateCalculator factory = getFactory(); 28 | assertEquals(myStart, factory.getDate("2017-06-15 10:15:00")); 29 | assertEquals(myStart, factory.getDate("start")); 30 | assertEquals(myStart, factory.getDate("START")); 31 | assertEquals(new DateTime("2017-06-15"), factory.getDate("today")); 32 | assertEquals(new DateTime("2017-06-14"), factory.getDate("today-1d")); 33 | assertEquals(new DateTime("2017-06-14"), factory.getDate("Today-1D")); 34 | assertEquals(new DateTime("2017-06-15 10:10:00"), factory.getDate("start-5m")); 35 | assertEquals(new DateTime("2017-06-15 10:10:00"), factory.getDate("-5m")); 36 | assertEquals(new DateTime("2017-06-14 10:15:00"), factory.getDate("-1d")); 37 | assertEquals(new DateTime("2017-06-14 11:15:00"), factory.getDate("-23h")); 38 | assertEquals(new DateTime("2017-06-14 17:35:35"), factory.getDate("last")); 39 | // assertEquals(new DateTime("2017-06-14 17:35:37"), factory.getDate("incident.start")); 40 | // assertEquals(new DateTime("2017-06-14 17:35:37"), factory.getDate("incident.start")); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/TableLoaderTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.junit.runners.Parameterized; 8 | import org.junit.runners.Parameterized.Parameters; 9 | 10 | import sndml.util.Metrics; 11 | 12 | @RunWith(Parameterized.class) 13 | public class TableLoaderTest { 14 | 15 | static class TestParam { 16 | final TestingProfile profile; 17 | final YamlFile yamlFile; 18 | TestParam(TestingProfile p, YamlFile f) { 19 | profile = p; 20 | yamlFile = f; 21 | } 22 | @Override 23 | public String toString() { 24 | return profile.toString() + ":" + yamlFile.toString(); 25 | } 26 | } 27 | 28 | final TestingProfile profile; 29 | final YamlFile yamlFile; 30 | 31 | public TableLoaderTest(TestParam param) { 32 | this.profile = param.profile; 33 | this.yamlFile = new YamlFile(param.yamlFile); 34 | TestManager.setProfile(this.getClass(), profile); 35 | } 36 | 37 | @Parameters(name = "{index}:{0}") 38 | public static TestParam[] getParams() { 39 | TestingProfile[] allProfiles = TestManager.allProfiles(); 40 | TestFolder folder = new TestFolder("TableLoaderTest"); 41 | int size = folder.listFiles().length; 42 | TestParam[] result = new TestParam[size]; 43 | int index = 0; 44 | for (TestingProfile profile : allProfiles) { 45 | for (YamlFile yamlFile : folder.yamlFiles()) { 46 | TestManager.bannerStart(TableLoaderTest.class, "Test", profile, yamlFile); 47 | result[index++] = new TestParam(profile, yamlFile); 48 | } 49 | } 50 | return result; 51 | } 52 | 53 | @Test 54 | public void test() throws Exception { 55 | YamlLoader loader = yamlFile.getLoader(profile); 56 | Metrics metrics = loader.loadTables(); 57 | int processed = metrics.getProcessed(); 58 | assertTrue(processed > 0); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/ServiceNowRequest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.net.URI; 4 | import org.slf4j.Logger; 5 | 6 | import sndml.util.Log; 7 | 8 | import org.apache.http.StatusLine; 9 | import org.apache.http.impl.client.CloseableHttpClient; 10 | 11 | public abstract class ServiceNowRequest { 12 | 13 | final CloseableHttpClient client; 14 | final URI uri; 15 | final HttpMethod method; 16 | protected StatusLine statusLine; 17 | protected int statusCode; 18 | protected String responseContentType; 19 | protected String requestText; 20 | protected String responseText; 21 | 22 | ServiceNowRequest(CloseableHttpClient client, URI uri, HttpMethod method) { 23 | this.client = client; 24 | this.uri = uri; 25 | this.method = method; 26 | } 27 | 28 | public URI getURI() { 29 | return uri; 30 | } 31 | 32 | public StatusLine getStatusLine() { 33 | return statusLine; 34 | } 35 | 36 | public int getStatusCode() { 37 | return statusLine.getStatusCode(); 38 | } 39 | 40 | public String dumpRequestText() { 41 | return requestText; 42 | } 43 | 44 | public String dumpResponseText() { 45 | return responseText; 46 | } 47 | 48 | public String dump() { 49 | StringBuilder text = new StringBuilder(); 50 | text.append(method.toString()); 51 | text.append(" "); 52 | text.append(uri.toString()); 53 | if (statusLine != null) { 54 | text.append("\n"); 55 | text.append(statusLine); 56 | } 57 | if (requestText != null && requestText.length() > 0) { 58 | text.append("\nREQUEST:\n"); 59 | text.append(dumpRequestText()); 60 | } 61 | if (responseText != null && responseText.length() > 0) { 62 | text.append("\nRESPONSE:\n"); 63 | text.append(dumpResponseText()); 64 | } 65 | return text.toString(); 66 | } 67 | 68 | public void logResponseError(Logger logger) { 69 | logger.error(Log.RESPONSE, dump()); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/XmlFormatter.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.jdom2.Document; 4 | import org.jdom2.Element; 5 | import org.jdom2.output.Format; 6 | import org.jdom2.output.XMLOutputter; 7 | 8 | /** 9 | * Thread-safe class used to convert a JDOM Element to a string. 10 | */ 11 | public class XmlFormatter { 12 | 13 | private static ThreadLocal rawFormatter = 14 | new ThreadLocal() { 15 | protected XMLOutputter initialValue() { 16 | return new XMLOutputter(Format.getRawFormat()); 17 | } 18 | }; 19 | 20 | private static ThreadLocal prettyFormatter = 21 | new ThreadLocal() { 22 | protected XMLOutputter initialValue() { 23 | return new XMLOutputter(Format.getPrettyFormat()); 24 | } 25 | }; 26 | 27 | /** 28 | * Returns a JDOM Element pretty formatted as an XML string. 29 | */ 30 | public static String format(Element element) { 31 | return format(element, true); 32 | } 33 | 34 | /** 35 | * Returns a JDOM Document pretty formatted as an XML string. 36 | */ 37 | public static String format(Document document) { 38 | return format(document, true); 39 | } 40 | 41 | /** 42 | * Returns a JDOM Element formatted as an XML string with the option of raw or pretty. 43 | */ 44 | public static String format(Element element, boolean pretty) { 45 | if (element == null) return null; 46 | XMLOutputter formatter = pretty ? 47 | prettyFormatter.get() : rawFormatter.get(); 48 | return formatter.outputString(element); 49 | } 50 | 51 | /** 52 | * Returns a JDOM Document formatted as an XML string with the option of raw or pretty. 53 | */ 54 | public static String format(Document document, boolean pretty) { 55 | if (document == null) return null; 56 | XMLOutputter formatter = pretty ? 57 | prettyFormatter.get() : rawFormatter.get(); 58 | return formatter.outputString(document); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordKey.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.util.Comparator; 4 | import java.util.regex.Pattern; 5 | 6 | /** 7 | * Thin wrapper for a sys_id (GUID). 8 | * This class is used to ensure proper parameter type resolution 9 | * for various methods. 10 | */ 11 | public class RecordKey implements Comparable, Comparator { 12 | 13 | static final Pattern pattern = Pattern.compile("[0-9a-f]{32}"); 14 | static final public int LENGTH = 32; 15 | 16 | final String value; 17 | 18 | public RecordKey(String value) { 19 | assert value != null; 20 | this.value = value; 21 | } 22 | 23 | public String toString() { 24 | return this.value; 25 | } 26 | 27 | @Override 28 | public int compareTo(RecordKey other) { 29 | return this.value.compareTo(other.value); 30 | } 31 | 32 | @Override 33 | public int compare(RecordKey key1, RecordKey key2) { 34 | return key1.value.compareTo(key2.value); 35 | } 36 | 37 | @Override 38 | public boolean equals(Object other) { 39 | if (other == null) return false; 40 | return this.value.equals(other.toString()); 41 | } 42 | 43 | public boolean greaterThan(RecordKey other) { 44 | // Note: Any value is greater than null 45 | if (other == null) return true; 46 | if (this.value.compareTo(other.value) > 0) return true; 47 | return false; 48 | } 49 | 50 | public boolean lessThan(RecordKey other) { 51 | // Note: Any value is less than null 52 | if (other == null) return true; 53 | if (this.value.compareTo(other.value)< 0) return true; 54 | return false; 55 | } 56 | 57 | public int hashCode() { 58 | return this.value.hashCode(); 59 | } 60 | 61 | public boolean isGUID() { 62 | return isGUID(value); 63 | } 64 | 65 | static public boolean isGUID(String v) { 66 | if (v == null) return false; 67 | if (v.length() != LENGTH) return false; 68 | return pattern.matcher(v).matches(); 69 | } 70 | 71 | } 72 | 73 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseDeleteWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.servicenow.RecordKey; 7 | import sndml.servicenow.RecordKeySet; 8 | import sndml.servicenow.TableRecord; 9 | import sndml.util.Log; 10 | import sndml.util.Metrics; 11 | import sndml.util.ProgressLogger; 12 | import sndml.servicenow.Table; 13 | 14 | public class DatabaseDeleteWriter extends DatabaseTableWriter { 15 | 16 | protected DatabaseDeleteStatement deleteStmt; 17 | 18 | public DatabaseDeleteWriter(DatabaseWrapper db, Table table, String sqlTableName, String writerName) 19 | throws IOException, SQLException { 20 | super(db, table, sqlTableName, writerName); 21 | } 22 | 23 | @Override 24 | public DatabaseDeleteWriter open(Metrics writerMetrics) throws SQLException, IOException { 25 | super.open(writerMetrics); 26 | deleteStmt = new DatabaseDeleteStatement(this.db, this.sqlTableName); 27 | // progressLogger.setOperation("Deleted"); 28 | return this; 29 | } 30 | 31 | @Override 32 | void writeRecord(TableRecord rec, Metrics writerMetrics) throws SQLException { 33 | assert rec.getTable().getName().equals("sys_audit_delete"); 34 | RecordKey key = rec.getKey("documentkey"); 35 | assert key != null; 36 | deleteRecord(key, writerMetrics); 37 | } 38 | 39 | void deleteRecords(RecordKeySet keys, Metrics writerMetrics, ProgressLogger progressLogger) 40 | throws SQLException { 41 | for (RecordKey key : keys) { 42 | deleteRecord(key, writerMetrics); 43 | } 44 | db.commit(); 45 | } 46 | 47 | void deleteRecord(RecordKey key, Metrics writerMetrics) throws SQLException { 48 | logger.info(Log.PROCESS, "Delete " + key); 49 | if (deleteStmt.deleteRecord(key)) { 50 | writerMetrics.incrementDeleted(); 51 | } 52 | else { 53 | logger.warn(Log.PROCESS, "Delete: Not found: " + key); 54 | writerMetrics.incrementSkipped(); 55 | } 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseInsertWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.sql.SQLIntegrityConstraintViolationException; 6 | import java.util.regex.Pattern; 7 | 8 | import sndml.servicenow.RecordKey; 9 | import sndml.servicenow.TableRecord; 10 | import sndml.util.Log; 11 | import sndml.util.Metrics; 12 | import sndml.servicenow.Table; 13 | 14 | public class DatabaseInsertWriter extends DatabaseTableWriter { 15 | 16 | protected DatabaseInsertStatement insertStmt; 17 | 18 | public DatabaseInsertWriter(DatabaseWrapper db, Table table, String sqlTableName, String writerName) 19 | throws IOException, SQLException { 20 | super(db, table, sqlTableName, writerName); 21 | } 22 | 23 | @Override 24 | public DatabaseInsertWriter open(Metrics writerMetrics) 25 | throws SQLException, IOException { 26 | super.open(writerMetrics); 27 | insertStmt = new DatabaseInsertStatement(this.db, this.sqlTableName, columns); 28 | return this; 29 | } 30 | 31 | Pattern primaryKeyViolation = 32 | Pattern.compile("\\b(primary key|unique constraint)\\b", Pattern.CASE_INSENSITIVE); 33 | 34 | @Override 35 | void writeRecord(TableRecord rec, Metrics writerMetrics) throws SQLException { 36 | RecordKey key = rec.getKey(); 37 | logger.debug(Log.PROCESS, "Insert " + key); 38 | try { 39 | insertStmt.insert(rec); 40 | writerMetrics.incrementInserted(); 41 | } 42 | catch (SQLIntegrityConstraintViolationException e) { 43 | logger.debug(Log.PROCESS, e.getClass().getName() + ": " + e.getMessage()); 44 | logger.warn(Log.PROCESS, "Failed/Skipped " + key); 45 | writerMetrics.incrementSkipped(); 46 | } 47 | catch (SQLException e) { 48 | logger.debug(Log.PROCESS, e.getClass().getName() + ": " + e.getMessage()); 49 | if (primaryKeyViolation.matcher(e.getMessage()).find()) { 50 | logger.warn(Log.PROCESS, "Failed/Skipped " + key); 51 | writerMetrics.incrementSkipped(); 52 | } 53 | else { 54 | throw e; 55 | } 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/SingleThreadScanner.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | 7 | import sndml.loader.ConfigParseException; 8 | import sndml.loader.Resources; 9 | import sndml.util.Log; 10 | 11 | /** 12 | * An Agent Scanner which does not utilize a thread pool and runs all jobs in the current thread. 13 | */ 14 | public class SingleThreadScanner extends AgentScanner { 15 | 16 | public SingleThreadScanner(Resources resources) { 17 | super(resources); 18 | } 19 | 20 | @Override 21 | protected int getErrorLimit() { 22 | return 0; 23 | } 24 | 25 | @Override 26 | public void scanUntilDone() throws IOException, ConfigParseException, SQLException, InterruptedException { 27 | String myname = this.getClass().getSimpleName() + ".scanUntilDone"; 28 | logger.debug(Log.INIT, String.format("%s begin %s", myname, agentName)); 29 | int jobcount; 30 | do { 31 | jobcount = scanOnce(); 32 | if (jobcount > 0) Thread.sleep(rescanDelayMillisec); 33 | } 34 | while (jobcount > 0); 35 | logger.debug(Log.FINISH, String.format("%s end", myname)); 36 | } 37 | 38 | /** 39 | * Run all jobs that are ready. Return the number of jobs run. 40 | * Note that when this function exits there may be new jobs ready to run, 41 | * but his function will not run them. 42 | * @throws SQLException 43 | */ 44 | @Override 45 | public int scanOnce() throws IOException, ConfigParseException, SQLException, InterruptedException { 46 | String myname = this.getClass().getSimpleName() + ".scanOnce"; 47 | Log.setJobContext(agentName); 48 | logger.debug(Log.INIT, String.format("%s begin %s", myname, agentName)); 49 | ArrayList joblist = getJobList(); 50 | if (joblist.size() > 0) { 51 | // Run the jobs one at a time 52 | for (AppJobRunner job : joblist) { 53 | logger.info(Log.INIT, "Running job " + job.number); 54 | job.call(); 55 | } 56 | Log.setGlobalContext(); 57 | } 58 | int result = joblist.size(); 59 | logger.debug(Log.FINISH, String.format("%s end %d", myname, result)); 60 | return result; 61 | 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/RowCountExceptionTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.AfterClass; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.junit.runners.Parameterized; 9 | import org.junit.runners.Parameterized.Parameters; 10 | import org.slf4j.Logger; 11 | 12 | import sndml.servicenow.*; 13 | 14 | @RunWith(Parameterized.class) 15 | public class RowCountExceptionTest { 16 | 17 | final TestingProfile profile; 18 | final TestFolder folder = new TestFolder(this.getClass()); 19 | final Logger logger = TestManager.getLogger(this.getClass()); 20 | 21 | @Parameters(name = "{index}:{0}") 22 | public static TestingProfile[] profiles() { 23 | return TestManager.getDatamartProfiles(); 24 | } 25 | 26 | @AfterClass 27 | public static void clear() throws Exception { 28 | TestManager.clearAll(); 29 | } 30 | 31 | public RowCountExceptionTest(TestingProfile profile) { 32 | TestManager.setProfile(this.getClass(), profile); 33 | this.profile = profile; 34 | } 35 | 36 | @Test(expected = TooManyRowsException.class) 37 | public void testTooManyRows() throws Exception { 38 | YamlFile yamlFile = folder.getYaml("too-many-rows"); 39 | TestManager.bannerStart(this.getClass(), "testTooManyRows", yamlFile); 40 | YamlLoader loader = yamlFile.getLoader(profile); 41 | JobRunner job = loader.getJob("incident_load"); 42 | assertNotNull(job); 43 | assertNotNull(job.getConfig()); 44 | Integer maxRows = job.getConfig().getMaxRows(); 45 | assertNotNull(maxRows); 46 | assertTrue(maxRows < 500); 47 | loader.loadTables(); 48 | fail("Exception not detected"); 49 | } 50 | 51 | @Test(expected = TooFewRowsException.class) 52 | public void testTooFewRows() throws Exception { 53 | YamlFile yamlFile = folder.getYaml("too-few-rows"); 54 | TestManager.bannerStart(this.getClass(), "testTooFewRows", yamlFile); 55 | YamlLoader loader = yamlFile.getLoader(profile); 56 | JobRunner job = loader.getJob("user_load"); 57 | assertNotNull(job); 58 | assertNotNull(job.getConfig()); 59 | Integer minRows = job.getConfig().getMinRows(); 60 | assertNotNull(minRows); 61 | assertTrue(minRows > 0); 62 | loader.loadTables(); 63 | fail("Exception not detected"); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/GetRunListRequest.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import com.fasterxml.jackson.databind.node.ArrayNode; 8 | import com.fasterxml.jackson.databind.node.ObjectNode; 9 | 10 | import sndml.loader.ConfigParseException; 11 | import sndml.servicenow.HttpMethod; 12 | import sndml.servicenow.JsonRequest; 13 | import sndml.servicenow.RecordKey; 14 | import sndml.util.Log; 15 | 16 | public class GetRunListRequest extends JsonRequest { 17 | 18 | final String agentName; 19 | final AppSession appSession; 20 | 21 | public GetRunListRequest(AppSession appSession, String agentName) { 22 | super(appSession, appSession.uriGetJobRunList(), HttpMethod.GET, null); 23 | this.appSession = appSession; 24 | this.agentName = agentName; 25 | } 26 | 27 | public RecordKey getAgentKey() throws IOException { 28 | if (!this.executed) this.execute(); 29 | if (logger.isDebugEnabled()) logger.debug(Log.RESPONSE, responseObj.toPrettyString()); 30 | String agent_sysid = this.getResult().get("agent").asText(); 31 | assert agent_sysid != null; 32 | assert agent_sysid.length() == 32; 33 | RecordKey result = new RecordKey(agent_sysid); 34 | return result; 35 | } 36 | 37 | public ArrayNode getRunList() throws IOException, ConfigParseException { 38 | ArrayNode runlist = null; 39 | Log.setJobContext(agentName); 40 | if (!this.executed) this.execute(); 41 | if (logger.isDebugEnabled()) logger.debug(Log.RESPONSE, responseObj.toPrettyString()); 42 | ObjectNode objResult = this.getResult(); 43 | if (objResult.has("runs")) { 44 | runlist = (ArrayNode) objResult.get("runs"); 45 | } 46 | if (runlist == null || runlist.size() == 0) { 47 | logger.debug(Log.INIT, "Nothing ready"); 48 | } 49 | else { 50 | logger.debug(Log.INIT, "Runlist=" + getNumbers(runlist)); 51 | } 52 | return runlist; 53 | } 54 | 55 | private static String getNumbers(ArrayNode runlist) { 56 | ArrayList numbers = new ArrayList(); 57 | for (JsonNode node : runlist) { 58 | assert node.isObject(); 59 | ObjectNode obj = (ObjectNode) node; 60 | numbers.add(obj.get("number").asText()); 61 | } 62 | return String.join(",", numbers); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/YamlFile.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | 7 | import sndml.util.DateTime; 8 | import sndml.util.ResourceException; 9 | 10 | /** 11 | * A file with YAML Loader Config instructions. 12 | * This class is used primarily for JUnit tests. 13 | */ 14 | @SuppressWarnings("serial") 15 | public class YamlFile extends File { 16 | 17 | public YamlFile(File file) { 18 | super(file.getPath()); 19 | assert this.exists() : file.getPath() + " does not exist"; 20 | assert this.canRead() : file.getPath() + " is not readable"; 21 | assert hasYamlExt(this) : file.getPath() + " does not have .yaml extension"; 22 | } 23 | 24 | public JobConfig getJobConfig(ConnectionProfile profile) 25 | throws ConfigParseException, IOException { 26 | ConfigFactory factory = new ConfigFactory(); 27 | JobConfig config = factory.yamlJob(profile, this); 28 | return config; 29 | } 30 | 31 | public JobRunner getJobRunner(Resources resources) 32 | throws ConfigParseException, IOException { 33 | JobFactory jf = new JobFactory(resources, DateTime.now()); 34 | return jf.yamlJob(this); 35 | } 36 | 37 | public YamlLoader getLoader(Resources resources) throws ConfigParseException, IOException { 38 | ConfigFactory factory = new ConfigFactory(); 39 | YamlLoaderConfig config = factory.loaderConfig(null, this); 40 | return new YamlLoader(resources, config); 41 | } 42 | 43 | public YamlLoader getLoader(ConnectionProfile profile) 44 | throws ConfigParseException, IOException, ResourceException, SQLException { 45 | // ConfigFactory factory = new ConfigFactory(); 46 | // YamlLoaderConfig config = factory.loaderConfig(null, this); 47 | // return new YamlLoader(new Resources(profile), config); 48 | return getLoader(new Resources(profile)); 49 | } 50 | 51 | /** 52 | * Return the name of this file to be displayed when running JUnit tests. 53 | */ 54 | @Override 55 | public String toString() { 56 | return this.getName(); 57 | } 58 | 59 | /** 60 | * Return true if a file has a ".yaml" or ".yml" file extension. 61 | */ 62 | public static boolean hasYamlExt(File file) { 63 | String path = file.getPath(); 64 | return path.endsWith(".yaml") || path.endsWith(".yml"); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/ColumnsTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.net.URISyntaxException; 6 | import java.sql.SQLException; 7 | 8 | import org.junit.AfterClass; 9 | import org.junit.Test; 10 | 11 | import sndml.util.Metrics; 12 | 13 | public class ColumnsTest { 14 | 15 | final TestingProfile profile; 16 | 17 | public ColumnsTest() throws SQLException, URISyntaxException { 18 | profile = TestManager.getDefaultProfile(); 19 | TestManager.setProfile(this.getClass(), profile); 20 | } 21 | 22 | @AfterClass 23 | public static void clear() throws Exception { 24 | TestManager.clearAll(); 25 | } 26 | 27 | @Test 28 | public void test() throws Exception { 29 | TestFolder folder = new TestFolder(this.getClass().getSimpleName()); 30 | Resources resources = new Resources(profile); 31 | DatabaseWrapper database = resources.getDatabaseWrapper(); 32 | DBUtil util = new DBUtil(database); 33 | YamlLoader loader1 = folder.getYaml("incident-include-columns").getLoader(profile); 34 | Metrics metrics1 = loader1.loadTables(); 35 | int processed = metrics1.getProcessed(); 36 | assertTrue(processed > 0); 37 | assertEquals(processed, util.sqlCount("select count(*) from incident")); 38 | assertEquals(processed, util.sqlCount("select count(*) from incident where short_description is null")); 39 | assertEquals(processed, util.sqlCount("select count(*) from incident where state is not null")); 40 | YamlLoader loader2 = folder.getYaml("load-incident-truncate").getLoader(profile); 41 | loader2.loadTables(); 42 | assertEquals(processed, util.sqlCount("select count(*) from incident")); 43 | assertEquals(processed, util.sqlCount("select count(*) from incident where short_description is not null")); 44 | assertEquals(processed, util.sqlCount("select count(*) from incident where state is not null")); 45 | /* 46 | // "exclude" is no longer supported! 47 | 48 | Loader loader3 = folder.getYaml("incident-exclude-columns").getLoader(profile); 49 | loader3.loadTables(); 50 | assertEquals(processed, util.sqlCount("select count(*) from incident")); 51 | assertEquals(processed, util.sqlCount("select count(*) from incident where short_description is null")); 52 | assertEquals(processed, util.sqlCount("select count(*) from incident where state is not null")); 53 | */ 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/Parameters.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.LinkedHashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 9 | import com.fasterxml.jackson.databind.node.ObjectNode; 10 | 11 | import sndml.servicenow.RecordKey; 12 | 13 | import org.apache.http.NameValuePair; 14 | import org.apache.http.message.BasicNameValuePair; 15 | 16 | /** 17 | * This class holds a list of name/value pairs. 18 | */ 19 | public class Parameters extends LinkedHashMap { 20 | 21 | private static final long serialVersionUID = 1L; 22 | 23 | public Parameters() { 24 | super(); 25 | } 26 | 27 | public Parameters(Parameters params) { 28 | super(); 29 | if (params != null) this.putAll(params); 30 | } 31 | 32 | public Parameters(String name, String value) { 33 | super(); 34 | super.put(name, value); 35 | } 36 | 37 | public Parameters(List list) { 38 | super(); 39 | for (NameValuePair nvp : list) { 40 | super.put(nvp.getName(), nvp.getValue()); 41 | } 42 | } 43 | 44 | public void add(String name, String value) { 45 | super.put(name, value); 46 | } 47 | 48 | public void add(Parameters params) { 49 | if (params == null) return; 50 | super.putAll(params); 51 | } 52 | 53 | public boolean contains(String name) { 54 | return super.containsKey(name); 55 | } 56 | 57 | public RecordKey getSysId() { 58 | String sysid = get("sys_id"); 59 | return (sysid == null) ? null : new RecordKey(sysid); 60 | } 61 | 62 | public String getNumber() { 63 | return get("number"); 64 | } 65 | 66 | public void add(NameValuePair nvp) { 67 | super.put(nvp.getName(), nvp.getValue()); 68 | } 69 | 70 | public List nvpList() { 71 | List result = new ArrayList(this.size()); 72 | for (Map.Entry entry : this.entrySet()) { 73 | NameValuePair nvp = new BasicNameValuePair(entry.getKey(), entry.getValue()); 74 | result.add(nvp); 75 | } 76 | return result; 77 | } 78 | 79 | public ObjectNode toJSON() { 80 | ObjectNode node = JsonNodeFactory.instance.objectNode(); 81 | for (Map.Entry entry : this.entrySet()) { 82 | node.put(entry.getKey(), entry.getValue()); 83 | } 84 | return node; 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/CompositeProgressLogger.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import sndml.agent.AppProgressLogger; 4 | import sndml.agent.JobCancelledException; 5 | import sndml.util.DatePartition; 6 | import sndml.util.Metrics; 7 | import sndml.util.ProgressLogger; 8 | 9 | public class CompositeProgressLogger extends ProgressLogger { 10 | 11 | public final Log4jProgressLogger textLogger; 12 | public final AppProgressLogger appLogger; 13 | 14 | public CompositeProgressLogger(Log4jProgressLogger textLogger, AppProgressLogger appLogger) { 15 | super(textLogger.getMetrics(), textLogger.getPart()); 16 | this.textLogger = textLogger; 17 | this.appLogger = appLogger; 18 | } 19 | 20 | // @Deprecated 21 | // public CompositeProgressLogger(TableReader reader, Action action, AppProgressLogger appLogger) { 22 | // this(reader, action, appLogger, null); 23 | // } 24 | // 25 | // @Deprecated 26 | // public CompositeProgressLogger(TableReader reader, Action action, 27 | // AppProgressLogger appLogger, DatePart part) { 28 | // super(appLogger.getMetrics(), part); 29 | // this.textLogger = new Log4jProgressLogger(reader, action); 30 | // this.appLogger = appLogger; 31 | // } 32 | 33 | @Override 34 | public CompositeProgressLogger newPartLogger(Metrics newMetrics, DatePartition newPart) { 35 | Log4jProgressLogger newTextLogger = 36 | (Log4jProgressLogger) textLogger.newPartLogger(newMetrics, newPart); 37 | AppProgressLogger newAppLogger = 38 | (AppProgressLogger) appLogger.newPartLogger(newMetrics, newPart); 39 | return new CompositeProgressLogger(newTextLogger, newAppLogger); 40 | } 41 | 42 | @Override 43 | public void logPrepare() { 44 | if (appLogger != null) appLogger.logPrepare(); 45 | if (textLogger != null) textLogger.logPrepare(); 46 | } 47 | 48 | @Override 49 | public void logStart() throws JobCancelledException { 50 | if (appLogger != null) appLogger.logStart(); 51 | if (textLogger != null) textLogger.logStart(); 52 | } 53 | 54 | @Override 55 | public void logProgress() throws JobCancelledException { 56 | if (appLogger != null) appLogger.logProgress(); 57 | if (textLogger != null) textLogger.logProgress(); 58 | } 59 | 60 | 61 | @Override 62 | public void logComplete() { 63 | if (appLogger != null) appLogger.logComplete(); 64 | if (textLogger != null) textLogger.logComplete(); 65 | 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/JsonRecord.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.util.Iterator; 4 | 5 | import com.fasterxml.jackson.databind.JsonNode; 6 | import com.fasterxml.jackson.databind.node.JsonNodeType; 7 | import com.fasterxml.jackson.databind.node.ObjectNode; 8 | 9 | import sndml.util.FieldNames; 10 | 11 | public class JsonRecord extends TableRecord { 12 | 13 | final ObjectNode root; 14 | 15 | public JsonRecord(Table table, ObjectNode obj) { 16 | super(table); 17 | this.root = obj; 18 | } 19 | 20 | @Override 21 | public String getValue(String fieldname) { 22 | JsonNode node = root.get(fieldname); 23 | if (node == null) return null; 24 | JsonNodeType nodetype = node.getNodeType(); 25 | switch (nodetype) { 26 | case MISSING: 27 | case NULL: 28 | return null; 29 | case STRING: 30 | String value = node.asText(); 31 | if (value.length() == 0) return null; 32 | return value; 33 | case OBJECT: 34 | if (node.has("value")) { 35 | value = node.get("value").asText(); 36 | if (value.length() == 0) return null; 37 | return value; 38 | } 39 | // fall through to error 40 | default: 41 | String msg = table.getName() + 42 | "." + this.getKey() + "." + fieldname + 43 | " type is " + nodetype.toString(); 44 | throw new JsonResponseError(msg); 45 | } 46 | } 47 | 48 | @Override 49 | public String getDisplayValue(String fieldname) { 50 | JsonNode node = root.get(fieldname); 51 | // REST Table API 52 | if (node.isObject()) { 53 | return node.get("display_value").asText(); 54 | } 55 | // JSONv2 API 56 | JsonNode dvNode = root.get("dv_" + fieldname); 57 | if (dvNode != null) { 58 | return dvNode.asText(); 59 | } 60 | return null; 61 | } 62 | 63 | @Override 64 | public Iterator keys() { 65 | return root.fieldNames(); 66 | } 67 | 68 | @Override 69 | public FieldNames getFieldNames() { 70 | FieldNames names = new FieldNames(); 71 | Iterator iter = root.fieldNames(); 72 | while (iter.hasNext()) { 73 | names.add(iter.next()); 74 | } 75 | return names; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return root.toString(); 81 | } 82 | 83 | @Override 84 | public String asText(boolean pretty) { 85 | if (pretty) 86 | return root.toPrettyString(); 87 | else 88 | return root.toString(); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/PropertiesEditor.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import java.io.IOException; 4 | import java.util.Properties; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | 8 | import org.apache.commons.io.IOUtils; 9 | import org.apache.commons.text.StringSubstitutor; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | public class PropertiesEditor { 14 | 15 | private static final Logger logger = LoggerFactory.getLogger(PropertiesEditor.class); 16 | 17 | /** 18 | * Substitute environment variables found in property values 19 | */ 20 | public Properties replaceEnvVariables(Properties props) { 21 | StringSubstitutor envMap = 22 | new org.apache.commons.text.StringSubstitutor(System.getenv()); 23 | for (String name : props.stringPropertyNames()) { 24 | String value = props.getProperty(name); 25 | String newValue = envMap.replace(value); 26 | if (!newValue.equals(value)) props.setProperty(name, newValue); 27 | } 28 | return props; 29 | } 30 | 31 | /** 32 | * If property value is enclosed in backtics then evaluate as a command 33 | */ 34 | public Properties replaceCommands(Properties props) throws IOException { 35 | final Pattern cmdPattern = Pattern.compile("^`(.+)`$"); 36 | for (String name : props.stringPropertyNames()) { 37 | String value = props.getProperty(name); 38 | // If property is in backticks then evaluate as a command 39 | Matcher cmdMatcher = cmdPattern.matcher(value); 40 | if (cmdMatcher.matches()) { 41 | logger.info(Log.INIT, "evaluate " + name); 42 | String command = cmdMatcher.group(1); 43 | value = evaluate(command); 44 | if (value == null || value.length() == 0) 45 | throw new AssertionError(String.format("Failed to evaluate \"%s\"", command)); 46 | logger.debug(Log.INIT, value); 47 | props.setProperty(name, value); 48 | } 49 | } 50 | return props; 51 | } 52 | 53 | /** 54 | * Pass a string to Runtime.exec() for evaluation 55 | * @param command - Command to be executed 56 | * @return Result of command with whitespace trimmed 57 | * @throws IOException 58 | */ 59 | public static String evaluate(String command) throws IOException { 60 | Process p = Runtime.getRuntime().exec(command); 61 | String output = IOUtils.toString(p.getInputStream(), "UTF-8").trim(); 62 | return output; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordList.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.util.*; 4 | 5 | import com.fasterxml.jackson.databind.node.ArrayNode; 6 | import com.fasterxml.jackson.databind.node.ObjectNode; 7 | 8 | /** 9 | * An array of Records. 10 | */ 11 | public class RecordList extends ArrayList { 12 | 13 | private static final long serialVersionUID = 1L; 14 | 15 | final protected Table table; 16 | 17 | public RecordList(Table table) { 18 | super(); 19 | this.table = table; 20 | } 21 | 22 | public RecordList(Table table, int size) { 23 | super(size); 24 | this.table = table; 25 | } 26 | 27 | public RecordList(Table table, ArrayNode array) { 28 | this(table, array.size()); 29 | for (int i = 0; i < array.size(); ++i) { 30 | ObjectNode entry = (ObjectNode) array.get(i); 31 | JsonRecord rec = new JsonRecord(table, entry); 32 | this.add(rec); 33 | } 34 | } 35 | 36 | public RecordIterator iterator() { 37 | return new RecordIterator(this); 38 | } 39 | 40 | public RecordKey maxKey() { 41 | RecordKey result = null; 42 | for (TableRecord rec : this) { 43 | RecordKey key = rec.getKey(); 44 | if (result == null || key.greaterThan(result)) result = key; 45 | } 46 | return result; 47 | } 48 | 49 | public RecordKey minKey() { 50 | RecordKey result = null; 51 | for (TableRecord rec : this) { 52 | RecordKey key = rec.getKey(); 53 | if (result == null || key.lessThan(result)) result = key; 54 | } 55 | return result; 56 | } 57 | 58 | /** 59 | * Extract all the values of a reference field from a list of records. 60 | * Null keys are not included in the list. 61 | * This function is private since it does not remove duplicates. 62 | * @param fieldname Name of a reference field 63 | * @return A list keys 64 | */ 65 | private RecordKeySet extractKeys(String fieldname) { 66 | RecordKeySet result = new RecordKeySet(this.size()); 67 | if (this.size() == 0) return result; 68 | for (TableRecord rec : this) { 69 | String value = rec.getValue(fieldname); 70 | if (value != null) { 71 | assert RecordKey.isGUID(value); 72 | result.add(new RecordKey(value)); 73 | } 74 | } 75 | return result; 76 | } 77 | 78 | /** 79 | * Extract the primary keys (sys_ids) from this list. 80 | * @return list of values as a @{link KeySet} 81 | */ 82 | public RecordKeySet extractKeys() { 83 | return extractKeys("sys_id"); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/Instance.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.net.MalformedURLException; 4 | import java.net.URI; 5 | import java.net.URISyntaxException; 6 | import java.net.URL; 7 | import java.util.Properties; 8 | 9 | import org.apache.http.HttpHost; 10 | import org.apache.http.client.utils.URIBuilder; 11 | 12 | import sndml.util.Parameters; 13 | 14 | /** 15 | * Holds the URL for a ServiceNow instance. 16 | * Does not hold connection credentials, cookies or session ID. 17 | */ 18 | public class Instance { 19 | 20 | final HttpHost host; 21 | final URL url; 22 | 23 | public Instance(URL url) { 24 | this.url = url; 25 | this.host = new HttpHost(url.getHost()); 26 | } 27 | 28 | public Instance(Properties props) { 29 | this(props.getProperty("instance")); 30 | } 31 | 32 | public Instance(String name) { 33 | assert name != null; 34 | assert name.length() > 0; 35 | try { 36 | this.url = getURL(name); 37 | } catch (MalformedURLException e) { 38 | throw new ServiceNowError(e); 39 | } 40 | this.host = new HttpHost(url.getHost()); 41 | } 42 | 43 | private URL getURL(String name) throws MalformedURLException { 44 | if (name == null || name.length() == 0) 45 | throw new AssertionError("Instance URL or name not provided"); 46 | if (name.matches("[\\w-]+")) { 47 | // name is the instance name; build the URL 48 | return new URL("https://" + name + ".service-now.com/"); 49 | } 50 | if (name.startsWith("https://")) { 51 | // name is the the full URL 52 | // make sure it ends with a slash 53 | if (!name.endsWith("/")) name += "/"; 54 | return new URL(name); 55 | } 56 | throw new AssertionError("Instance URL not valid: " + name); 57 | } 58 | 59 | public URI getURI(String path) { 60 | return getURI(path, null); 61 | } 62 | 63 | public URI getURI(String path, Parameters params) { 64 | assert path != null; 65 | assert path.length() > 0; 66 | URI result; 67 | try { 68 | String base = url.toString() + path; 69 | URIBuilder builder = new URIBuilder(base); 70 | if (params != null) builder.addParameters(params.nvpList()); 71 | result = builder.build(); 72 | } 73 | catch (URISyntaxException e) { 74 | throw new ServiceNowError(e); 75 | } 76 | return result; 77 | } 78 | 79 | public URL getURL() { 80 | return this.url; 81 | } 82 | 83 | public HttpHost getHost() { 84 | return this.host; 85 | } 86 | 87 | public String toString() { 88 | return url.toString(); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/FieldDefinition.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import sndml.util.FieldNames; 4 | 5 | /** 6 | * Contains data type information for a single field in a ServiceNow table. 7 | * @author Giles Lewis 8 | * 9 | */ 10 | public class FieldDefinition { 11 | 12 | private final Table table; 13 | private final String name; 14 | private final String type; 15 | private final Integer max_length; 16 | private final String ref_table; 17 | public static final FieldNames DICT_FIELDS = new FieldNames("element,internal_type,max_length,reference"); 18 | 19 | /** 20 | * Construct a FieldDefinition from sys_dictionary record. 21 | * 22 | * @param table - The table in which this field appears. 23 | * @param dictrec - The sys_dictionary record that describes this field. 24 | */ 25 | protected FieldDefinition(Table table, TableRecord dictrec) { 26 | this(table, dictrec.getValue("element"), dictrec.getValue("internal_type"), 27 | dictrec.getInteger("max_length"), dictrec.getValue("reference")); 28 | } 29 | 30 | public FieldDefinition(Table table, String name, String type, Integer len, String ref) { 31 | if (name == null) 32 | throw new AssertionError(String.format( 33 | "Missing name for field in \"%s\". Check sys_dictionary read permissions.", 34 | table.getName())); 35 | if (type == null) 36 | throw new AssertionError(String.format( 37 | "Field \"%s.%s\" has no type. Check sys_dictionary read permissions.", 38 | table.getName(), name)); 39 | this.table = table; 40 | this.name = name; 41 | this.type = type; 42 | this.max_length = len; 43 | this.ref_table = ref; 44 | } 45 | 46 | /** 47 | * Return the table 48 | */ 49 | public Table getTable() { 50 | return table; 51 | } 52 | 53 | /** 54 | * Return the name of this field. 55 | */ 56 | public String getName() { 57 | return name; 58 | } 59 | 60 | /** 61 | * Return the type of this field. 62 | */ 63 | public String getType() { 64 | return type; 65 | } 66 | 67 | /** 68 | * Return the length of this field. 69 | */ 70 | public int getLength() { 71 | return max_length; 72 | } 73 | 74 | /** 75 | * If this is a reference field then return the name of the 76 | * referenced table. Otherwise return null. 77 | */ 78 | public String getReference() { 79 | return ref_table; 80 | } 81 | 82 | /** 83 | * Return true if the field is a reference field. 84 | * The value of a reference field is always a {@link RecordKey} (sys_id). 85 | */ 86 | public boolean isReference() { 87 | return (ref_table != null && ref_table.length() > 0); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AgentMain.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import org.apache.commons.cli.CommandLine; 4 | import org.slf4j.Logger; 5 | 6 | import sndml.loader.ConnectionProfile; 7 | import sndml.loader.Main; 8 | import sndml.loader.Resources; 9 | import sndml.servicenow.RecordKey; 10 | import sndml.util.Log; 11 | 12 | //Q: Why does this class exist? 13 | //A: Because many of the classes in the package are not public. 14 | 15 | public class AgentMain extends Main { 16 | 17 | static private RecordKey agentKey; 18 | static final Logger logger = Log.getLogger(AgentMain.class); 19 | 20 | /** 21 | * Called from {@link Main#main()} 22 | */ 23 | public static void main(CommandLine cmd, Resources resources) throws Exception { 24 | // Note: resources is actually a static protected variable in Main; 25 | // thus we could access it even if it were not a parameter 26 | assert resources != null; 27 | // agentName is a static variable defined in Main 28 | if (agentName == null) 29 | throw new AssertionError( 30 | "No value for property: " + ConnectionProfile.APP_AGENT_PROPERTY); 31 | AppSession appSession = resources.getAppSession(); 32 | agentKey = appSession.getAgentKey(); 33 | assert agentKey != null; 34 | 35 | if (cmd.hasOption(optScan)) { 36 | // Scan once 37 | AgentDaemon daemon = new AgentDaemon(resources); 38 | logger.info(Log.INIT, "Scanning agent: " + AgentDaemon.getAgentName()); 39 | daemon.scanUntilDone(); 40 | } 41 | else if (cmd.hasOption(optDaemon)) { 42 | // Scan forever 43 | AgentDaemon daemon = new AgentDaemon(resources); 44 | logger.info(Log.INIT, "Starting daemon: " + AgentDaemon.getAgentName()); 45 | daemon.run(); 46 | } 47 | else if (cmd.hasOption(optJobRun)) { 48 | // Run a single job 49 | String sys_id = cmd.getOptionValue("jobrun"); 50 | RecordKey jobkey = new RecordKey(sys_id); 51 | AppConfigFactory factory = new AppConfigFactory(resources); 52 | AppJobConfig jobconfig = factory.appJobConfig(jobkey); 53 | AppJobRunner jobrunner = new AppJobRunner(resources, jobconfig); 54 | jobrunner.call(); 55 | } 56 | else if (cmd.hasOption(optServer)) { 57 | // Run as an HTTP Server 58 | AgentHttpServer server = new AgentHttpServer(resources); 59 | server.run(); 60 | } 61 | else { 62 | throw new IllegalStateException(); // should never be here 63 | } 64 | 65 | } 66 | 67 | RecordKey getAgentKey() { 68 | return agentKey; 69 | } 70 | 71 | /** 72 | * Return value of profile property "app.agent" or null if not defined. 73 | */ 74 | public static String getAgentName() { 75 | // agentName is declared in {@link sndml.loader.Main} 76 | return agentName; 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/YamlLoaderConfig.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | 6 | import com.fasterxml.jackson.annotation.JsonIgnore; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import sndml.util.DateTime; 12 | import sndml.util.Log; 13 | 14 | public class YamlLoaderConfig { 15 | 16 | @JsonIgnore final DateTime start = DateTime.now(); 17 | @JsonIgnore File metricsFolder = null; 18 | 19 | @JsonProperty("threads") public Integer threads; 20 | @JsonProperty("pagesize") public Integer pageSize; 21 | @JsonProperty("metrics") public String metricsFileName = null; 22 | 23 | @JsonProperty("tables") 24 | public ArrayList tables; // = new java.util.ArrayList(); 25 | 26 | private Logger logger = LoggerFactory.getLogger(YamlLoaderConfig.class); 27 | 28 | 29 | public YamlLoaderConfig() { 30 | } 31 | 32 | void setMetricsFolder(File metricsFolder) { 33 | this.metricsFolder = metricsFolder; 34 | if (metricsFolder != null) 35 | logger.debug(Log.INIT, "metricsFolder=" + metricsFolder); 36 | } 37 | 38 | File getMetricsFile() { 39 | if (metricsFileName == null) return null; 40 | if (metricsFolder == null) return new File(metricsFileName); 41 | if (metricsFileName.startsWith("/")) return new File(metricsFileName); 42 | return new File(metricsFolder, metricsFileName); 43 | } 44 | 45 | java.util.List getJobs() { 46 | return this.tables; 47 | } 48 | 49 | void updateFields(ConnectionProfile profile) throws ConfigParseException { 50 | File metricsFile = getMetricsFile(); 51 | DateCalculator dateFactory = 52 | (metricsFile != null && metricsFile.canRead()) ? 53 | dateFactory = new DateCalculator(start, metricsFile) : 54 | new DateCalculator(start); 55 | for (JobConfig table : tables) { 56 | table.initialize(profile, dateFactory); 57 | } 58 | } 59 | 60 | void validate() throws ConfigParseException { 61 | if (tables.size() < 1) throw new ConfigParseException("No tables"); 62 | for (JobConfig table : tables) { 63 | table.validate(); 64 | } 65 | } 66 | 67 | /* 68 | * Used for JUnit tests 69 | */ 70 | JobConfig getJobByName(String name) { 71 | assert name != null; 72 | for (JobConfig job : tables) { 73 | if (name.equals(job.getName())) return job; 74 | } 75 | return null; 76 | } 77 | 78 | int getThreads() { 79 | return this.threads==null ? 0 : this.threads.intValue(); 80 | } 81 | 82 | Integer getPageSize() { 83 | return pageSize; 84 | } 85 | 86 | /** 87 | * Return the DateTime that this object was initialized. 88 | */ 89 | DateTime getStart() { 90 | return start; 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/GetRecordsTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import static org.junit.Assert.*; 4 | import java.io.IOException; 5 | 6 | import org.junit.*; 7 | import org.junit.runner.RunWith; 8 | import org.junit.runners.Parameterized; 9 | import org.junit.runners.Parameterized.Parameters; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import sndml.loader.TestManager; 14 | import sndml.loader.TestingProfile; 15 | 16 | @RunWith(Parameterized.class) 17 | public class GetRecordsTest { 18 | 19 | @Parameters(name = "{index}:{0}") 20 | public static TestingProfile[] profiles() { 21 | return TestManager.getProfiles("mydevjson mydevsoap mydevrest"); 22 | } 23 | 24 | final Logger logger = LoggerFactory.getLogger(this.getClass()); 25 | final TestingProfile profile; 26 | final Session session; 27 | 28 | public GetRecordsTest(TestingProfile profile) throws IOException { 29 | TestManager.setProfile(this.getClass(), profile); 30 | this.profile = profile; 31 | this.session = profile.newReaderSession(); 32 | } 33 | 34 | @Test 35 | public void testGetSingleRecord() throws Exception { 36 | TestManager.bannerStart("testGetSingleRecord"); 37 | // logger.info(Log.TEST, String.format("%s %s", profile, "testGetSingleRecord")); 38 | Table tbl = session.table("cmn_department"); 39 | RecordList recs = tbl.api().getRecords("id", TestManager.getProperty("some_department_id")); 40 | assertTrue(recs.size() == 1); 41 | RecordKey sysid = recs.get(0).getKey(); 42 | assertTrue(RecordKey.isGUID(sysid.toString())); 43 | TableRecord rec0 = tbl.api().getRecord(sysid); 44 | assertTrue(rec0.getKey().equals(sysid)); 45 | } 46 | 47 | @Test 48 | public void testGetEmptyRecordset() throws Exception { 49 | TestManager.bannerStart("testGetSingleRecord"); 50 | // logger.info(Log.TEST, String.format("%s %s", profile, "testGetEmptyRecordset")); 51 | Table tbl = session.table("sys_user"); 52 | RecordList recs = tbl.api().getRecords("name", "Zebra Elephant"); 53 | assertTrue(recs.size() == 0); 54 | } 55 | 56 | @Test 57 | public void getGoodKey() throws Exception { 58 | TestManager.bannerStart("getGoodKey"); 59 | String goodKey = TestManager.getProperty("some_incident_sys_id"); 60 | Table tbl = session.table("incident"); 61 | TableRecord rec = tbl.getRecord(new RecordKey(goodKey)); 62 | assertNotNull(rec); 63 | assertEquals(goodKey, rec.getValue("sys_id")); 64 | } 65 | 66 | @Test 67 | public void testGetBadKey() throws Exception { 68 | TestManager.bannerStart("testGetBadKey"); 69 | String badKey = "00000000000000000000000000000000"; 70 | Table tbl = session.table("incident"); 71 | TableRecord rec = tbl.getRecord(new RecordKey(badKey)); 72 | assertNull(rec); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/FieldValues.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.TimeZone; 7 | 8 | /** 9 | *

This object contains a collection of name/value pairs. 10 | * It is used to insert or update ServiceNow tables.

11 | * 12 | *

Example:

13 | * 
14 |  * FieldValues values = new FieldValues();
15 |  * values.set("short_description", short_description);
16 |  * values.set("assignment_group", assignment_group);
17 |  * table.insert(values);
18 |  *
19 | * 20 | */ 21 | public class FieldValues extends Parameters { 22 | 23 | private static final long serialVersionUID = 1L; 24 | 25 | public FieldValues() { 26 | super(); 27 | } 28 | 29 | public FieldValues(String name, String value) { 30 | super(); 31 | put(name, value); 32 | } 33 | 34 | public FieldValues set(String name, String value) { 35 | put(name, value); 36 | return this; 37 | } 38 | 39 | public FieldValues set(String name, boolean value) { 40 | return set(name, value ? "1" : "0"); 41 | } 42 | 43 | public FieldValues set(String name, int value) { 44 | return set(name, Integer.toString(value)); 45 | } 46 | 47 | public FieldValues set(String name, DateTime value) { 48 | return set(name, value.toString()); 49 | } 50 | 51 | final static int SECONDS_PER_DAY = 24*60*60; 52 | 53 | static ThreadLocal dateTimeFormat = 54 | new ThreadLocal() { 55 | protected DateFormat initialValue() { 56 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 57 | df.setTimeZone(TimeZone.getTimeZone("GMT")); 58 | return df; 59 | } 60 | }; 61 | 62 | public FieldValues setDuration(String name, Integer seconds) { 63 | if (seconds == null) return setNull(name); 64 | int days = seconds.intValue() / SECONDS_PER_DAY; 65 | int sec = seconds.intValue() - (days * SECONDS_PER_DAY); 66 | long millisec = sec * 1000; 67 | Date dt = new Date(millisec); 68 | DateFormat tf = dateTimeFormat.get(); 69 | String all = tf.format(dt); 70 | String dur = String.valueOf(days) + " " + all.substring(11); 71 | return set(name, dur); 72 | } 73 | 74 | /** 75 | * Copy the contents of another {@link FieldValues} object into this one. 76 | * @param newvalues The values to be applied to this object. 77 | * @return The modified object. 78 | */ 79 | public FieldValues set(FieldValues newvalues) { 80 | for (String name : newvalues.keySet()) { 81 | set(name, newvalues.get(name).toString()); 82 | } 83 | return this; 84 | } 85 | 86 | /** 87 | * Set a field value to null or empty. 88 | */ 89 | public FieldValues setNull(String name) { 90 | put(name, null); 91 | return this; 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/SoapKeySetTableReader.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import sndml.agent.JobCancelledException; 7 | import sndml.util.FieldNames; 8 | import sndml.util.Log; 9 | import sndml.util.Metrics; 10 | import sndml.util.Parameters; 11 | import sndml.util.ProgressLogger; 12 | 13 | /** 14 | * A {@link TableReader} which attempts to read a set of records 15 | * by first getting a list of all the keys. 16 | */ 17 | 18 | @Deprecated 19 | public class SoapKeySetTableReader extends TableReader { 20 | 21 | protected final SoapTableAPI soapAPI; 22 | private RecordKeySet allKeys; 23 | 24 | static final int DEFAULT_PAGE_SIZE = 200; 25 | 26 | public SoapKeySetTableReader(Table table) { 27 | super(table); 28 | soapAPI = table.soap(); 29 | } 30 | 31 | public int getDefaultPageSize() { 32 | return DEFAULT_PAGE_SIZE; 33 | } 34 | 35 | @Override 36 | public void prepare(RecordWriter writer, Metrics metrics, ProgressLogger progress) 37 | throws IOException, InterruptedException, SQLException { 38 | beginPrepare(writer, metrics, progress); 39 | EncodedQuery query = getQuery(); 40 | logger.debug(Log.INIT, "initialize query=" + query); 41 | allKeys = soapAPI.getKeys(query); 42 | endPrepare(allKeys.size()); 43 | logger.debug(Log.INIT, String.format("expected=%d", getExpected())); 44 | } 45 | 46 | public Integer getExpected() { 47 | assert allKeys != null : "TableReader not initialized"; 48 | return allKeys.size(); 49 | } 50 | 51 | public Metrics call() throws IOException, InterruptedException, SQLException, JobCancelledException { 52 | progress.logStart(); 53 | assert writer != null; 54 | assert allKeys != null; 55 | assert pageSize > 0; 56 | int fromIndex = 0; 57 | int totalRows = allKeys.size(); 58 | int rowCount = 0; 59 | while (fromIndex < totalRows) { 60 | int toIndex = fromIndex + pageSize; 61 | if (toIndex > totalRows) toIndex = totalRows; 62 | RecordKeySet slice = allKeys.getSlice(fromIndex, toIndex); 63 | EncodedQuery sliceQuery = new EncodedQuery(table, slice); 64 | Parameters params = new Parameters(); 65 | params.add("__encoded_query", sliceQuery.toString()); 66 | if (viewName != null) params.add("__use_view", viewName); 67 | RecordList recs = soapAPI.getRecords(params, this.displayValue); 68 | incrementInput(recs.size()); 69 | writer.processRecords(recs, metrics, progress); 70 | rowCount += recs.size(); 71 | logger.info(String.format("processed %d / %d rows", rowCount, totalRows)); 72 | fromIndex += pageSize; 73 | } 74 | progress.logComplete(); 75 | return metrics; 76 | } 77 | 78 | @Override 79 | public TableReader setFields(FieldNames names) { 80 | throw new UnsupportedOperationException(); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/AppConfigFactory.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | 6 | import org.slf4j.Logger; 7 | 8 | import com.fasterxml.jackson.core.JsonProcessingException; 9 | import com.fasterxml.jackson.databind.JsonNode; 10 | import com.fasterxml.jackson.databind.node.ObjectNode; 11 | 12 | import sndml.loader.ConfigFactory; 13 | import sndml.loader.ConfigParseException; 14 | import sndml.loader.ConnectionProfile; 15 | import sndml.loader.DateCalculator; 16 | import sndml.loader.Resources; 17 | import sndml.servicenow.RecordKey; 18 | import sndml.util.Log; 19 | 20 | public class AppConfigFactory extends ConfigFactory { 21 | 22 | final Resources resources; 23 | final AppSession appSession; 24 | final ConnectionProfile profile; 25 | final DateCalculator dateCalculator; 26 | 27 | Logger logger = Log.getLogger(AppConfigFactory.class); 28 | 29 | public AppConfigFactory(Resources resources) { 30 | super(); 31 | this.resources = resources; 32 | this.appSession = resources.getAppSession(); 33 | this.profile = resources.getProfile(); 34 | this.dateCalculator = new DateCalculator(); 35 | } 36 | 37 | public AppJobConfig jobConfig(ConnectionProfile profile, JsonNode node) throws ConfigParseException { 38 | DateCalculator dateFactory = new DateCalculator(); 39 | AppJobConfig config; 40 | try { 41 | config = jsonMapper.treeToValue(node, AppJobConfig.class); 42 | } catch (JsonProcessingException e) { 43 | throw new ConfigParseException(e.getMessage()); 44 | } 45 | config.initializeAndValidate(profile, dateFactory); 46 | logger.debug(Log.INIT, "jobConfig: " + config.toString()); 47 | return config; 48 | } 49 | 50 | public AppJobConfig appJobConfig(RecordKey runKey) throws ConfigParseException, IOException { 51 | URI uriGetRun = appSession.uriGetJobRunConfig(runKey); 52 | ObjectNode node = appSession.httpGet(uriGetRun); 53 | AppJobConfig config; 54 | try { 55 | config = jsonMapper.treeToValue(node, AppJobConfig.class); 56 | config.initialize(profile, dateCalculator); 57 | config.validate(); 58 | } catch (JsonProcessingException e) { 59 | throw new ConfigParseException(e.getMessage()); 60 | } 61 | return config; 62 | } 63 | 64 | // TODO can this procedure use AppSession.httpGet ? 65 | // private ObjectNode getRun(RecordKey jobKey) throws IOException, ConfigParseException { 66 | // Log.setJobContext(appSession.getAgentName()); 67 | // URI uriGetRun = appSession.uriGetJobRunConfig(jobKey); 68 | // JsonRequest request = new JsonRequest(appSession, uriGetRun, HttpMethod.GET, null); 69 | // logger.info(Log.INIT, uriGetRun.toString()); 70 | // ObjectNode response = request.execute(); 71 | // logger.debug(Log.RESPONSE, response.toPrettyString()); 72 | // ObjectNode objResult = (ObjectNode) response.get("result"); 73 | // return objResult; 74 | // } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/DatePartitionSet.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | /** 4 | * A {@link DatePartitionSet} divides a {@link DateTimeRange} into partitions 5 | * based on the {@link PartitionInterval}. 6 | *

7 | * Each {@link DatePartition} is a {@link DateTimeRange} which begins and ends on an interval boundary. 8 | * Thus, the first and last partitions may exceed the boundaries of the original range. 9 | *

10 | * The {@link #iterator()} method returns the partitions in reverse chronological order. 11 | * In other words, the most recent partition is returned first. 12 | */ 13 | public class DatePartitionSet implements Iterable { 14 | 15 | private final DateTimeRange range; 16 | private final PartitionInterval interval; 17 | 18 | public DatePartitionSet(DateTimeRange range, PartitionInterval interval) { 19 | this.range = range; 20 | this.interval = interval; 21 | if (range == null) { 22 | // this is empty 23 | } 24 | else { 25 | if (range.getStart() == null) 26 | throw new IllegalArgumentException("start date is null"); 27 | if (range.getEnd() == null) 28 | throw new IllegalArgumentException("end date is null"); 29 | if (range.getEnd().compareTo(range.getStart()) < 0) 30 | throw new IllegalArgumentException("end date is before start date"); 31 | } 32 | } 33 | 34 | public boolean isEmpty() { 35 | return range == null; 36 | } 37 | 38 | public DateTimeRange getRange() { 39 | return this.range; 40 | } 41 | 42 | public PartitionInterval getInterval() { 43 | return this.interval; 44 | } 45 | 46 | /** 47 | * Compute the number of partitions in this {@link DatePartitionSet}. 48 | * 49 | * @return number of partitions 50 | */ 51 | public int computeSize() { 52 | if (range == null) return 0; 53 | if (range.start == null) return 0; 54 | if (range.end == null ) return 0; 55 | if (range.end.compareTo(range.start) <= 0) return 0; // end is before start (should be impossible) 56 | if (interval == null) return 0; 57 | DateTime end = range.end.ceiling(interval); 58 | assert end.compareTo(range.end) >= 0 : "computeSize bad ceiling"; 59 | DateTime start = end.decrementBy(interval); 60 | assert start.compareTo(end) < 0 : "computeSize bad decrement"; 61 | int size = 1; 62 | while (start.compareTo(range.start) > 0) { 63 | size += 1; 64 | end = start; 65 | start = end.decrementBy(interval); 66 | assert start.compareTo(end) < 0; 67 | } 68 | return size; 69 | } 70 | 71 | public String toString() { 72 | if (range == null) 73 | return "empty"; 74 | else 75 | return range.toString() + " by " + interval.toString(); 76 | } 77 | 78 | @Override 79 | /** 80 | * Process the ranges in a partition beginning with the most recent 81 | * and ending with the earliest. 82 | */ 83 | public DatePartitionIterator iterator() { 84 | return new DatePartitionIterator(this); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/test/java/sndml/servicenow/TableWSDLTest.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.junit.*; 4 | import java.io.IOException; 5 | import java.util.List; 6 | import org.slf4j.Logger; 7 | 8 | import sndml.loader.TestManager; 9 | 10 | import static org.junit.Assert.*; 11 | 12 | public class TableWSDLTest { 13 | 14 | final Logger logger = TestManager.getLogger(TableWSDLTest.class); 15 | final Session session; 16 | 17 | public TableWSDLTest() { 18 | session = TestManager.getDefaultProfile().newReaderSession(); 19 | } 20 | 21 | TableWSDL getWSDL(String tablename) throws IOException { 22 | Table table = session.table(tablename); 23 | TableWSDL wsdl = table.getWSDL(); 24 | return wsdl; 25 | } 26 | 27 | @Test 28 | public void testGoodTable() throws Exception { 29 | String tablename = "incident"; 30 | TableWSDL wsdl = getWSDL(tablename); 31 | List columns = wsdl.getReadFieldNames(); 32 | int count = columns.size(); 33 | logger.info(tablename + " has " + count + " columns"); 34 | assert(count > 60); 35 | } 36 | 37 | @Test (expected = InvalidTableNameException.class) 38 | public void testBadTableName() throws Exception { 39 | String tablename = "incidentxxx"; 40 | @SuppressWarnings("unused") 41 | TableWSDL wsdl = getWSDL(tablename); 42 | fail(); 43 | } 44 | 45 | @Test 46 | public void testDefaultWSDL() throws Exception { 47 | String tablename = "incident"; 48 | TableWSDL wsdl = getWSDL(tablename); 49 | assertTrue(wsdl.canReadField("sys_updated_on")); 50 | assertFalse(wsdl.canReadField("dv_assigned_to")); 51 | assertFalse(wsdl.canReadField("createdxxxxx")); 52 | assertTrue(wsdl.canWriteField("short_description")); 53 | assertFalse(wsdl.canWriteField("short_descriptionxxx")); 54 | } 55 | 56 | @Test 57 | public void testDisplayValues() throws Exception { 58 | String tablename = "incident"; 59 | TableWSDL wsdl = new TableWSDL(session, tablename, true); 60 | assertTrue(wsdl.canReadField("sys_updated_on")); 61 | assertTrue(wsdl.canReadField("dv_assigned_to")); 62 | assertFalse(wsdl.canReadField("createdxxxxx")); 63 | assertTrue(wsdl.canWriteField("short_description")); 64 | assertFalse(wsdl.canWriteField("short_descriptionxxx")); 65 | } 66 | 67 | @Test 68 | public void testDisplayValues2() throws Exception { 69 | String tablename = "incident"; 70 | TableWSDL wsdl = new TableWSDL(session, tablename, true); 71 | assertTrue(wsdl.canReadField("sys_updated_on")); 72 | assertTrue(wsdl.canReadField("dv_assigned_to")); 73 | assertFalse(wsdl.canReadField("createdxxxxx")); 74 | } 75 | 76 | @Test 77 | public void testSysTemplate() throws Exception { 78 | String tablename = "sys_template"; 79 | TableWSDL wsdl = new TableWSDL(session, tablename, true); 80 | assertTrue(wsdl.canReadField("sys_created_on")); 81 | assertTrue(wsdl.canReadField("sys_updated_on")); 82 | assertTrue(wsdl.canReadField("short_description")); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /docs/_layouts/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {% seo %} 7 | 8 | 9 | 10 | 11 | 12 | 13 | {% include head-custom.html %} 14 | 15 | 16 | 17 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |

41 | 42 |
43 | {{ content }} 44 | 45 | 59 |
60 | 61 | -------------------------------------------------------------------------------- /src/main/java/sndml/util/DateTimeRange.java: -------------------------------------------------------------------------------- 1 | package sndml.util; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.databind.node.ArrayNode; 5 | 6 | /** 7 | * A {@link DateTime} range which is open at the beginning but closed at the end. 8 | * In other words, 9 | * the start date is included but the end date is NOT included. 10 | */ 11 | public class DateTimeRange { 12 | 13 | protected final DateTime start; 14 | protected final DateTime end; 15 | 16 | public DateTimeRange(DateTime start, DateTime end) { 17 | this.start = start; 18 | this.end = end; 19 | } 20 | 21 | public DateTimeRange(String start, String end) { 22 | assert end != null : "null end invalid for constructor with Strings"; 23 | assert start != null : "null start invalid for constructor with Strings"; 24 | this.start = new DateTime(start); 25 | this.end = new DateTime(end); 26 | } 27 | 28 | public static DateTimeRange all() { 29 | return new DateTimeRange((DateTime) null, (DateTime) null); 30 | } 31 | 32 | /** 33 | * Determine whether a date falls within the {@link DateTimeRange}. 34 | * The range is open at the beginning and closed at the end. 35 | * In other words, 36 | * the start date is included but the end date is NOT included. 37 | */ 38 | public boolean contains(DateTime date) { 39 | assert date != null : "null date cannot be tested"; 40 | if (start != null && date.compareTo(start) < 0) return false; 41 | if (end != null && date.compareTo(end) >= 0) return false; 42 | return true; 43 | } 44 | 45 | public DateTime getStart() { 46 | return start; 47 | } 48 | 49 | public DateTime getEnd() { 50 | return end; 51 | } 52 | 53 | public boolean hasStart() { 54 | return start != null; 55 | } 56 | 57 | public boolean hasEnd() { 58 | return end != null; 59 | } 60 | 61 | @Override 62 | public boolean equals(Object obj) { 63 | DateTimeRange other = (DateTimeRange) obj; 64 | return start.equals(other.start) && end.equals(other.end); 65 | } 66 | 67 | /** 68 | * Determine the overlap of two date ranges 69 | */ 70 | public DateTimeRange intersect(DateTimeRange other) { 71 | DateTime start = 72 | other != null && other.hasStart() && ( 73 | this.getStart() == null || other.getStart().after(this.getStart())) ? 74 | other.getStart() : this.getStart(); 75 | DateTime end = 76 | other != null && other.hasEnd() && ( 77 | this.getEnd() == null || other.getEnd().before(this.getEnd())) ? 78 | other.getEnd() : this.getEnd(); 79 | return new DateTimeRange(start, end); 80 | } 81 | 82 | public String toString() { 83 | return String.format("[%s,%s]", 84 | hasStart() ? start.toString() : "null", 85 | hasEnd() ? end.toString() : "null"); 86 | } 87 | 88 | public ArrayNode toJsonNode() { 89 | ObjectMapper mapper = new ObjectMapper(); 90 | ArrayNode node = mapper.createArrayNode(); 91 | node.add(hasStart() ? start.toString() : null); 92 | node.add(hasEnd() ? end.toString() : null); 93 | return node; 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/RecordKeySet.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Hashtable; 5 | import java.util.Set; 6 | 7 | 8 | import com.fasterxml.jackson.databind.JsonNode; 9 | import com.fasterxml.jackson.databind.node.ArrayNode; 10 | 11 | /** 12 | * Holds a list of {@link RecordKey} (i.e. sys_id) values 13 | * as returned from a getKeys SOAP Web Services call. 14 | * 15 | */ 16 | public class RecordKeySet extends ArrayList { 17 | 18 | private static final long serialVersionUID = 1L; 19 | 20 | public RecordKeySet() { 21 | super(); 22 | } 23 | 24 | public RecordKeySet(int size) { 25 | super(size); 26 | } 27 | 28 | public RecordKeySet(ArrayNode array) { 29 | this(array.size()); 30 | for (int i = 0; i < array.size(); ++i) { 31 | JsonNode ele = array.get(i); 32 | this.add(new RecordKey(ele.asText())); 33 | } 34 | } 35 | 36 | public RecordKeySet(Set set) { 37 | super(set.size()); 38 | for (RecordKey key : set) { 39 | this.add(key); 40 | } 41 | } 42 | 43 | /** 44 | * @return the complete list as a comma separated list of sys_ids. 45 | */ 46 | public String toString() { 47 | StringBuffer result = new StringBuffer(); 48 | for (int i = 0; i < size(); ++i) { 49 | if (i > 0) result.append(","); 50 | result.append(get(i).toString()); 51 | } 52 | return result.toString(); 53 | } 54 | 55 | /** 56 | * Returns a subset of the list as comma separated string. 57 | * Used to construct encoded queries. 58 | * The number of entries returned is (toIndex - fromIndex). 59 | * An exception may occur if toIndex less than 0 or fromIndex greater than size(). 60 | * 61 | * @param startIndex Zero based starting index (inclusive). 62 | * @param endIndex Zero based ending index (exclusive). 63 | * @return A list of keys. 64 | */ 65 | public RecordKeySet getSlice(int startIndex, int endIndex) { 66 | RecordKeySet result = new RecordKeySet(endIndex - startIndex); 67 | int size = size(); 68 | for (int i = startIndex; i < endIndex && i < size; ++i) { 69 | result.add(get(i)); 70 | } 71 | return result; 72 | } 73 | 74 | public RecordKey maxValue() { 75 | RecordKey result = null; 76 | for (RecordKey key : this) { 77 | if (result == null || key.greaterThan(result)) result = key; 78 | } 79 | return result; 80 | } 81 | 82 | public RecordKey minValue() { 83 | RecordKey result = null; 84 | for (RecordKey key : this) { 85 | if (result == null || key.lessThan(result)) result = key; 86 | } 87 | return result; 88 | } 89 | 90 | /** 91 | * Return the number of unique values in this list of keys. 92 | */ 93 | @Deprecated 94 | int uniqueCount() { 95 | Hashtable hash = new Hashtable(this.size()); 96 | for (RecordKey key : this) { 97 | hash.put(key, true); 98 | } 99 | return hash.size(); 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/sndml/agent/MultiThreadScanner.java: -------------------------------------------------------------------------------- 1 | package sndml.agent; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | 6 | import sndml.loader.*; 7 | import sndml.util.Log; 8 | 9 | public class MultiThreadScanner extends AgentScanner { 10 | 11 | final WorkerPool workerPool; 12 | 13 | public MultiThreadScanner(Resources resources, WorkerPool workerPool) { 14 | super(resources); 15 | this.workerPool = workerPool; 16 | } 17 | 18 | @Override 19 | protected int getErrorLimit() { 20 | // Allow getrunlist to fail two times in a row, but not three 21 | return 3; 22 | } 23 | 24 | @Override 25 | protected AppJobRunner createJob(AppJobConfig jobConfig) { 26 | Resources workerResources = resources.workerCopy(); 27 | AppJobRunner job = new ScannerJobRunner(this, workerResources, jobConfig); 28 | return job; 29 | } 30 | 31 | @Override 32 | public void scanUntilDone() throws IOException, ConfigParseException, InterruptedException { 33 | String myname = this.getClass().getSimpleName() + ".scanUntilDone"; 34 | logger.debug(Log.INIT, String.format("%s begin %s", myname, agentName)); 35 | boolean done = false; 36 | while (!done) { 37 | int jobcount = scanOnce(); 38 | if (jobcount == 0) { 39 | done = true; 40 | } 41 | else { 42 | // wait for workerPool to become idle 43 | int loopCounter = 0; 44 | int activeTasks = workerPool.activeTaskCount(); 45 | if (activeTasks == 0) done = true; 46 | while (activeTasks > 0) { 47 | // print message every 20 seconds 48 | if (++loopCounter % 20 == 0) 49 | logger.info(Log.PROCESS, 50 | String.format("scanUntilDone: %d active workers", activeTasks)); 51 | Thread.sleep(1000); 52 | activeTasks = workerPool.activeTaskCount(); 53 | } 54 | logger.info(Log.PROCESS, "scanUntilDone: no active workers"); 55 | Thread.sleep(rescanDelayMillisec); 56 | } 57 | } 58 | logger.debug(Log.FINISH, String.format("%s end", myname)); 59 | if (logger.isDebugEnabled()) workerPool.dumpJobList(); 60 | } 61 | 62 | /** 63 | *

Submit for execution all jobs that are ready. 64 | * Return the number of jobs submitted.

65 | * 66 | *

Note: This function does NOT wait for jobs to complete.

67 | */ 68 | 69 | @Override 70 | public int scanOnce() throws IOException, ConfigParseException { 71 | String myname = this.getClass().getSimpleName() + ".scanOnce"; 72 | Log.setJobContext(agentName); 73 | logger.debug(Log.INIT, String.format("%s begin %s", myname, agentName)); 74 | ArrayList joblist = getJobList(); 75 | if (joblist.size() > 0) { 76 | // Schedule all jobs for future execution 77 | // Do not wait for them to complete 78 | // Each job will open its own Session and Database connection 79 | for (AppJobRunner job : joblist) { 80 | workerPool.submit(job); 81 | } 82 | } 83 | Log.setGlobalContext(); 84 | int result = joblist.size(); 85 | logger.debug(Log.FINISH, String.format("%s end jobs=%d", myname, result)); 86 | return result; 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/RefreshTest1.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.After; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.junit.runners.Parameterized; 10 | import org.junit.runners.Parameterized.Parameters; 11 | import org.slf4j.Logger; 12 | 13 | import sndml.servicenow.*; 14 | import sndml.util.DateTime; 15 | import sndml.util.FieldValues; 16 | import sndml.util.Log; 17 | import sndml.util.Metrics; 18 | 19 | @RunWith(Parameterized.class) 20 | public class RefreshTest1 { 21 | 22 | final TestingProfile profile; 23 | final Logger logger = TestManager.getLogger(this.getClass()); 24 | 25 | @Parameters(name = "{index}:{0}") 26 | public static TestingProfile[] profiles() { 27 | return TestManager.getDatamartProfiles(); 28 | } 29 | 30 | public RefreshTest1(TestingProfile profile) throws Exception { 31 | this.profile = profile; 32 | TestManager.setProfile(this.getClass(), profile); 33 | } 34 | 35 | @Before 36 | public void setUp() throws Exception { 37 | } 38 | 39 | @After 40 | public void tearDown() throws Exception { 41 | } 42 | 43 | @Test 44 | public void testRefresh() throws Exception { 45 | Resources resources = new Resources(profile); 46 | String tableName = "incident"; 47 | DBUtil db = new DBUtil(profile); 48 | JobFactory jf = new JobFactory(resources, DateTime.now()); 49 | TableAPI api = profile.newReaderSession().table(tableName).api(); 50 | TestManager.banner(logger, "Load"); 51 | db.dropTable(tableName); 52 | JobRunner create = jf.yamlJob("{source: incident, action: create}"); 53 | create.call(); 54 | assertTrue(db.tableExists(tableName)); 55 | JobRunner load = jf.yamlJob("{source: incident, action: load, created: 2020-01-01}"); 56 | Metrics loadMetrics = load.call(); 57 | int count1 = db.sqlCount(tableName, null); 58 | assertTrue(count1 > 0); 59 | assertEquals(count1, loadMetrics.getInserted()); 60 | assertNotNull(loadMetrics.getStarted()); 61 | TestManager.banner(logger, "Insert"); 62 | FieldValues values = new FieldValues(); 63 | String descr1 = String.format( 64 | "%s %s", this.getClass().getSimpleName(), 65 | loadMetrics.getStarted().toString()); 66 | values.put("short_description", descr1); 67 | values.put("cmdb_ci", TestManager.getProperty("some_ci")); 68 | RecordKey key = api.insertRecord(values).getKey(); 69 | TestManager.sleep(2); 70 | TestManager.banner(logger, "Refresh"); 71 | String yaml = String.format( 72 | "{source: incident, action: refresh, since: %s}", 73 | loadMetrics.getStarted().toString()); 74 | logger.info(Log.TEST, yaml); 75 | JobRunner refresh = jf.yamlJob(yaml); 76 | Metrics refreshMetrics = refresh.call(); 77 | assertEquals(1, refreshMetrics.getInserted()); 78 | assertEquals(0, refreshMetrics.getUpdated()); 79 | int count2 = db.sqlCount(tableName, null); 80 | assertEquals(count1 + 1, count2); 81 | api.deleteRecord(key); 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/RefreshTest2.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import org.junit.After; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.junit.runners.Parameterized; 10 | import org.junit.runners.Parameterized.Parameters; 11 | import org.slf4j.Logger; 12 | 13 | import sndml.servicenow.*; 14 | import sndml.util.DateTime; 15 | import sndml.util.FieldValues; 16 | import sndml.util.Log; 17 | import sndml.util.Metrics; 18 | 19 | @RunWith(Parameterized.class) 20 | public class RefreshTest2 { 21 | 22 | final Resources resources; 23 | final Logger logger = TestManager.getLogger(this.getClass()); 24 | 25 | @Parameters(name = "{index}:{0}") 26 | public static TestingProfile[] profiles() { 27 | return TestManager.getDatamartProfiles(); 28 | } 29 | 30 | public RefreshTest2(TestingProfile profile) throws Exception { 31 | TestManager.setProfile(this.getClass(), profile); 32 | this.resources = TestManager.getResources(); 33 | 34 | } 35 | 36 | @Before 37 | public void setUp() throws Exception { 38 | } 39 | 40 | @After 41 | public void tearDown() throws Exception { 42 | } 43 | 44 | @Test 45 | public void testRefresh() throws Exception { 46 | String tableName = "incident"; 47 | DBUtil db = new DBUtil(resources); 48 | JobFactory jf = new JobFactory(resources, DateTime.now()); 49 | TableAPI api = resources.getReaderSession().table(tableName).api(); 50 | TestManager.banner(logger, "Load"); 51 | db.dropTable(tableName); 52 | JobRunner create = jf.yamlJob("{source: incident, action: create}"); 53 | create.call(); 54 | assertTrue(db.tableExists(tableName)); 55 | JobRunner load = jf.yamlJob("{source: incident, action: load, created: 2020-01-01}"); 56 | Metrics loadMetrics = load.call(); 57 | int count1 = db.sqlCount(tableName, null); 58 | assertTrue(count1 > 0); 59 | assertEquals(count1, loadMetrics.getInserted()); 60 | assertNotNull(loadMetrics.getStarted()); 61 | TestManager.banner(logger, "Insert"); 62 | FieldValues values = new FieldValues(); 63 | String descr1 = String.format( 64 | "%s %s", this.getClass().getSimpleName(), 65 | loadMetrics.getStarted().toString()); 66 | values.put("short_description", descr1); 67 | values.put("cmdb_ci", TestManager.getProperty("some_ci")); 68 | RecordKey key = api.insertRecord(values).getKey(); 69 | TestManager.sleep(2); 70 | TestManager.banner(logger, "Refresh"); 71 | String yaml = String.format( 72 | "{source: incident, action: refresh, since: last, last: %s}", 73 | loadMetrics.getStarted().toString()); 74 | logger.info(Log.TEST, yaml); 75 | JobRunner refresh = jf.yamlJob(yaml); 76 | Metrics refreshMetrics = refresh.call(); 77 | assertEquals(1, refreshMetrics.getInserted()); 78 | assertEquals(0, refreshMetrics.getUpdated()); 79 | int count2 = db.sqlCount(tableName, null); 80 | assertEquals(count1 + 1, count2); 81 | api.deleteRecord(key); 82 | 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/sndml/loader/LoaderConfigTest.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import sndml.util.DateTime; 4 | 5 | import static org.junit.Assert.*; 6 | import org.junit.*; 7 | 8 | import java.io.File; 9 | import java.io.StringReader; 10 | import org.slf4j.Logger; 11 | 12 | public class LoaderConfigTest { 13 | 14 | final Logger logger = TestManager.getLogger(this.getClass()); 15 | final TestingProfile profile = TestManager.getDefaultProfile(); 16 | final TestFolder folder = new TestFolder(this.getClass().getSimpleName()); 17 | final ConfigFactory factory = new ConfigFactory(); 18 | 19 | 20 | @AfterClass 21 | public static void clear() throws Exception { 22 | TestManager.clearAll(); 23 | } 24 | 25 | @Test 26 | @Ignore 27 | // This syntax is no longer supported 28 | public void testSimpleOld() throws Exception { 29 | String yaml = "tables: [core_company, incident]"; 30 | YamlLoaderConfig config = factory.loaderConfig(profile, new StringReader(yaml)); 31 | assertEquals(2, config.getJobs().size()); 32 | assertEquals("core_company", config.getJobs().get(0).getName()); 33 | assertEquals("incident", config.getJobs().get(1).getName()); 34 | assertEquals(false, config.getJobs().get(0).getTruncate()); 35 | } 36 | 37 | public void testSimpleNew() throws Exception { 38 | String yaml = "tables: [{source: core_company}, {source: incident}]"; 39 | YamlLoaderConfig config = factory.loaderConfig(profile, new StringReader(yaml)); 40 | assertEquals(2, config.getJobs().size()); 41 | assertEquals("core_company", config.getJobs().get(0).getName()); 42 | assertEquals("incident", config.getJobs().get(1).getName()); 43 | assertEquals(false, config.getJobs().get(0).getTruncate()); 44 | } 45 | 46 | @Test 47 | public void testGoodConfig1() throws Exception { 48 | File config1 = folder.getYaml("multi-table-load"); 49 | YamlLoaderConfig config = factory.loaderConfig(profile, config1); 50 | DateTime start = config.getStart(); 51 | DateTime today = DateTime.today(); 52 | assertEquals(8, config.getJobs().size()); 53 | assertEquals("sys_user", config.getJobByName("sys_user").getTarget()); 54 | assertEquals("rm_story", config.getJobByName("rm_story").getTarget()); 55 | assertEquals(new DateTime("2017-01-01"), config.getJobByName("rm_story").getCreatedRange(null).getStart()); 56 | assertEquals(start, config.getJobByName("rm_story").getCreatedRange(null).getEnd()); 57 | assertEquals(today, config.getJobByName("cmdb_ci_service").getSince()); 58 | } 59 | 60 | @Test 61 | public void testGoodSync1() throws Exception { 62 | File goodConfig = folder.getYaml("incident-sync"); 63 | YamlLoaderConfig config = factory.loaderConfig(profile, goodConfig); 64 | assertNotNull(config); 65 | } 66 | 67 | /* 68 | @Test 69 | public void test_createIncident() throws JacksonException { 70 | ObjectNode obj = TestManager.yaml("{action: create, source: incident, drop: true}"); 71 | JobConfig config = factory.jobConfig(obj); 72 | assertEquals(JobAction.CREATE, config.getAction()); 73 | assertFalse(config.getTruncate()); 74 | assertTrue(config.dropTable); 75 | } 76 | */ 77 | 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/sndml/servicenow/XmlRecord.java: -------------------------------------------------------------------------------- 1 | package sndml.servicenow; 2 | 3 | import org.jdom2.*; 4 | 5 | import sndml.util.FieldNames; 6 | 7 | import java.util.*; 8 | 9 | 10 | /** 11 | * Contains an XML document (in the form of a JDOM Element) which 12 | * has been retrieved from ServiceNow. 13 | * The only way to obtain one of these is from a {@link SoapTableAPI} method. 14 | * 15 | * @author Giles Lewis 16 | */ 17 | public class XmlRecord extends TableRecord { 18 | 19 | final protected Element element; 20 | final protected Namespace ns; 21 | 22 | public XmlRecord(Table table, Element element) 23 | throws SoapResponseException { 24 | super(table); 25 | this.element = element; 26 | this.ns = element.getNamespace(); 27 | } 28 | 29 | @Override 30 | public String getValue(String fieldname) { 31 | String result = element.getChildText(fieldname, ns); 32 | if (result == null) return null; 33 | if (result.length() == 0) return null; 34 | return result; 35 | } 36 | 37 | @Override 38 | public String getDisplayValue(String fieldname) { 39 | return getValue("dv_" + fieldname); 40 | } 41 | 42 | /** 43 | * Return a clone of the JDOM Element underlying this object. 44 | * The name of the returned element is the table name. 45 | */ 46 | Element getElement() { 47 | Element result = element.clone(); 48 | result.setName(table.getName()); 49 | return result; 50 | } 51 | 52 | /** 53 | * The number of XML elements in this record, 54 | * which may be fewer than getTable().getSchema().numFields(). 55 | */ 56 | public int numFields() { 57 | return element.getContentSize(); 58 | } 59 | 60 | /** 61 | * Returns the underlying JDOM Element as a formatted XML string. 62 | * Use for debugging and diagnostics. 63 | * 64 | * Deprecated: Use asText() 65 | */ 66 | @Deprecated 67 | public String getXML() { 68 | return getXML(false); 69 | } 70 | 71 | /** 72 | * Returns the underlying JDOM Element as a formatted XML string. 73 | * Use for debugging and diagnostics. 74 | * 75 | * Deprecated: Use asText() 76 | */ 77 | @Deprecated 78 | public String getXML(boolean pretty) { 79 | return XmlFormatter.format(element, pretty); 80 | } 81 | 82 | @Override 83 | public Iterator keys() { 84 | return getFieldNames().iterator(); 85 | } 86 | 87 | 88 | @Override 89 | public FieldNames getFieldNames() { 90 | FieldNames result = new FieldNames(); 91 | for (Element field : element.getChildren()) { 92 | result.add(field.getName()); 93 | } 94 | return result; 95 | } 96 | 97 | public LinkedHashMap getAllFields() { 98 | LinkedHashMap result = new LinkedHashMap(); 99 | for (Element field : element.getChildren()) { 100 | String name = field.getName(); 101 | String value = field.getText(); 102 | result.put(name, value); 103 | } 104 | return result; 105 | } 106 | 107 | @Override 108 | public String asText(boolean pretty) { 109 | return XmlFormatter.format(element, pretty); 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/sndml/loader/DatabaseTableWriter.java: -------------------------------------------------------------------------------- 1 | package sndml.loader; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import org.slf4j.Logger; 7 | 8 | import sndml.agent.JobCancelledException; 9 | import sndml.servicenow.TableRecord; 10 | import sndml.util.Log; 11 | import sndml.util.Metrics; 12 | import sndml.util.ProgressLogger; 13 | import sndml.util.ResourceException; 14 | import sndml.servicenow.RecordList; 15 | import sndml.servicenow.RecordWriter; 16 | import sndml.servicenow.Table; 17 | 18 | /** 19 | *

A class which knows how to process records retrieved from ServiceNow.

20 | *

There are three subclasses: one for each of the three database operations.

21 | *
    22 | *
  • {@link DatabaseInsertWriter}
    • 23 | *
  • {@link DatabaseUpdateWriter}
    • 24 | *
  • {@link DatabaseDeleteWriter}
    • 25 | *
26 | * 27 | */ 28 | public abstract class DatabaseTableWriter extends RecordWriter { 29 | 30 | final protected DatabaseWrapper db; 31 | final protected Table table; 32 | final protected String sqlTableName; 33 | 34 | protected ColumnDefinitions columns; 35 | 36 | final Logger logger = Log.getLogger(this.getClass()); 37 | /** 38 | * Abstract class which knows how to write records to a SQL database 39 | * using the {@link processRecords} method. 40 | * Implementations must override the {@link writeRecord} method. 41 | * 42 | * @param db Database connection 43 | * @param table ServiceNow table 44 | * @param sqlTableName name of the table in the SQL database 45 | * @param writerName used only for logging 46 | */ 47 | public DatabaseTableWriter(DatabaseWrapper db, Table table, String sqlTableName, String writerName) 48 | throws IOException, SQLException { 49 | super(); 50 | assert db != null; 51 | assert table != null; 52 | assert sqlTableName != null; 53 | this.db = db; 54 | this.table = table; 55 | this.sqlTableName = sqlTableName; 56 | Log.setTableContext(this.table); 57 | } 58 | 59 | @Override 60 | public DatabaseTableWriter open(Metrics metrics) throws SQLException, IOException { 61 | assert metrics != null; 62 | super.open(metrics); 63 | columns = new ColumnDefinitions(this.db, this.table, this.sqlTableName); 64 | metrics.start(); 65 | return this; 66 | } 67 | 68 | @Override 69 | public void close(Metrics metrics) { 70 | try { 71 | db.commit(); 72 | } catch (SQLException e) { 73 | throw new ResourceException(e); 74 | } 75 | metrics.finish(); 76 | super.close(metrics); 77 | } 78 | 79 | @Override 80 | public synchronized void processRecords( 81 | RecordList recs, Metrics metrics, ProgressLogger progressLogger) 82 | throws JobCancelledException, IOException, SQLException { 83 | assert metrics != null; 84 | assert progressLogger != null; 85 | for (TableRecord rec : recs) { 86 | logger.trace(Log.PROCESS, String.format( 87 | "processing %s %s", rec.getKey(), rec.getCreatedTimestamp())); 88 | writeRecord(rec, metrics); 89 | } 90 | db.commit(); 91 | progressLogger.logProgress(); 92 | } 93 | 94 | abstract void writeRecord(TableRecord rec, Metrics writerMetrics) throws SQLException; 95 | 96 | } 97 | --------------------------------------------------------------------------------