├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── codeql.yml ├── .gitignore ├── LICENSE ├── README.md ├── csv-validator-cmd ├── README.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── appassembler-output.xml │ ├── resources-filtered │ │ └── Dockerfile │ └── scala │ │ └── uk │ │ └── gov │ │ └── nationalarchives │ │ └── csv │ │ └── validator │ │ └── cmd │ │ └── CsvValidatorCmdApp.scala │ └── test │ ├── resources │ └── uk │ │ └── gov │ │ └── nationalarchives │ │ └── csv │ │ └── validator │ │ └── cmd │ │ ├── acceptance │ │ ├── standardRulesFailMetaData.csv │ │ └── standardRulesSchema.csvs │ │ ├── badSchema.csvs │ │ ├── metaData.csv │ │ ├── schema.csvs │ │ ├── warning.csv │ │ ├── warning.csvs │ │ └── windows-1252.csv │ └── scala │ └── uk │ └── gov │ └── nationalarchives │ └── csv │ └── validator │ └── cmd │ ├── CsvValidatorCmdAppSpec.scala │ └── TestResources.scala ├── csv-validator-core ├── pom.xml └── src │ ├── main │ └── scala │ │ └── uk │ │ └── gov │ │ └── nationalarchives │ │ └── csv │ │ └── validator │ │ ├── AllErrorsMetaDataValidator.scala │ │ ├── FailFastMetaDataValidator.scala │ │ ├── MetaDataValidator.scala │ │ ├── Util.scala │ │ ├── api │ │ └── CsvValidator.scala │ │ ├── metadata │ │ └── MetaData.scala │ │ ├── package.scala │ │ └── schema │ │ ├── Rule.scala │ │ ├── Schema.scala │ │ ├── SchemaParser.scala │ │ ├── SchemaValidator.scala │ │ ├── TraceableParsers.scala │ │ ├── package.scala │ │ ├── v1_0 │ │ ├── Rule.scala │ │ ├── SchemaParser.scala │ │ └── SchemaValidator.scala │ │ ├── v1_1 │ │ ├── Rule.scala │ │ ├── Schema.scala │ │ ├── SchemaParser.scala │ │ └── SchemaValidator.scala │ │ └── v1_2 │ │ ├── Schema.scala │ │ └── SchemaParser.scala │ └── test │ ├── resources │ └── uk │ │ └── gov │ │ └── nationalarchives │ │ └── csv │ │ └── validator │ │ ├── acceptance │ │ ├── andyPass.csvs │ │ ├── anyRuleFailMetaData.csv │ │ ├── anyRulePassMetaData.csv │ │ ├── anyRuleSchema.csvs │ │ ├── benPass.csvs │ │ ├── bigMetaData.csv │ │ ├── bigSchema.csvs │ │ ├── bom.csv │ │ ├── bom.csvs │ │ ├── caseSensitiveFiles.csv │ │ ├── caseSensitiveFiles.csvs │ │ ├── caseSensitiveFilesChecksum.csv │ │ ├── caseSensitiveFilesChecksum.csvs │ │ ├── concat.csvs │ │ ├── concat4.csvs │ │ ├── concat4Fail.csv │ │ ├── concat4Pass.csv │ │ ├── concatFail.csv │ │ ├── concatPass.csv │ │ ├── dp │ │ │ ├── regexRuleFailMetaData.csv │ │ │ ├── regexRulePassMetaData.csv │ │ │ └── regexRuleSchema.csvs │ │ ├── duplicateColumnIdsFailSchema.csvs │ │ ├── duplicateColumnIdsMetaData.csv │ │ ├── duplicateColumnIdsPassSchema.csvs │ │ ├── fileExistsCrossRefPassMetaData.csv │ │ ├── fileExistsCrossRefSchema.csvs │ │ ├── fileExistsPassMetaData.csv │ │ ├── fileExistsSchema.csvs │ │ ├── fileExistsSchemaWithBadBasePath.csvs │ │ ├── identicalEmptyMetaData.csv │ │ ├── identicalFailMetaData.csv │ │ ├── identicalHeaderMetaData.csv │ │ ├── identicalHeaderSchema.csvs │ │ ├── identicalPassMetaData.csv │ │ ├── identicalSchema.csvs │ │ ├── ifElseRuleSchema.csvs │ │ ├── ifRuleFailElseMetaData.csv │ │ ├── ifRuleFailThenMetaData.csv │ │ ├── ifRulePassMetaData.csv │ │ ├── ifRuleSchema.csvs │ │ ├── ignoreCasePassMetaData.csv │ │ ├── ignoreCaseSchema.csvs │ │ ├── inRuleCrossReferenceFailMetaData.csv │ │ ├── inRuleCrossReferencePassMetaData.csv │ │ ├── inRuleCrossReferenceSchema.csvs │ │ ├── inRuleFailMetaData.csv │ │ ├── inRulePassMetaData.csv │ │ ├── inRuleSchema.csvs │ │ ├── multipleErrorsMetaData.csv │ │ ├── noext.csvs │ │ ├── noextFail.csv │ │ ├── noextPass.csv │ │ ├── notempty.csv │ │ ├── notempty.csvs │ │ ├── optionalFailMetaData.csv │ │ ├── optionalPassMetaData.csv │ │ ├── optionalSchema.csvs │ │ ├── orWithFourRulesFailMetaData.csv │ │ ├── orWithFourRulesPassMetaData.csv │ │ ├── orWithFourRulesSchema.csvs │ │ ├── orWithTwoRulesFailMetaData.csv │ │ ├── orWithTwoRulesPassMetaData.csv │ │ ├── orWithTwoRulesSchema.csvs │ │ ├── rangeRuleFailMetaData.csv │ │ ├── rangeRuleFailSchema.csvs │ │ ├── rangeRuleInvalidSchema.csvs │ │ ├── rangeRulePassMetaData.csv │ │ ├── rangeRuleSchema.csvs │ │ ├── redacted.csvs │ │ ├── redactedFail.csv │ │ ├── redactedPass.csv │ │ ├── regexRuleFailMetaData.csv │ │ ├── regexRulePassMetaData.csv │ │ ├── regexRuleSchema.csvs │ │ ├── regexRuleSchemaWithNoHeaderSet.csvs │ │ ├── regexRuleSchemaWithoutNoHeaderSet.csvs │ │ ├── rule1_0.csvs │ │ ├── rulesFailMetaData.csv │ │ ├── rulesSchema.csvs │ │ ├── separated1.csvs │ │ ├── separated1.dsv │ │ ├── separated2-1.csvs │ │ ├── separated2.csvs │ │ ├── separated2.tsv │ │ ├── separated3.csvs │ │ ├── separated3.dsv │ │ ├── separated4-1.csvs │ │ ├── separated4.csvs │ │ ├── separated4.tsv │ │ ├── standardRulesFailMetaData.csv │ │ ├── standardRulesPassMetaData.csv │ │ ├── standardRulesSchema.csvs │ │ ├── switch1RuleFailMetaData.csv │ │ ├── switch1RulePassMetaData.csv │ │ ├── switch1RuleSchema.csvs │ │ ├── switch2RuleFailMetaData.csv │ │ ├── switch2RulePassMetaData.csv │ │ ├── switch2RuleSchema.csvs │ │ ├── totalColumnsFailMetaData.csv │ │ ├── totalColumnsSchema.csvs │ │ ├── twoRuleSchema.csvs │ │ ├── twoRuleSchemaFail.csvs │ │ ├── twoRulesFailMetaData.csv │ │ ├── twoRulesPassMetaData.csv │ │ ├── uriDecode.csvs │ │ ├── uriDecodeFail.csv │ │ ├── uriDecodePass.csv │ │ ├── uriDecodeWithCharset.csvs │ │ ├── uriDecodeWithCharsetPass.csv │ │ ├── warnings.csv │ │ ├── warnings.csvs │ │ ├── warningsAndErrors.csv │ │ ├── xsdDateTime.csvs │ │ ├── xsdDateTimeFail.csv │ │ ├── xsdDateTimePass.csv │ │ ├── xsdDateTimeRange.csvs │ │ ├── xsdDateTimeRangeFail.csv │ │ ├── xsdDateTimeRangePass.csv │ │ ├── xsdDateTimeTz.csvs │ │ ├── xsdDateTimeTzFail.csv │ │ ├── xsdDateTimeTzPass.csv │ │ ├── xsdDateTimeTzRange.csvs │ │ ├── xsdDateTimeTzRangeFail.csv │ │ └── xsdDateTimeTzRangePass.csv │ │ ├── api │ │ ├── metaData.csv │ │ ├── metaDataWithALongCellLength.csv │ │ ├── metadataMultipleLineBreaksInCell.csv │ │ ├── schema.csvs │ │ └── windows-1252.csv │ │ ├── badSchema.csvs │ │ ├── fileCountTestFiles │ │ ├── threeFiles │ │ │ ├── file1.jp2 │ │ │ ├── file2.jp2 │ │ │ └── file3.jp2 │ │ └── threeFilesinSubDir │ │ │ ├── file1a.jp2 │ │ │ ├── threeFilesSub1 │ │ │ └── file2a.jp2 │ │ │ └── threeFilesSub2 │ │ │ └── file3a.jp2 │ │ ├── integrityCheck │ │ ├── WO_95 │ │ │ ├── closure_v2.csv │ │ │ ├── closure_v2.csv.sha256 │ │ │ ├── content │ │ │ │ ├── 16 │ │ │ │ │ ├── 1 │ │ │ │ │ │ ├── 16_1_0001.jp2 │ │ │ │ │ │ └── 16_1_0003.jp2 │ │ │ │ │ └── 2 │ │ │ │ │ │ └── 16_2_0001.jp2 │ │ │ │ └── 39 │ │ │ │ │ └── 1 │ │ │ │ │ └── 39_1_0001.jp2 │ │ │ ├── tech_acq_metadata_v1_WO95Y14B000.csvs │ │ │ ├── tech_acq_metadata_v1_WO95Y14B000_v1.2.csvs │ │ │ ├── tech_acq_metadata_v1_WO95Y14B003.csv │ │ │ └── tech_acq_metadata_v1_WO95Y14B003.csv.sha256 │ │ ├── content │ │ │ ├── content │ │ │ │ ├── file1 │ │ │ │ ├── file2 │ │ │ │ └── file3 │ │ │ ├── integrityCheckMetaData.csv │ │ │ └── integrityCheckSchema.csvs │ │ ├── header │ │ │ ├── content │ │ │ │ ├── file1 │ │ │ │ ├── file2 │ │ │ │ └── file3 │ │ │ ├── integrityCheckDefaultIncludeFolderSchema.csvs │ │ │ ├── integrityCheckMetaData-missing-files.csv │ │ │ ├── integrityCheckMetaData.csv │ │ │ ├── integrityCheckMetaDataTooManyFiles.csv │ │ │ └── integrityCheckSchema.csvs │ │ └── noheader │ │ │ ├── badIntegrityCheckSchema.csvs │ │ │ ├── content │ │ │ ├── file1 │ │ │ ├── file2 │ │ │ └── file3 │ │ │ ├── integrityCheckMetaData.csv │ │ │ └── integrityCheckSchema.csvs │ │ ├── invalidTotalColumnsSchema.csvs │ │ └── schema │ │ ├── v1_0 │ │ ├── checksum.csvs │ │ ├── must Exist With Spaces For Rule.csvs │ │ ├── mustExistFor#Rule.csvs │ │ └── mustExistForRule.csvs │ │ └── v1_1 │ │ └── integrityCheck │ │ └── folder1 │ │ └── content │ │ ├── file#2.txt │ │ └── file1.txt │ └── scala │ └── uk │ └── gov │ └── nationalarchives │ └── csv │ └── validator │ ├── MetaDataValidatorAcceptanceSpec.scala │ ├── MetaDataValidatorBigFileSpec.scala │ ├── MetaDataValidatorBusinessAcceptanceSpec.scala │ ├── MetaDataValidatorChecksumSpec.scala │ ├── MetaDataValidatorFileCountSpec.scala │ ├── MetaDataValidatorIntegrityCheckSpec.scala │ ├── MetaDataValidatorSpec.scala │ ├── NotEmptyBugTest.scala │ ├── TestHelper.scala │ ├── TestResources.scala │ ├── UtilSpec.scala │ ├── api │ ├── CsvValidatorFileEncodingSpec.scala │ ├── CsvValidatorMaxCharsPerCellSpec.scala │ └── CsvValidatorSpec.scala │ └── schema │ ├── DateRulesSpec.scala │ ├── SchemaSpecBase.scala │ ├── v1_0 │ ├── AndRuleSpec.scala │ ├── ChecksumRuleSpec.scala │ ├── EmptyRuleSpec.scala │ ├── EndsRuleSpec.scala │ ├── FileCountSpec.scala │ ├── FileExistsRuleSpec.scala │ ├── FileWildcardSearchSpec.scala │ ├── IfRuleSpec.scala │ ├── InRuleSpec.scala │ ├── IsRuleSpec.scala │ ├── LengthRuleSpec.scala │ ├── NoArgRulesSpec.scala │ ├── NotEmptyRuleSpec.scala │ ├── NotRuleSpec.scala │ ├── OrRuleSpec.scala │ ├── RangeRuleSpec.scala │ ├── SchemaParserColumnDefinitionsSpecs.scala │ ├── SchemaParserColumnDirectivesSpec.scala │ ├── SchemaParserGlobalDirectivesSpec.scala │ ├── SchemaParserRulesSpec.scala │ ├── SchemaParserSpecs.scala │ ├── SchemaParserTotalColumnsSpec.scala │ ├── StartsRuleSpec.scala │ ├── UniqueMultiRuleSpec.scala │ └── UniqueRuleSpec.scala │ ├── v1_1 │ ├── AnyRuleSpec.scala │ ├── CaseRuleSpec.scala │ ├── IdenticalRuleSpec.scala │ ├── IntegrityCheckRuleSpec.scala │ ├── RangeRuleSpec.scala │ ├── SchemaParserSpec.scala │ ├── SchemaParserVersionSpec.scala │ ├── SchemaSpec.scala │ └── SwitchRuleSpec.scala │ └── v1_2 │ └── SchemaSpec.scala ├── csv-validator-distribution ├── README.md ├── assembly-windows-with-jre.xml ├── assembly-without-jre.xml ├── bin │ ├── csv-validator-cmd │ ├── csv-validator-cmd.bat │ ├── csv-validator-gui │ ├── csv-validator-gui.bat │ └── running-csv-validator.txt ├── pom.xml └── src │ └── assembly │ ├── filter-windows-with-jre.properties │ └── filter-without-jre.properties ├── csv-validator-java-api ├── pom.xml └── src │ ├── main │ ├── java │ │ └── uk │ │ │ └── gov │ │ │ └── nationalarchives │ │ │ └── csv │ │ │ └── validator │ │ │ └── api │ │ │ └── java │ │ │ ├── CsvValidator.java │ │ │ ├── ErrorMessage.java │ │ │ ├── FailMessage.java │ │ │ ├── ProgressCallback.java │ │ │ ├── Request.java │ │ │ ├── Result.java │ │ │ ├── Substitution.java │ │ │ └── WarningMessage.java │ └── scala │ │ └── uk │ │ └── gov │ │ └── nationalarchives │ │ └── csv │ │ └── validator │ │ └── api │ │ └── java │ │ └── CsvValidatorJavaBridge.scala │ └── test │ └── java │ └── uk │ └── gov │ └── nationalarchives │ └── csv │ └── validator │ └── api │ └── java │ └── CsvValidatorTest.java ├── csv-validator-parent └── pom.xml ├── csv-validator-ui ├── README.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── appassembler-output.xml │ └── scala │ ├── scala │ └── swing │ │ └── PopupMenu.scala │ └── uk │ └── gov │ └── nationalarchives │ └── csv │ └── validator │ └── ui │ ├── CsvValidatorUi.scala │ ├── DesignGridImplicits.scala │ ├── SJXHelpers.scala │ └── ScalaSwingHelpers.scala └── pom.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/caseSensitiveFilesChecksum.csvs -crlf 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: maven 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - master 7 | jobs: 8 | build: 9 | name: Build and Test 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | os: [ubuntu-latest, macos-latest, windows-latest] 14 | jdk: [21] 15 | runs-on: ${{ matrix.os }} 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Set up JDK 19 | uses: actions/setup-java@v4 20 | with: 21 | java-version: ${{ matrix.jdk }} 22 | distribution: temurin 23 | - name: Cache Maven packages 24 | uses: actions/cache@v4 25 | with: 26 | path: ~/.m2 27 | key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} 28 | restore-keys: ${{ runner.os }}-m2 29 | - name: Maven Build 30 | run: mvn -V -B -DskipTests=true install 31 | - name: Maven Test 32 | run: mvn -B verify 33 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL Advanced" 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - master 7 | schedule: 8 | - cron: '30 8 * * 1' 9 | 10 | jobs: 11 | analyze: 12 | name: Analyze (${{ matrix.language }}) 13 | # Runner size impacts CodeQL analysis time. To learn more, please see: 14 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 15 | # - https://gh.io/supported-runners-and-hardware-resources 16 | # - https://gh.io/using-larger-runners (GitHub.com only) 17 | # Consider using larger runners or machines with greater resources for possible analysis time improvements. 18 | runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} 19 | permissions: 20 | # required for all workflows 21 | security-events: write 22 | 23 | # required to fetch internal or private CodeQL packs 24 | packages: read 25 | 26 | # only required for workflows in private repositories 27 | actions: read 28 | contents: read 29 | 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | include: 34 | - language: java-kotlin 35 | build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too. 36 | # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' 37 | # Use `c-cpp` to analyze code written in C, C++ or both 38 | # Use 'java-kotlin' to analyze code written in Java, Kotlin or both 39 | # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both 40 | # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, 41 | # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. 42 | # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how 43 | # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages 44 | steps: 45 | - name: Checkout repository 46 | uses: actions/checkout@v4 47 | 48 | # Add any setup steps before running the `github/codeql-action/init` action. 49 | # This includes steps like installing compilers or runtimes (`actions/setup-node` 50 | # or others). This is typically only required for manual builds. 51 | # - name: Setup runtime (example) 52 | # uses: actions/setup-example@v1 53 | 54 | # Initializes the CodeQL tools for scanning. 55 | - name: Initialize CodeQL 56 | uses: github/codeql-action/init@v3 57 | with: 58 | languages: ${{ matrix.language }} 59 | build-mode: ${{ matrix.build-mode }} 60 | # If you wish to specify custom queries, you can do so here or in a config file. 61 | # By default, queries listed here will override any specified in a config file. 62 | # Prefix the list here with "+" to use these queries and those in the config file. 63 | 64 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 65 | # queries: security-extended,security-and-quality 66 | 67 | # If the analyze step fails for one of the languages you are analyzing with 68 | # "We were unable to automatically build your code", modify the matrix above 69 | # to set the build mode to "manual" for that language. Then modify this step 70 | # to build your code. 71 | # ℹ️ Command-line programs to run using the OS shell. 72 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 73 | - if: matrix.build-mode == 'manual' 74 | shell: bash 75 | run: | 76 | echo 'If you are using a "manual" build mode for one or more of the' \ 77 | 'languages you are analyzing, replace this with the commands to build' \ 78 | 'your code, for example:' 79 | echo ' make bootstrap' 80 | echo ' make release' 81 | exit 1 82 | 83 | - name: Perform CodeQL Analysis 84 | uses: github/codeql-action/analyze@v3 85 | with: 86 | category: "/language:${{matrix.language}}" 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | dist/* 6 | target/ 7 | lib_managed/ 8 | src_managed/ 9 | project/boot/ 10 | project/plugins/project/ 11 | 12 | # Scala-IDE specific 13 | .scala_dependencies 14 | 15 | # IntelliJ specific 16 | .idea/* 17 | *.iml 18 | .bsp/ 19 | 20 | # MacOS specific 21 | **/.DS_Store 22 | -------------------------------------------------------------------------------- /csv-validator-cmd/README.md: -------------------------------------------------------------------------------- 1 | CSV Validator - Command Line 2 | ============================ 3 | 4 | A simple command line utility for running the CSV Validator. 5 | 6 | Designed and tested on both Windows and Linux/Unix/Mac platforms. 7 | 8 | 9 | Basic Usage 10 | ----------- 11 | 12 | The following command will show the usage for the application: 13 | 14 | ```bash 15 | $ validate 16 | ``` 17 | 18 | which results in: 19 | 20 | ```bash 21 | $ validate 22 | CSV Validator - Command Line 23 | Usage: validate [options] 24 | 25 | -f | --fail-fast 26 | Stops on the first validation error rather than reporting all errors 27 | -p:= | --path:= 28 | Allows you to substitute a file path (or part of) in the CSV for a different file path 29 | 30 | The path to the CSV file to validate 31 | 32 | The path to the CSV Schema file to use for validation 33 | --disable-utf8-validation 34 | Disable UTF-8 validation for CSV files 35 | --max-chars-per-cell 36 | Maximum number of chars allowed in cell (is set to 4096 by default) 37 | --skip-file-checks 38 | Skip integrity, checksum and file existence checks 39 | --show-progress 40 | Show progress 41 | --help 42 | Prints this usage text 43 | 44 | ``` 45 | 46 | 47 | Windows Users 48 | ------------- 49 | 50 | Instead of using `validate` simply use `validate.bat` 51 | 52 | 53 | Building from Source Code 54 | ------------------------- 55 | 56 | We are using the Maven build system, executing `mvn package` will produce a distribution of the command line application in both `target/csv-validator-cmd-?-application` and `target/csv-validator-cmd-?-application.zip`. Note the '?' will be replaced by the version number of the product. -------------------------------------------------------------------------------- /csv-validator-cmd/src/main/assembly/appassembler-output.xml: -------------------------------------------------------------------------------- 1 | 11 | 14 | application 15 | 16 | dir 17 | zip 18 | 19 | 20 | true 21 | 22 | 23 | 24 | 25 | 26 | ${project.build.directory}/appassembler 27 | 28 | bin/${project.artifactId} 29 | 30 | 755 31 | / 32 | 33 | 34 | 35 | 36 | ${project.build.directory}/appassembler/bin 37 | 38 | bin/${project.artifactId} 39 | 40 | /bin 41 | 42 | 43 | 44 | 45 | ${project.build.directory}/appassembler 46 | 47 | bin 48 | 49 | / 50 | 51 | 52 | 53 | 54 | 55 | 56 | ../LICENSE 57 | / 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/main/resources-filtered/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2013, The National Archives 3 | # https://www.nationalarchives.gov.uk 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla Public 6 | # License, v. 2.0. If a copy of the MPL was not distributed with this 7 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | # 9 | 10 | FROM cgr.dev/chainguard/jre:openjdk-17 11 | 12 | # Copy CSV Validator 13 | COPY LICENSE /csv-validator/LICENSE 14 | COPY lib /csv-validator/lib 15 | 16 | 17 | # Build-time metadata as defined at http://label-schema.org 18 | # and used by autobuilder @hooks/build 19 | LABEL org.label-schema.build-date=${maven.build.timestamp} \ 20 | org.label-schema.description="${project.description}" \ 21 | org.label-schema.name="CSV Validator" \ 22 | org.label-schema.schema-version="1.0" \ 23 | org.label-schema.url="${project.url}" \ 24 | org.label-schema.vcs-ref=${build-commit-abbrev} \ 25 | org.label-schema.vcs-url="${project.scm.url}" \ 26 | org.label-schema.vendor="The National Archives" 27 | 28 | ENV CLASSPATH=/csv-validator/lib/* 29 | 30 | ENV JAVA_OPTS \ 31 | -Dfile.encoding=UTF8 \ 32 | -Dsun.jnu.encoding=UTF-8 \ 33 | -Djava.awt.headless=true \ 34 | -XX:+UseNUMA \ 35 | -XX:+UseZGC \ 36 | -XX:+UseStringDeduplication \ 37 | -XX:+UseContainerSupport \ 38 | -XX:MaxRAMPercentage=${JVM_MAX_RAM_PERCENTAGE:-75.0} \ 39 | -XX:+ExitOnOutOfMemoryError 40 | 41 | ENTRYPOINT [ "java", \ 42 | "uk.gov.nationalarchives.csv.validator.cmd.CsvValidatorCmdApp"] 43 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/acceptance/standardRulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | http:##datagov.nationalarchives.gov.uk#66#WO#409#9999#0#aaaaaaaa-aaaa-4aaa-9eee-0123456789ab,2002-999-30T09:00:10,02-99-30,99/00/0009,99:00:889,aaaaaaaab-aaaab-4aaa-9eee-0123456789ab,12-0912459 -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/acceptance/standardRulesSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 7 @noHeader 3 | uri: uri 4 | xDateTime: xDateTime 5 | xDate: xDate 6 | ukDate: ukDate 7 | xTime: xTime 8 | uuid4: uuid4 9 | positiveInteger: positiveInteger 10 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/badSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @TotalColum 2 3 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/metaData.csv: -------------------------------------------------------------------------------- 1 | col1,col2 2 | col1,col2 -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/schema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Age: 5 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/warning.csv: -------------------------------------------------------------------------------- 1 | department,division 2 | WO,13 3 | BT,13 4 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/warning.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 3 | department: is("WO") @warning 4 | division: is("13") 5 | -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/windows-1252.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-cmd/src/test/resources/uk/gov/nationalarchives/csv/validator/cmd/windows-1252.csv -------------------------------------------------------------------------------- /csv-validator-cmd/src/test/scala/uk/gov/nationalarchives/csv/validator/cmd/TestResources.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.cmd 10 | 11 | import org.specs2.mutable.Specification 12 | import uk.gov.nationalarchives.csv.validator.FILE_SEPARATOR 13 | 14 | import java.nio.file.Paths 15 | 16 | trait TestResources { 17 | 18 | spec: Specification => 19 | 20 | /** 21 | * Utility function to get an absolute path to a resource 22 | * in the same class as a spec test 23 | * 24 | * @param resource The path of the resource relative to the spec test 25 | */ 26 | def resourcePath(resource: String) : String = Paths.get(baseResourcePkgPath).resolve(resource).toAbsolutePath.toString 27 | 28 | def baseResourcePkgPath : String = Paths.get(basePath).resolve(spec.getClass.getPackage.getName.replace('.', '/')).toAbsolutePath.toString 29 | 30 | def basePath : String = { 31 | val url = spec.getClass.getClassLoader.getResource(".") 32 | Paths.get(url.toURI).toAbsolutePath.toString 33 | } 34 | 35 | def relBasePath : String = basePath.replace(System.getProperty("user.dir") + FILE_SEPARATOR, "") 36 | def relBaseResourcePkgPath : String = Paths.get(relBasePath).resolve(spec.getClass.getPackage.getName.replace('.', '/')).toString 37 | def relResourcePath(resource: String) : String = Paths.get(relBaseResourcePkgPath).resolve(resource).toString 38 | } 39 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/AllErrorsMetaDataValidator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import cats.syntax.all._ 12 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 13 | import uk.gov.nationalarchives.csv.validator.schema.{Optional, Rule, Schema, Warning} 14 | 15 | import scala.annotation.tailrec 16 | 17 | trait AllErrorsMetaDataValidator extends MetaDataValidator { 18 | 19 | override def validateRows( 20 | rows: Iterator[Row], 21 | schema: Schema, 22 | rowCallback: MetaDataValidation[Any] => Unit 23 | ): Boolean = { 24 | 25 | @tailrec 26 | def inner(passing: Boolean) : Boolean = { 27 | if(!rows.hasNext) { 28 | passing 29 | } else { 30 | val row = rows.next() 31 | val result = validateRow(row, schema, Some(rows.hasNext)) 32 | rowCallback(result) 33 | inner(passing && !containsErrors(result)) 34 | } 35 | } 36 | 37 | inner(true) 38 | } 39 | 40 | 41 | override protected def rules(row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): MetaDataValidation[List[Any]] = { 42 | val cells: (Int) => Option[Cell] = row.cells.lift 43 | val v = schema.columnDefinitions.zipWithIndex.map { 44 | case (columnDefinition, columnIndex) => 45 | validateCell(columnIndex, cells, row, schema, mayBeLast) 46 | } 47 | 48 | v.sequence[MetaDataValidation, Any] 49 | } 50 | 51 | override protected def rulesForCell(columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): MetaDataValidation[Any] = { 52 | 53 | val columnDefinition = schema.columnDefinitions(columnIndex) 54 | 55 | def isWarningDirective: Boolean = columnDefinition.directives.contains(Warning()) 56 | def isOptionDirective: Boolean = columnDefinition.directives.contains(Optional()) 57 | 58 | if(row.cells(columnIndex).value.trim.isEmpty && isOptionDirective) true.validNel 59 | else columnDefinition.rules.map{rule => 60 | rule.evaluate(columnIndex, row, schema, mayBeLast) 61 | }.map{ ruleResult:Rule#RuleValidation[Any] => { 62 | if(isWarningDirective) toWarnings(ruleResult, row.lineNumber, columnIndex) else toErrors(ruleResult, row.lineNumber, columnIndex) 63 | }}.sequence[MetaDataValidation, Any] 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/FailFastMetaDataValidator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import cats.data.Validated.{Invalid => Failure} 12 | import cats.syntax.all._ 13 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 14 | import uk.gov.nationalarchives.csv.validator.schema._ 15 | 16 | import scala.annotation.tailrec 17 | 18 | trait FailFastMetaDataValidator extends MetaDataValidator { 19 | 20 | //TODO(AR) work on removing use of `Any` 21 | 22 | override def validateRows( 23 | rows: Iterator[Row], 24 | schema: Schema, 25 | rowCallback: MetaDataValidation[Any] => Unit 26 | ): Boolean = { 27 | 28 | @tailrec 29 | def inner(passing: Boolean) : Boolean = { 30 | if(!rows.hasNext) { 31 | passing 32 | } else { 33 | val row = rows.next() 34 | val result = validateRow(row, schema, Some(rows.hasNext)) 35 | rowCallback(result) 36 | if(!containsErrors(result)) 37 | inner(passing) 38 | else 39 | false 40 | } 41 | } 42 | 43 | inner(true) 44 | } 45 | 46 | override protected def rules(row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): MetaDataValidation[List[Any]] = { 47 | val cells: (Int) => Option[Cell] = row.cells.lift 48 | 49 | @tailrec 50 | def validateRules(columnDefinitions: List[(ColumnDefinition, Int)], accum: List[MetaDataValidation[Any]] = List.empty) : List[MetaDataValidation[Any]] = { 51 | columnDefinitions match { 52 | case Nil => 53 | if (accum.isEmpty) { 54 | List(true.validNel[FailMessage]) 55 | } else { 56 | accum.reverse 57 | } 58 | 59 | case (columnDefinition, columnIndex) :: tail => 60 | validateCell(columnIndex, cells, row, schema, mayBeLast) match { 61 | case failure @ Failure(_) if(!schema.columnDefinitions(columnIndex).directives.contains(Warning())) => 62 | validateRules(List.empty, failure :: accum) //stop on first failure which is not a warning 63 | case result => 64 | validateRules(tail, result :: accum) 65 | } 66 | } 67 | } 68 | 69 | val v = validateRules(schema.columnDefinitions.zipWithIndex) 70 | v.sequence[MetaDataValidation, Any] 71 | } 72 | 73 | override protected def rulesForCell(columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): MetaDataValidation[Any] = { 74 | val columnDefinition = schema.columnDefinitions(columnIndex) 75 | 76 | def isWarningDirective: Boolean = columnDefinition.directives.contains(Warning()) 77 | def isOptionDirective: Boolean = columnDefinition.directives.contains(Optional()) 78 | 79 | @tailrec 80 | def validateRulesForCell(rules: List[Rule]): MetaDataValidation[Any] = rules match { 81 | case Nil => true.validNel[FailMessage] 82 | case rule :: tail => rule.evaluate(columnIndex, row, schema, mayBeLast) match { 83 | case e@Failure(_) => toErrors(e, row.lineNumber, columnIndex) 84 | case _ => validateRulesForCell(tail) 85 | } 86 | } 87 | 88 | def validateAllRulesForCell(rules: List[Rule]): MetaDataValidation[Any] = rules.map(_.evaluate(columnIndex, row, schema, mayBeLast)).map(toWarnings(_, row.lineNumber, columnIndex)).sequence[MetaDataValidation, Any] 89 | 90 | if(row.cells(columnIndex).value.trim.isEmpty && isOptionDirective) true.validNel 91 | else if(isWarningDirective) validateAllRulesForCell(columnDefinition.rules) 92 | else validateRulesForCell(columnDefinition.rules) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/metadata/MetaData.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.metadata 10 | 11 | case class Row(cells: List[Cell], lineNumber: Int) 12 | 13 | case class Cell(value: String) 14 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv 10 | 11 | import java.nio.charset.StandardCharsets 12 | 13 | import scala.sys 14 | 15 | package object validator { 16 | 17 | /** 18 | * End-of-Line 19 | */ 20 | val EOL = sys.props("line.separator") 21 | 22 | val ENCODING = StandardCharsets.UTF_8 23 | 24 | /** 25 | * Separator in file paths i.e. '\' or '/' 26 | */ 27 | val FILE_SEPARATOR = sys.props("file.separator").head 28 | 29 | val WINDOWS_FILE_SEPARATOR = '\\' 30 | 31 | val UNIX_FILE_SEPARATOR = '/' 32 | 33 | val URI_PATH_SEPARATOR = '/' 34 | 35 | val CSV_RFC1480_SEPARATOR: Char = ',' 36 | 37 | val CSV_RFC1480_QUOTE_CHARACTER: Char = '"' 38 | 39 | val CSV_RFC1480_QUOTE_ESCAPE_CHARACTER: Char = '"' 40 | 41 | val CSV_RFC1480_LINE_SEPARATOR = Array('\r', '\n') 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/Schema.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema 10 | 11 | import org.apache.commons.io.FilenameUtils 12 | import uk.gov.nationalarchives.csv.validator.metadata.Row 13 | import util.parsing.input.Positional 14 | 15 | case class Schema(globalDirectives: List[GlobalDirective], columnDefinitions: List[ColumnDefinition], version: String = Schema.version) 16 | 17 | object Schema { 18 | val version = "1.2" 19 | } 20 | 21 | abstract class GlobalDirective(val name: String) extends Positional 22 | 23 | case class Separator(separatorChar: Char) extends GlobalDirective("separator") 24 | 25 | case class Quoted() extends GlobalDirective("quoted") 26 | 27 | case class TotalColumns(numberOfColumns: BigInt) extends GlobalDirective("totalColumns") 28 | 29 | case class PermitEmpty() extends GlobalDirective("permitEmpty") 30 | 31 | case class NoHeader() extends GlobalDirective("noHeader") 32 | 33 | case class IgnoreColumnNameCase() extends GlobalDirective("ignoreColumnNameCase") 34 | 35 | trait ColumnIdentifier { 36 | val value: String 37 | override def toString = value 38 | } 39 | 40 | case class NamedColumnIdentifier(name: String) extends ColumnIdentifier { 41 | val value = name 42 | } 43 | 44 | case class OffsetColumnIdentifier(offset: BigInt) extends ColumnIdentifier { 45 | val value = offset.toString 46 | } 47 | 48 | case class ColumnDefinition(id: ColumnIdentifier, rules: List[Rule] = Nil, directives: List[ColumnDirective] = Nil) extends Positional 49 | 50 | trait ArgProvider { 51 | 52 | def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] 53 | 54 | def toError: String 55 | } 56 | 57 | case class ColumnReference(ref: ColumnIdentifier) extends ArgProvider { 58 | 59 | def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = { 60 | val referencedIndex = schema.columnDefinitions.indexWhere(_.id == ref) 61 | row.cells.lift(referencedIndex).map(_.value) 62 | } 63 | 64 | @throws[IndexOutOfBoundsException] 65 | def referenceValueEx(columnIndex: Int, row: Row, schema: Schema): String = { 66 | referenceValue(columnIndex, row, schema).getOrElse{ 67 | throw new ArrayIndexOutOfBoundsException(s"Could not access reference column $ref at [$columnIndex:${row.lineNumber}]") 68 | } 69 | } 70 | 71 | def toError ="$" + ref 72 | } 73 | 74 | case class Literal(value: Option[String]) extends ArgProvider { 75 | 76 | def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = value 77 | 78 | def toError = value.map("\"" + _ + "\"").getOrElse("") 79 | } 80 | 81 | 82 | 83 | trait ColumnDirective extends Positional 84 | 85 | case class Optional() extends ColumnDirective { 86 | override def toString = "optional" 87 | } 88 | 89 | case class MatchIsFalse() extends ColumnDirective { 90 | override def toString = "matchIsFalse" 91 | } 92 | 93 | case class Warning() extends ColumnDirective { 94 | override def toString = "warning" 95 | } 96 | 97 | case class IgnoreCase() extends ColumnDirective { 98 | override def toString = "ignoreCase" 99 | } 100 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaValidator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema 10 | 11 | import org.gfccollective.semver.SemVer 12 | import uk.gov.nationalarchives.csv.validator.schema.v1_0.{SchemaValidator => SchemaValidator1_0} 13 | import uk.gov.nationalarchives.csv.validator.schema.v1_1.{SchemaValidator => SchemaValidator1_1} 14 | 15 | /** 16 | * Utility object to validate schema consistency 17 | */ 18 | object SchemaValidator { 19 | /** 20 | * Validate the schema against certain rules depending on the schema Version 21 | * @param schema 22 | * @return 23 | */ 24 | def validate(schema: Schema): String = { 25 | val g: List[GlobalDirective] = schema.globalDirectives 26 | val c: List[ColumnDefinition] = schema.columnDefinitions 27 | 28 | schema.version match { 29 | case "1.1" => SchemaValidator1_1(g,c) 30 | case _ => SchemaValidator1_0(g,c) 31 | } 32 | } 33 | 34 | def versionValid(version: String): Option[String] = { 35 | if (SemVer(version) > SemVer(Schema.version)) 36 | Some(s"Invalid schema version. This version of the csv validator supports only ${Schema.version} and below.") 37 | else 38 | None 39 | } 40 | } 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/TraceableParsers.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema 10 | 11 | import scala.util.parsing.combinator.Parsers 12 | import scala.language.implicitConversions 13 | 14 | /** 15 | * TraceableParsers provides the Traceable 16 | * Parser `::=` which allows you to name parsers 17 | * and also to trace their parse attempts. 18 | * 19 | * The `::=` symbol was chosen as it aligns with 20 | * the syntax of EBNF which also leads to the 21 | * nice conincidence that your parser definitions 22 | * will likely look closer to their EBNF counterpart 23 | * definitions. 24 | * 25 | * @author Adam Retter 26 | */ 27 | trait TraceableParsers extends Parsers { 28 | 29 | val trace: Boolean //whether tracing should be enabled or not 30 | 31 | class TraceableParser[+T](parser: Parser[T]) extends Parser[T] { 32 | def apply(in: Input): ParseResult[T] = { 33 | if(trace) { 34 | val first = in.first 35 | val pos = in.pos 36 | val offset = in.offset 37 | val parseResult = parser.apply(in) 38 | println(s"""${parser.toString()}.apply for token "$first" at position $pos offset $offset returns "$parseResult"""") 39 | parseResult 40 | } else { 41 | parser.apply(in) 42 | } 43 | } 44 | } 45 | 46 | /** 47 | * Implicit which converts a name and a parser into a TraceableParser 48 | * 49 | * e.g. Give the EBNF: 50 | * 
MyParser ::= "SomeValue"?
 51 | * 52 | * You would write the Scala code: 53 | * 
val myParser : Option[Parser[String]] = "MyParser" ::= "SomeValue"?
 54 | */ 55 | implicit def toTraceableParser(name: String) = new { 56 | def ::=[T](p: Parser[T]) = new TraceableParser(p.named(name)) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | package object schema { 12 | val Uuid4Regex = "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" 13 | //val UriRegex = """([A-Za-z0-9]+:\/\/)?([a-zA-Z0-9]+(\.[a-zA-Z0-9]+)*)?(\/|(\/([A-Za-z0-9\:@!\$&'\(\}\*\+\-_,;=~\.]+|(%[A-F0-9]{2})+))*)(\?[A-Za-z0-9]+=[A-Za-z0-9]+(&[A-Za-z0-9]+=[A-Za-z0-9]+)*)?""" 14 | 15 | val XsdDateTimeRegex = xsdDateNoTzComponentRegex + "T" + XsdTimeOptionalTimeZoneRegex 16 | val XsdDateTimeWithTimeZoneRegex = xsdDateNoTzComponentRegex + "T" + XsdTimeWithTimeZoneRegex 17 | val XsdDateRegex = xsdDateNoTzComponentRegex + xsdOptionalTzComponentRegex 18 | lazy val XsdTimeOptionalTimeZoneRegex = xsdTimeNoTzComponentRegex + xsdOptionalTzComponentRegex 19 | lazy val XsdTimeWithTimeZoneRegex = xsdTimeNoTzComponentRegex + xsdTzComponentRegex 20 | 21 | private lazy val xsdDateNoTzComponentRegex = """-?[0-9]{4}-(((0(1|3|5|7|8)|1(0|2))-(0[1-9]|(1|2)[0-9]|3[0-1]))|((0(4|6|9)|11)-(0[1-9]|(1|2)[0-9]|30))|(02-(0[1-9]|(1|2)[0-9])))""" 22 | private lazy val xsdTimeNoTzComponentRegex = """([0-1][0-9]|2[0-4]):(0[0-9]|[1-5][0-9]):(0[0-9]|[1-5][0-9])(\.[0-9]{3})?""" 23 | private lazy val xsdOptionalTzComponentRegex = """((\+|-)([0-1][0-9]|2[0-4]):(0[0-9]|[1-5][0-9])|Z)?""" 24 | private lazy val xsdTzComponentRegex = """((\+|-)([0-1][0-9]|2[0-4]):(0[0-9]|[1-5][0-9])|Z)""" 25 | 26 | val UkDateRegex = """(((0[1-9]|(1|2)[0-9]|3[0-1])\/(0(1|3|5|7|8)|1(0|2)))|((0[1-9]|(1|2)[0-9]|30)\/(0(4|6|9)|11))|((0[1-9]|(1|2)[0-9])\/02))\/[0-9]{4}""" 27 | val PositiveIntegerRegex = "[0-9]+" 28 | val UpperCaseRegex = "^[\\p{Lu}\\p{N}\\p{P}\\s]*$" 29 | val LowerCaseRegex = "^[\\p{Ll}\\p{N}\\p{P}\\s]*$" 30 | val UkDateFormat = "dd/MM/YYYY" 31 | val PartUkDateRegex = """(([0\?][1-9\?])|([1-2\?][0-9\?])|([3\?][0-1\?])|\*)\/(January|February|March|April|May|June|July|August|September|October|November|December|\?|\*)\/([0-9\?]{4}|\*)""" 32 | 33 | 34 | type FilePathBase = String 35 | type FileName = String 36 | } 37 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/Schema.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import org.apache.commons.io.FilenameUtils 12 | import uk.gov.nationalarchives.csv.validator.metadata.Row 13 | import uk.gov.nationalarchives.csv.validator.schema._ 14 | 15 | case class NoExt(value: ArgProvider) extends ArgProvider { 16 | 17 | def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = value.referenceValue(columnIndex, row, schema).map(FilenameUtils.removeExtension(_)) 18 | 19 | def toError = "noExt(" + value.toError + ")" 20 | 21 | } 22 | 23 | case class Concat(args: ArgProvider*) extends ArgProvider { 24 | 25 | def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = concat(args.map(_.referenceValue(columnIndex, row, schema))) 26 | 27 | def concat(ms: Seq[Option[String]]): Option[String] = 28 | if (ms.forall(_.isEmpty)) 29 | None 30 | else { 31 | Some(ms.foldLeft(""){case (acc,elem) => acc + elem.getOrElse("")}) 32 | } 33 | 34 | def toError = "concat(" + args.map(_.toError).mkString(", ") + ")" 35 | } 36 | 37 | -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaValidator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import uk.gov.nationalarchives.csv.validator._ 12 | import uk.gov.nationalarchives.csv.validator.schema.v1_0.{SchemaValidator => SchemaValidator1_0} 13 | import uk.gov.nationalarchives.csv.validator.schema.{ColumnDefinition, GlobalDirective, Rule} 14 | 15 | /** 16 | * Set of rules to validate the schema (compatible with the CSV Schema 1.1) 17 | * Overrides 1.0 rules for backward compatibilities and update only the rules who needs to be 18 | * @author Valy Dia 19 | */ 20 | class SchemaValidator extends SchemaValidator1_0 { 21 | 22 | override protected def rangeValid(columnDefinitions: List[ColumnDefinition]): Option[String] = { 23 | def rangeCheck(rule: Rule): Option[String] = rule match { 24 | case RangeRule(None,None) => Some(s"""Invalid range in 'range(*,*)' at least one value needs to be defined""") 25 | case RangeRule(Some(min),Some(max)) => if (min > max) Some(s"""Invalid range, minimum greater than maximum in: 'range($min,$max)' at line: ${rule.pos.line}, column: ${rule.pos.column}""") else None 26 | case _ => None 27 | } 28 | 29 | val v = for { 30 | cd <- columnDefinitions 31 | rule <- cd.rules 32 | message = rangeCheck(rule) 33 | if (message.isDefined) 34 | } yield { 35 | val errormessage = rule match { 36 | case range:RangeRule => message.getOrElse("") 37 | case _ => s"""Column: ${cd.id}: Invalid range, minimum greater than maximum: at line: ${rule.pos.line}, column: ${rule.pos.column}""" 38 | } 39 | s"""Column: ${cd.id}: """ + errormessage 40 | 41 | } 42 | 43 | if (v.isEmpty) None else Some(v.mkString(EOL)) 44 | } 45 | 46 | } 47 | 48 | object SchemaValidator { 49 | def apply(g: List[GlobalDirective], c: List[ColumnDefinition]): String = { 50 | val parser = new SchemaValidator() 51 | parser.validate(g,c) 52 | } 53 | } -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_2/Schema.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_2 10 | 11 | import uk.gov.nationalarchives.csv.validator.metadata.Row 12 | import uk.gov.nationalarchives.csv.validator.schema.{Schema, ArgProvider} 13 | import java.net.{URLDecoder => JURLDecoder} 14 | 15 | case class UriDecode(value: ArgProvider, charset: Option[ArgProvider]) extends ArgProvider { 16 | 17 | val DefaultCharset = "UTF-8" 18 | 19 | override def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = value.referenceValue(columnIndex, row, schema).map( value => { 20 | 21 | val codepage = charset.flatMap(x => x.referenceValue(columnIndex, row, schema)).getOrElse(DefaultCharset) 22 | 23 | JURLDecoder.decode(value,codepage) 24 | 25 | }) 26 | 27 | override def toError: String = "uriDecode(" + value.toError + ")" 28 | } -------------------------------------------------------------------------------- /csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_2/SchemaParser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_2 10 | 11 | import scala.language.reflectiveCalls 12 | import uk.gov.nationalarchives.csv.validator.schema.{Literal, ArgProvider} 13 | import uk.gov.nationalarchives.csv.validator.schema.v1_1.{SchemaParser => SchemaParser1_1} 14 | 15 | trait SchemaParser extends SchemaParser1_1 { 16 | 17 | /** 18 | * [59] StringProvider ::= ColumnRef | StringLiteral 19 | */ 20 | override lazy val stringProvider: PackratParser[ArgProvider] = "StringProvider" ::= noExt | concat | urlDecode | columnRef | stringLiteral ^^ { 21 | s => Literal(Some(s)) 22 | } 23 | 24 | lazy val urlDecode: PackratParser[ArgProvider] = "UriDecode" ::= "uriDecode(" ~> stringProvider ~ opt("," ~> stringProvider) <~ ")" ^^ { 25 | case value ~ charset => UriDecode(value, charset) 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/andyPass.csvs: -------------------------------------------------------------------------------- 1 | somePa55w0rd -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/anyRuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,value1 2 | Andy,value2 3 | Andy,value3 4 | Andy,value4 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/anyRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,value1 2 | Andy,value2 3 | Andy,value3 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/anyRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 @noHeader 3 | Name: 4 | SomeAnyRule: any("value1","value2","value3") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/benPass.csvs: -------------------------------------------------------------------------------- 1 | somePa55w0rd -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/bigSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | AlphaNumeric: 4 | Numeric: regex("[0-9]+") 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/bom.csv: -------------------------------------------------------------------------------- 1 | column 2 | col -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/bom.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 1 3 | column: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/caseSensitiveFiles.csv: -------------------------------------------------------------------------------- 1 | filename,comment 2 | caseSensitiveFiles.csv,this one is correct 3 | casesensitivefiles.csv,this one should fail validation 4 | CASESENSITIVEFILES.csv, this one should also fail validation -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/caseSensitiveFiles.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 3 | filename: fileExists("$$acceptancePath$$") 4 | comment: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/caseSensitiveFilesChecksum.csv: -------------------------------------------------------------------------------- 1 | filename,checksum,comment 2 | caseSensitiveFilesChecksum.csvs,41424313f6052b7f062358ed38640b6e,this one is correct 3 | casesensitivefileschecksum.csvs,41424313f6052b7f062358ed38640b6e,this one should fail validation 4 | CASESENSITIVEFILESCHECKSUM.csvs,41424313f6052b7f062358ed38640b6e,this one should also fail validation -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/caseSensitiveFilesChecksum.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | filename: 4 | checksum: checksum(file("$$acceptancePath$$", $filename), "MD5") 5 | comment: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concat.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 3 | c1: 4 | c2: 5 | c3: is(concat($c1,$c2)) 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concat4.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 5 3 | c1: 4 | c2: 5 | c3: 6 | c4: 7 | c5: is(concat($c1,$c2,"hello",$c4)) 8 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concat4Fail.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3,c4,c5 2 | a,b,c,d,abhellod 3 | aa,bb,cc,dd,aabbccdd -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concat4Pass.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3,c4,c5 2 | a,b,c,d,abhellod 3 | aa,bb,cc,dd,aabbhellodd 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concatFail.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3 2 | hello ,world,hello world 3 | the tree, is blue,the tree is blue 4 | aaaa,bbbbb,ccccc -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/concatPass.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3 2 | hello ,world,hello world 3 | the tree, is blue,the tree is blue -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/dp/regexRuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Andy,twenty 2 | Ben,39 3 | Nicki,seven 4 | Welch,-two 5 | Nicola,29 6 | Tracey,100 and 50 7 | Vicki,40 8 | David,22 9 | Hilda,500 years old 10 | Penelope,63 11 | Nicki Gilhooly,05 12 | Viola,-333 13 | Bryan,66 14 | Mildred,54 15 | Frog,12 16 | Nicki Welch,880 17 | Cat,22 18 | Rob,33 19 | Ian Henderson,+54 20 | Ian Ireland,54 21 | Allie,01 22 | Jamie,900 23 | Graham Brown,1000 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/dp/regexRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Andy,28 2 | Ben,39 3 | Nicki,27 4 | Welch,28 5 | Nicola,29 6 | Tracey,100 7 | Vicki,40 8 | David,22 9 | Hilda,500 10 | Penelope,63 11 | Nicki Gilhooly,05 12 | Viola,333 13 | Bryan,66 14 | Mildred,54 15 | Frog,12 16 | Nicki Welch,880 17 | Cat,22 18 | Rob,33 19 | Ian Henderson,54 20 | Ian Ireland,54 21 | Allie,01 22 | Jamie,900 23 | Graham Brown,1000 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/dp/regexRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Age: regex("[0-9]+") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/duplicateColumnIdsFailSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 6 3 | Age: 4 | Country: 5 | Country: 6 | Name: 7 | Country: 8 | Age: 9 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/duplicateColumnIdsMetaData.csv: -------------------------------------------------------------------------------- 1 | Age,Age,Age,Age,Age,Age 2 | dummy,dummy,dummy,dummy,dummy,dummy -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/duplicateColumnIdsPassSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 6 3 | Age: 4 | Country: 5 | Address: 6 | Name: 7 | DateOfBirth: 8 | FavouriteColour: 9 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/fileExistsCrossRefPassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,$$acceptancePath$$,fileExistsSchema.csvs 2 | Andy,$$acceptancePath$$,fileExistsSchema.csvs -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/fileExistsCrossRefSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 @noHeader 3 | Name: 4 | PhotoDir: 5 | Photo: fileExists($PhotoDir) 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/fileExistsPassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,39,benPass.csvs 2 | Andy,22,andyPass.csvs -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/fileExistsSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 @noHeader 3 | Name: 4 | Age: 5 | PasswordFile: fileExists("$$acceptancePath$$") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/fileExistsSchemaWithBadBasePath.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 @noHeader 3 | Name: 4 | Age: 5 | PasswordFile: fileExists("src/test/resources/uk/gov/nationalarchives") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalEmptyMetaData.csv: -------------------------------------------------------------------------------- 1 | ben, 2 | andy, 3 | david, -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalFailMetaData.csv: -------------------------------------------------------------------------------- 1 | ben,abc 2 | andy,abc 3 | david,fff -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalHeaderMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,FullName 2 | ben,abc 3 | andy,abc 4 | david,abc -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalHeaderSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | FullName: identical 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalPassMetaData.csv: -------------------------------------------------------------------------------- 1 | ben,abc 2 | andy,abc 3 | david,abc -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/identicalSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 @noHeader 3 | Name: 4 | FullName: identical 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ifElseRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | SomeIfRule: if($Name/starts("hello"),is("hello world"),upperCase) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ifRuleFailElseMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeIfRule 2 | hello,hello world 3 | world,SDWGRGWR 4 | 1234,EFQWeGW -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ifRuleFailThenMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeIfRule 2 | hello,hello world1 3 | world,SDWGRGWR 4 | 1234,EFQWEGW -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ifRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeIfRule 2 | hello,hello world 3 | world,SDWGRGWR 4 | 1234,EFQWEGW -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ifRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | SomeIfRule: if($Name/starts("hello"),is("hello world")) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ignoreCasePassMetaData.csv: -------------------------------------------------------------------------------- 1 | ben,Ben Parker 2 | andy hicks,Andy Hicks 3 | david,David Ainslie -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/ignoreCaseSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: in($FullName) @ignoreCase 4 | FullName: 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRuleCrossReferenceFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,Ben Parker 2 | Dave,David Ainslie 3 | Andy,Andy Hicks -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRuleCrossReferencePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,Ben Parker 2 | Andy,Andy Hicks -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRuleCrossReferenceSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | FirstName: in($FullName) 4 | FullName: 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,valuenotinrule 2 | Andy,beinthis 3 | David,thisonewillfailtoo -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,valuemust 2 | Andy,beinthis -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/inRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | SomeInRule: in("thevaluemustbeinthisstring") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/multipleErrorsMetaData.csv: -------------------------------------------------------------------------------- 1 | Andy,twenty 2 | Ben,thirty -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/noext.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | identifier: 4 | noext: is(noExt($identifier)) 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/noextFail.csv: -------------------------------------------------------------------------------- 1 | identifier,noext 2 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D 3 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R.pdf,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R 4 | file:/a/b/c.txt,file:/a/b/c.txt -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/noextPass.csv: -------------------------------------------------------------------------------- 1 | identifier,noext 2 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D 3 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R.pdf,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R 4 | file:/a/b/c.txt,file:/a/b/c -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/notempty.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3,c4 2 | aaa,cell1-2,bbb,cell1-4 3 | cell2-1,,bbb,cell2-4 4 | cell3-1,,cell3-3, 5 | aaa,cell4-2,cell4-3, -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/notempty.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 4 3 | c1: if($c2/notEmpty, is("aaa")) 4 | c2: 5 | c3: if($c4/notEmpty, is("bbb")) 6 | c4: 7 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/optionalFailMetaData.csv: -------------------------------------------------------------------------------- 1 | BP,Benjamin Parker 2 | Andy,Andy Hicks 3 | ,David Ainslie -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/optionalPassMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,Ben Parker 2 | Andy,Andy Hicks 3 | ,David Ainslie -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/optionalSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: in($FullName) @optional 4 | FullName: 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithFourRulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | John,England 2 | Jack,ngland 3 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithFourRulesPassMetaData.csv: -------------------------------------------------------------------------------- 1 | John,England 2 | Hans,Germany 3 | Pierre,France 4 | Julie,Uk 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithFourRulesSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Country: in("EnglandUkBritain") or in("France") or in("Germany") regex("[A-Z].+") 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithTwoRulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Bob,England 2 | Julia,351 3 | Hank,1 4 | MarvinTheMartian,Andromeda9 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithTwoRulesPassMetaData.csv: -------------------------------------------------------------------------------- 1 | Bob,England 2 | Julia,351 3 | Hank,1 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/orWithTwoRulesSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | CountryOrCountryCode: regex("[A-Z][a-z]+") or regex("[0-9]+") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rangeRuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,Year_of_birth 2 | Andy,1985 3 | Ben,1909 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rangeRuleFailSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | Year_of_birth: range(*,*) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rangeRuleInvalidSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | Year_of_birth: range(100,1) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rangeRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,Year_of_birth 2 | Andy,1985 3 | Ben,1970 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rangeRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | Year_of_birth: range(1910,*) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/redacted.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 3 | identifier: if($original_identifier/notEmpty,is(concat(noExt($original_identifier),"_R.pdf"))) 4 | file_name: 5 | original_identifier: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/redactedFail.csv: -------------------------------------------------------------------------------- 1 | identifier,file_name,original_identifier 2 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc,TNA Digital Preservation Strategy v0.3[A1031178].doc, 3 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R1.pdf,TNA Digital Preservation Strategy v0.3[A1031178]_R1.pdf,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/redactedPass.csv: -------------------------------------------------------------------------------- 1 | identifier,file_name,original_identifier 2 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc,TNA Digital Preservation Strategy v0.3[A1031178].doc, 3 | file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D_R.pdf,TNA Digital Preservation Strategy v0.3[A1031178]_R.pdf,file:/some/folder/TNA%20Digital%20Preservation%20Strategy%20v0.3%5BA1031178%5D.doc -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/regexRuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Andy,twenty 2 | Ben,39 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/regexRulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,Age 2 | Andy,28 3 | Ben,39 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/regexRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 3 | Name: 4 | Age: regex("[0-9]+") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/regexRuleSchemaWithNoHeaderSet.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Age: regex("[0-9]+") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/regexRuleSchemaWithoutNoHeaderSet.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Age: regex("[0-9]+") -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Andy,Andy Hicks 2 | ben,Ben Parker 3 | bob,Big Bob 4 | 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/rulesSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: regex("[A-Z][a-z]+") in($FullName) 4 | FullName: 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated1.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator '$' 4 | col1: notEmpty 5 | col2: notEmpty 6 | col3: notEmpty 7 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated1.dsv: -------------------------------------------------------------------------------- 1 | col1$col2$col3 2 | a$b$c 3 | x$y$z -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated2-1.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator '\t' 4 | col1: notEmpty 5 | col2: notEmpty 6 | col3: notEmpty 7 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated2.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator TAB 4 | col1: notEmpty 5 | col2: notEmpty 6 | col3: notEmpty 7 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated2.tsv: -------------------------------------------------------------------------------- 1 | col1 col2 col3 2 | a b c 3 | x y z -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated3.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator '$' 4 | @quoted 5 | col1: notEmpty 6 | col2: notEmpty 7 | col3: notEmpty -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated3.dsv: -------------------------------------------------------------------------------- 1 | col1$col2$col3 2 | "a"$"b"$"c" 3 | "x"$"y"$"z" -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated4-1.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator '\t' 4 | @quoted 5 | col1: notEmpty 6 | col2: notEmpty 7 | col3: notEmpty 8 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated4.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 3 | @separator TAB 4 | @quoted 5 | col1: notEmpty 6 | col2: notEmpty 7 | col3: notEmpty 8 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/separated4.tsv: -------------------------------------------------------------------------------- 1 | col1 col2 col3 2 | "a" "b" "c" 3 | "x" "y" "z" -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/standardRulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | http:##datagov.nationalarchives.gov.uk#66#WO#409#9999#0#aaaaaaaa-aaaa-4aaa-9eee-0123456789ab,2002-999-30T09:00:10,02-99-30,99/00/0009,99:00:889,aaaaaaaab-aaaab-4aaa-9eee-0123456789ab,12-0912459 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/standardRulesPassMetaData.csv: -------------------------------------------------------------------------------- 1 | http://datagov.nationalarchives.gov.uk/66/WO/409/9999/0/aaaaaaaa-aaaa-4aaa-9eee-0123456789ab,2002-12-12T09:00:10,2002-01-30,13/02/1970,23:30:20,aaaaaaaa-aaaa-4aaa-9eee-0123456789ab,120912459 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/standardRulesSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 7 @noHeader 3 | uri: uri 4 | xDateTime: xDateTime 5 | xDate: xDate 6 | ukDate: ukDate 7 | xTime: xTime 8 | uuid4: uuid4 9 | positiveInteger: positiveInteger 10 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch1RuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeSwitchRule 2 | hello,hello world1 3 | world,SDWGRGWR 4 | 1234,EFQWEGW -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch1RulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeSwitchRule 2 | hello,hello world 3 | world,SDWGRGWR 4 | 1234,EFQWEGW -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch1RuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | SomeSwitchRule: switch(($Name/starts("hello"),is("hello world"))) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch2RuleFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeSwitchRule 2 | hello,hello world1 3 | HELLO,HELLO WORLD1 4 | 1234,45637 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch2RulePassMetaData.csv: -------------------------------------------------------------------------------- 1 | Name,SomeSwitchRule 2 | hello,hello world 3 | HELLO,HELLO WORLD 4 | 1234,45637 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/switch2RuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 2 3 | Name: 4 | SomeSwitchRule: switch(($Name/starts("hello"),is("hello world")),($Name/starts("HELLO"),is("HELLO WORLD"))) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/totalColumnsFailMetaData.csv: -------------------------------------------------------------------------------- 1 | oneThing 2 | oneThing,anotherThing 3 | oneThingAgain,anotherthingAgain -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/totalColumnsSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 1 @noHeader 3 | Thing: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/twoRuleSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 3 @noHeader 3 | Name: 4 | Age: regex("[0-9]+[a-z]+") in("99yearstoday") 5 | CrossRef: regex("[a-z]+") in($Name) 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/twoRuleSchemaFail.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: in($FullName) regex("[a-z]+") 4 | FullName: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/twoRulesFailMetaData.csv: -------------------------------------------------------------------------------- 1 | Ben,ben parker 2 | Dave,david ainslie -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/twoRulesPassMetaData.csv: -------------------------------------------------------------------------------- 1 | some year,99years,year 2 | some date,99years,date -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecode.csvs: -------------------------------------------------------------------------------- 1 | version 1.2 2 | @totalColumns 2 @noHeader 3 | identifier: 4 | filename: in(uriDecode($identifier)) -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeFail.csv: -------------------------------------------------------------------------------- 1 | file:/some/folder/some%21file.txt,some file.txt -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodePass.csv: -------------------------------------------------------------------------------- 1 | file:/some/folder/some%20file.txt,some file.txt -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeWithCharset.csvs: -------------------------------------------------------------------------------- 1 | version 1.2 2 | @totalColumns 3 @noHeader 3 | identifier: 4 | filename: in(uriDecode($identifier, $charset)) 5 | charset: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeWithCharsetPass.csv: -------------------------------------------------------------------------------- 1 | file:/some/folder/some%20file.txt,some file.txt,UTF-8 2 | file:/some/folder/text%9Atext.txt,textštext.txt,windows-1252 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/warnings.csv: -------------------------------------------------------------------------------- 1 | department,division 2 | WO,13 3 | BT,13 4 | ED,13 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/warnings.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 3 | department: is("WO") @warning 4 | division: is("13") 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/warningsAndErrors.csv: -------------------------------------------------------------------------------- 1 | department,division 2 | WO,13 3 | BT,13 4 | ED,13 5 | WO,15 6 | WO,16 7 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTime.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 1 3 | date: xDateTime 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeFail.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2002-12-30T09:00:10 3 | 2013-03-22 4 | 2012-01-01T00:00:00 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimePass.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2002-12-30T09:00:10 3 | 2013-03-22T11:33:40+00:00 4 | 2012-01-01T00:00:00 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeRange.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 1 3 | date: xDateTime(2012-01-01T01:00:00,2013-01-01T01:00:00) 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeRangeFail.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2012-01-01T01:00:00 3 | 2014-01-01T01:00:00 4 | 2013-01-01T00:00:00 5 | 2013-01-01T00:00:00 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeRangePass.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2012-01-01T01:00:00 3 | 2012-12-12T00:00:00 4 | 2013-01-01T00:00:00 5 | 2013-01-01T00:00:00 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTz.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 1 3 | date: xDateTimeTz 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTzFail.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2002-12-30T09:00:10-01:00 3 | 2013-03-22T11:33:40+01:00 4 | 2012-01-01T00:00:00Z 5 | 2012-01-01T00:00:00 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTzPass.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2002-12-30T09:00:10-01:00 3 | 2013-03-22T11:33:40+01:00 4 | 2012-01-01T00:00:00Z 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTzRange.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 1 3 | date: xDateTimeTz(2012-01-01T01:00:00+00:00,2013-01-01T01:00:00+00:00) 4 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTzRangeFail.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2012-01-01T01:00:00+00:00 3 | 2014-01-01T01:00:00+00:00 4 | 2013-01-01T00:00:00+00:00 5 | 2013-01-01T00:00:00+00:00 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/xsdDateTimeTzRangePass.csv: -------------------------------------------------------------------------------- 1 | date 2 | 2012-01-01T01:00:00+00:00 3 | 2012-12-12T00:00:00+00:00 4 | 2013-01-01T00:00:00+00:00 5 | 2013-01-01T00:00:00+00:00 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/metaData.csv: -------------------------------------------------------------------------------- 1 | col1,col2 2 | col1,col2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/metaDataWithALongCellLength.csv: -------------------------------------------------------------------------------- 1 | col1,col2 2 | row1Col1,row1Col2LongCellLength -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/metadataMultipleLineBreaksInCell.csv: -------------------------------------------------------------------------------- 1 | col1,"this 2 | cell 3 | has 4 | multiple 5 | line 6 | breaks 7 | in 8 | it" 9 | col1, col2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/schema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @totalColumns 2 @noHeader 3 | Name: 4 | Age: 5 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/windows-1252.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/windows-1252.csv -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/badSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.0 2 | @TotalColum 2 3 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file1.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file1.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file2.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file2.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file3.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFiles/file3.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/file1a.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/file1a.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/threeFilesSub1/file2a.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/threeFilesSub1/file2a.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/threeFilesSub2/file3a.jp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/fileCountTestFiles/threeFilesinSubDir/threeFilesSub2/file3a.jp2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/closure_v2.csv: -------------------------------------------------------------------------------- 1 | identifier,folder,last_modified_date,closure_period,foi_exemption_code,foi_exemption_asserted,description_public,description_alternate,closure_type 2 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/closure_v2.csv.sha256: -------------------------------------------------------------------------------- 1 | 3f01fab1f9be32b5a5aea4dc4496ebc23c6b60198aa60fe3eec8eb3b58915f27 closure_v2.csv 2 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content/16/1/16_1_0001.jp2: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content/16/1/16_1_0003.jp2: -------------------------------------------------------------------------------- 1 | 2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content/16/2/16_2_0001.jp2: -------------------------------------------------------------------------------- 1 | 3 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content/39/1/39_1_0001.jp2: -------------------------------------------------------------------------------- 1 | 4 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/tech_acq_metadata_v1_WO95Y14B003.csv: -------------------------------------------------------------------------------- 1 | batch_code,department,division,series,sub_series,sub_sub_series,piece,item,ordinal,description,covering_date,legal_status,held_by,file_uuid,file_path,file_checksum,resource_uri,scan_operator,scan_id,scan_location,image_resolution,image_width,image_height,image_tonal_resolution,image_format,image_compression,image_colour_space,image_split,image_split_ordinal,image_split_other_uuid,image_crop,image_deskew,comments 2 | WO95Y14B003,WO,13,95,1,1,16,1,1,Branches and Services: General Staff,1918 Jan-1918 Feb,Public Record,"The National Archives, Kew",54d71705-99ab-4d9a-8216-9e216561a0b6,file:///WO_95/content/16/1/16_1_0001.jp2,6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b,http://datagov.nationalarchives.gov.uk/66/WO/95/16/54d71705-99ab-4d9a-8216-9e216561a0b6,PALLAS,57056,"The National Archives, Kew, Richmond, Surrey, TW9 4DU",300,2764,4165,24-bit colour,x-fmt/392,6,sRGB,no,,,auto,yes, 3 | WO95Y14B003,WO,13,95,1,1,16,1,3,Branches and Services: General Staff,1918 Jan-1918 Feb,Public Record,"The National Archives, Kew",5481287b-74e8-4573-bffc-35ac63d70c97,file:///WO_95/content/16/1/16_1_0003.jp2,d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35,http://datagov.nationalarchives.gov.uk/66/WO/95/16/5481287b-74e8-4573-bffc-35ac63d70c97,PALLAS,57056,"The National Archives, Kew, Richmond, Surrey, TW9 4DU",300,2606,4123,24-bit colour,x-fmt/392,6,sRGB,no,,,auto,yes, 4 | WO95Y14B003,WO,13,95,1,1,16,2,1,Branches and Services: General Staff,1918 Jan-1918 Jan,Public Record,"The National Archives, Kew",c787151b-a861-4d08-9412-d2a63c3ff214,file:///WO_95/content/16/2/16_2_0001.jp2,4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce,http://datagov.nationalarchives.gov.uk/66/WO/95/16/c787151b-a861-4d08-9412-d2a63c3ff214,PALLAS,57056,"The National Archives, Kew, Richmond, Surrey, TW9 4DU",300,2773,4115,24-bit colour,x-fmt/392,6,sRGB,no,,,auto,yes, 5 | WO95Y14B003,WO,13,95,1,1,39,1,1,Branches and Services: Quarter-Master General,1918 June-1918 June,Public Record,"The National Archives, Kew",4b3ebdc1-0de2-48fc-9d69-133b6341ac22,file:///WO_95/content/39/1/39_1_0001.jp2,4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a,http://datagov.nationalarchives.gov.uk/66/WO/95/39/4b3ebdc1-0de2-48fc-9d69-133b6341ac22,PONTUS,57059,"The National Archives, Kew, Richmond, Surrey, TW9 4DU",300,4110,2719,24-bit colour,x-fmt/392,6,sRGB,no,,,auto,yes, 6 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/tech_acq_metadata_v1_WO95Y14B003.csv.sha256: -------------------------------------------------------------------------------- 1 | 24084b1c16a82501161a5bb93dd1d69a66acdd9022c4c255427562e5422155fa tech_acq_metadata_v1_WO95Y14B003.csv 2 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/content/file1: -------------------------------------------------------------------------------- 1 | rwgwryheahy35th3aw5h35h3wa5h5h -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/content/file2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/content/file2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/content/file3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/content/file3 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/integrityCheckMetaData.csv: -------------------------------------------------------------------------------- 1 | identifier,filename,size 2 | file:///T:/WORK/RF_5/content/file1,file1,30 3 | file:///T:/WORK/RF_5/content/file2,file2,0 4 | file:///T:/WORK/RF_5/content/file3,file3,0 5 | file:///T:/WORK/RF_5/content/,content,10 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/content/integrityCheckSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 3 | identifier: integrityCheck("includeFolder") 4 | filename: 5 | size: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file1: -------------------------------------------------------------------------------- 1 | rwgwryheahy35th3aw5h35h3wa5h5h -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file3 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckDefaultIncludeFolderSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 3 | identifier: fileExists integrityCheck("excludeFolder") 4 | filename: 5 | size: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckMetaData-missing-files.csv: -------------------------------------------------------------------------------- 1 | identifier,filename,size 2 | file:///T:/WORK/RF_5/content/file1,file1,30 3 | file:///T:/WORK/RF_5/content/file3,file3,0 4 | file:///T:/WORK/RF_5/content/,content,10 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckMetaData.csv: -------------------------------------------------------------------------------- 1 | identifier,filename,size 2 | file:///T:/WORK/RF_5/content/file1,file1,30 3 | file:///T:/WORK/RF_5/content/file2,file2,0 4 | file:///T:/WORK/RF_5/content/file3,file3,0 5 | file:///T:/WORK/RF_5/content/,content,10 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckMetaDataTooManyFiles.csv: -------------------------------------------------------------------------------- 1 | identifier,filename,size 2 | identifier,filename,size 3 | file:///T:/WORK/RF_5/content/file1,file1,30 4 | file:///T:/WORK/RF_5/content/file2,file2,0 5 | file:///T:/WORK/RF_5/content/file3,file3,0 6 | file:///T:/WORK/RF_5/content/file4,file4,0 7 | file:///T:/WORK/RF_5/content/,content,10 8 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 3 | identifier: integrityCheck("includeFolder") 4 | filename: 5 | size: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/badIntegrityCheckSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 @noHeader 3 | identifier: fileExists integrityCheck("badincludeFolder") 4 | filename: 5 | size: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content/file1: -------------------------------------------------------------------------------- 1 | rwgwryheahy35th3aw5h35h3wa5h5h -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content/file2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content/file2 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content/file3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content/file3 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/integrityCheckMetaData.csv: -------------------------------------------------------------------------------- 1 | file:///T:/WORK/RF_5/content/file1,file1,30 2 | file:///T:/WORK/RF_5/content/file2,file2,0 3 | file:///T:/WORK/RF_5/content/file3,file3,0 4 | file:///T:/WORK/RF_5/content/,content,10 -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/integrityCheckSchema.csvs: -------------------------------------------------------------------------------- 1 | version 1.1 2 | @totalColumns 3 @noHeader 3 | identifier: fileExists integrityCheck("includeFolder") 4 | filename: 5 | size: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/invalidTotalColumnsSchema.csvs: -------------------------------------------------------------------------------- 1 | @totalColumns 3 2 | Col1: 3 | Col2: -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/checksum.csvs: -------------------------------------------------------------------------------- 1 | Do not modify or remove -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/must Exist With Spaces For Rule.csvs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/must Exist With Spaces For Rule.csvs -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/mustExistFor#Rule.csvs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/mustExistFor#Rule.csvs -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/mustExistForRule.csvs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_0/mustExistForRule.csvs -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_1/integrityCheck/folder1/content/file#2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_1/integrityCheck/folder1/content/file#2.txt -------------------------------------------------------------------------------- /csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_1/integrityCheck/folder1/content/file1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digital-preservation/csv-validator/bc7d4b4fd7423b3f2708bc8de24ee7244c966ff5/csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/schema/v1_1/integrityCheck/folder1/content/file1.txt -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBigFileSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.schema.Schema 15 | import uk.gov.nationalarchives.csv.validator.api.{CsvValidator, TextFile} 16 | import uk.gov.nationalarchives.csv.validator.api.CsvValidator.SubstitutePath 17 | import cats.data.Validated 18 | import java.nio.file.Paths 19 | 20 | @RunWith(classOf[JUnitRunner]) 21 | class MetaDataValidatorBigFileSpec extends Specification with TestResources { 22 | 23 | val base = acceptancePath 24 | 25 | "Big file" should { 26 | 27 | "succeed with no stack overflow for all errors" in { 28 | val v = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 29 | def parse(filePath: String): Schema = v.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 30 | 31 | v.validate(TextFile(Paths.get(base).resolve("bigMetaData.csv")), parse(base + "/bigSchema.csvs"), None) must beLike { case Validated.Valid(_) => ok } 32 | } 33 | 34 | "succeed with no stack overflow for fail fast" in { 35 | val v = new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 36 | def parse(filePath: String): Schema = v.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 37 | 38 | v.validate(TextFile(Paths.get(base).resolve("bigMetaData.csv")), parse(base + "/bigSchema.csvs"), None) must beLike { case Validated.Valid(_) => ok } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBusinessAcceptanceSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import cats.data.Validated 15 | import uk.gov.nationalarchives.csv.validator.schema.Schema 16 | import uk.gov.nationalarchives.csv.validator.api.{CsvValidator, TextFile} 17 | 18 | import java.nio.file.Paths 19 | 20 | @RunWith(classOf[JUnitRunner]) 21 | class MetaDataValidatorBusinessAcceptanceSpec extends Specification with TestResources { 22 | 23 | val base = resourcePath("acceptance/dp") 24 | 25 | val v: CsvValidator = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 26 | import v.{validate, parseSchema} 27 | 28 | def parse(filePath: String): Schema = parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 29 | 30 | "Regex rule" should { 31 | 32 | "succeed" in { 33 | validate(TextFile(Paths.get(base).resolve("regexRulePassMetaData.csv")), parse(base + "/regexRuleSchema.csvs"), None) must beLike { 34 | case Validated.Valid(_) => ok 35 | } 36 | } 37 | 38 | "fail" in { 39 | validate(TextFile(Paths.get(base).resolve("regexRuleFailMetaData.csv")), parse(base + "/regexRuleSchema.csvs"), None) must beLike { 40 | case Validated.Invalid(_) => ok 41 | } 42 | } 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/TestHelper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | object TestHelper { 12 | 13 | implicit class RichString(str: String) { 14 | 15 | /** Strip all carriage returns in order to facilitate cross-platform/encoding comparisons 16 | */ 17 | def removeCR: String = str.replace("\r", "") 18 | } 19 | 20 | implicit class RichFailMessages(fm: FailMessage) { 21 | def removeCR: FailMessage = { 22 | fm.copy( 23 | message = fm.message.removeCR 24 | ) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/TestResources.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import org.specs2.mutable.Specification 12 | 13 | import java.nio.file.Paths 14 | 15 | trait TestResources { 16 | 17 | spec: Specification => 18 | 19 | /** 20 | * Utility function to get an absolute path to a resource 21 | * in the same class as a spec test 22 | * 23 | * @param resource The path of the resource relative to the spec test 24 | */ 25 | def resourcePath(resource: String) : String = Paths.get(baseResourcePkgPath).resolve(resource).toAbsolutePath.toString 26 | 27 | def baseResourcePkgPath : String = Paths.get(basePath).resolve(spec.getClass.getPackage.getName.replace('.', '/')).toAbsolutePath.toString 28 | 29 | def basePath : String = { 30 | val url = spec.getClass.getClassLoader.getResource(".") 31 | Paths.get(url.toURI).toAbsolutePath.toString 32 | } 33 | 34 | def relBasePath : String = basePath.replace(System.getProperty("user.dir") + FILE_SEPARATOR, "") 35 | def relBaseResourcePkgPath : String = Paths.get(relBasePath).resolve(spec.getClass.getPackage.getName.replace('.', '/')).toString 36 | def relResourcePath(resource: String) : String = Paths.get(relBaseResourcePkgPath).resolve(resource).toString 37 | 38 | def toUriPath(path: String) = path.replace('\\', '/') 39 | 40 | val acceptancePath = resourcePath("acceptance") 41 | val integrityCheckPath = resourcePath("integrityCheck") 42 | 43 | val threeFilesInSubDirPath = resourcePath("fileCountTestFiles/threeFilesinSubDir") 44 | val threeFilesPath = resourcePath("fileCountTestFiles/threeFiles") 45 | 46 | val schemaPath1_0 = resourcePath("schema/v1_0") 47 | val checksumPath = resourcePath("schema/v1_0/checksum.csvs") 48 | } 49 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/UtilSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | 15 | import java.nio.file.Paths 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class UtilSpec extends Specification with TestResources { 19 | 20 | "Util" should { 21 | val base = resourcePath("integrityCheck") 22 | 23 | "check containAll for list" in { 24 | val l1 = List(1,2,3,4) 25 | val l2 = List(1,2,3) 26 | 27 | Util.containAll(l1,l2) mustEqual true 28 | Util.containAll(l2,l1) mustEqual false 29 | } 30 | 31 | 32 | "get set A minus set B" in { 33 | val l1 = Set(1,2,3,4) 34 | val l2 = Set(1,2,3) 35 | val l3 = Set(1,2,3,5) 36 | 37 | Util.minus(l1,l2) mustEqual Set(4) 38 | Util.minus(l2,l1) mustEqual Set() 39 | Util.minus(l1,l3) mustEqual Set(4) 40 | } 41 | 42 | "get difference between set A and set B" in { 43 | val l1 = Set(1,2,3,4) 44 | val l2 = Set(1,2,3) 45 | val l3 = Set(1,2,3,5) 46 | 47 | Util.diff(l1,l2) mustEqual Set(4) 48 | Util.diff(l2,l1) mustEqual Set(4) 49 | Util.diff(l1,l3) mustEqual Set(4,5) 50 | } 51 | 52 | "list file in folder" in { 53 | 54 | val apiFiles = Util.findAllFiles(true, Paths.get(acceptancePath)) 55 | 56 | apiFiles must haveLength(127) 57 | 58 | apiFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/acceptance/twoRulesPassMetaData.csv")) 59 | 60 | apiFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/acceptance/dp/regexRuleSchema.csvs")) 61 | 62 | apiFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/acceptance/dp")) 63 | 64 | apiFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/acceptance")) 65 | 66 | val integrityCheckFiles = Util.findAllFiles(true, Paths.get(base)) 67 | 68 | integrityCheckFiles must haveLength(43) 69 | 70 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckSchema.csvs")) 71 | 72 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file1")) 73 | 74 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content")) 75 | 76 | 77 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content")) 78 | 79 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content")) 80 | 81 | integrityCheckFiles must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck")) 82 | 83 | val integrityCheckFilesNoFolder = Util.findAllFiles(false, Paths.get(base)) 84 | 85 | integrityCheckFilesNoFolder must haveLength(29) 86 | 87 | integrityCheckFilesNoFolder must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/header/content/file1")) 88 | 89 | integrityCheckFilesNoFolder must contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/header/integrityCheckSchema.csvs")) 90 | 91 | integrityCheckFilesNoFolder must not contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/noheader/content")) 92 | 93 | integrityCheckFilesNoFolder must not contain (Paths.get(s"$basePath/uk/gov/nationalarchives/csv/validator/integrityCheck/WO_95/content")) 94 | 95 | 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorFileEncodingSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.TestResources 15 | import cats.data.Validated 16 | import uk.gov.nationalarchives.csv.validator.AllErrorsMetaDataValidator 17 | import uk.gov.nationalarchives.csv.validator.schema.Schema 18 | 19 | import java.nio.file.Paths 20 | 21 | /** 22 | * Created by rhubner on 11/12/15. 23 | */ 24 | @RunWith(classOf[JUnitRunner]) 25 | class CsvValidatorFileEncodingSpec extends Specification with TestResources { 26 | 27 | "Validation" should { 28 | 29 | val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 30 | def parse(filePath: String): Schema = app.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 31 | 32 | "fail for non UTF-8 file" in { 33 | app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("windows-1252.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None) must beLike { 34 | case Validated.Invalid(_) => ok 35 | } 36 | } 37 | 38 | 39 | "not fail for valid UTF-8 file" in { 40 | app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaData.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None) must beLike { 41 | case Validated.Valid(_) => ok 42 | } 43 | } 44 | 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorMaxCharsPerCellSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.schema.Schema 15 | import uk.gov.nationalarchives.csv.validator.{AllErrorsMetaDataValidator, FailMessage, TestResources, ValidationError} 16 | 17 | import java.nio.file.Paths 18 | 19 | 20 | @RunWith(classOf[JUnitRunner]) 21 | class CsvValidatorMaxCharsPerCellSpec extends Specification with TestResources { 22 | 23 | "Validation" should { 24 | 25 | def app(maxChars: Int=4096) = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions: List[(String, String)] = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell: Int = maxChars } 26 | def parse(filePath: String, maxChars: Int=4096): Schema = app(maxChars).parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 27 | 28 | "fail if the number of characters in a cell in the header is more than the maxCharsPerCell number and indicate the column number" in { 29 | val maxCharsAllowed = 2 30 | val validatedNel = app(maxCharsAllowed).validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaData.csv")), parse(baseResourcePkgPath + "/schema.csvs", maxCharsAllowed), None).swap 31 | validatedNel.toList.head.toList must beEqualTo( 32 | List(FailMessage(ValidationError,"java.lang.Exception: The number of characters in column 1 of the header row is larger than the maximum number of characters allowed in a cell (2); increase this limit and re-run.",None,None)) 33 | ) 34 | } 35 | 36 | "fail if the number of characters in a cell in a non-header row is more than the maxCharsPerCell number and indicate the column name" in { 37 | val maxCharsAllowed = 15 38 | val validatedNel = app(maxCharsAllowed).validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaDataWithALongCellLength.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None).swap 39 | validatedNel.toList.head.toList must beEqualTo( 40 | List(FailMessage(ValidationError,"java.lang.Exception: The number of characters in the cell located at row: 1, column: col2, is larger than the maximum number of characters allowed in a cell (15); increase this limit and re-run.",None,None)) 41 | ) 42 | } 43 | 44 | "not fail if the number of characters in a cell is less than the maxCharsPerCell number" in { 45 | val validatedNel = app().validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaData.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None) 46 | validatedNel.toList must beEqualTo(List(())) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api 10 | 11 | import java.io.StringReader 12 | import org.junit.runner.RunWith 13 | import org.specs2.mutable.Specification 14 | import org.specs2.runner.JUnitRunner 15 | import cats.data.Validated 16 | import uk.gov.nationalarchives.csv.validator._ 17 | import uk.gov.nationalarchives.csv.validator.schema.Schema 18 | import uk.gov.nationalarchives.csv.validator.api.CsvValidator.SubstitutePath 19 | 20 | import java.nio.file.Paths 21 | 22 | @RunWith(classOf[JUnitRunner]) 23 | class CsvValidatorSpec extends Specification with TestResources { 24 | 25 | "Parsing schema" should { 26 | val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 27 | 28 | "report position on parse fail" in { 29 | 30 | val schema = 31 | """version 1.0 32 | |@totalColumns 1 33 | |Name: regox("A") 34 | """.stripMargin 35 | 36 | app.parseAndValidate(new StringReader(schema)) must beLike { 37 | case Validated.Invalid(msgs) => 38 | msgs.toList mustEqual List(FailMessage(SchemaDefinitionError, 39 | "[3.7] failure: Invalid column definition" + EOL 40 | + EOL 41 | + """Name: regox("A")""" + EOL 42 | + " ^")) 43 | } 44 | } 45 | } 46 | 47 | "Validation" should { 48 | val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 49 | 50 | def parse(filePath: String): Schema = app.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) 51 | 52 | "succeed for valid schema and metadata file" in { 53 | app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaData.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None) must beLike { 54 | case Validated.Valid(_) => ok 55 | } 56 | } 57 | 58 | "succeed for valid @totalColumns in schema and metadata file" in { 59 | app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metaData.csv")), parse(baseResourcePkgPath + "/schema.csvs"), None) must beLike { 60 | case Validated.Valid(_) => ok 61 | } 62 | } 63 | 64 | val callback = new ProgressCallback { 65 | var processed = -1 66 | var total = -2 67 | override def update(complete: Percentage): Unit = throw new NotImplementedError() 68 | 69 | override def update(_total: Int, _processed: Int): Unit = { 70 | total = _total 71 | processed = _processed 72 | } 73 | } 74 | 75 | "have a total (of rows) equal to the actual number of rows in the metadata file even if there are multiple line breaks in a cell" in { 76 | app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metadataMultipleLineBreaksInCell.csv")), parse(baseResourcePkgPath + "/schema.csvs"), Some(callback)) must beLike { 77 | case Validated.Valid(_) => ok 78 | } 79 | callback.total must beEqualTo(2) 80 | } 81 | } 82 | } 83 | 84 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaSpecBase.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema 10 | 11 | import org.specs2.mutable.Specification 12 | import uk.gov.nationalarchives.csv.validator.schema.v1_0.NotEmptyRule 13 | 14 | 15 | trait SchemaSpecBase extends Specification { 16 | 17 | object TestSchemaParser extends SchemaParser { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false; val maxCharsPerCell = 4096 } 18 | 19 | def buildSchema1_0(globalDirective: GlobalDirective*)(columnDefinition: ColumnDefinition*) = 20 | Schema(globalDirective.toList, columnDefinition.toList, "1.0") 21 | 22 | def buildSchema1_1(globalDirective: GlobalDirective*)(columnDefinition: ColumnDefinition*) = 23 | Schema(globalDirective.toList, columnDefinition.toList, "1.1") 24 | 25 | def buildSchema1_2(globalDirective: GlobalDirective*)(columnDefinition: ColumnDefinition*) = 26 | Schema(globalDirective.toList, columnDefinition.toList, "1.2") 27 | 28 | def namedColumn(name: String) = ColumnDefinition(NamedColumnIdentifier(name)) 29 | 30 | def nonEmptyColumn(name: String): ColumnDefinition = 31 | ColumnDefinition(NamedColumnIdentifier(name), List(NotEmptyRule()), List()) 32 | 33 | } 34 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/EmptyRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema.{ColumnDefinition, NamedColumnIdentifier, Schema, TotalColumns} 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class EmptyRuleSpec extends Specification { 21 | 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "EmptyRule" should { 25 | 26 | "Succeed if cell is empty" in { 27 | val emptyRule = EmptyRule() 28 | emptyRule.evaluate(0, Row(List(Cell("")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 29 | } 30 | 31 | "Fail if cell is NOT empty" in { 32 | val emptyRule = EmptyRule() 33 | emptyRule.evaluate(0, Row(List(Cell("something")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 34 | case Validated.Invalid(messages) => messages.head mustEqual """empty fails for row: 1, column: column1, value: "something"""" 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/EndsRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import cats.data.Validated 12 | import org.junit.runner.RunWith 13 | import org.specs2.mutable.Specification 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 16 | import uk.gov.nationalarchives.csv.validator.schema._ 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class EndsRuleSpec extends Specification { 20 | 21 | "EndsRule with a string literal behaviour" should { 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "succeed if cell ends with endsRule value" in { 25 | val endsRule = EndsRule(Literal(Some("world"))) 26 | 27 | endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 28 | } 29 | 30 | "fail if cell does not end with endsRule value" in { 31 | val endsRule = EndsRule(Literal(Some("hello world"))) 32 | endsRule.evaluate(0, Row(List(Cell("hello")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 33 | case Validated.Invalid(messages) => messages.head mustEqual """ends("hello world") fails for row: 1, column: column1, value: "hello"""" 34 | } 35 | } 36 | 37 | "succeed if endsRule is the same as value" in { 38 | val endsRule = EndsRule(Literal(Some("hello world"))) 39 | endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 40 | } 41 | 42 | "succeed if endsRule's column reference does exist" in { 43 | val endsRule = EndsRule(ColumnReference(NamedColumnIdentifier("column1"))) 44 | 45 | endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 46 | } 47 | 48 | "fail if endsRule's column reference doesn't exist" in { 49 | val endsRule = EndsRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn"))) 50 | 51 | endsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 52 | case Validated.Invalid(messages) => messages.head mustEqual """ends($nonExistentColumn) fails for row: 1, column: column1, value: "hello world today"""" 53 | } 54 | } 55 | 56 | "succeed with @ignoreCase" in { 57 | val endsRule = EndsRule(Literal(Some("hello world"))) 58 | endsRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true) 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/IfRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | import cats.data.Validated 18 | 19 | /** 20 | * User: Jim Collins 21 | * Date: 3/7/13 22 | */ 23 | @RunWith(classOf[JUnitRunner]) 24 | class IfRuleSpec extends Specification { 25 | 26 | "IfRule" should { 27 | val globalDirsOne = List(TotalColumns(1)) 28 | val schema: Schema = Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1")))) 29 | 30 | val row: Row = Row(List(Cell("hello world")), 1) 31 | 32 | def startsRule(prefix: String): StartsRule = StartsRule(Literal(Some(prefix))) 33 | def endsRules(suffix: String): List[EndsRule] = List(EndsRule(Literal(Some(suffix)))) 34 | 35 | "condition and 'if body' is true and no 'else'" in { 36 | val ifRule = IfRule(startsRule("hello world"),endsRules("world"), None) 37 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List(true)) 38 | } 39 | 40 | "condition is true and no else, 'if body' rule fails" in { 41 | val ifRule = IfRule(startsRule("hello world"),endsRules("hello"), None) 42 | ifRule.evaluate(0, row, schema) must beLike { 43 | case Validated.Invalid(messages) => messages.head mustEqual "ends(\"hello\") fails for row: 1, column: column1, value: \"hello world\"" 44 | } 45 | } 46 | 47 | "condition is true and there is an 'else' that is true and 'if' body that is false" in { 48 | val ifRule = IfRule(startsRule("hello"),endsRules("hello"), Some(endsRules("world"))) 49 | ifRule.evaluate(0, row, schema) must beLike { 50 | case Validated.Invalid(messages) => messages.head mustEqual "ends(\"hello\") fails for row: 1, column: column1, value: \"hello world\"" 51 | } 52 | } 53 | 54 | "condition is false and there is no 'else' and 'if' body that is true" in { 55 | val ifRule = IfRule(startsRule("sello"),endsRules("world"), None) 56 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List()) 57 | } 58 | 59 | "condition is false and there is an 'else' that is true and body that is true" in { 60 | val ifRule = IfRule(StartsRule(Literal(Some("sello"))),endsRules("world"), Some(endsRules("world"))) 61 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List(true)) 62 | } 63 | 64 | "condition is false and there is an 'else' that is false and 'if' body that is true" in { 65 | val ifRule = IfRule(startsRule("sello"),endsRules("world"), Some(endsRules("hello"))) 66 | ifRule.evaluate(0, row, schema) must beLike { 67 | case Validated.Invalid(messages) => messages.head mustEqual "ends(\"hello\") fails for row: 1, column: column1, value: \"hello world\"" 68 | } 69 | } 70 | 71 | "condition is false and 'else' has nested 'if' that is false with no 'else'" in { 72 | val ifRule = IfRule(startsRule("sello"),endsRules("world"), 73 | Some(List( 74 | IfRule(startsRule("sello"),endsRules("world"), None) 75 | )) 76 | ) 77 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List(List())) 78 | } 79 | 80 | "condition is false and 'else' has nested 'if' that is true with no 'else'" in { 81 | val ifRule = IfRule(startsRule("sello"),endsRules("world"), 82 | Some(List( 83 | IfRule(startsRule("hello"),endsRules("world"), None) 84 | )) 85 | ) 86 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List(List(true))) 87 | } 88 | 89 | "nested if'" in { 90 | val ifRule = IfRule( 91 | startsRule("u"),endsRules("def"), 92 | Some(List( 93 | IfRule( 94 | startsRule("u") 95 | ,endsRules("world"), Some(endsRules("world"))) 96 | )) 97 | ) 98 | ifRule.evaluate(0, row, schema) mustEqual Validated.Valid(List(List(true))) 99 | } 100 | 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/InRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class InRuleSpec extends Specification { 21 | 22 | "InRule with a string literal behaviour" should { 23 | val globalDirsOne = List(TotalColumns(1)) 24 | 25 | "succeed if inRule is embedded in value" in { 26 | val inRule = InRule(Literal(Some("hello world today"))) 27 | inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 28 | } 29 | 30 | "fail if inRule is not in value" in { 31 | val inRule = InRule(Literal(Some("hello world"))) 32 | 33 | inRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 34 | case Validated.Invalid(messages) => messages.head mustEqual """in("hello world") fails for row: 1, column: column1, value: "hello world today"""" 35 | } 36 | } 37 | 38 | "succeed if inRule is the same as value" in { 39 | val inRule = InRule(Literal(Some("hello world"))) 40 | inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 41 | } 42 | 43 | "succeed if inRule's column reference does exist" in { 44 | val inRule = InRule(ColumnReference(NamedColumnIdentifier("column1"))) 45 | 46 | inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 47 | } 48 | 49 | "fail if inRule's column reference doesn't exist" in { 50 | val inRule = InRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn"))) 51 | 52 | inRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 53 | case Validated.Invalid(messages) => messages.head mustEqual """in($nonExistentColumn) fails for row: 1, column: column1, value: "hello world today"""" 54 | } 55 | } 56 | 57 | "succeed with @ignoreCase" in { 58 | val inRule = InRule(Literal(Some("hello world"))) 59 | inRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true) 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/IsRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import cats.data.Validated 12 | import org.junit.runner.RunWith 13 | import org.specs2.mutable.Specification 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 16 | import uk.gov.nationalarchives.csv.validator.schema._ 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class IsRuleSpec extends Specification { 20 | 21 | "IsRule with a string literal behaviour" should { 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "succeed if isRule is the same as value" in { 25 | val isRule = IsRule(Literal(Some("hello world"))) 26 | isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 27 | } 28 | 29 | "fail if isRule is not the same as value" in { 30 | val isRule = IsRule(Literal(Some("completely different value"))) 31 | isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 32 | case Validated.Invalid(messages) => messages.head mustEqual """is("completely different value") fails for row: 1, column: column1, value: "hello world"""" 33 | } 34 | } 35 | 36 | "fail if isRule is embedded in value" in { 37 | val isRule = IsRule(Literal(Some("hello world today"))) 38 | isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 39 | case Validated.Invalid(messages) => messages.head mustEqual """is("hello world today") fails for row: 1, column: column1, value: "hello world"""" 40 | } 41 | } 42 | 43 | "fail if isRule is not in value" in { 44 | val isRule = IsRule(Literal(Some("hello world"))) 45 | isRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 46 | case Validated.Invalid(messages) => messages.head mustEqual """is("hello world") fails for row: 1, column: column1, value: "hello world today"""" 47 | } 48 | } 49 | 50 | "succeed with @ignoreCase" in { 51 | val isRule = IsRule(Literal(Some("hello world"))) 52 | isRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true) 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/NotEmptyRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema.{ColumnDefinition, NamedColumnIdentifier, Schema, TotalColumns} 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class NotEmptyRuleSpec extends Specification { 21 | 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "NotEmptyRule" should { 25 | 26 | "File if cell is empty" in { 27 | val notEmptyRule = NotEmptyRule() 28 | notEmptyRule.evaluate(0, Row(List(Cell("")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 29 | case Validated.Invalid(messages) => messages.head mustEqual """notEmpty fails for row: 1, column: column1, value: """"" 30 | } 31 | 32 | 33 | } 34 | 35 | "Succeed if cell is NOT empty" in { 36 | val notEmptyRule = NotEmptyRule() 37 | notEmptyRule.evaluate(0, Row(List(Cell("something")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/NotRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import cats.data.Validated 12 | import org.junit.runner.RunWith 13 | import org.specs2.mutable.Specification 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 16 | import uk.gov.nationalarchives.csv.validator.schema._ 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class NotRuleSpec extends Specification { 20 | 21 | "NotRule with a string literal behaviour" should { 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "succeed if notRule is not the same as value" in { 25 | val notRule = NotRule(Literal(Some("completely different value"))) 26 | notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 27 | } 28 | 29 | "succeed if notRule is the similar to value" in { 30 | val notRule = NotRule(Literal(Some("hello world "))) 31 | notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 32 | } 33 | 34 | "fail if notRule is the same as value" in { 35 | val notRule = NotRule(Literal(Some("hello world"))) 36 | notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 37 | case Validated.Invalid(messages) => messages.head mustEqual """not("hello world") fails for row: 1, column: column1, value: "hello world"""" 38 | } 39 | } 40 | 41 | "fail with @ignoreCase" in { 42 | val notRule = NotRule(Literal(Some("hello world"))) 43 | notRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) must beLike { 44 | case Validated.Invalid(messages) => messages.head mustEqual """not("hello world") fails for row: 1, column: column1, value: "hello WORLD"""" 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/RangeRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | import cats.data.Validated 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class RangeRuleSpec extends Specification { 20 | 21 | "RangeRule" should { 22 | val globalDirectives = List(TotalColumns(1)) 23 | val schema = Schema(globalDirectives, List(ColumnDefinition(NamedColumnIdentifier("Country")))) 24 | 25 | "fail when non numeric number passed" in { 26 | val rangeRule = new RangeRule(1,2) 27 | 28 | rangeRule.evaluate(0, Row(List(Cell("Germany")), 1), schema) must beLike { 29 | case Validated.Invalid(messages) => messages.toList mustEqual List("""range(1,2) fails for row: 1, column: Country, value: "Germany"""") 30 | } 31 | } 32 | 33 | "pass when we test integer boundaries" in { 34 | val rangeRule = new RangeRule(Int.MinValue,(Int.MaxValue)) 35 | 36 | rangeRule.evaluate(0, Row(List(Cell((Int.MaxValue).toString)), 1), schema) mustEqual Validated.Valid(true) 37 | } 38 | 39 | "fail when we test small decimal outside range" in { 40 | val rangeRule = new RangeRule(0.01,0.1) 41 | 42 | rangeRule.evaluate(0, Row(List(Cell(("0.00999999999999999999999999999999"))), 1), schema) must beLike { 43 | case Validated.Invalid(messages) => messages.toList mustEqual List("""range(0.01,0.1) fails for row: 1, column: Country, value: "0.00999999999999999999999999999999"""") 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserColumnDirectivesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import java.io.StringReader 12 | import org.junit.runner.RunWith 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.{FailMessage, SchemaDefinitionError} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | import uk.gov.nationalarchives.csv.validator.TestHelper._ 17 | import cats.data.{NonEmptyList, Validated} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class SchemaParserColumnDirectivesSpec extends SchemaSpecBase { 21 | 22 | import TestSchemaParser._ 23 | 24 | "Schema" should { 25 | 26 | "succeed for a @optional column directive" in { 27 | val schema = """version 1.0 28 | |@totalColumns 1 29 | |column1: @optional""".stripMargin 30 | 31 | parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("column1"), _, List(Optional()))), v),_) => ok } 32 | } 33 | 34 | "succeed for a @ignoreCase column directive" in { 35 | val schema = """version 1.0 36 | @totalColumns 1 37 | column1: @ignoreCase""".stripMargin 38 | 39 | parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("column1"), _, List(IgnoreCase()))), v),_) => ok } 40 | } 41 | 42 | "fail for duplicate column directives" in { 43 | val schema = """version 1.0 44 | |@totalColumns 1 45 | |column1: @ignoreCase @ignoreCase""".stripMargin 46 | 47 | parseAndValidate(new StringReader(schema)) must beLike { case Validated.Invalid(msgs) => msgs.map(_.removeCR) mustEqual NonEmptyList.one(FailMessage(SchemaDefinitionError, 48 | """[3.23] failure: Invalid column definition 49 | | 50 | |column1: @ignoreCase @ignoreCase 51 | | ^""".stripMargin.removeCR)) } 52 | } 53 | 54 | "fail for multiple duplicate column directives" in { 55 | val schema = """version 1.0 56 | |@totalColumns 1 57 | |column1: @ignoreCase @optional @ignoreCase @optional""".stripMargin 58 | 59 | parseAndValidate(new StringReader(schema)) must beLike { case Validated.Invalid(msgs) => msgs.map(_.removeCR) mustEqual NonEmptyList.one(FailMessage(SchemaDefinitionError, 60 | """[3.33] failure: Invalid column definition 61 | | 62 | |column1: @ignoreCase @optional @ignoreCase @optional 63 | | ^""".stripMargin.removeCR)) } 64 | } 65 | 66 | "fail for duplicate column directives on different columns" in { 67 | val schema = """version 1.0 68 | |@totalColumns 3 69 | |column1: @ignoreCase @optional @ignoreCase @optional 70 | |column2: @optional @ignoreCase 71 | |column3: @ignoreCase @ignoreCase @optional @optional""".stripMargin 72 | 73 | parseAndValidate(new StringReader(schema)) must beLike { case Validated.Invalid(msgs) => msgs.map(_.removeCR) mustEqual NonEmptyList.one(FailMessage(SchemaDefinitionError, 74 | """[3.33] failure: Invalid column definition 75 | | 76 | |column1: @ignoreCase @optional @ignoreCase @optional 77 | | ^""".stripMargin.removeCR)) } 78 | } 79 | 80 | } 81 | 82 | "Schema ordering" should { 83 | "allow any ordering of column directives - optional before ignore case" in { 84 | val schema = """version 1.0 85 | |@totalColumns 1 86 | |column1: @optional @ignoreCase""".stripMargin 87 | 88 | parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("column1"), _, Optional() :: IgnoreCase() :: Nil)), _),_) => ok } 89 | } 90 | 91 | "allow any ordering of column directives - ignore case before optional" in { 92 | val schema = """version 1.0 93 | @totalColumns 1 94 | column1: @ignoreCase @optional""".stripMargin 95 | 96 | parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("column1"), _, IgnoreCase() :: Optional() :: Nil)), _), _) => ok } 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserGlobalDirectivesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import java.io.StringReader 12 | 13 | import org.junit.runner.RunWith 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class SchemaParserGlobalDirectivesSpec extends SchemaSpecBase { 19 | 20 | import TestSchemaParser._ 21 | 22 | "Schema" should { 23 | 24 | "succeed for a @totalColumns global directive" in { 25 | val schema = 26 | """version 1.0 27 | |@totalColumns 1 28 | |column1: """.stripMargin 29 | 30 | parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_)), List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _),_) => ok } 31 | } 32 | 33 | "with @totalColumns and @noHeader global directives" should { 34 | "succeed on seperate lines" in { 35 | val schema = 36 | """version 1.0 37 | |@totalColumns 1 38 | |@noHeader 39 | |column1: """.stripMargin 40 | 41 | parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _), _) => ok} 42 | } 43 | 44 | "succeed on same line" in { 45 | val schema = 46 | """version 1.0 47 | |@totalColumns 1 @noHeader 48 | |column1: """.stripMargin 49 | 50 | parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _), _) => ok} 51 | } 52 | } 53 | 54 | "@noHeader and @ignoreColumnNameCase global directives (mutually exclusive)" should { 55 | "fail for @noHeader followed by @ignoreColumnNameCase" in { 56 | val schema = 57 | """version 1.0 58 | |@noHeader 59 | |@ignoreColumnNameCase 60 | |column1: """.stripMargin 61 | 62 | parse(new StringReader(schema)) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 63 | }.pendingUntilFixed("Need to improve error messages") 64 | 65 | "fail for @ignoreColumnNameCase followed by @noHeader" in { 66 | val schema = 67 | """version 1.0 68 | |@ignoreColumnNameCase 69 | |@noHeader 70 | |column1: """.stripMargin 71 | 72 | parse(new StringReader(schema)) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 73 | }.pendingUntilFixed("Need to improve error messages") 74 | } 75 | 76 | "succeed with no global directives" in { 77 | val schema = 78 | """version 1.0 79 | |column1: """.stripMargin 80 | 81 | parse(new StringReader(schema)) must beLike { case Success(Schema(Nil, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _), _) => ok} 82 | } 83 | 84 | "succeed with @permitEmpty" in { 85 | val schema = 86 | """version 1.0 87 | |@permitEmpty 88 | |column1: """.stripMargin 89 | 90 | parse(new StringReader(schema)) must beLike { case Success(Schema(List(PermitEmpty()), List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _), _) => ok} 91 | } 92 | 93 | "succeed with @permitEmpty and @noHeader" in { 94 | val schema = 95 | """version 1.0 96 | |@permitEmpty 97 | |@noHeader 98 | |column1: """.stripMargin 99 | 100 | parse(new StringReader(schema)) must beLike { case Success(Schema(List(PermitEmpty(), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, Nil)), _), _) => ok} 101 | } 102 | 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserTotalColumnsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import java.io.StringReader 12 | 13 | import org.junit.runner.RunWith 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class SchemaParserTotalColumnsSpec extends SchemaSpecBase { 19 | 20 | import TestSchemaParser._ 21 | 22 | "Schema" should { 23 | 24 | "fail for TotalColumns with missing value" in { 25 | parse(new StringReader("version 1.0\n@totalColumns")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 26 | } 27 | 28 | "fail for incorrect TotalColumns field name" in { 29 | parse(new StringReader("version 1.0\n@ToalColumns 23")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 30 | } 31 | 32 | "fail for incorrect TotalColumns field name with no value" in { 33 | parse(new StringReader("version 1.0\n@TtalColumn")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 34 | } 35 | 36 | "fail for TotalColumns field name incorrect case" in { 37 | parse(new StringReader("version 1.0\n@TotalColumns 65")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 38 | } 39 | 40 | "fail for TotalColumns of zero" in { 41 | parse(new StringReader("version 1.0\n@totalColumns 0")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 42 | } 43 | 44 | "fail for TotalColumns with negative integer" in { 45 | parse(new StringReader("version 1.0\n@totalColumns -23")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 46 | } 47 | 48 | "fail for TotalColumns with non integer" in { 49 | parse(new StringReader("version 1.0\n@totalColumns 132.45")) must beLike { case Failure(message, _) => message mustEqual "Invalid column definition" } 50 | } 51 | 52 | "fail for TotalColumns with non numeric" in { 53 | parse(new StringReader("version 1.0\n@totalColumns blah")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/StartsRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import cats.data.Validated 12 | import org.junit.runner.RunWith 13 | import org.specs2.mutable.Specification 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 16 | import uk.gov.nationalarchives.csv.validator.schema._ 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class StartsRuleSpec extends Specification { 20 | 21 | "StartsRule with a string literal behaviour" should { 22 | val globalDirsOne = List(TotalColumns(1)) 23 | 24 | "succeed if cell starts with startsRule value" in { 25 | val startsRule = StartsRule(Literal(Some("hello world"))) 26 | 27 | startsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 28 | } 29 | 30 | "fail if cell does not start with startsRule value" in { 31 | val startsRule = StartsRule(Literal(Some("hello world today"))) 32 | startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 33 | case Validated.Invalid(messages) => messages.head mustEqual """starts("hello world today") fails for row: 1, column: column1, value: "hello world"""" 34 | } 35 | } 36 | 37 | "succeed if startsRule is the same as value" in { 38 | val startsRule = StartsRule(Literal(Some("hello world"))) 39 | startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 40 | } 41 | 42 | "succeed if startsRule's column reference does exist" in { 43 | val startsRule = StartsRule(ColumnReference(NamedColumnIdentifier("column1"))) 44 | 45 | startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) 46 | } 47 | 48 | "fail if startsRule's column reference doesn't exist" in { 49 | val startsRule = StartsRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn"))) 50 | 51 | startsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { 52 | case Validated.Invalid(messages) => messages.head mustEqual """starts($nonExistentColumn) fails for row: 1, column: column1, value: "hello world today"""" 53 | } 54 | } 55 | 56 | "succeed with @ignoreCase" in { 57 | val startsRule = StartsRule(Literal(Some("hello world"))) 58 | startsRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true) 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/UniqueMultiRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class UniqueMultiRuleSpec extends Specification { 21 | 22 | "unique multi rule" should { 23 | 24 | "succeed if all column values are distinct" in { 25 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name")), ColumnDefinition(NamedColumnIdentifier("Legs")))) 26 | val rule = UniqueMultiRule( ColumnReference(NamedColumnIdentifier("Legs")) :: Nil ) 27 | 28 | rule.evaluate(0, Row(Cell("r2d2") :: Cell("3") :: Nil, 1), schema) 29 | rule.evaluate(0, Row(Cell("c3po") :: Cell("2") :: Nil, 2), schema) must beLike { case Validated.Valid(_) => ok } 30 | } 31 | 32 | "succeed if duplicate column but 2nd is distinct" in { 33 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name")), ColumnDefinition(NamedColumnIdentifier("Legs")))) 34 | val rule = UniqueMultiRule( ColumnReference(NamedColumnIdentifier("Legs")) :: Nil ) 35 | 36 | rule.evaluate(0, Row(Cell("r2d2") :: Cell("3") :: Nil, 1), schema) 37 | rule.evaluate(0, Row(Cell("r2d2") :: Cell("2") :: Nil, 2), schema) must beLike { case Validated.Valid(_) => ok } 38 | } 39 | 40 | "fail if there are duplicate on all columns column values" in { 41 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name")), ColumnDefinition(NamedColumnIdentifier("Legs")), ColumnDefinition(NamedColumnIdentifier("Color")))) 42 | val rule = UniqueMultiRule( ColumnReference(NamedColumnIdentifier("Legs")) :: ColumnReference(NamedColumnIdentifier("Color")) :: Nil ) 43 | 44 | rule.evaluate(0, Row(Cell("r2d2") :: Cell("3") :: Cell("blue") :: Nil, 1), schema) 45 | rule.evaluate(0, Row(Cell("r2d2") :: Cell("3") :: Cell("blue") :: Nil, 2), schema) must beLike { 46 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("unique( $Legs, $Color ) fails for row: 2, column: Name, value: \"r2d2\" (original at row: 1)") 47 | } 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/UniqueRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_0 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class UniqueRuleSpec extends Specification { 21 | 22 | "unique rule" should { 23 | 24 | "pass if all column values are distinct" in { 25 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name")))) 26 | val rule = UniqueRule() 27 | 28 | rule.evaluate(0, Row(Cell("Jim") :: Nil, 1), schema) 29 | rule.evaluate(0, Row(Cell("Ben") :: Nil, 2), schema) must beLike { case Validated.Valid(_) => ok } 30 | } 31 | 32 | "fail if there are duplicate column values" in { 33 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name")))) 34 | val rule = UniqueRule() 35 | 36 | rule.evaluate(0, Row(Cell("Jim") :: Nil, 1), schema) 37 | rule.evaluate(0, Row(Cell("Ben") :: Nil, 2), schema) 38 | 39 | rule.evaluate(0, Row(Cell("Jim") :: Nil, 3), schema) must beLike { 40 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("unique fails for row: 3, column: Name, value: \"Jim\" (original at row: 1)") 41 | } 42 | } 43 | 44 | "fail if columns differ only in case with @ignoreCase" in { 45 | val rule = UniqueRule() 46 | val schema = Schema(List(TotalColumns(1)), List(ColumnDefinition(NamedColumnIdentifier("Name"), rule :: Nil, IgnoreCase() :: Nil))) 47 | 48 | rule.evaluate(0, Row(Cell("Ben") :: Nil, 1), schema) 49 | 50 | rule.evaluate(0, Row(Cell("BEN") :: Nil, 2), schema) must beLike { 51 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("unique fails for row: 2, column: Name, value: \"BEN\" (original at row: 1)") 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/AnyRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class AnyRuleSpec extends Specification { 21 | 22 | "AnyRule with a string literal behaviour" should { 23 | val globalDirsOne = List(TotalColumns(1)) 24 | val schema: Schema = Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1")))) 25 | 26 | "succeed if it matches the only any rule value" in { 27 | val anyRule = AnyRule(List(Literal(Some("hello world")))) 28 | anyRule.evaluate(0, Row(List(Cell("hello world")), 1), schema) mustEqual Validated.Valid(true) 29 | } 30 | 31 | "succeed if it matches the one of any rule value" in { 32 | val anyRule = AnyRule(List(Literal(Some("hello world")), Literal(Some("value2")), Literal(Some("value3")))) 33 | anyRule.evaluate(0, Row(List(Cell("hello world")), 1), schema) mustEqual Validated.Valid(true) 34 | anyRule.evaluate(0, Row(List(Cell("value2")), 2), schema) mustEqual Validated.Valid(true) 35 | anyRule.evaluate(0, Row(List(Cell("value3")), 3), schema) mustEqual Validated.Valid(true) 36 | } 37 | 38 | 39 | 40 | "fail if it doesn't matches" in { 41 | val anyRule = AnyRule(List(Literal(Some("hello world")))) 42 | 43 | anyRule.evaluate(0, Row(List(Cell("hello world today")), 1), schema) must beLike { 44 | case Validated.Invalid(messages) => messages.head mustEqual """any("hello world") fails for row: 1, column: column1, value: "hello world today"""" 45 | } 46 | } 47 | 48 | "succeed with @ignoreCase" in { 49 | val anyRule = AnyRule(List(Literal(Some("hello world")))) 50 | anyRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true) 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/IdenticalRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema.{ColumnDefinition, NamedColumnIdentifier, Schema, TotalColumns} 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class IdenticalRuleSpec extends Specification { 21 | 22 | "IdenticalRule" should { 23 | val globalDirsTwo = List(TotalColumns(2)) 24 | 25 | "succeed when there is only one non empty row" in { 26 | val identicalRule = IdenticalRule() 27 | val schema = Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2")))) 28 | identicalRule.evaluate(0, Row(List(Cell("row1"),Cell("abc")), 1), schema) mustEqual Validated.Valid(true) 29 | } 30 | 31 | "succeed when all rows in the column are equal" in { 32 | val identicalRule = IdenticalRule() 33 | val schema = Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2")))) 34 | identicalRule.evaluate(0, Row(List(Cell("row1"),Cell("abc")), 1), schema) mustEqual Validated.Valid(true) 35 | identicalRule.evaluate(0, Row(List(Cell("row1"),Cell("efd")), 2), schema) mustEqual Validated.Valid(true) 36 | identicalRule.evaluate(0, Row(List(Cell("row1"),Cell("ghi")), 3), schema) mustEqual Validated.Valid(true) 37 | } 38 | 39 | "fail when rows in the column are different" in { 40 | val identicalRule1 = IdenticalRule() 41 | val schema = Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2")))) 42 | identicalRule1.evaluate(1, Row(List(Cell("row1"),Cell("abc")), 1), schema) mustEqual Validated.Valid(true) 43 | identicalRule1.evaluate(1, Row(List(Cell("row1"),Cell("efd")), 2), schema) must beLike { 44 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 2, column: column2, value: \"efd\"") 45 | } 46 | identicalRule1.evaluate(1, Row(List(Cell("row1"),Cell("ghi")), 3), schema) must beLike { 47 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 3, column: column2, value: \"ghi\"") 48 | } 49 | 50 | val identicalRule2 = IdenticalRule() 51 | identicalRule2.evaluate(0, Row(List(Cell("row1"),Cell("abc")), 1), schema) mustEqual Validated.Valid(true) 52 | identicalRule2.evaluate(0, Row(List(Cell("row1"),Cell("efd")), 2), schema) mustEqual Validated.Valid(true) 53 | identicalRule2.evaluate(0, Row(List(Cell("row2"),Cell("ghi")), 3), schema) must beLike { 54 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 3, column: column1, value: \"row2\"") 55 | } 56 | } 57 | 58 | "fail when rows are empty" in { 59 | val identicalRule = IdenticalRule() 60 | val schema = Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2")))) 61 | identicalRule.evaluate(0, Row(List(Cell(""),Cell("abc")), 1), schema) must beLike { 62 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 1, column: column1, value: \"\"") 63 | } 64 | identicalRule.evaluate(0, Row(List(Cell(""),Cell("efd")), 2), schema) must beLike { 65 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 2, column: column1, value: \"\"") 66 | } 67 | identicalRule.evaluate(0, Row(List(Cell(""),Cell("ghi")), 3), schema) must beLike { 68 | case Validated.Invalid(msgs) => msgs.toList mustEqual List("identical fails for row: 3, column: column1, value: \"\"") 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/RangeRuleSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema.{ColumnDefinition, NamedColumnIdentifier, Schema, TotalColumns} 16 | 17 | import cats.data.Validated 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class RangeRuleSpec extends Specification { 21 | 22 | "RangeRule" should { 23 | val globalDirectives = List(TotalColumns(1)) 24 | val schema = Schema(globalDirectives, List(ColumnDefinition(NamedColumnIdentifier("Country")))) 25 | 26 | "fail when non numeric number passed" in { 27 | val rangeRule = new RangeRule(Some(1),Some(2)) 28 | 29 | rangeRule.evaluate(0, Row(List(Cell("Germany")), 1), schema) must beLike { 30 | case Validated.Invalid(messages) => messages.toList mustEqual List("""range(1,2) fails for row: 1, column: Country, value: "Germany"""") 31 | } 32 | } 33 | 34 | "pass when we test integer boundaries" in { 35 | val rangeRule = new RangeRule(Some(Int.MinValue),Some(Int.MaxValue)) 36 | 37 | rangeRule.evaluate(0, Row(List(Cell((Int.MaxValue).toString)), 1), schema) mustEqual Validated.Valid(true) 38 | } 39 | 40 | "pass when we test integer boundaries with no max limit" in { 41 | val rangeRule = new RangeRule(Some(0) ,None) 42 | 43 | rangeRule.evaluate(0, Row(List(Cell(BigDecimal(Int.MaxValue).toString)), 1), schema) mustEqual Validated.Valid(true) 44 | } 45 | 46 | "pass when we test integer boundaries with no min limit" in { 47 | val rangeRule = new RangeRule(None,Some(0) ) 48 | 49 | rangeRule.evaluate(0, Row(List(Cell(BigDecimal(Int.MinValue).toString)), 1), schema) mustEqual Validated.Valid(true) 50 | } 51 | 52 | "fail when we test small decimal outside range" in { 53 | val rangeRule = new RangeRule(Some(0.01),Some(0.1)) 54 | 55 | rangeRule.evaluate(0, Row(List(Cell(("0.00999999999999999999999999999999"))), 1), schema) must beLike { 56 | case Validated.Invalid(messages) => messages.toList mustEqual List("""range(0.01,0.1) fails for row: 1, column: Country, value: "0.00999999999999999999999999999999"""") 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaParserSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import java.io.StringReader 12 | 13 | import org.junit.runner.RunWith 14 | import org.specs2.runner.JUnitRunner 15 | import uk.gov.nationalarchives.csv.validator.schema._ 16 | import uk.gov.nationalarchives.csv.validator.schema.v1_0.IsRule 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class SchemaParserSpec extends SchemaSpecBase { 20 | 21 | import TestSchemaParser._ 22 | 23 | "Schema" should { 24 | 25 | "succeed for valid minimal schema" in { 26 | val columnDefinitions = List(new ColumnDefinition(NamedColumnIdentifier("column1")), new ColumnDefinition(NamedColumnIdentifier("column2"),List(IsRule(NoExt(ColumnReference(NamedColumnIdentifier("column1"))))),List()), new ColumnDefinition(NamedColumnIdentifier("column3"))) 27 | 28 | val schema = 29 | """version 1.1 30 | |@totalColumns 3 31 | |@noHeader 32 | |column1: 33 | |column2: is(noExt($column1)) 34 | |column3:""".stripMargin 35 | 36 | parse(new StringReader(schema)) must beLike { case Success(parsedSchema, _) => parsedSchema mustEqual buildSchema1_1(TotalColumns(3), NoHeader())(columnDefinitions:_*) } 37 | } 38 | 39 | "succeed for valid minimal schema" in { 40 | val columnDefinitions = List(new ColumnDefinition(NamedColumnIdentifier("c1")), new ColumnDefinition(NamedColumnIdentifier("c2")),new ColumnDefinition(NamedColumnIdentifier("c3"),List(IsRule(Concat(ColumnReference(NamedColumnIdentifier("c1")), ColumnReference(NamedColumnIdentifier("c2"))))),List())) 41 | 42 | val schema = 43 | """version 1.1 44 | |@totalColumns 3 45 | |c1: 46 | |c2: 47 | |c3: is(concat($c1,$c2))""".stripMargin 48 | 49 | parse(new StringReader(schema)) must beLike { case Success(parsedSchema, _) => parsedSchema mustEqual buildSchema1_1(TotalColumns(3))(columnDefinitions:_*) } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_1 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.mutable.Specification 13 | import org.specs2.runner.JUnitRunner 14 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 15 | import uk.gov.nationalarchives.csv.validator.schema.{TotalColumns, SchemaSpecBase, Literal, Schema} 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class SchemaSpec extends SchemaSpecBase { 19 | 20 | "NoExt Arg provider" should { 21 | 22 | "remove extension from string" in { 23 | 24 | val result = NoExt(Literal(Some("somestringwithextension.txt"))).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 25 | 26 | result must_=== Some("somestringwithextension") 27 | } 28 | 29 | "leave a string without extension unchanged" in { 30 | 31 | val result = NoExt(Literal(Some("somestringwithoutextension"))).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 32 | 33 | result must_=== Some("somestringwithoutextension") 34 | } 35 | 36 | } 37 | 38 | "Concat Arg provider" should { 39 | 40 | " append literal together" in { 41 | 42 | val result1 = Concat(Literal(Some("aaaaa")), Literal(None)).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 43 | 44 | result1 must_=== Some("aaaaa") 45 | 46 | val result2 = Concat(Literal(None), Literal(Some("aaaaa"))).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 47 | 48 | result2 must_=== Some("aaaaa") 49 | 50 | val result3 = Concat(Literal(Some("aaaaa")), Literal(Some("bbbbb"))).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 51 | 52 | result3 must_=== Some("aaaaabbbbb") 53 | 54 | val result4 = Concat(Literal(None), Literal(None)).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_1(TotalColumns(0))()) 55 | 56 | result4 must_=== None 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_2/SchemaSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.schema.v1_2 10 | 11 | import org.junit.runner.RunWith 12 | import org.specs2.runner.JUnitRunner 13 | import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row} 14 | import uk.gov.nationalarchives.csv.validator.schema.{TotalColumns, Literal, SchemaSpecBase} 15 | import uk.gov.nationalarchives.csv.validator.schema.v1_1.NoExt 16 | 17 | /** 18 | * Created by rhubner on 4/26/16. 19 | */ 20 | @RunWith(classOf[JUnitRunner]) 21 | class SchemaSpec extends SchemaSpecBase { 22 | 23 | "UriDecode Arg provider" should { 24 | 25 | "decode url parameter" in { 26 | 27 | val result = UriDecode(Literal(Some("text%20text")), None).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_2(TotalColumns(0))()) 28 | 29 | result must beSome("text text") 30 | 31 | } 32 | 33 | "decode URL parameter with different charset" in { 34 | val result = UriDecode(Literal(Some("text%9Atext")), Some(Literal(Some("windows-1252")))).referenceValue(1, Row(List(Cell("Germany")), 1), buildSchema1_2(TotalColumns(0))()) 35 | 36 | result must beSome("text\u0161text") 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /csv-validator-distribution/README.md: -------------------------------------------------------------------------------- 1 | CSV Validator Application 2 | ========================= 3 | 4 | This module assembles the csv-validator deliverable in a zip format. 5 | 6 | --- 7 | ### Structure of the generated ZIP file 8 | The zip, which includes bundled JRE, has following structure. 9 | 10 | ``` 11 | csv-validator-application- 12 | | 13 | +--- csv-validator-gui.bat 14 | | 15 | +--- csv-validator-cmd.bat 16 | | 17 | +--- csv-validator-cmd-.jar 18 | | 19 | +--- csv-validator-ui-.jar 20 | | 21 | +--- LICENSE 22 | | 23 | +-- [lib] 24 | | | 25 | | +--- 26 | | 27 | +-- [jre] 28 | | 29 | +--- 30 | 31 | ``` 32 | 33 | The zip file without JRE has a similar structure as above, except the `[JRE]` folder and the scripts are made as 34 | shell scripts rather than Windows batch files 35 | 36 | ``` 37 | csv-validator-application- 38 | | 39 | +--- csv-validator-gui 40 | | 41 | +--- csv-validator-cmd 42 | | 43 | +--- csv-validator-cmd-.jar 44 | | 45 | +--- csv-validator-ui-.jar 46 | | 47 | +--- LICENSE 48 | | 49 | +-- [lib] 50 | | 51 | +--- 52 | 53 | ``` 54 | -------------------------------------------------------------------------------- /csv-validator-distribution/assembly-windows-with-jre.xml: -------------------------------------------------------------------------------- 1 | 11 | 14 | bin-win64-with-jre 15 | 16 | zip 17 | 18 | false 19 | 20 | 21 | false 22 | runtime 23 | lib 24 | 25 | uk.gov.nationalarchives:csv-validator-cmd 26 | uk.gov.nationalarchives:csv-validator-core 27 | uk.gov.nationalarchives:csv-validator-ui 28 | 29 | 644 30 | 31 | 32 | false 33 | runtime 34 | lib 35 | 36 | uk.gov.nationalarchives:csv-validator-cmd 37 | uk.gov.nationalarchives:csv-validator-core 38 | 39 | 644 40 | 41 | 42 | false 43 | runtime 44 | 45 | 46 | uk.gov.nationalarchives:csv-validator-cmd 47 | uk.gov.nationalarchives:csv-validator-ui 48 | 49 | 644 50 | 51 | 52 | 53 | 54 | 55 | ${project.build.outputDirectory} 56 | 57 | 58 | bin 59 | / 60 | 61 | csv-validator-gui 62 | csv-validator-cmd 63 | 64 | 644 65 | 66 | 67 | target/jre-windows 68 | jre 69 | 70 | 71 | 72 | 73 | 74 | bin/csv-validator-gui.bat 75 | / 76 | true 77 | 644 78 | 79 | 80 | bin/csv-validator-cmd.bat 81 | / 82 | true 83 | 755 84 | 85 | 86 | bin/running-csv-validator.txt 87 | / 88 | true 89 | 644 90 | 91 | 92 | ../LICENSE 93 | / 94 | true 95 | 644 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /csv-validator-distribution/assembly-without-jre.xml: -------------------------------------------------------------------------------- 1 | 11 | 14 | bin 15 | 16 | zip 17 | 18 | false 19 | 20 | 21 | false 22 | runtime 23 | lib 24 | 25 | uk.gov.nationalarchives:csv-validator-cmd 26 | uk.gov.nationalarchives:csv-validator-core 27 | uk.gov.nationalarchives:csv-validator-ui 28 | 29 | 644 30 | 31 | 32 | false 33 | runtime 34 | lib 35 | 36 | uk.gov.nationalarchives:csv-validator-cmd 37 | uk.gov.nationalarchives:csv-validator-core 38 | 39 | 644 40 | 41 | 42 | false 43 | runtime 44 | 45 | 46 | uk.gov.nationalarchives:csv-validator-cmd 47 | uk.gov.nationalarchives:csv-validator-ui 48 | 49 | 644 50 | 51 | 52 | 53 | 54 | 55 | ${project.build.outputDirectory} 56 | 57 | 58 | 59 | 60 | 61 | bin/csv-validator-gui.bat 62 | / 63 | true 64 | 755 65 | 66 | 67 | bin/csv-validator-cmd.bat 68 | / 69 | true 70 | 755 71 | 72 | 73 | bin/csv-validator-gui 74 | / 75 | true 76 | 755 77 | 78 | 79 | bin/csv-validator-cmd 80 | / 81 | true 82 | 755 83 | 84 | 85 | bin/running-csv-validator.txt 86 | / 87 | true 88 | 644 89 | 90 | ../LICENSE 91 | / 92 | true 93 | 644 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /csv-validator-distribution/bin/csv-validator-cmd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (c) 2013, The National Archives 4 | # https://www.nationalarchives.gov.uk 5 | # 6 | # This Source Code Form is subject to the terms of the Mozilla Public 7 | # License, v. 2.0. If a copy of the MPL was not distributed with this 8 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | # 10 | 11 | # Location of this script at runtime 12 | SCRIPT=$0 13 | 14 | # Resolve absolute and relative symlinks 15 | while [ -h "$SCRIPT" ]; do 16 | LS=$( ls -ld "$SCRIPT" ) 17 | LINK=$( expr "$LS" : '.*-> \(.*\)$' ) 18 | if expr "$LINK" : '/.*' > /dev/null; then 19 | SCRIPT="$LINK" 20 | else 21 | SCRIPT="$( dirname "$SCRIPT" )/$LINK" 22 | fi 23 | done 24 | 25 | # Store absolute location 26 | CWD=$( pwd ) 27 | APP_HOME="$( cd "$(dirname "$SCRIPT" )" && pwd )" 28 | cd "$CWD" 29 | 30 | # Max memory to use for the JVM. Uncomment the following line and set the desired value for max memory. Alternatively, 31 | # you can set the max memory in terminal before launching the script by setting the environment variable for that shell 32 | # e.g. to set it to 1024m, use `export csvValidatorMemory=1024` 33 | # csvValidatorMemory=1024 34 | 35 | # Create param for runtime options. 36 | RUNTIME_OPTIONS="-Xmx1024m" 37 | 38 | if [ -n "$csvValidatorMemory" ] 39 | then 40 | RUNTIME_OPTIONS="-Xmx"$csvValidatorMemory"m" 41 | fi 42 | 43 | # Detect if we are running on a Mac: 44 | OS=$( uname ) 45 | if [ "Darwin" = "$OS" ]; then 46 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Xdock:name=CSV-Validator" 47 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Dcom.apple.mrj.application.growbox.intrudes=false" 48 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Dcom.apple.mrj.application.live-resize=true" 49 | fi 50 | 51 | # Launch the CLI application 52 | java $RUNTIME_OPTIONS -jar "$APP_HOME/csv-validator-cmd-${project.version}.jar" "$@" -------------------------------------------------------------------------------- /csv-validator-distribution/bin/csv-validator-cmd.bat: -------------------------------------------------------------------------------- 1 | @REM 2 | @REM Copyright (c) 2013, The National Archives 3 | @REM https://www.nationalarchives.gov.uk 4 | @REM 5 | @REM This Source Code Form is subject to the terms of the Mozilla Public 6 | @REM License, v. 2.0. If a copy of the MPL was not distributed with this 7 | @REM file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | @REM 9 | 10 | @ECHO OFF 11 | 12 | REM Maximum memory: 13 | REM --------------- 14 | REM This is the maximum memory CSV-Validator can use in megabytes. 15 | REM You can edit this script and set the maximum memory on the line below to set the maximum memory after the "=". 16 | REM If you do not want to make a permanent change but wish to set max memory each time, in the command prompt, before 17 | REM launching the batch file, set the max memory variable (e.g. SET csvValidatorMemory=1024) 18 | 19 | REM Edit the following line, remove 'REM' from beginning and set a specific maximum memory to change it permanently. 20 | REM SET csvValidatorMemory=1024 21 | 22 | REM Location of this script at runtime 23 | SET SCRIPT_HOME=%~dp0 24 | 25 | REM Create param for runtime option and set default max memory: 26 | SET RUNTIME_OPTIONS="-Xmx1024m" 27 | 28 | IF "%csvValidatorMemory%"=="" GOTO LaunchProgram 29 | SET RUNTIME_OPTIONS="-Xmx%csvValidatorMemory%m" 30 | 31 | :LaunchProgram 32 | "${JRE_BIN_PATH}java" %RUNTIME_OPTIONS% -jar "%SCRIPT_HOME%/csv-validator-cmd-${project.version}.jar" %* -------------------------------------------------------------------------------- /csv-validator-distribution/bin/csv-validator-gui: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (c) 2013, The National Archives 4 | # https://www.nationalarchives.gov.uk 5 | # 6 | # This Source Code Form is subject to the terms of the Mozilla Public 7 | # License, v. 2.0. If a copy of the MPL was not distributed with this 8 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | # 10 | 11 | # Location of this script at runtime 12 | SCRIPT=$0 13 | 14 | # Resolve absolute and relative symlinks 15 | while [ -h "$SCRIPT" ]; do 16 | LS=$( ls -ld "$SCRIPT" ) 17 | LINK=$( expr "$LS" : '.*-> \(.*\)$' ) 18 | if expr "$LINK" : '/.*' > /dev/null; then 19 | SCRIPT="$LINK" 20 | else 21 | SCRIPT="$( dirname "$SCRIPT" )/$LINK" 22 | fi 23 | done 24 | 25 | # Store absolute location 26 | CWD=$( pwd ) 27 | APP_HOME="$( cd "$(dirname "$SCRIPT" )" && pwd )" 28 | cd "$CWD" 29 | 30 | # Max memory to use for the JVM. Uncomment the following line and set the desired value for max memory. Alternatively, 31 | # you can set the max memory in terminal before launching the script by setting the environment variable for that shell 32 | # e.g. to set it to 1024m, use `export csvValidatorMemory=1024` 33 | # csvValidatorMemory=1024 34 | 35 | # Create param for runtime options. 36 | RUNTIME_OPTIONS="-Xmx1024m" 37 | 38 | if [ -n "$csvValidatorMemory" ] 39 | then 40 | RUNTIME_OPTIONS="-Xmx"$csvValidatorMemory"m" 41 | fi 42 | 43 | # Detect if we are running on a Mac: 44 | OS=$( uname ) 45 | if [ "Darwin" = "$OS" ]; then 46 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Xdock:name=CSV-Validator" 47 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Dcom.apple.mrj.application.growbox.intrudes=false" 48 | RUNTIME_OPTIONS=$RUNTIME_OPTIONS" -Dcom.apple.mrj.application.live-resize=true" 49 | fi 50 | 51 | # Launch the UI application 52 | java $RUNTIME_OPTIONS -jar "$APP_HOME/csv-validator-ui-${project.version}.jar" -------------------------------------------------------------------------------- /csv-validator-distribution/bin/csv-validator-gui.bat: -------------------------------------------------------------------------------- 1 | @REM 2 | @REM Copyright (c) 2013, The National Archives 3 | @REM https://www.nationalarchives.gov.uk 4 | @REM 5 | @REM This Source Code Form is subject to the terms of the Mozilla Public 6 | @REM License, v. 2.0. If a copy of the MPL was not distributed with this 7 | @REM file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | @REM 9 | 10 | @ECHO OFF 11 | 12 | REM Maximum memory: 13 | REM --------------- 14 | REM This is the maximum memory CSV-Validator can use in megabytes. 15 | REM You can edit this script and set the maximum memory on the line below to set the maximum memory after the "=". 16 | REM If you do not want to make a permanent change but wish to set max memory each time, in the command prompt, before 17 | REM launching the batch file, set the max memory variable (e.g. SET csvValidatorMemory=1024) 18 | 19 | REM Edit the following line, remove 'REM' from beginning and set a specific maximum memory to change it permanently. 20 | REM SET csvValidatorMemory=1024 21 | 22 | REM Location of this script at runtime 23 | SET SCRIPT_HOME=%~dp0 24 | 25 | REM Create param for runtime option and set default max memory: 26 | SET RUNTIME_OPTIONS="-Xmx1024m" 27 | 28 | IF "%csvValidatorMemory%"=="" GOTO LaunchProgram 29 | SET RUNTIME_OPTIONS="-Xmx%csvValidatorMemory%m" 30 | 31 | :LaunchProgram 32 | REM For GUI, we are launching the java process using `start` so it launches a separate command prompt. 33 | start "" "${JRE_BIN_PATH}javaw" %RUNTIME_OPTIONS% -jar "%SCRIPT_HOME%/csv-validator-ui-${project.version}.jar" -------------------------------------------------------------------------------- /csv-validator-distribution/bin/running-csv-validator.txt: -------------------------------------------------------------------------------- 1 | This document explains how CSV validator can be run using the launch scripts available in the distribution package. 2 | 3 | To launch the application as a GUI or CLI, follow the steps given below: 4 | 5 | On Windows, 6 | * To launch the GUI, 7 | * open command prompt 8 | * navigate to the folder where the zip is expanded 9 | * type `csv-validator-gui.bat` 10 | 11 | * To launch the CLI, 12 | * open command prompt 13 | * navigate to the folder where the zip is expanded 14 | * type `csv-validator-cmd.bat ` 15 | * you can type `csv-validator-cmd.bat --help` to see other CLI options 16 | 17 | On Linux / Mac, 18 | * To launch the GUI, 19 | * open terminal window 20 | * navigate to the folder where the zip is expanded 21 | * type `./csv-validator-gui` 22 | 23 | * To launch the CLI, 24 | * open terminal window 25 | * navigate to the folder where the zip is expanded 26 | * type `./csv-validator-cmd ` 27 | * you can type `./csv-validator-cmd --help` to see other CLI options 28 | 29 | ======================================== 30 | 31 | Sometimes, it is necessary to make more heap memory available to the CSV Validator, this is typically the case when the 32 | files being validated are large and the memory needed for processing falls short. In such case, before launching the 33 | CSV Validator, you can configure the maximum heap memory available to the Java process which runs the CSV Validator. 34 | 35 | Both, the Windows batch files and the shell scripts have the ability to configure maximum memory used by the JVM 36 | which runs CSV Validator. If you find that you are getting an error related to not having enough heap memory, you can 37 | increase the memory by following the steps given below: 38 | 39 | NOTE: The following examples set the available max heap memory to 2048MB (2GB). You can change the value to suit 40 | your own need. Also note, that the following way of increasing available memory only affects the scripts when launched 41 | within the same window, if you open another window, you will need to set the max memory in that window as well. 42 | 43 | On Windows, 44 | * Open a command prompt 45 | * type `set csvValidatorMemory=2048` 46 | * Without closing the command prompt, within the same window, launch the batch file 47 | 48 | On Linux, 49 | * Open a terminal window 50 | * type `export csvValidatorMemory=2048` 51 | * Without closing the terminal window, within the same window, call the shell script 52 | 53 | -------- 54 | 55 | In case you want to permanently change the maximum memory used, you can do so by editing the batch or shell scripts 56 | and updating the relevant line in those scripts. The instructions for updating the lines are given within the scripts. 57 | 58 | -------------------------------------------------------------------------------- /csv-validator-distribution/src/assembly/filter-windows-with-jre.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2013, The National Archives 3 | # https://www.nationalarchives.gov.uk 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla Public 6 | # License, v. 2.0. If a copy of the MPL was not distributed with this 7 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | # 9 | 10 | JRE_BIN_PATH=%SCRIPT_HOME%/jre/bin/ -------------------------------------------------------------------------------- /csv-validator-distribution/src/assembly/filter-without-jre.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2013, The National Archives 3 | # https://www.nationalarchives.gov.uk 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla Public 6 | # License, v. 2.0. If a copy of the MPL was not distributed with this 7 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | # 9 | 10 | JRE_BIN_PATH= -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/ErrorMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | /** 12 | * Error Message (failure) 13 | */ 14 | public class ErrorMessage extends FailMessage { 15 | 16 | public ErrorMessage(final String message, final int lineNumber, final int columnIndex) { 17 | super(message, lineNumber, columnIndex); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/FailMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | /** 12 | * Abstract type for CSV Schema Parser and CSV Validation errors 13 | */ 14 | public abstract class FailMessage { 15 | private final String message; 16 | private final int lineNumber; 17 | private final int columnIndex; 18 | 19 | /** 20 | * @param message The failure message 21 | * @param lineNumber The original line number in the file 22 | * @param columnIndex Zero-based column index 23 | */ 24 | public FailMessage(final String message, final int lineNumber, final int columnIndex) { 25 | this.message = message; 26 | this.lineNumber = lineNumber; 27 | this.columnIndex = columnIndex; 28 | } 29 | 30 | public String getMessage() { 31 | return message; 32 | } 33 | 34 | public int getLineNumber() 35 | { 36 | return lineNumber; 37 | } 38 | 39 | public int getColumnIndex() 40 | { 41 | return columnIndex; 42 | } 43 | } -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/ProgressCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | /** 12 | * Interface for receiving progress information 13 | */ 14 | public interface ProgressCallback { 15 | 16 | public void update(float complete); 17 | } 18 | -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/Request.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | import java.util.ArrayList; 12 | 13 | public interface Request { 14 | boolean failFast = false; 15 | ArrayList pathSubstitutionsList = new ArrayList<>(); 16 | boolean enforceCaseSensitivePathChecks = false; 17 | boolean trace = false; 18 | ProgressCallback progress = null; 19 | boolean skipFileChecks = false; 20 | int maxCharsPerCellLimit = 4096; 21 | } 22 | -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/Result.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | import java.util.ArrayList; 12 | 13 | public interface Result { 14 | ArrayList errors = new ArrayList<>(); 15 | Request validatorRequest = null; 16 | } 17 | -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/Substitution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | /** 12 | * Path substitutions are used where part of a 13 | * file path is to be replaced by something else 14 | * 15 | * Useful for when CSV files have hard-coded absolute paths 16 | * to files that accompany the CSV file 17 | */ 18 | public class Substitution { 19 | private final String from; 20 | private final String to; 21 | 22 | /** 23 | * Path Substitution 24 | * 25 | * @param from Part of a path to replace 26 | * @param to The replacement to place into the path 27 | */ 28 | public Substitution(final String from, final String to) { 29 | this.from = from; 30 | this.to = to; 31 | } 32 | 33 | /** 34 | * Substitution from 35 | * 36 | * @return The replacement to use in the substitution 37 | */ 38 | public String getFrom() { 39 | return from; 40 | } 41 | 42 | /** 43 | * Substitution to 44 | * 45 | * @return The string to replace in the substitution 46 | */ 47 | public String getTo() { 48 | return to; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /csv-validator-java-api/src/main/java/uk/gov/nationalarchives/csv/validator/api/java/WarningMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.api.java; 10 | 11 | /** 12 | * Warning message (failure) 13 | */ 14 | public class WarningMessage extends FailMessage { 15 | 16 | public WarningMessage(final String message, final int lineNumber, final int columnIndex) { 17 | super(message, lineNumber, columnIndex); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /csv-validator-ui/README.md: -------------------------------------------------------------------------------- 1 | CSV Validator - Graphical User Interface 2 | ======================================== 3 | 4 | A simple Swing GUI for running the CSV Validator. 5 | 6 | Designed and tested on both Windows and Linux/Unix/Mac platforms. 7 | 8 | 9 | Basic Usage 10 | ----------- 11 | 12 | ```bash 13 | $ validate-gui 14 | ``` 15 | 16 | 17 | Windows Users 18 | ------------- 19 | 20 | Instead of using `validate-gui` simply use `validate-gui.bat` 21 | 22 | 23 | Building from Source Code 24 | ------------------------- 25 | 26 | We are using the Maven build system, executing `mvn package` will produce a distribution of the command line application in both `target/csv-validator-ui-?-application` and `target/csv-validator-ui-?-application.zip`. Note the '?' will be replaced by the version number of the product. -------------------------------------------------------------------------------- /csv-validator-ui/src/main/assembly/appassembler-output.xml: -------------------------------------------------------------------------------- 1 | 11 | 14 | application 15 | 16 | dir 17 | zip 18 | 19 | 20 | true 21 | 22 | 23 | 24 | 25 | 26 | ${project.build.directory}/appassembler 27 | 28 | bin/${project.artifactId} 29 | 30 | 755 31 | / 32 | 33 | 34 | 35 | 36 | ${project.build.directory}/appassembler/bin 37 | 38 | bin/${project.artifactId} 39 | 40 | /bin 41 | 42 | 43 | 44 | 45 | ${project.build.directory}/appassembler 46 | 47 | bin 48 | 49 | / 50 | 51 | 52 | 53 | 54 | 55 | 56 | ../LICENSE 57 | / 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /csv-validator-ui/src/main/scala/scala/swing/PopupMenu.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package scala.swing 10 | 11 | import javax.swing.JPopupMenu 12 | 13 | /** 14 | * Implementation of JPopupMenu for scala.swing 15 | */ 16 | class PopupMenu(title0: String) extends SequentialContainer.Wrapper { self: PopupMenu => 17 | lazy val peer: JPopupMenu = new JPopupMenu(title0) 18 | 19 | def this() = this(null) 20 | } 21 | 22 | /** 23 | * Some simple implicits to enable 24 | * easier integration of PopupMenu with the existing 25 | * scala.swing classes 26 | */ 27 | object PopupMenuImplicits { 28 | implicit def componentPopupMenu(component: Component): { def popupMenu(menu: PopupMenu) : Unit } = new { 29 | def popupMenu(menu: PopupMenu) : Unit = { 30 | component.peer.setComponentPopupMenu(menu.peer) 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/DesignGridImplicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.ui 10 | 11 | import scala.language.implicitConversions 12 | import net.java.dev.designgridlayout.{ISpannableGridRow, IRowCreator, IGridRow, IRow} 13 | import swing.{Label, Component} 14 | 15 | /** 16 | * Some simple Implicits to ease 17 | * working with DesignGridLayout 18 | * from scala.swing 19 | */ 20 | object DesignGridImplicits { 21 | 22 | implicit def iRowAddComponent(iRow: IRow): { def add(component: Component) : IRow } = new { 23 | def add(component: Component) : IRow = iRow.add(component.peer) 24 | } 25 | 26 | implicit def iGridRowAddComponentSpan(iGridRow: IGridRow): { def add(component: Component, gridSpan : Int) : IGridRow } = new { 27 | def add(component: Component, gridSpan: Int) : IGridRow = iGridRow.add(component.peer, gridSpan) 28 | } 29 | 30 | implicit def iRowCreatorGridLabel(iRowCreator: IRowCreator): { def grid(label: Label) : ISpannableGridRow } = new { 31 | def grid(label: Label) : ISpannableGridRow = iRowCreator.grid(label.peer) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/SJXHelpers.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013, The National Archives 3 | * https://www.nationalarchives.gov.uk 4 | * 5 | * This Source Code Form is subject to the terms of the Mozilla Public 6 | * License, v. 2.0. If a copy of the MPL was not distributed with this 7 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | */ 9 | package uk.gov.nationalarchives.csv.validator.ui 10 | 11 | import scala.swing._ 12 | import org.jdesktop.swingx.{JXCollapsiblePane, JXFrame, JXTaskPane, JXTaskPaneContainer} 13 | import javax.swing.JFrame 14 | import scala.swing.event.{ActionEvent, ButtonClicked} 15 | import uk.gov.nationalarchives.csv.validator.ui.SJXTaskPane.ViewStateChanged 16 | 17 | /** 18 | * scala.swing compatible implementation of JXFrame 19 | */ 20 | class SJXFrame(gc: java.awt.GraphicsConfiguration = null) extends Frame(gc) { 21 | override lazy val peer: JFrame with InterfaceMixin2 = new JXFrame(gc) with InterfaceMixin2 with SuperMixin 22 | } 23 | 24 | /** 25 | * scala.swing compatible implementation of JXTaskPaneContainer 26 | */ 27 | class SJXTaskPaneContainer extends Panel { 28 | override lazy val peer = new JXTaskPaneContainer 29 | def add(taskPane: SJXTaskPane) = peer.add(taskPane.peer) 30 | } 31 | 32 | /** 33 | * scala.swing compatible implementation of JXTaskPane 34 | */ 35 | class SJXTaskPane(title: String) extends Panel with Publisher { 36 | override lazy val peer = new JXTaskPane(title) 37 | peer.addPropertyChangeListener("collapsed", ScalaSwingHelpers.PropertyChangeListener { e => 38 | publish(ViewStateChanged(SJXTaskPane.this)) 39 | }) 40 | 41 | def this() = this("") 42 | def this(title : String, collapsed : Boolean) = { 43 | this(title) 44 | this.collapsed = collapsed 45 | } 46 | def add(component: Component) = peer.add(component.peer) 47 | def collapsed = peer.isCollapsed 48 | def collapsed_= (c : Boolean) : Unit = { 49 | peer.setCollapsed(c) 50 | } 51 | } 52 | 53 | object SJXTaskPane { 54 | 55 | case class ViewStateChanged(override val source: SJXTaskPane) extends ActionEvent(source) 56 | 57 | def onViewStateChanged(action: => Unit): Reactions.Reaction = { 58 | case evt: ViewStateChanged => 59 | action 60 | } 61 | } -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | uk.gov.nationalarchives 7 | csv-validator-parent 8 | 1.4.1-SNAPSHOT 9 | csv-validator-parent 10 | 11 | 12 | 13 | csv-validator 14 | pom 15 | 16 | csv-validator 17 | 18 | 19 | scm:git:https://github.com/digital-preservation/csv-validator.git 20 | scm:git:https://github.com/digital-preservation/csv-validator.git 21 | scm:git:https://github.com/digital-preservation/csv-validator.git 22 | HEAD 23 | 24 | 25 | 26 | csv-validator-parent 27 | csv-validator-core 28 | csv-validator-cmd 29 | csv-validator-ui 30 | csv-validator-java-api 31 | csv-validator-distribution 32 | 33 | 34 | 35 | --------------------------------------------------------------------------------