├── .classpath ├── .gitignore ├── LICENSE ├── README.md ├── bag-utils ├── pom.xml └── src │ └── main │ ├── edu │ └── georgetown │ │ └── library │ │ └── fileAnalyzer │ │ ├── cmd │ │ └── AIPToAPTCmd.java │ │ └── util │ │ ├── AIPDirToAPTHelper.java │ │ ├── AIPToAPTHelper.java │ │ ├── AIPZipToAPTHelper.java │ │ ├── APTrustHelper.java │ │ ├── FABagHelper.java │ │ ├── FABagHelperData.java │ │ ├── IncompleteSettingsException.java │ │ ├── InvalidFilenameException.java │ │ ├── InvalidMetadataException.java │ │ ├── TarBagHelper.java │ │ ├── TarUtil.java │ │ └── ZipBagHelper.java │ └── resources │ └── log4j.properties ├── core-utils ├── pom.xml └── src │ └── main │ └── edu │ └── georgetown │ └── library │ └── fileAnalyzer │ └── util │ ├── FileAnalyzerURIResolver.java │ ├── XMLUtil.java │ └── ZipUtil.java ├── core ├── pom.xml └── src │ └── main │ ├── gov │ └── nara │ │ └── nwts │ │ └── ftapp │ │ ├── ActionResult.java │ │ ├── BatchAnalyzer.java │ │ ├── BatchImporter.java │ │ ├── FTDriver.java │ │ ├── FileAnalyzer.java │ │ ├── FileAnalyzerMod.java │ │ ├── FileTraversal.java │ │ ├── MyDirectoryFilter.java │ │ ├── MyFilenameFilter.java │ │ ├── ResultFilter.java │ │ ├── Timer.java │ │ ├── YN.java │ │ ├── counter │ │ ├── Cell.java │ │ ├── CellCheck.java │ │ ├── CheckResult.java │ │ ├── ColSumCounterCheck.java │ │ ├── CounterCheck.java │ │ ├── CounterData.java │ │ ├── CounterRec.java │ │ ├── CounterStat.java │ │ ├── DateCounterCheck.java │ │ ├── IntCounterCheck.java │ │ ├── IntOptCounterCheck.java │ │ ├── ListCounterCheck.java │ │ ├── PatternCounterCheck.java │ │ ├── REV.java │ │ ├── RPT.java │ │ ├── ReportType.java │ │ ├── RowSumCounterCheck.java │ │ ├── StaticCounterCheck.java │ │ └── SumCounterCheck.java │ │ ├── counterReport │ │ ├── BookReport1R4.java │ │ ├── BookReport2.java │ │ ├── BookReport2R4.java │ │ ├── DatabaseReport1.java │ │ ├── DatabaseReport1R4.java │ │ ├── DatabaseReport3.java │ │ ├── JournalReport1.java │ │ ├── JournalReport1R4.java │ │ └── UnknownReport.java │ │ ├── crud │ │ ├── CRUD.java │ │ ├── CRUDDetails.java │ │ └── package.html │ │ ├── filetest │ │ ├── ActionRegistry.java │ │ ├── BaseNameMatch.java │ │ ├── CountByType.java │ │ ├── CounterValidation.java │ │ ├── DefaultFileTest.java │ │ ├── DigitalDerivatives.java │ │ ├── DirMatch.java │ │ ├── DirTypeNameMatch.java │ │ ├── FileTest.java │ │ ├── ListDirectories.java │ │ ├── ListFiles.java │ │ ├── LowercaseTest.java │ │ ├── NameChecksum.java │ │ ├── NameMatch.java │ │ ├── NameValidationTest.java │ │ ├── RandomFileTest.java │ │ ├── ReadChecksum.java │ │ └── package.html │ │ ├── filter │ │ ├── AVFileTestFilter.java │ │ ├── CSVFilter.java │ │ ├── CounterFilter.java │ │ ├── CounterFilterXls.java │ │ ├── DefaultFileTestFilter.java │ │ ├── ExcelFilter.java │ │ ├── FileTestFilter.java │ │ ├── ImageFileTestFilter.java │ │ ├── Jp2FileTestFilter.java │ │ ├── JpegFileTestFilter.java │ │ ├── MrcFilter.java │ │ ├── ODSFilter.java │ │ ├── OdsCsvFilter.java │ │ ├── PdfFileTestFilter.java │ │ ├── TarFilter.java │ │ ├── TiffFileTestFilter.java │ │ ├── TiffJpegFileTestFilter.java │ │ ├── TsvFilter.java │ │ ├── TxtFilter.java │ │ ├── XmlFilter.java │ │ ├── ZipFilter.java │ │ └── package.html │ │ ├── ftprop │ │ ├── DefaultFTProp.java │ │ ├── FTProp.java │ │ ├── FTPropDate.java │ │ ├── FTPropEnum.java │ │ ├── FTPropFile.java │ │ ├── FTPropInt.java │ │ ├── FTPropString.java │ │ ├── InitializationStatus.java │ │ ├── InvalidInputException.java │ │ └── package.html │ │ ├── gui │ │ ├── CriteriaPanel.java │ │ ├── DetailsPanel.java │ │ ├── DirSelect.java │ │ ├── DirSelectChooser.java │ │ ├── DirectoryTable.java │ │ ├── FileCatalog.java │ │ ├── FileSelect.java │ │ ├── FileSelectChooser.java │ │ ├── FilterPanel.java │ │ ├── GuiFileTraversal.java │ │ ├── GuiFileTraversalSW.java │ │ ├── ImportPanel.java │ │ ├── MergePanel.java │ │ ├── MyBorderPanel.java │ │ ├── MyPanel.java │ │ ├── MyProgress.java │ │ ├── MyTableModel.java │ │ ├── ProgressPanel.java │ │ ├── StatsTable.java │ │ ├── SummaryPanel.java │ │ ├── TableSaver.java │ │ └── package.html │ │ ├── importer │ │ ├── CountKey.java │ │ ├── DefaultImporter.java │ │ ├── DelimitedFileImporter.java │ │ ├── DelimitedFileReader.java │ │ ├── DelimitedFileWriter.java │ │ ├── Importer.java │ │ ├── ImporterRegistry.java │ │ ├── MultiParser.java │ │ ├── Parser.java │ │ └── package.html │ │ ├── nameValidation │ │ ├── CustomPattern.java │ │ ├── DirAnalysis.java │ │ ├── InvalidManualPattern.java │ │ ├── NameValidationPattern.java │ │ ├── RenameDetails.java │ │ ├── RenamePassFail.java │ │ ├── RenameStatus.java │ │ ├── RenameablePattern.java │ │ ├── ValidPattern.java │ │ └── package.html │ │ ├── package.html │ │ └── stats │ │ ├── CRUDStats.java │ │ ├── ChecksumStats.java │ │ ├── CountStats.java │ │ ├── DirStats.java │ │ ├── FileCountStats.java │ │ ├── Money.java │ │ ├── NameStats.java │ │ ├── NameValidationStats.java │ │ ├── RandomStats.java │ │ ├── Randomizer.java │ │ ├── Stats.java │ │ ├── StatsGenerator.java │ │ ├── StatsItem.java │ │ ├── StatsItemConfig.java │ │ ├── StatsItemEnum.java │ │ └── package.html │ └── overview.html ├── demo ├── pom.xml └── src │ └── main │ ├── edu │ └── georgetown │ │ └── library │ │ └── fileAnalyzer │ │ ├── BAG_TYPE.java │ │ ├── DemoBatchAnalyzer.java │ │ ├── DemoBatchImporter.java │ │ ├── DemoFileAnalyzer.java │ │ ├── dateValidation │ │ ├── DateValidationPattern.java │ │ ├── DateValidationResult.java │ │ ├── DateValidationStatus.java │ │ ├── DateValidator.java │ │ └── DefaultDateValidationPattern.java │ │ ├── filetest │ │ ├── AIPDirToAPT.java │ │ ├── AIPToAPT.java │ │ ├── AIPZipToAPT.java │ │ ├── CounterValidationXls.java │ │ ├── CreateAPTrustBag.java │ │ ├── CreateBag.java │ │ ├── DemoActionRegistry.java │ │ ├── DemoFileTest.java │ │ ├── ImageProperties.java │ │ ├── MarcItemInventory.java │ │ ├── PageCount.java │ │ ├── VerifyAPTrustBagTar.java │ │ ├── VerifyBag.java │ │ ├── VerifyBagTar.java │ │ ├── VerifyBagZip.java │ │ ├── YearbookNameValidationTest.java │ │ └── iiif │ │ │ ├── CSVInputFile.java │ │ │ ├── CollItemFolders.java │ │ │ ├── CreateDateProjectTranslate.java │ │ │ ├── CreateIIIFManifest.java │ │ │ ├── DefaultInput.java │ │ │ ├── DefaultInputFile.java │ │ │ ├── DefaultManifestProjectTranslate.java │ │ │ ├── DefaultManifestProjectTranslateEnum.java │ │ │ ├── EADFolderTranslate.java │ │ │ ├── EADFolderTranslateSubjectsOnly.java │ │ │ ├── FileSystemProjectTranslate.java │ │ │ ├── FolderProjectTranslate.java │ │ │ ├── IIIFCanvasWrapper.java │ │ │ ├── IIIFEnums.java │ │ │ ├── IIIFJSONWrapper.java │ │ │ ├── IIIFLookup.java │ │ │ ├── IIIFManifest.java │ │ │ ├── IIIFProp.java │ │ │ ├── InputFileException.java │ │ │ ├── ItemFolderProjectTranslate.java │ │ │ ├── ManifestDimensions.java │ │ │ ├── ManifestGeneratePropFile.java │ │ │ ├── ManifestProjectTranslate.java │ │ │ ├── ManifestProjectTranslateEnum.java │ │ │ ├── MetadataInputFile.java │ │ │ ├── MetadataInputFileBuilder.java │ │ │ ├── PropertyFile.java │ │ │ ├── README.md │ │ │ ├── RESTResponseInputFile.java │ │ │ ├── RangePath.java │ │ │ ├── XMLInputFile.java │ │ │ └── manifestGenerate.prop │ │ ├── importer │ │ ├── DemoImporter.java │ │ ├── DemoImporterRegistry.java │ │ ├── EncodingCheck.java │ │ ├── ExcelReader.java │ │ ├── MarcInventory.java │ │ ├── MarcRecValidator.java │ │ ├── MarcSerializer.java │ │ ├── OAIImporter.java │ │ └── OutsourcedMarcRecValidator.java │ │ └── stats │ │ ├── BagStatsItems.java │ │ └── DetailedBagStatsItems.java │ └── resources │ └── edu │ └── georgetown │ └── library │ └── fileAnalyzer │ └── importer │ └── marc.xsl ├── doc ├── NARA File Analyzer and Metadata Harvester.doc ├── TestPlan.html └── TestPlan.txt ├── dspace ├── .gitignore ├── pom.xml └── src │ └── main │ ├── edu │ └── georgetown │ │ └── library │ │ └── fileAnalyzer │ │ ├── DSpaceBatchAnalyzer.java │ │ ├── DSpaceBatchImporter.java │ │ ├── DSpaceFileAnalyzer.java │ │ ├── filetest │ │ ├── DSpaceActionRegistry.java │ │ ├── DSpaceDAT.java │ │ ├── IngestInventory.java │ │ ├── IngestValidate.java │ │ ├── ProquestQC.java │ │ └── ProquestToIngest.java │ │ ├── filter │ │ └── GUImageFileTestFilter.java │ │ ├── importer │ │ ├── CSVBatcher.java │ │ ├── DSpaceImporterRegistry.java │ │ ├── DSpaceMetadata2Marc.java │ │ ├── EAD2DAO.java │ │ ├── EAD2DC.java │ │ ├── Field.java │ │ ├── IngestFolderCreate.java │ │ ├── ItemUpdateFolderCreate.java │ │ ├── ItemUpdateFolderCreateAddBitstream.java │ │ ├── ItemUpdateFolderCreateDeleteBitstream.java │ │ ├── MetadataRegPropFile.java │ │ ├── ReadMetadataRegistry.java │ │ └── Schema.java │ │ └── proquestXsl │ │ ├── GUProquestURIResolver.java │ │ └── MarcUtil.java │ └── resources │ └── edu │ └── georgetown │ └── library │ └── fileAnalyzer │ ├── ead-dao.xsl │ ├── ead.xsl │ └── proquestXsl │ ├── common.xsl │ ├── dc2marc.xsl │ ├── lcsh.xml │ ├── marc.xsl │ ├── proquest.xsl │ ├── proquest2ingest-dc.xsl │ ├── proquest2ingest-local.xsl │ └── proquest2marc.xsl └── pom.xml /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #ignore compiled source 2 | *.class 3 | .project 4 | .settings/* 5 | .idea/* 6 | bin/* 7 | export/* 8 | gulibrary/* 9 | core/target/* 10 | core-utils/target/* 11 | bag-utils/target/* 12 | dspace/target/* 13 | demo/target/* 14 | gulibrary/target/* 15 | image/target/* 16 | image/lib/* 17 | /bin 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Georgetown University Library Information Technology 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## PURPOSE 2 | Project Page: http://georgetown-university-libraries.github.io/File-Analyzer/ 3 | 4 | The File Analyzer and Metadata Harvester is a general purpose desktop (and command line) 5 | tool designed to automate simple, file-based operations. The File Analyzer assembles a toolkit of tasks a user can perform. 6 | 7 | The tasks that have been written into the File Analyzer code base have been optimized for use by libraries, archives, 8 | and other cultural heritage institutions. 9 | 10 | File Analyzer Wiki: https://github.com/Georgetown-University-Libraries/File-Analyzer/wiki 11 | 12 | #### Demonstration Videos 13 | [![Demonstration Video](https://i.ytimg.com/vi/kVi_k-HdH_4/1.jpg)](http://www.youtube.com/watch?v=kVi_k-HdH_4) 14 | [![Demonstration Video](https://i.ytimg.com/vi/1I8n60ZrwHo/1.jpg)](http://www.youtube.com/watch?v=1I8n60ZrwHo) 15 | [![Demonstration Video](https://i.ytimg.com/vi/5zYA04P0HPk/default.jpg)](http://www.youtube.com/watch?v=5zYA04P0HPk) 16 | 17 | ## History 18 | 19 | This code has been derived from the NARA File Analyzer and Metadata Harvester which is available at 20 | https://github.com/usnationalarchives/File-Analyzer. 21 | 22 | ## PREREQUISITES 23 | - JDK 1.8 or higher (for build) 24 | - JRE 1.8 or higher (for runtime) 25 | - (If you need to run with Java 6, see Releases for an older version) 26 | - Maven (or you will need to compile the modules manually) 27 | 28 | ## INSTALLATION 29 | - Clone this code to your computer 30 | - Run mvn install 31 | - Detailed Installation Instructions: https://github.com/Georgetown-University-Libraries/File-Analyzer/wiki/Installation-instructions 32 | 33 | ## DEPLOYMENTS 34 | This code will build 3 flavors of the File Analyzer. 35 | 36 | ### Core File Analyzer 37 | * All code runs from a self-extracting jar file 38 | 39 | ### DSpace File Analyzer 40 | * This version of the file analyzer is a self-extracting jar file that references the core file analyzer jar file. 41 | * It contains tools for automating the creation of DSpace ingestion folders 42 | 43 | ### Demo File Analyzer 44 | * This version contains extensions illustrating various capabilities of the File Analyzer. 45 | * This version of the file analyzer is a self-extracting jar file that references both the core and dspace file analyzer jar files. 46 | * This version of the application uses features of Apache Tika, BagIt, and Marc4j 47 | 48 | *** 49 | [![Georgetown University Library IT Code Repositories](https://raw.githubusercontent.com/Georgetown-University-Libraries/georgetown-university-libraries.github.io/master/LIT-logo-small.png)Georgetown University Library IT Code Repositories](http://georgetown-university-libraries.github.io/) 50 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/AIPDirToAPTHelper.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.io.IOException; 6 | 7 | import gov.loc.repository.bagit.Bag; 8 | 9 | public class AIPDirToAPTHelper extends AIPToAPTHelper { 10 | 11 | File outdir; 12 | public AIPDirToAPTHelper() { 13 | } 14 | 15 | @Override 16 | public int fillBag(File f, APTrustHelper aptHelper) throws FileNotFoundException, IOException, InvalidMetadataException, InvalidFilenameException { 17 | f = testForAptCompliantFilenames(f, aptHelper.allowSourceRename); 18 | 19 | Bag bag = aptHelper.getBag(); 20 | for(File cf: f.listFiles()) { 21 | bag.addFileToPayload(cf); 22 | } 23 | findMetadata(f, aptHelper); 24 | return bag.getPayload().size(); 25 | } 26 | 27 | public void findMetadata(File f, APTrustHelper aptHelper) throws IOException, InvalidMetadataException { 28 | if (f.isDirectory()) { 29 | for(File cf: f.listFiles()) { 30 | findMetadata(cf, aptHelper); 31 | } 32 | } else { 33 | if (f.getName().equals(METSXML)) { 34 | aptHelper.parseMetsFile(f); 35 | } else if (f.getName().startsWith("ead.")) { 36 | aptHelper.parseEadFile(f); 37 | } else if (f.getName().startsWith("hierarchy.")) { 38 | aptHelper.parseHierarchyFile(f); 39 | } 40 | 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/FABagHelperData.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.File; 4 | 5 | import gov.loc.repository.bagit.Bag; 6 | import gov.loc.repository.bagit.BagFactory; 7 | 8 | public class FABagHelperData { 9 | public File parent; 10 | public File source; 11 | public File newBag; 12 | public Bag bag; 13 | public BagFactory bf; 14 | 15 | public FABagHelperData() { 16 | } 17 | } -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/IncompleteSettingsException.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | public class IncompleteSettingsException extends Exception { 4 | private static final long serialVersionUID = 1L; 5 | 6 | public IncompleteSettingsException(String message) { 7 | super(message); 8 | } 9 | }; 10 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/InvalidFilenameException.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | public class InvalidFilenameException extends Exception { 4 | private static final long serialVersionUID = 1L; 5 | 6 | InvalidFilenameException(String s) { 7 | super(s); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/InvalidMetadataException.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | public class InvalidMetadataException extends Exception { 4 | private static final long serialVersionUID = 1L; 5 | 6 | InvalidMetadataException(String s) { 7 | super(s); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/TarBagHelper.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | import edu.georgetown.library.fileAnalyzer.util.TarUtil; 7 | 8 | public class TarBagHelper extends FABagHelper { 9 | 10 | public TarBagHelper(File source) { 11 | super(source); 12 | } 13 | 14 | @Override public void writeBagFile() throws IOException, IncompleteSettingsException { 15 | super.writeBagFile(); 16 | 17 | data.newBag = TarUtil.tarFolderAndDeleteFolder(data.newBag); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /bag-utils/src/main/edu/georgetown/library/fileAnalyzer/util/ZipBagHelper.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | import gov.loc.repository.bagit.writer.impl.ZipWriter; 7 | 8 | public class ZipBagHelper extends FABagHelper { 9 | 10 | public ZipBagHelper(File source) { 11 | super(source); 12 | } 13 | 14 | @Override public void createBagFile() throws IncompleteSettingsException { 15 | validate(); 16 | data.newBag = new File(data.parent, data.source.getName() + "_bag.zip"); 17 | } 18 | 19 | @Override public void writeBagFile() throws IOException, IncompleteSettingsException { 20 | validate(); 21 | if (data.newBag == null) throw new IncompleteSettingsException("Bag File must be created - call createBagFile()"); 22 | ZipWriter writer = new ZipWriter(data.bf); 23 | writer.setBagDir(data.source.getName()); 24 | data.bag.write(writer, data.newBag); 25 | data.bag.close(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /bag-utils/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to DEBUG and its only appender to A1. 2 | log4j.rootLogger=INFO, A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n 10 | -------------------------------------------------------------------------------- /core-utils/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | edu.georgetown.library 7 | FileAnalyzer 8 | 2.0 9 | 10 | 11 | edu.georgetown.library 12 | FileAnalyzerCoreUtils 13 | 2.0 14 | jar 15 | 16 | FileAnalyzerCoreUtils 17 | http://maven.apache.org 18 | 19 | 20 | 21 | UTF-8 22 | 23 | 24 | 25 | 26 | 27 | org.apache.maven.plugins 28 | maven-compiler-plugin 29 | 2.3.2 30 | 31 | 1.8 32 | 1.8 33 | 34 | 35 | 36 | org.apache.maven.plugins 37 | maven-jar-plugin 38 | 2.4 39 | 40 | 41 | target 42 | target/classes 43 | ${project.artifactId}-${project.version} 44 | src/main 45 | 46 | 47 | -------------------------------------------------------------------------------- /core-utils/src/main/edu/georgetown/library/fileAnalyzer/util/FileAnalyzerURIResolver.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.InputStream; 4 | 5 | import javax.xml.transform.Source; 6 | import javax.xml.transform.TransformerException; 7 | import javax.xml.transform.URIResolver; 8 | import javax.xml.transform.stream.StreamSource; 9 | 10 | public class FileAnalyzerURIResolver implements URIResolver { 11 | public Source resolve(String href, String base) throws TransformerException { 12 | InputStream is = FileAnalyzerURIResolver.class.getClassLoader().getResourceAsStream(href); 13 | if (is == null) is = FileAnalyzerURIResolver.class.getClassLoader().getResourceAsStream("resources" + href); 14 | return new StreamSource(is); 15 | } 16 | public InputStream getInputStream(String href, String base) throws TransformerException { 17 | InputStream is = FileAnalyzerURIResolver.class.getClassLoader().getResourceAsStream(href); 18 | if (is == null) is = FileAnalyzerURIResolver.class.getClassLoader().getResourceAsStream("resources/" + href); 19 | return is; 20 | } 21 | public static FileAnalyzerURIResolver INSTANCE = new FileAnalyzerURIResolver(); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /core-utils/src/main/edu/georgetown/library/fileAnalyzer/util/ZipUtil.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.util; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.FileNotFoundException; 6 | import java.io.FileOutputStream; 7 | import java.io.IOException; 8 | import java.util.zip.ZipEntry; 9 | import java.util.zip.ZipOutputStream; 10 | 11 | public class ZipUtil { 12 | public static void zipFolder(File folder) throws FileNotFoundException, IOException { 13 | File zip = new File(folder.getParentFile(), folder.getName() + ".zip"); 14 | try(ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zip))) { 15 | ZipUtil.zipSubDirectory("", folder, zout); 16 | } 17 | } 18 | /* 19 | * From http://stackoverflow.com/questions/2403830/recursively-zip-a-directory-containing-any-number-of-files-and-subdirectories-in 20 | */ 21 | public static void zipSubDirectory(String basePath, File dir, ZipOutputStream zout) throws IOException { 22 | byte[] buffer = new byte[4096]; 23 | File[] files = dir.listFiles(); 24 | for (File file : files) { 25 | if (file.isDirectory()) { 26 | String path = basePath + file.getName() + "/"; 27 | zout.putNextEntry(new ZipEntry(path)); 28 | zipSubDirectory(path, file, zout); 29 | zout.closeEntry(); 30 | } else { 31 | FileInputStream fin = new FileInputStream(file); 32 | zout.putNextEntry(new ZipEntry(basePath + file.getName())); 33 | int length; 34 | while ((length = fin.read(buffer)) > 0) { 35 | zout.write(buffer, 0, length); 36 | } 37 | zout.closeEntry(); 38 | fin.close(); 39 | } 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | edu.georgetown.library 7 | FileAnalyzer 8 | 2.0 9 | 10 | 11 | edu.georgetown.library 12 | CoreFileAnalyzer 13 | 2.0 14 | jar 15 | 16 | CoreFileAnalyzer 17 | http://maven.apache.org 18 | 19 | 20 | 21 | UTF-8 22 | 23 | 24 | 25 | 26 | edu.georgetown.library 27 | FileAnalyzerCoreUtils 28 | 2.0 29 | 30 | 31 | 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-compiler-plugin 36 | 2.3.2 37 | 38 | 1.8 39 | 1.8 40 | 41 | 42 | 43 | org.apache.maven.plugins 44 | maven-jar-plugin 45 | 2.4 46 | 47 | 48 | 49 | gov.nara.nwts.ftapp.FileAnalyzer 50 | 51 | 52 | 53 | 54 | 55 | target 56 | target/classes 57 | ${project.artifactId}-${project.version} 58 | src/main 59 | 60 | 61 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ActionResult.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.stats.Stats; 4 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 5 | 6 | import java.io.File; 7 | import java.util.TreeMap; 8 | /** 9 | * This class contains details about the completion of a {@link gov.nara.nwts.ftapp.filetest.FileTest} or a {@link gov.nara.nwts.ftapp.importer.Importer} 10 | * @author TBrady 11 | * 12 | */ 13 | public class ActionResult { 14 | /** 15 | * Unique name to assign to the results of the action 16 | */ 17 | public String name; 18 | /** 19 | * Action name + input 20 | */ 21 | public String longname; 22 | /** 23 | * Descriptive name of the action that was performed 24 | */ 25 | public String action; 26 | /** 27 | * Root directory or file on which the action was performed 28 | */ 29 | public File root; 30 | /** 31 | * Descriptive structure defining the components of the action results 32 | */ 33 | public StatsItemConfig details; 34 | /** 35 | * Container of the stats generated by the action 36 | */ 37 | public TreeMaptypes; 38 | /** 39 | * Indicates whether or not the task ran to completion or if the task was terminated manually or terminated by the file count threshold 40 | */ 41 | public boolean completed; 42 | /** 43 | * Time in milliseconds that the action took to complete 44 | */ 45 | public double duration; 46 | 47 | public ActionResult(File root, String name, String action, StatsItemConfig details, TreeMaptypes, boolean completed, double duration) { 48 | this.root = root; 49 | this.name = name; 50 | String s = (root.getParentFile() == null) ? "" : root.getParentFile().getName(); 51 | this.longname = action + " for .../" + s + "/" + root.getName(); 52 | this.action = action; 53 | this.details = details; 54 | this.types = types; 55 | this.completed = completed; 56 | this.duration = duration; 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/FileAnalyzer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Driver for the GUI version of the file analyzer with file modification rules disabled. 9 | * This application was originally created by Terry Brady in NARA's Digitization Services Branch. 10 | * @author TBrady 11 | * 12 | */ 13 | public class FileAnalyzer { 14 | 15 | public static void main(String[] args) { 16 | if (args.length > 0) 17 | new DirectoryTable(new File(args[0]),false); 18 | else 19 | new DirectoryTable(null,false); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/FileAnalyzerMod.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Driver for the GUI version of the file analyzer with file modification rules enabled. 9 | * This application was originally created by Terry Brady in NARA's Digitization Services Branch. 10 | * @author TBrady 11 | * 12 | */ 13 | public class FileAnalyzerMod { 14 | 15 | public static void main(String[] args) { 16 | if (args.length > 0) 17 | new DirectoryTable(new File(args[0]),true); 18 | else 19 | new DirectoryTable(null,true); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/MyDirectoryFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | 6 | /** 7 | * Filter for identifying directories when traversing a directory tree 8 | * @author TBrady 9 | * 10 | */ 11 | public class MyDirectoryFilter implements FilenameFilter { 12 | boolean ignorePeriods; 13 | public MyDirectoryFilter(boolean ignorePeriods){ 14 | this.ignorePeriods = ignorePeriods; 15 | } 16 | public boolean accept(File dir, String filename) { 17 | 18 | if (ignorePeriods) { 19 | if (filename.contains(".")) return false; 20 | } 21 | return (new File(dir,filename)).isDirectory(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ResultFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.util.Vector; 4 | 5 | /** 6 | * Container for the various filters that may be applied to a set of results allowing results to be filtered by one or more columns. 7 | * @author TBrady 8 | * 9 | */ 10 | public class ResultFilter { 11 | class ResultFilterItem { 12 | int col; 13 | String val; 14 | 15 | ResultFilterItem(int col, String val) { 16 | this.col = col; 17 | this.val = val; 18 | } 19 | } 20 | Vector resfilters; 21 | public ResultFilter() { 22 | resfilters = new Vector(); 23 | } 24 | 25 | public void add(int col, String val) { 26 | resfilters.add(new ResultFilterItem(col, val)); 27 | } 28 | 29 | public boolean evaluate(Vector obj) { 30 | for(ResultFilterItem rfi: resfilters){ 31 | if (rfi.col < obj.size()) { 32 | Object o = obj.get(rfi.col); 33 | if (o == null) o = ""; 34 | if (!rfi.val.equals(o.toString())){ 35 | return false; 36 | } 37 | } 38 | } 39 | return true; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/Timer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.util.Date; 4 | /** 5 | * Helper class to report on the duration of a File Analyzer action. 6 | * @author TBrady 7 | * 8 | */ 9 | public class Timer { 10 | Date start; 11 | Date end; 12 | 13 | public Timer() { 14 | start = new Date(); 15 | } 16 | 17 | public void end() { 18 | end = new Date(); 19 | } 20 | 21 | public double getDuration() { 22 | if (end == null) 23 | end(); 24 | return (end.getTime() - start.getTime())*.001; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/YN.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | /** 4 | * Simple enumeration when reporting statistics 5 | * @author TBrady 6 | * 7 | */ 8 | public enum YN implements Comparable { 9 | Y, 10 | N; 11 | } 12 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/CellCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class CellCheck { 7 | ArrayList cells = new ArrayList(); 8 | 9 | CounterCheck check; 10 | CellCheck(CounterCheck check) { 11 | this.check = check; 12 | } 13 | CellCheck(CounterCheck check, Cell cell) { 14 | this.check = check; 15 | cells.add(cell); 16 | } 17 | CellCheck(CounterCheck check, List acell) { 18 | this.check = check; 19 | for(Cell cell: acell){ 20 | cells.add(cell); 21 | } 22 | } 23 | List performCheck(CounterData cd) { 24 | ArrayList results = new ArrayList(); 25 | for(Cell cell: cells) { 26 | CheckResult res = check.performCheck(cd, cell, cd.getCellValue(cell)); 27 | results.add(res); 28 | 29 | if (res.stat != CounterStat.VALID) { 30 | if (res.newVal != null) { 31 | CounterData.setCellValue(cd.getFix(false), res.cell, res.newVal); 32 | } else { 33 | if (check.ignoreVal) { 34 | res.setIgnoreVal(true); 35 | } 36 | } 37 | } 38 | 39 | if (res.stat.ordinal() >= CounterStat.ERROR.ordinal()) { 40 | break; 41 | } 42 | } 43 | return results; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/CheckResult.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class CheckResult { 4 | CounterRec rec; 5 | public Cell cell; 6 | public CounterStat stat; 7 | public String message = ""; 8 | public String newVal; 9 | public boolean ignoreVal; 10 | 11 | CheckResult(Cell cell, CounterStat stat) { 12 | this.rec = CounterRec.CELL; 13 | this.cell = cell; 14 | this.stat = stat; 15 | } 16 | 17 | CheckResult(CounterStat stat) { 18 | this.rec = CounterRec.FILE; 19 | this.stat = stat; 20 | } 21 | 22 | CheckResult setMessage(String message) { 23 | this.message = message; 24 | return this; 25 | } 26 | 27 | CheckResult setNewVal(String newVal) { 28 | this.newVal = newVal; 29 | return this; 30 | } 31 | 32 | CheckResult setIgnoreVal(boolean b) { 33 | ignoreVal = b; 34 | return this; 35 | } 36 | 37 | static CheckResult createFileStatus(CounterStat stat) { 38 | return new CheckResult(stat); 39 | } 40 | static CheckResult createCellStatus(Cell cell, CounterStat stat) { 41 | return new CheckResult(cell, stat); 42 | } 43 | static CheckResult createCellValid(Cell cell) { 44 | return createCellStatus(cell, CounterStat.VALID); 45 | } 46 | static CheckResult createCellWarning(Cell cell, String message) { 47 | return createCellStatus(cell, CounterStat.WARNING).setMessage(message); 48 | } 49 | static CheckResult createCellInvalid(Cell cell, String message) { 50 | return createCellStatus(cell, CounterStat.INVALID).setMessage(message); 51 | } 52 | static CheckResult createCellInvalidCase(Cell cell, String message) { 53 | return createCellStatus(cell, CounterStat.WARNING_CASE).setMessage(message); 54 | } 55 | static CheckResult createCellInvalidPunct(Cell cell, String message) { 56 | return createCellStatus(cell, CounterStat.WARNING_PUNCT).setMessage(message); 57 | } 58 | static CheckResult createCellInvalidTrim(Cell cell, String message) { 59 | return createCellStatus(cell, CounterStat.WARNING_TRIM).setMessage(message); 60 | } 61 | static CheckResult createCellInvalidDate(Cell cell, String message) { 62 | return createCellStatus(cell, CounterStat.WARNING_DATE).setMessage(message); 63 | } 64 | static CheckResult createCellInvalidSum(Cell cell, String message) { 65 | return createCellStatus(cell, CounterStat.INVALID_SUM).setMessage(message); 66 | } 67 | static CheckResult createCellError(Cell cell, String message) { 68 | return createCellStatus(cell, CounterStat.ERROR).setMessage(message); 69 | } 70 | 71 | } -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/ColSumCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class ColSumCounterCheck extends SumCounterCheck { 4 | String val; 5 | int sr; 6 | int er; 7 | public ColSumCounterCheck(int sr, int er, String message) { 8 | super(message); 9 | this.sr = sr; 10 | this.er = er; 11 | } 12 | 13 | public int getRangeSum(CounterData cd, Cell cell) { 14 | int rangesum = 0; 15 | for(int r=sr; r<=er; r++) { 16 | rangesum += getIntValue(cd.getCellValue(new Cell(r, cell.col)), 0); 17 | } 18 | return rangesum; 19 | } 20 | 21 | } 22 | 23 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/CounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class CounterCheck { 4 | String message = "Cell does not match specifications"; 5 | CounterStat stat = CounterStat.INVALID; 6 | boolean allowNull = false; 7 | boolean ignoreVal = false; 8 | 9 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 10 | return CheckResult.createCellValid(cell); 11 | } 12 | 13 | public CounterCheck setMessage(String message) { 14 | this.message = message; 15 | return this; 16 | } 17 | 18 | public CounterCheck setCounterStat(CounterStat stat) { 19 | this.stat = stat; 20 | return this; 21 | } 22 | 23 | public CounterCheck setAllowNull(boolean b) { 24 | allowNull = b; 25 | return this; 26 | } 27 | 28 | public CounterCheck setIgnoreVal(boolean b) { 29 | this.ignoreVal = b; 30 | return this; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/CounterRec.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public enum CounterRec { 4 | FILE, 5 | CELL 6 | } 7 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/CounterStat.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public enum CounterStat { 4 | VALID, //Cell fonforms to rule 5 | WARNING, //Cell does not conform to spec, but issue occurrs so frequently it is being passed 6 | WARNING_CASE, //Cell has case mismatch 7 | WARNING_PUNCT, //Cell has trailing punct 8 | WARNING_TRIM, //Cell has extra whitespace 9 | WARNING_DATE, //Cell date is not in proper format 10 | INVALID_BLANK, //Non blank expected 11 | INVALID_SUM, //Invalid sum found 12 | INVALID, //Cell does not conform to rule 13 | JSTOR, //Common JSTOR format errors 14 | SHIFT_2_COL, //Common error, need to shift data table by 2 COL 15 | ERROR, //Processing error interpreting rule 16 | UNSUPPORTED_REPORT, //Code not yet implemented for this report type 17 | UNKNOWN_REPORT_TYPE, //Report type does not match know types 18 | UNSUPPORTED_FILE, //File cannot be broken into cells for processing 19 | } 20 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/DateCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | import java.text.ParseException; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.GregorianCalendar; 7 | 8 | class DateCounterCheck extends CounterCheck { 9 | String val; 10 | String fmtDisp; 11 | String fmtParse; 12 | SimpleDateFormat dfParse; 13 | SimpleDateFormat dfDisp; 14 | static SimpleDateFormat defdf = new SimpleDateFormat("MM/dd/yyyy"); 15 | static SimpleDateFormat def1df = new SimpleDateFormat("dd-MMM-yyyy"); 16 | static SimpleDateFormat def2df = new SimpleDateFormat("yy-MMM"); 17 | static Date y2000 = new GregorianCalendar(2000,1,1).getTime(); 18 | static { 19 | defdf.set2DigitYearStart(y2000); 20 | def2df.set2DigitYearStart(y2000); 21 | } 22 | 23 | public DateCounterCheck(String fmtDisp, String fmtParse, String message) { 24 | this.fmtDisp = fmtDisp; 25 | this.fmtParse = fmtParse; 26 | this.message = message; 27 | this.dfParse = new SimpleDateFormat(fmtParse); 28 | this.dfParse.set2DigitYearStart(y2000); 29 | this.dfDisp = new SimpleDateFormat(fmtDisp); 30 | } 31 | 32 | public DateCounterCheck(String fmtDisp, String message) { 33 | this(fmtDisp, fmtDisp, message); 34 | } 35 | public Date getDate(String cellval, SimpleDateFormat cdf) { 36 | Date date = null; 37 | try { 38 | date = cdf.parse(cellval); 39 | } catch (ParseException e) { 40 | } 41 | return date; 42 | } 43 | 44 | public Date getDate(String cellval) { 45 | Date date = getDate(cellval, dfParse); 46 | if (date == null) { 47 | date = getDate(cellval, defdf); 48 | } 49 | if (date == null) { 50 | date = getDate(cellval, def1df); 51 | } 52 | if (date == null) { 53 | date = getDate(cellval, def2df); 54 | } 55 | return date; 56 | } 57 | 58 | @Override 59 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 60 | if (cellval == null) { 61 | return CheckResult.createCellInvalid(cell, message + ". Null cell value"); 62 | } 63 | if (cellval.isEmpty()) { 64 | return CheckResult.createCellInvalid(cell, message + "Empty cell value"); 65 | } 66 | Date date = getDate(cellval); 67 | 68 | if (date == null) { 69 | return CheckResult.createCellInvalid(cell, "Date parse error"); 70 | } 71 | 72 | String s = dfDisp.format(date); 73 | if (s.equals(cellval)) { 74 | return CheckResult.createCellValid(cell); 75 | } 76 | 77 | return CheckResult.createCellInvalidDate(cell, "Date in inproper format").setNewVal(s); 78 | } 79 | 80 | } 81 | 82 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/IntCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class IntCounterCheck extends CounterCheck { 4 | String val; 5 | public IntCounterCheck(String message) { 6 | this.message = message; 7 | } 8 | 9 | public Integer getIntValue(String cellval) { 10 | cellval = (cellval == null) ? "" : cellval.replaceAll("\\.\\d+$", ""); 11 | try { 12 | return Integer.parseInt(cellval); 13 | } catch(NumberFormatException e) { 14 | return null; 15 | } 16 | } 17 | 18 | public Integer getIntValue(String cellval, int def) { 19 | cellval = (cellval == null) ? "" : cellval.replaceAll("\\.\\d+$", ""); 20 | try { 21 | return Integer.parseInt(cellval); 22 | } catch(NumberFormatException e) { 23 | return def; 24 | } 25 | } 26 | 27 | @Override 28 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 29 | if (getIntValue(cellval) == null) { 30 | return CheckResult.createCellInvalid(cell, "Cells should be numeric"); 31 | } 32 | 33 | return CheckResult.createCellValid(cell); 34 | } 35 | 36 | } 37 | 38 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/IntOptCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class IntOptCounterCheck extends IntCounterCheck { 4 | 5 | public IntOptCounterCheck(String message) { 6 | super(message); 7 | } 8 | 9 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 10 | if (cellval.isEmpty()) return CheckResult.createCellValid(cell); 11 | return super.performCheck(cd, cell, cellval); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/ListCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | import java.util.HashMap; 4 | 5 | public class ListCounterCheck extends CounterCheck { 6 | HashMap valueMap; 7 | public ListCounterCheck(String[] vals) { 8 | message = "Cell label does not contain expected value"; 9 | this.valueMap = new HashMap(); 10 | for(String s: vals) { 11 | this.valueMap.put(s, s); 12 | } 13 | } 14 | 15 | public void addAlternative(String s, String rep) { 16 | valueMap.put(s, rep); 17 | } 18 | 19 | @Override 20 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 21 | if (cellval == null) cellval = ""; 22 | String mapv = valueMap.get(cellval); 23 | if (mapv == null) { 24 | mapv = valueMap.get(cellval.trim()); 25 | } 26 | if (mapv == null) { 27 | return CheckResult.createCellStatus(cell, stat).setMessage(message); 28 | } 29 | if (!mapv.equals(cellval)) { 30 | if (mapv.equals(cellval.trim())) { 31 | return CheckResult.createCellStatus(cell, stat).setMessage("Cell value contains extra white space").setNewVal(mapv); 32 | } 33 | return CheckResult.createCellStatus(cell, stat).setMessage(message).setNewVal(mapv); 34 | } 35 | 36 | return CheckResult.createCellValid(cell); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/PatternCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | 6 | class PatternCounterCheck extends CounterCheck { 7 | Pattern patt; 8 | Pattern fixmatch; 9 | String rep; 10 | public PatternCounterCheck(Pattern patt) { 11 | this.patt = patt; 12 | } 13 | public PatternCounterCheck(Pattern patt, Pattern fixmatch, String rep) { 14 | this.patt = patt; 15 | this.rep = rep; 16 | this.fixmatch = fixmatch; 17 | } 18 | @Override 19 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 20 | if (patt == null || cellval == null) { 21 | return CheckResult.createCellInvalid(cell, message); 22 | } 23 | Matcher m = patt.matcher(cellval); 24 | if (m.matches()) { 25 | return CheckResult.createCellValid(cell); 26 | } 27 | 28 | if (fixmatch != null && rep != null) { 29 | m = fixmatch.matcher(cellval); 30 | String newVal = m.replaceAll(rep); 31 | m = patt.matcher(newVal); 32 | if (m.matches()) { 33 | return CheckResult.createCellStatus(cell, CounterStat.WARNING).setMessage(message).setNewVal(newVal); 34 | } else { 35 | return CheckResult.createCellInvalid(cell, message); 36 | } 37 | } 38 | 39 | CheckResult res = CheckResult.createCellStatus(cell, stat).setMessage(message); 40 | return res; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/REV.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public enum REV { 4 | NA,R1,R3,R4; 5 | 6 | public static REV find(String s) { 7 | for(REV rev: REV.values()) { 8 | if (rev.name().equals(s)) return rev; 9 | } 10 | return REV.NA; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/RowSumCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | public class RowSumCounterCheck extends SumCounterCheck { 4 | String val; 5 | int sc; 6 | int ec; 7 | public RowSumCounterCheck(int sc, int ec, String message) { 8 | super(message); 9 | this.sc = sc; 10 | this.ec = ec; 11 | } 12 | 13 | public int getRangeSum(CounterData cd, Cell cell) { 14 | int rangesum = 0; 15 | for(int c=sc; c<=ec; c++) { 16 | rangesum += getIntValue(cd.getCellValue(new Cell(cell.row,c)), 0); 17 | } 18 | return rangesum; 19 | } 20 | 21 | } 22 | 23 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/StaticCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | public class StaticCounterCheck extends CounterCheck { 6 | String val; 7 | Pattern noReplace; 8 | public StaticCounterCheck(String val) { 9 | this(val, null); 10 | } 11 | 12 | public StaticCounterCheck(String val, Pattern pNoReplace) { 13 | this.val = val; 14 | this.message = "Expected value: " + val; 15 | this.noReplace = pNoReplace; 16 | } 17 | 18 | @Override 19 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 20 | if (cellval == null) { 21 | if (allowNull) { 22 | return CheckResult.createCellValid(cell); 23 | } else { 24 | return CheckResult.createCellInvalid(cell, message + ": Cell is empty"); 25 | } 26 | } 27 | 28 | if (val.equals(cellval)) { 29 | return CheckResult.createCellValid(cell); 30 | } 31 | 32 | if (val.equalsIgnoreCase(cellval)) { 33 | return CheckResult.createCellInvalidCase(cell, message).setNewVal(val); 34 | } 35 | if (val.equals(cellval.trim())) { 36 | return CheckResult.createCellInvalidTrim(cell, message).setNewVal(val); 37 | } 38 | if (val.equalsIgnoreCase(cellval.trim())) { 39 | return CheckResult.createCellInvalidCase(cell, message).setNewVal(val); 40 | } 41 | if (val.equalsIgnoreCase(cellval.replaceAll("[:\\s]+$", ""))) { 42 | return CheckResult.createCellInvalidPunct(cell, message).setNewVal(val); 43 | } 44 | 45 | if (noReplace != null) { 46 | if (noReplace.matcher(cellval).matches()) { 47 | return CheckResult.createCellStatus(cell, stat).setMessage(message); 48 | } 49 | } 50 | 51 | CheckResult res = CheckResult.createCellStatus(cell, stat).setMessage(message).setNewVal(val); 52 | return res; 53 | } 54 | 55 | } 56 | 57 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counter/SumCounterCheck.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counter; 2 | 3 | class SumCounterCheck extends IntCounterCheck { 4 | String val; 5 | 6 | public SumCounterCheck(String message) { 7 | super(message); 8 | } 9 | 10 | @Override 11 | public CheckResult performCheck(CounterData cd, Cell cell, String cellval) { 12 | CheckResult res = super.performCheck(cd, cell, cellval); 13 | if (res.stat != CounterStat.VALID) return res; 14 | 15 | int cursum = getIntValue(cellval); 16 | 17 | int rangesum = getRangeSum(cd, cell); 18 | 19 | if (cursum == rangesum) { 20 | return CheckResult.createCellValid(cell); 21 | } 22 | return CheckResult.createCellInvalidSum(cell, message + ". Invalid range sum (expected:" + rangesum + "; current:" + cursum+")").setNewVal(""+rangesum); 23 | } 24 | 25 | public int getRangeSum(CounterData cd, Cell cell) { 26 | return 0; 27 | } 28 | 29 | } 30 | 31 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/BookReport1R4.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.REV; 5 | import gov.nara.nwts.ftapp.counter.ReportType; 6 | import gov.nara.nwts.ftapp.counter.StaticCounterCheck; 7 | 8 | public class BookReport1R4 extends ReportType { 9 | public static final String NAME = "Book Report 1"; 10 | public BookReport1R4() { 11 | super(NAME, REV.R4, "Number of Successful Title Requests by Month and Title"); 12 | } 13 | 14 | public static String[] COLS = {"","Publisher","Platform","Book DOI","Proprietary Identifier","ISBN","ISSN"}; 15 | 16 | public String[] getCols() {return COLS;} 17 | 18 | @Override public void initCustom(CounterData data) { 19 | addCheck("A9", new StaticCounterCheck("Total for all titles")); 20 | checkColHeader(data); 21 | checkGrid(data); 22 | 23 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 24 | addCheckRange(ReportType.ISBN, getDataRow(), 5, data.getLastRow(), 5); 25 | addCheckRange(ReportType.ISSN, getDataRow(), 6, data.getLastRow(), 6); 26 | } 27 | 28 | public boolean isSupported() { 29 | return true; 30 | } 31 | public boolean hasTotalRow() { 32 | return true; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/BookReport2.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.REV; 5 | import gov.nara.nwts.ftapp.counter.ReportType; 6 | 7 | public class BookReport2 extends ReportType { 8 | public static final String NAME = "Book Report 2"; 9 | public BookReport2() { 10 | super(NAME, REV.R1, "Number of Successful Section Requests by Month and Title"); 11 | } 12 | 13 | public static String[] COLS = {"","Publisher","Platform","ISBN","ISSN"}; 14 | public String[] getCols() {return COLS;} 15 | 16 | @Override public void initCustom(CounterData data) { 17 | checkColHeader(data); 18 | checkGrid(data); 19 | 20 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 21 | addCheckRange(ReportType.ISBN, getDataRow(), 3, data.getLastRow(), 3); 22 | addCheckRange(ReportType.ISSN, getDataRow(), 4, data.getLastRow(), 4); 23 | } 24 | 25 | public boolean isSupported() { 26 | return true; 27 | } 28 | public boolean hasTotalRow() { 29 | return true; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/BookReport2R4.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.REV; 5 | import gov.nara.nwts.ftapp.counter.ReportType; 6 | import gov.nara.nwts.ftapp.counter.StaticCounterCheck; 7 | 8 | public class BookReport2R4 extends ReportType { 9 | public static final String NAME = "Book Report 2"; 10 | public BookReport2R4() { 11 | super(NAME, REV.R4, "Number of Successful Section Requests by Month and Title"); 12 | } 13 | 14 | public static String[] COLS = {"","Publisher","Platform","Book DOI","Proprietary Identifier","ISBN","ISSN"}; 15 | 16 | public String[] getCols() {return COLS;} 17 | 18 | @Override public void initCustom(CounterData data) { 19 | addCheck("A9", new StaticCounterCheck("Total for all titles")); 20 | addCheck("B2", new StaticCounterCheck("Section Type:")); 21 | addCheck("B3", ReportType.NONBLANK); 22 | checkColHeader(data); 23 | checkGrid(data); 24 | 25 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 26 | addCheckRange(ReportType.ISBN, getDataRow(), 5, data.getLastRow(), 5); 27 | addCheckRange(ReportType.ISSN, getDataRow(), 6, data.getLastRow(), 6); 28 | } 29 | 30 | public boolean isSupported() { 31 | return true; 32 | } 33 | public boolean hasTotalRow() { 34 | return true; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/DatabaseReport1.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.ListCounterCheck; 5 | import gov.nara.nwts.ftapp.counter.REV; 6 | import gov.nara.nwts.ftapp.counter.ReportType; 7 | 8 | public class DatabaseReport1 extends ReportType { 9 | public static final String NAME = "Database Report 1"; 10 | public DatabaseReport1(REV rev, String title) { 11 | super(NAME, rev, title); 12 | } 13 | public DatabaseReport1() { 14 | super(NAME, REV.R3, "Total Searches and Sessions by Month and Database"); 15 | } 16 | 17 | public static String[] FIELDS = {"Total searches run","Searches-federated and automated","Total sessions","Sessions-federated and automated"}; 18 | public static String[] COLS = {"","Publisher","Platform",""}; 19 | public String[] getCols() {return COLS;} 20 | 21 | @Override public void initCustom(CounterData data) { 22 | ListCounterCheck labels = new ListCounterCheck(FIELDS); 23 | labels.addAlternative("searches", "Total searches run"); 24 | labels.addAlternative("sessions", "Total sessions"); 25 | addCheckRange(labels, getDataRow(), getFirstDataCol()-1, data.getLastRow(), getFirstDataCol()-1); 26 | checkFieldData(data, getDataRow()); 27 | this.checkColHeader(data); 28 | 29 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 30 | } 31 | 32 | public boolean isSupported() { 33 | return true; 34 | } 35 | public boolean hasTotalRow() { 36 | return false; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/DatabaseReport1R4.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.ListCounterCheck; 5 | import gov.nara.nwts.ftapp.counter.REV; 6 | import gov.nara.nwts.ftapp.counter.ReportType; 7 | 8 | public class DatabaseReport1R4 extends DatabaseReport1 { 9 | public static final String NAME = "Database Report 1"; 10 | public DatabaseReport1R4() { 11 | super(REV.R4, "Total Searches, Result Clicks and Record Views by Month and Database"); 12 | } 13 | 14 | public static final String[] FIELDS = {"Regular Searches", "Searches-federated and automated", "Result Clicks", "Record Views"}; 15 | public static String[] COLS = {"Database","Publisher","Platform","User Activity"}; 16 | public String[] getCols() {return COLS;} 17 | 18 | @Override public void initCustom(CounterData data) { 19 | checkColHeader(data); 20 | ListCounterCheck labels = new ListCounterCheck(FIELDS); 21 | labels.addAlternative("searches", "Total searches run"); 22 | labels.addAlternative("sessions", "Total sessions"); 23 | addCheckRange(labels, getDataRow(), getTotalCol(data)-1, data.getLastRow(), getTotalCol(data)-1); 24 | checkFieldData(data, getDataRow()); 25 | 26 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 27 | } 28 | 29 | public boolean isSupported() { 30 | return true; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/DatabaseReport3.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.ListCounterCheck; 5 | import gov.nara.nwts.ftapp.counter.REV; 6 | import gov.nara.nwts.ftapp.counter.ReportType; 7 | 8 | public class DatabaseReport3 extends ReportType { 9 | public static final String NAME = "Database Report 3"; 10 | public DatabaseReport3(REV rev, String title) { 11 | super(NAME, rev, title); 12 | } 13 | public DatabaseReport3() { 14 | super(NAME, REV.R3, "Total Searches and Sessions by Month and Service."); 15 | } 16 | public static String[] COLS = {"","Platform",""}; 17 | public String[] getCols() {return COLS;} 18 | 19 | @Override public void initCustom(CounterData data) { 20 | ListCounterCheck labels = new ListCounterCheck(DatabaseReport1.FIELDS); 21 | labels.addAlternative("searches", "Total searches run"); 22 | labels.addAlternative("sessions", "Total sessions"); 23 | addCheckRange(labels, getDataRow(), getFirstDataCol()-1, data.getLastRow(), getFirstDataCol()-1); 24 | checkFieldData(data, getDataRow()); 25 | this.checkColHeader(data); 26 | 27 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 1, data.getLastRow(), 1); //Plat 28 | } 29 | 30 | public boolean isSupported() { 31 | return true; 32 | } 33 | public boolean hasTotalRow() { 34 | return false; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/JournalReport1.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.REV; 5 | import gov.nara.nwts.ftapp.counter.ReportType; 6 | import gov.nara.nwts.ftapp.counter.StaticCounterCheck; 7 | 8 | public class JournalReport1 extends ReportType { 9 | public static final String NAME = "Journal Report 1"; 10 | public JournalReport1(REV rev) { 11 | super(NAME, rev, "Number of Successful Full-text Article Requests by Month and Journal"); 12 | } 13 | public JournalReport1() { 14 | this(REV.R3); 15 | } 16 | 17 | public static String[] COLS = {"","Publisher","Platform","Print ISSN","Online ISSN"}; 18 | public String[] getCols() {return COLS;} 19 | public static String[] TCOLS = {"YTD Total","YTD HTML","YTD PDF"}; 20 | public String[] getTotalCols() {return TCOLS;} 21 | 22 | @Override public void initCustom(CounterData data) { 23 | addCheck("A6", new StaticCounterCheck("Total for all journals")); 24 | addCheck("C6", ReportType.NONBLANK); 25 | addCheck("D6", ReportType.BLANK); 26 | addCheck("E6", ReportType.BLANK); 27 | 28 | addCheckRange(ReportType.NB_JOURNAL, getDataRow(), 0, data.getLastRow(), 0); //Journal 29 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //Plat 30 | addCheckRange(ReportType.ISSN, getDataRow(), 3, data.getLastRow(), 4); //Print 31 | 32 | checkColHeader(data); 33 | checkGrid(data); 34 | } 35 | 36 | public boolean isSupported() { 37 | return true; 38 | } 39 | 40 | public boolean hasTotalRow() {return true;} 41 | 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/JournalReport1R4.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.CounterData; 4 | import gov.nara.nwts.ftapp.counter.REV; 5 | import gov.nara.nwts.ftapp.counter.ReportType; 6 | import gov.nara.nwts.ftapp.counter.StaticCounterCheck; 7 | 8 | public class JournalReport1R4 extends JournalReport1 { 9 | public JournalReport1R4() { 10 | super(REV.R4); 11 | } 12 | 13 | public static String[] COLS = {"Journal","Publisher","Platform","Journal DOI","Proprietary Identifier","Print ISSN","Online ISSN"}; 14 | public String[] getCols() {return COLS;} 15 | public static String[] TCOLS = {"Reporting Period Total","Reporting Period HTML","Reporting Period PDF"}; 16 | public String[] getTotalCols() {return TCOLS;} 17 | 18 | @Override public void initCustom(CounterData data) { 19 | addCheck("A9", new StaticCounterCheck("Total for all journals")); 20 | //B9, publisher/vendor, may be blank 21 | addCheck("C9", ReportType.NONBLANK); //platform 22 | addCheck("D9", ReportType.BLANK); 23 | addCheck("E9", ReportType.BLANK); 24 | addCheck("F9", ReportType.BLANK); 25 | addCheck("G9", ReportType.BLANK); 26 | 27 | addCheckRange(ReportType.NB_JOURNAL, getDataRow(), 0, data.getLastRow(), 0); //journal 28 | addCheckRange(ReportType.NB_PLATFORM, getDataRow(), 2, data.getLastRow(), 2); //plat 29 | addCheckRange(ReportType.ISSN, getDataRow(), 5, data.getLastRow(), 6); 30 | 31 | checkColHeader(data); 32 | checkGrid(data); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/counterReport/UnknownReport.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.counterReport; 2 | 3 | import gov.nara.nwts.ftapp.counter.REV; 4 | import gov.nara.nwts.ftapp.counter.ReportType; 5 | 6 | public class UnknownReport extends ReportType { 7 | public UnknownReport() { 8 | super("Unknown", REV.NA, ""); 9 | } 10 | 11 | public boolean isSupported() { 12 | return false; 13 | } 14 | public boolean hasTotalRow() { 15 | return false; 16 | } 17 | 18 | public static String[] COLS = {""}; 19 | public String[] getCols() { 20 | return COLS; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/crud/CRUD.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.crud; 2 | 3 | /** 4 | *Categorizes the resulting status of a database action. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum CRUD implements Comparable { 9 | NA, 10 | Created, 11 | Replaced, 12 | Updated, 13 | Deleted, 14 | Skipped, 15 | NotFound, 16 | AlreadyExists; 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/crud/CRUDDetails.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.crud; 2 | 3 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 4 | /** 5 | * Stores the results of a database action. 6 | * @author TBrady 7 | * 8 | */ 9 | public class CRUDDetails { 10 | CRUD action; 11 | RenamePassFail status; 12 | int id; 13 | String note; 14 | public CRUDDetails(CRUD action, RenamePassFail status, int id, String note){ 15 | this.action = action; 16 | this.status = status; 17 | this.id = id; 18 | this.note = note; 19 | } 20 | public String toString() { 21 | return action.toString()+" "+ status.toString() +". ID=" + id+". "+((note==null)?"":note); 22 | } 23 | } 24 | 25 | 26 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/crud/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | The Create, Replace, Update, Delete package is intended to contain helper classes when using the File Analyzer as a database ingest tool. This package has not yet been fully developed. 4 | 5 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/ActionRegistry.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.util.Vector; 6 | /** 7 | * Activates the FileTest objects that will be made available to a user. 8 | * In the event of a load failure of an individual class, the runtime loading of those classes can be disabled in this method. 9 | * @author TBrady 10 | * 11 | */ 12 | public class ActionRegistry extends Vector { 13 | 14 | private static final long serialVersionUID = 1L; 15 | boolean modifyAllowed = true; 16 | 17 | public ActionRegistry(FTDriver dt, boolean modifyAllowed) { 18 | this.modifyAllowed = modifyAllowed; 19 | add(new CountByType(dt)); 20 | add(new ListFiles(dt)); 21 | add(new ListDirectories(dt)); 22 | add(new NameMatch(dt)); 23 | add(new BaseNameMatch(dt)); 24 | add(new NameChecksum(dt)); 25 | add(new ReadChecksum(dt)); 26 | add(new DirMatch(dt)); 27 | add(new DirTypeNameMatch(dt)); 28 | add(new RandomFileTest(dt)); 29 | FileTest next = new LowercaseTest(dt, null); 30 | add(next); 31 | if (modifyAllowed()) { 32 | add(new LowercaseTest(dt, next)); 33 | } 34 | add(new DigitalDerivatives(dt)); 35 | add(new CounterValidation(dt)); 36 | } 37 | 38 | /** 39 | * By design, File Analyzer actions should not be destructive. 40 | * When implementing routines that will automatically rename items (vs validating file names), the registry can be configured to only load those actions for specific users. 41 | * Also, the base File Analyzer module can be initiated with or without modification enabled. 42 | * This check is only enforced if the FileTest honors this flag. 43 | * 44 | */ 45 | public boolean modifyAllowed() { 46 | return modifyAllowed; 47 | } 48 | public void removeFT(Class c) { 49 | for(FileTest ft: this) { 50 | if (c.isInstance(ft)) { 51 | this.remove(ft); 52 | break; 53 | } 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/BaseNameMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.NameStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 7 | 8 | import java.io.File; 9 | /** 10 | * Match files based on the base file name (without a file extension). 11 | * @author TBrady 12 | * 13 | */ 14 | class BaseNameMatch extends DefaultFileTest { 15 | 16 | public BaseNameMatch(FTDriver dt) { 17 | super(dt); 18 | } 19 | 20 | public String toString() { 21 | return "Match By Base Name"; 22 | } 23 | public String getKey(File f) { 24 | String s = f.getName(); 25 | String[] sa = s.split("\\."); 26 | if (sa.length > 1) s = s.substring(0,s.length()-sa[sa.length-1].length()-1); 27 | return s; 28 | } 29 | 30 | public String getShortName(){return "Base Name";} 31 | 32 | public Object fileTest(File f) { 33 | return null; 34 | } 35 | public Stats createStats(String key){ 36 | return NameStats.Generator.INSTANCE.create(key); 37 | } 38 | public StatsItemConfig getStatsDetails() { 39 | return NameStats.details; 40 | } 41 | public void initFilters() { 42 | initAllFilters(); 43 | } 44 | 45 | public String getDescription() { 46 | return "This test reports on file size by base name (no extension) regardless of the directory in which a file name is found."; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/CountByType.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.FileCountStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 7 | 8 | import java.io.File; 9 | /** 10 | * Count items by file extension; this is the most basic and easy to understand File Analyzer rule. 11 | * @author TBrady 12 | * 13 | */ 14 | class CountByType extends DefaultFileTest { 15 | 16 | public CountByType(FTDriver dt) { 17 | super(dt); 18 | } 19 | 20 | public Object fileTest(File f) { 21 | return null; 22 | } 23 | 24 | public String toString() { 25 | return "Count Files By Type"; 26 | } 27 | 28 | public Stats createStats(String key){ 29 | return FileCountStats.Generator.INSTANCE.create(key); 30 | } 31 | public StatsItemConfig getStatsDetails() { 32 | return FileCountStats.details; 33 | } 34 | public String getShortName(){return "By Type";} 35 | public void initFilters() { 36 | initAllFilters(); 37 | } 38 | 39 | public String getDescription() { 40 | return "This test counts the number of files found by file extension.\n" + 41 | "A report will be generated listing the number of files found for each extension" + 42 | " as well as a cumulative number of bytes for files of each type."; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/DirMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.DirStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Create FileAnalyzer statistics by directory. 12 | * @author TBrady 13 | * 14 | */ 15 | class DirMatch extends DefaultFileTest { 16 | 17 | public DirMatch(FTDriver dt) { 18 | super(dt); 19 | } 20 | 21 | public String toString() { 22 | return "Match By Path"; 23 | } 24 | public String getKey(File f) { 25 | return getKey(f, f.getParentFile()); 26 | } 27 | 28 | public String getKey(File f, Object parentdir) { 29 | String key = ""; 30 | if (parentdir instanceof File) { 31 | key = ((File)parentdir).getAbsolutePath().substring(getRoot().getAbsolutePath().length()); 32 | } 33 | return key; 34 | } 35 | 36 | public String getShortName(){return "Path";} 37 | 38 | public Object fileTest(File f) { 39 | return null; 40 | } 41 | public Stats createStats(String key){ 42 | return DirStats.Generator.INSTANCE.create(key); 43 | } 44 | public StatsItemConfig getStatsDetails() { 45 | return DirStats.details; 46 | } 47 | public void initFilters() { 48 | initAllFilters(); 49 | } 50 | 51 | public String getDescription() { 52 | return "This test counts the number of items found in a specific directory. This test will also compute cumulative totals found for each directory that is scanned."; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/FileTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.List; 5 | import java.util.regex.Pattern; 6 | 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.stats.Stats; 9 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 10 | import gov.nara.nwts.ftapp.filter.FileTestFilter; 11 | import gov.nara.nwts.ftapp.ftprop.FTProp; 12 | import gov.nara.nwts.ftapp.ftprop.InitializationStatus; 13 | 14 | /** 15 | * Contract defining the behavior of a File Analyzer custom rule. 16 | * @author TBrady 17 | * 18 | */ 19 | public interface FileTest { 20 | public String toString(); 21 | public String getDescription(); 22 | public String getExt(File f); 23 | public String getKey(File f); 24 | public String getKey(File f, Object o); 25 | public boolean isTestable(File f); 26 | public Object fileTest(File f); 27 | public Stats getStats(File f); 28 | public Stats getStats(String key); 29 | 30 | public List getFilters(); 31 | 32 | public Stats createStats(String key); 33 | public StatsItemConfig getStatsDetails(); 34 | public String getShortName(); 35 | public String getShortNameNormalized(); 36 | 37 | public FileTestFilter getDefaultFilter(); 38 | 39 | public File getRoot(); 40 | 41 | void initFilters(); 42 | 43 | public boolean isTestFiles(); 44 | public boolean isTestDirectory(); 45 | public boolean isTestDirectory(File f); 46 | public boolean processRoot(); 47 | public Pattern getDirectoryPattern(); 48 | 49 | public void refineResults(); 50 | public InitializationStatus init(); 51 | 52 | public void progress(int count); 53 | public FileTest resetOption(); 54 | public List getPropertyList(); 55 | public FTDriver getFTDriver(); 56 | public Object getProperty(String name); 57 | public void setProperty(String name, String str); 58 | } 59 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/ListDirectories.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.FTDriver; 6 | import gov.nara.nwts.ftapp.stats.Stats; 7 | import gov.nara.nwts.ftapp.stats.StatsGenerator; 8 | import gov.nara.nwts.ftapp.stats.StatsItem; 9 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 10 | import gov.nara.nwts.ftapp.stats.StatsItemEnum; 11 | 12 | /** 13 | * List the full path for a dirctory; this can be used as input for the FileAnalzyer batch capability. 14 | * @author TBrady 15 | * 16 | */ 17 | class ListDirectories extends DefaultFileTest { 18 | public static enum DataStatsItems implements StatsItemEnum { 19 | Key(StatsItem.makeStringStatsItem("Folder", 200)), 20 | Data(StatsItem.makeStatsItem(Object.class, "Name", 300).setInitVal("")); 21 | 22 | StatsItem si; 23 | DataStatsItems(StatsItem si) {this.si=si;} 24 | public StatsItem si() {return si;} 25 | } 26 | 27 | public static enum Generator implements StatsGenerator { 28 | INSTANCE; 29 | public Stats create(String key) { 30 | return new Stats(details, key) { 31 | public Object compute(File f, FileTest fileTest) { 32 | Object o = fileTest.fileTest(f); 33 | setVal(DataStatsItems.Data, o); 34 | return o; 35 | } 36 | 37 | }; 38 | } 39 | } 40 | public static StatsItemConfig details = StatsItemConfig.create(DataStatsItems.class); 41 | 42 | public ListDirectories(FTDriver dt) { 43 | super(dt); 44 | } 45 | 46 | public String toString() { 47 | return "List Dir"; 48 | } 49 | 50 | public Object fileTest(File f) { 51 | return f.getName(); 52 | } 53 | 54 | public String getKey(File f) { 55 | String path = f.getAbsolutePath(); 56 | return path; 57 | } 58 | 59 | public Stats createStats(String key) { 60 | return Generator.INSTANCE.create(key); 61 | } 62 | 63 | public StatsItemConfig getStatsDetails() { 64 | return ListDirectories.details; 65 | } 66 | 67 | public String getShortName() { 68 | return "DIR"; 69 | } 70 | 71 | 72 | public String getDescription() { 73 | return "This rule will generate a listing of the unique directory names found within a specific directory.\n" + 74 | "The purpose of this rule is to generate an tracking list when performing a similar batch process on a collection of directories."; 75 | } 76 | 77 | public boolean isTestDirectory() { 78 | return true; 79 | } 80 | public boolean isTestFiles() { 81 | return false; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/ListFiles.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.FTDriver; 6 | import gov.nara.nwts.ftapp.stats.Stats; 7 | import gov.nara.nwts.ftapp.stats.StatsGenerator; 8 | import gov.nara.nwts.ftapp.stats.StatsItem; 9 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 10 | import gov.nara.nwts.ftapp.stats.StatsItemEnum; 11 | 12 | /** 13 | * List the full path for a dirctory; this can be used as input for the FileAnalzyer batch capability. 14 | * @author TBrady 15 | * 16 | */ 17 | class ListFiles extends DefaultFileTest { 18 | public static enum DataStatsItems implements StatsItemEnum { 19 | Path(StatsItem.makeStringStatsItem("Folder", 500)); 20 | 21 | StatsItem si; 22 | DataStatsItems(StatsItem si) {this.si=si;} 23 | public StatsItem si() {return si;} 24 | } 25 | 26 | public static enum Generator implements StatsGenerator { 27 | INSTANCE; 28 | public Stats create(String key) { 29 | return new Stats(details, key) { 30 | public Object compute(File f, FileTest fileTest) { 31 | Object o = fileTest.fileTest(f); 32 | setVal(DataStatsItems.Path, o); 33 | return o; 34 | } 35 | 36 | }; 37 | } 38 | } 39 | public static StatsItemConfig details = StatsItemConfig.create(DataStatsItems.class); 40 | 41 | public ListFiles(FTDriver dt) { 42 | super(dt); 43 | } 44 | 45 | public String toString() { 46 | return "List Files"; 47 | } 48 | 49 | public Object fileTest(File f) { 50 | return f.getName(); 51 | } 52 | 53 | public String getKey(File f) { 54 | String path = f.getAbsolutePath(); 55 | return path; 56 | } 57 | 58 | public Stats createStats(String key) { 59 | return Generator.INSTANCE.create(key); 60 | } 61 | 62 | public StatsItemConfig getStatsDetails() { 63 | return ListFiles.details; 64 | } 65 | 66 | public String getShortName() { 67 | return "Files"; 68 | } 69 | 70 | 71 | public String getDescription() { 72 | return "This rule will generate a listing of the full path to every file it finds.\n" + 73 | "The purpose of this tool is to generate a file list for import into in other applications."; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/LowercaseTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | 6 | import gov.nara.nwts.ftapp.FTDriver; 7 | import gov.nara.nwts.ftapp.nameValidation.RenameablePattern; 8 | import gov.nara.nwts.ftapp.nameValidation.ValidPattern; 9 | /** 10 | * Filename validation rule to ensure that filenames are lowercase. 11 | * @author TBrady 12 | * 13 | */ 14 | class LowercaseTest extends NameValidationTest { 15 | 16 | public LowercaseTest(FTDriver dt, FileTest nextTest) { 17 | super(dt, new ValidPattern("^[^A-Z]*$", false),nextTest, "Lowercase","Lowercase"); 18 | testPatterns.add(new RenameablePattern(".*", false){ 19 | public String getMessage(File f, Matcher m) { 20 | return ""; 21 | } 22 | 23 | public File getNewFile(File f, Matcher m) { 24 | return new File(f.getParentFile(), f.getName().toLowerCase()); 25 | } 26 | 27 | }); 28 | } 29 | 30 | public String getDescription() { 31 | return "This test will check that all files are named with only lowercase characters." 32 | + getNameValidationDisclaimer(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/NameMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.NameStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 7 | 8 | import java.io.File; 9 | /** 10 | * Match files by file name (with file extension); this can be used to verify the completeness of a copy operation. 11 | * @author TBrady 12 | * 13 | */ 14 | class NameMatch extends DefaultFileTest { 15 | 16 | public NameMatch(FTDriver dt) { 17 | super(dt); 18 | } 19 | 20 | public String toString() { 21 | return "Match By Name"; 22 | } 23 | public String getKey(File f) { 24 | return f.getName(); 25 | } 26 | 27 | public String getShortName(){return "Name";} 28 | 29 | public Object fileTest(File f) { 30 | return null; 31 | } 32 | public Stats createStats(String key){ 33 | return NameStats.Generator.INSTANCE.create(key); 34 | } 35 | public StatsItemConfig getStatsDetails() { 36 | return NameStats.details; 37 | } 38 | public void initFilters() { 39 | initAllFilters(); 40 | } 41 | 42 | public String getDescription() { 43 | return "This test reports on file size by name regardless of the directory in which a file name is found."; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filetest/package.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Georgetown-University-Libraries/File-Analyzer/e9fb1a3d5d5c7f12b0d579c9e70af43c8b26011a/core/src/main/gov/nara/nwts/ftapp/filetest/package.html -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/AVFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for AV files 5 | * @author TBrady 6 | * 7 | */ 8 | public class AVFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*(\\.wav|\\.mov|\\.mp3|\\.avi|\\.dpx|\\.mxf)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "AV Files";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/CSVFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for comma-separated text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class CSVFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".csv"; 11 | } 12 | public String getName(){return "CSV";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/CounterFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for excel files 5 | * @author TBrady 6 | * 7 | */ 8 | public class CounterFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".*\\.(csv|txt)$"; 11 | } 12 | public boolean isReSuffix() { 13 | return true; 14 | } 15 | public String getName(){return "Counter";} 16 | 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/CounterFilterXls.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for excel files 5 | * @author TBrady 6 | * 7 | */ 8 | public class CounterFilterXls extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".*\\.(xls|xlsx|csv|txt)$"; 11 | } 12 | public boolean isReSuffix() { 13 | return true; 14 | } 15 | public String getName(){return "Counter";} 16 | 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/DefaultFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Default filter which accepts all files 5 | * @author TBrady 6 | * 7 | */ 8 | public class DefaultFileTestFilter implements FileTestFilter { 9 | 10 | public String getName() { 11 | return "All Files"; 12 | } 13 | public String getContains() { 14 | return ""; 15 | } 16 | public String getPrefix() { 17 | return ""; 18 | } 19 | 20 | public String getExclusion() { 21 | return ""; 22 | } 23 | 24 | public String getSuffix() { 25 | return ""; 26 | } 27 | public boolean isRePrefix() { 28 | return false; 29 | } 30 | public boolean isReContains() { 31 | return false; 32 | } 33 | public boolean isReSuffix() { 34 | return false; 35 | } 36 | public boolean isReExclusion() { 37 | return false; 38 | } 39 | 40 | public String getShortName() { 41 | return getName(); 42 | } 43 | 44 | public String getShortNameNormalized() { 45 | return getShortName().replaceAll("[\\s&]",""); 46 | } 47 | public String getShortNameFormatted() { 48 | StringBuffer buf = new StringBuffer(); 49 | buf.append(getShortNameNormalized()); 50 | buf.append(" "); 51 | return buf.substring(0,20); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/ExcelFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for excel files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ExcelFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".*\\.(xls|xlsx)$"; 11 | } 12 | public boolean isReSuffix() { 13 | return true; 14 | } 15 | public String getName(){return "Excel";} 16 | 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/FileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Contract that a File Test Filter must support 5 | * @author TBrady 6 | * 7 | */ 8 | public interface FileTestFilter { 9 | public String getName(); 10 | 11 | public String getPrefix(); 12 | public String getContains(); 13 | public String getSuffix(); 14 | public String getExclusion(); 15 | 16 | public boolean isRePrefix(); 17 | public boolean isReContains(); 18 | public boolean isReSuffix(); 19 | public boolean isReExclusion(); 20 | 21 | public String getShortName(); 22 | public String getShortNameFormatted(); 23 | public String getShortNameNormalized(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/ImageFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for images files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ImageFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(tiff?|gif|jpg|png|bmp)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "Images";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/Jp2FileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | import gov.nara.nwts.ftapp.filter.DefaultFileTestFilter; 4 | 5 | /** 6 | * Filter for TIF or JPG files 7 | * @author TBrady 8 | * 9 | */ 10 | public class Jp2FileTestFilter extends DefaultFileTestFilter { 11 | 12 | public String getSuffix() { 13 | return ".*\\.(jp2)$"; 14 | } 15 | public boolean isReSuffix() { 16 | return true; 17 | } 18 | public String getName(){return "JP2";} 19 | 20 | } 21 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/JpegFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for JPG files 5 | * @author TBrady 6 | * 7 | */ 8 | public class JpegFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".jpg"; 12 | } 13 | public String getPrefix() { 14 | return "^[^\\.].*"; 15 | } 16 | public boolean isRePrefix() { 17 | return true; 18 | } 19 | public String getName(){return "Jpg";} 20 | 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/MrcFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class MrcFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".mrc"; 11 | } 12 | public String getName(){return "MRC";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/ODSFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for Open Office Spreadsheet files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ODSFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".ods"; 11 | } 12 | public String getName(){return "Open Office Spreadsheet";} 13 | 14 | public String getShortName(){return "ODS";} 15 | } 16 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/OdsCsvFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for open office spreadsheet files OR Comma separated files 5 | * @author TBrady 6 | * 7 | */ 8 | public class OdsCsvFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(ods|csv)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "ODS/CSV";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/PdfFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | import gov.nara.nwts.ftapp.filter.DefaultFileTestFilter; 4 | 5 | /** 6 | * Filter for TIF or JPG files 7 | * @author TBrady 8 | * 9 | */ 10 | public class PdfFileTestFilter extends DefaultFileTestFilter { 11 | 12 | public String getSuffix() { 13 | return ".*\\.(pdf)$"; 14 | } 15 | public boolean isReSuffix() { 16 | return true; 17 | } 18 | public String getName(){return "PDF";} 19 | 20 | } 21 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/TarFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for XML files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TarFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".tar"; 11 | } 12 | public String getName(){return "TAR";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/TiffFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for TIF files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TiffFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(tiff?)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getPrefix() { 17 | return "^[^\\.].*"; 18 | } 19 | public boolean isRePrefix() { 20 | return true; 21 | } 22 | public String getName(){return "Tiffs";} 23 | 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/TiffJpegFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for TIF or JPG files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TiffJpegFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(tiff?|jpg)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "Tif Jpeg";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/TsvFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TsvFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".tsv"; 11 | } 12 | public String getName(){return "TSV";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/TxtFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TxtFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".txt"; 11 | } 12 | public String getName(){return "TXT";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/XmlFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for XML files 5 | * @author TBrady 6 | * 7 | */ 8 | public class XmlFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".xml"; 11 | } 12 | public String getName(){return "XML";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/ZipFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for XML files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ZipFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".zip"; 11 | } 12 | public String getName(){return "ZIP";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/filter/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | A File Test Filter is a string-based or regular-expression based file name matcher that will determine files that will be considered for analysis by a FileTest. The File Analyzer user interface will allow a user to further refine the filter criteria for more specific analysis. 4 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ftprop/FTProp.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import java.io.File; 4 | 5 | import javax.swing.JComponent; 6 | 7 | /** 8 | * Contract that a File Test Property must satisfy in both the GUI and Command Line form of the File Analyzer 9 | * @author TBrady 10 | * 11 | */ 12 | public interface FTProp { 13 | public String getName(); 14 | public String describe(); 15 | public JComponent getEditor(); 16 | public Object getDefault(); 17 | public Object validate(String s); 18 | public Object getValue(); 19 | public void setValue(Object obj); 20 | 21 | public String getShortName(); 22 | public String getShortNameNormalized(); 23 | 24 | public InitializationStatus initValidation(File refFile); 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ftprop/FTPropDate.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import java.text.SimpleDateFormat; 4 | 5 | import javax.swing.JFormattedTextField; 6 | 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | 9 | public class FTPropDate extends FTPropString { 10 | public static SimpleDateFormat df = new SimpleDateFormat("MM/dd/yyyy"); 11 | public FTPropDate(FTDriver ft, String prefix, String name, 12 | String shortname, String description, Object def) { 13 | super(ft, prefix, name, shortname, description + " (MM/dd/yyyy)", def); 14 | } 15 | 16 | public void createTextField() { 17 | tf = new JFormattedTextField(FTPropDate.df); 18 | tf.setText(this.def.toString()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ftprop/FTPropEnum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import java.awt.event.ItemEvent; 4 | import java.awt.event.ItemListener; 5 | 6 | import gov.nara.nwts.ftapp.FTDriver; 7 | 8 | import javax.swing.JComboBox; 9 | import javax.swing.JComponent; 10 | 11 | /** 12 | * File Test Property object presenting enumerated values as a choice 13 | * @author TBrady 14 | * 15 | */ 16 | 17 | public class FTPropEnum extends DefaultFTProp { 18 | JComboBox combo; 19 | 20 | public FTPropEnum(FTDriver ft, String prefix, String name, String shortname, String description, Object[]vals, Object def) { 21 | super(ft, prefix, name, shortname, description, def); 22 | init(vals); 23 | combo = new JComboBox(); 24 | initCombo(vals); 25 | combo.addItemListener(new ItemListener(){ 26 | public void itemStateChanged(ItemEvent arg0) { 27 | Object obj = combo.getSelectedItem(); 28 | if (obj == null) return; 29 | if (FTPropEnum.this.ft.hasPreferences()){ 30 | FTPropEnum.this.ft.setPreference(getPrefString(), combo.getSelectedItem().toString()); 31 | } 32 | } 33 | }); 34 | } 35 | 36 | public void initCombo(Object[] vals) { 37 | for(Object obj: vals) { 38 | combo.addItem(obj); 39 | } 40 | setValue(def); 41 | } 42 | 43 | public JComponent getEditor() { 44 | return combo; 45 | } 46 | 47 | public Object getValue() { 48 | return combo.getSelectedItem(); 49 | } 50 | 51 | public Object validate(String s) { 52 | for(int i=0; i 0) sb.append("\n"); 17 | sb.append(s); 18 | showMessage = true; 19 | } 20 | public void addFailMessage(String s) { 21 | addMessage(s); 22 | failTest = true; 23 | } 24 | public void addMessage(InitializationStatus iStat) { 25 | if (iStat.hasFailTest()) { 26 | addFailMessage(iStat.getMessage()); 27 | } else if (iStat.hasMessage()) { 28 | addMessage(iStat.getMessage()); 29 | } 30 | } 31 | public void addMessage(Exception ex) { 32 | addFailMessage(ex.getMessage()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ftprop/InvalidInputException.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | public class InvalidInputException extends Exception { 3 | private static final long serialVersionUID = 1376656719264462949L; 4 | public InvalidInputException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/ftprop/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | A File Test Property is an optional parameter that will alter the behavior of a File Test. Where possible, File Test Properties should be avoided in order to simplify the user experience. As new File Test Properties are created, care must be taken to ensure that they will behave in both the GUI mode and the Command Line mode. 4 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/DirSelect.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import javax.swing.JFileChooser; 4 | import javax.swing.JFrame; 5 | import javax.swing.JTextField; 6 | 7 | /** 8 | * Displays a directory selection dialog; results will be saved to a specified text field. 9 | * @author TBrady 10 | * 11 | */ 12 | class DirSelect extends FileSelect { 13 | private static final long serialVersionUID = 1L; 14 | 15 | public void configureChooser() { 16 | jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); 17 | } 18 | DirSelect(JFrame parent, JTextField result, String title) { 19 | super(parent, result, title); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/DirSelectChooser.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.event.ActionEvent; 4 | import java.util.prefs.Preferences; 5 | 6 | import javax.swing.JFrame; 7 | 8 | /** 9 | * Widget that will display a chosen directory and will open a separate window allowing a directory to be selected. 10 | * @author TBrady 11 | * 12 | */ 13 | class DirSelectChooser extends FileSelectChooser { 14 | private static final long serialVersionUID = 1L; 15 | 16 | DirSelectChooser(JFrame parent, String title, String def) { 17 | this(parent, title, null, null, def); 18 | } 19 | DirSelectChooser(JFrame parent, String title, Preferences p, String key, String def) { 20 | super(parent, title, p, key, def); 21 | } 22 | 23 | public void actionPerformed(ActionEvent arg0) { 24 | new DirSelect(parent, tf, title); 25 | } 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/FileCatalog.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import javax.swing.JFileChooser; 5 | import javax.swing.JFrame; 6 | import javax.swing.JLabel; 7 | import javax.swing.JPanel; 8 | 9 | /** 10 | * UI component allowing the selction of the root directory for a File Test. 11 | * Note: this pre-dates the creation of the {@link FileSelectChooser} and {@link DirSelectChooser} classes but accomplishes a similar function 12 | * @author TBrady 13 | * 14 | */ 15 | class FileCatalog extends JFrame { 16 | private static final long serialVersionUID = 1L; 17 | 18 | DirectoryTable dt; 19 | FileCatalog(DirectoryTable dt) { 20 | super("File Analyzer: Set Input Directory"); 21 | this.dt = dt; 22 | setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 23 | JPanel p = new JPanel(new BorderLayout()); 24 | add(p); 25 | JFileChooser jfc = new JFileChooser() { 26 | private static final long serialVersionUID = 1L; 27 | public void cancelSelection() { 28 | FileCatalog.this.dispose(); 29 | } 30 | public void approveSelection() { 31 | FileCatalog.this.setVisible(false); 32 | FileCatalog.this.dt.criteriaPanel.rootLabel.setText(getSelectedFile().getAbsolutePath()); 33 | FileCatalog.this.dt.setSelectedFile(); 34 | FileCatalog.this.dispose(); 35 | } 36 | }; 37 | if (dt.root != null){ 38 | jfc.setCurrentDirectory(dt.root); 39 | } 40 | jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); 41 | p.add(jfc, BorderLayout.CENTER); 42 | p.add(new JLabel("Please select the directory that you wish to catalog"),BorderLayout.NORTH); 43 | pack(); 44 | setVisible(true); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/FileSelect.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import java.io.File; 5 | 6 | import javax.swing.JDialog; 7 | import javax.swing.JFileChooser; 8 | import javax.swing.JFrame; 9 | import javax.swing.JLabel; 10 | import javax.swing.JPanel; 11 | import javax.swing.JTextField; 12 | 13 | /** 14 | * Displays a File selection dialog; results will be saved to a specified text field. 15 | * @author TBrady 16 | * 17 | */ 18 | class FileSelect extends JDialog { 19 | private static final long serialVersionUID = 1L; 20 | JTextField result; 21 | JFileChooser jfc; 22 | 23 | 24 | public void configureChooser() { 25 | jfc.setFileSelectionMode(JFileChooser.FILES_ONLY); 26 | } 27 | 28 | FileSelect(JFrame parent, JTextField result, String title) { 29 | super(parent, title); 30 | this.setModal(true); 31 | this.result = result; 32 | setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE); 33 | JPanel p = new JPanel(new BorderLayout()); 34 | add(p); 35 | jfc = new JFileChooser() { 36 | private static final long serialVersionUID = 1L; 37 | 38 | public void cancelSelection() { 39 | FileSelect.this.dispose(); 40 | } 41 | 42 | public void approveSelection() { 43 | FileSelect.this.setVisible(false); 44 | FileSelect.this.result.setText(this.getSelectedFile() 45 | .getAbsolutePath()); 46 | FileSelect.this.dispose(); 47 | } 48 | }; 49 | String root = FileSelect.this.result.getText(); 50 | if (root != null) { 51 | jfc.setCurrentDirectory(new File(root)); 52 | } 53 | p.add(jfc, BorderLayout.CENTER); 54 | p.add(new JLabel(title), BorderLayout.NORTH); 55 | configureChooser(); 56 | pack(); 57 | setVisible(true); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/FileSelectChooser.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.FlowLayout; 4 | import java.awt.event.ActionEvent; 5 | import java.awt.event.ActionListener; 6 | import java.util.prefs.Preferences; 7 | 8 | import javax.swing.JButton; 9 | import javax.swing.JFrame; 10 | import javax.swing.JPanel; 11 | import javax.swing.JTextField; 12 | import javax.swing.event.DocumentEvent; 13 | import javax.swing.event.DocumentListener; 14 | 15 | /** 16 | * Widget that will display a chosen file and will open a separate window allowing a file to be selected. 17 | * @author TBrady 18 | * 19 | */ 20 | public class FileSelectChooser extends JPanel implements ActionListener, DocumentListener { 21 | private static final long serialVersionUID = 1L; 22 | private Preferences p; 23 | protected JFrame parent; 24 | public JTextField tf; 25 | private JButton button; 26 | private JButton buttonX; 27 | protected String title; 28 | private String key; 29 | 30 | public FileSelectChooser(JFrame parent, String title, String def) { 31 | this(parent, title, null, null, def); 32 | } 33 | public FileSelectChooser(JFrame parent, String title, Preferences p, String key, String def) { 34 | this.p = p; 35 | this.parent = parent; 36 | this.title = title; 37 | this.key = key; 38 | setLayout(new FlowLayout(FlowLayout.LEFT)); 39 | String val = (p == null) ? def : p.get(key, def); 40 | tf = new JTextField(val, 40); 41 | tf.setEditable(false); 42 | add(tf); 43 | button = new JButton("..."); 44 | add(button); 45 | button.addActionListener(this); 46 | buttonX = new JButton("X"); 47 | add(buttonX); 48 | buttonX.addActionListener(new ActionListener(){ 49 | public void actionPerformed(ActionEvent e) { 50 | tf.setText(""); 51 | }}); 52 | tf.getDocument().addDocumentListener(this); 53 | } 54 | 55 | public void change() { 56 | if (p != null) { 57 | p.put(key, tf.getText()); 58 | } 59 | } 60 | 61 | public void actionPerformed(ActionEvent arg0) { 62 | new FileSelect(parent, tf, title); 63 | } 64 | public void changedUpdate(DocumentEvent arg0) { 65 | change(); 66 | } 67 | public void insertUpdate(DocumentEvent arg0) { 68 | change(); 69 | } 70 | public void removeUpdate(DocumentEvent arg0) { 71 | change(); 72 | } 73 | 74 | 75 | } 76 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/FilterPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.filter.FileTestFilter; 4 | 5 | import javax.swing.JCheckBox; 6 | import javax.swing.JPanel; 7 | import javax.swing.JTextField; 8 | /** 9 | * Panel presenting the various filter values that can be overridden from the user interface 10 | * @author TBrady 11 | * 12 | */ 13 | class FilterPanel extends MyPanel { 14 | private static final long serialVersionUID = 1L; 15 | JTextField prefix; 16 | JTextField contains; 17 | JTextField suffix; 18 | JTextField exclusion; 19 | 20 | JCheckBox rePrefix; 21 | JCheckBox reContains; 22 | JCheckBox reSuffix; 23 | JCheckBox reExclusion; 24 | 25 | FilterPanel(DirectoryTable dt, FileTestFilter filter) { 26 | JPanel p = addPanel("Filename Prefix Filter"); 27 | prefix = new JTextField(filter.getPrefix(), 50); 28 | p.add(prefix); 29 | rePrefix = new JCheckBox(); 30 | rePrefix.setText("Regex"); 31 | rePrefix.setSelected(filter.isRePrefix()); 32 | p.add(rePrefix); 33 | 34 | p = addPanel("Filename Contains Filter"); 35 | contains = new JTextField(filter.getContains(), 50); 36 | p.add(contains); 37 | reContains = new JCheckBox(); 38 | reContains.setText("Regex"); 39 | reContains.setSelected(filter.isReContains()); 40 | p.add(reContains); 41 | 42 | p = addPanel("Filename Suffix Filter"); 43 | suffix = new JTextField(filter.getSuffix(), 50); 44 | p.add(suffix); 45 | reSuffix = new JCheckBox(); 46 | reSuffix.setText("Regex"); 47 | reSuffix.setSelected(filter.isReSuffix()); 48 | p.add(reSuffix); 49 | 50 | p = addPanel("Filename Exclusion Filter"); 51 | exclusion = new JTextField(filter.getExclusion(), 50); 52 | p.add(exclusion); 53 | reExclusion = new JCheckBox(); 54 | reExclusion.setText("Regex"); 55 | reExclusion.setSelected(filter.isReExclusion()); 56 | p.add(reExclusion); 57 | 58 | dt.criteriaPanel.filterTabs.add(filter.getName(), this); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/GuiFileTraversalSW.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.util.Iterator; 4 | import java.util.List; 5 | 6 | import javax.swing.JOptionPane; 7 | import javax.swing.SwingWorker; 8 | import javax.swing.table.DefaultTableModel; 9 | 10 | /** 11 | * Worker thread that will perform an actual file test and send periodic status updates to the GUI thread. 12 | * Note: this class is the primary reason for running Java 1.6 or higher. It handles GUI actions elegantly. 13 | * @author TBrady 14 | * 15 | */ 16 | class GuiFileTraversalSW extends SwingWorker { 17 | GuiFileTraversal traversal; 18 | DirectoryTable dt; 19 | 20 | public GuiFileTraversalSW(DirectoryTable dt, DefaultTableModel tm) { 21 | this.dt = dt; 22 | traversal = new GuiFileTraversal(this, tm); 23 | } 24 | 25 | public void publish(String s) { 26 | super.publish(s); 27 | } 28 | 29 | protected String doInBackground() throws Exception { 30 | try { 31 | publish("starting"); 32 | traversal.traverseFile(); 33 | publish("Complete"); 34 | return "complete.."; 35 | } catch (Throwable e) { 36 | e.printStackTrace(); 37 | this.cancel(true); 38 | return "err..."; 39 | } 40 | } 41 | 42 | protected void process(List messages) { 43 | if (traversal.myprogress.processing) { 44 | dt.progressPanel.progress.setValue(traversal.myprogress.dircount+1); 45 | } 46 | for(Iteratori=messages.iterator(); i.hasNext(); ){ 47 | String s = i.next(); 48 | if (s.equals("")) continue; 49 | dt.report(s); 50 | } 51 | 52 | } 53 | 54 | 55 | public void done() { 56 | 57 | if (traversal.iStat != null) { 58 | if (traversal.iStat.hasMessage()) { 59 | if (traversal.iStat.hasFailTest()) { 60 | JOptionPane.showMessageDialog(dt.frame, traversal.iStat.getMessage(), "Property Error - Cannot Run Task", JOptionPane.ERROR_MESSAGE); 61 | } else { 62 | JOptionPane.showMessageDialog(dt.frame, traversal.iStat.getMessage(), "Property Warning", JOptionPane.WARNING_MESSAGE); 63 | } 64 | 65 | } 66 | } 67 | if (isCancelled()) { 68 | publish("Processing cancelled"); 69 | } 70 | dt.report((traversal.myprogress.dircount+1) + " of " + traversal.myprogress.dirfound + " processed"); 71 | } 72 | 73 | 74 | } 75 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/MyBorderPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | 5 | import javax.swing.JPanel; 6 | 7 | /** 8 | * GUI helper class to add a panel with a border layout 9 | * @author TBrady 10 | * 11 | */ 12 | class MyBorderPanel extends MyPanel { 13 | private static final long serialVersionUID = 1L; 14 | MyBorderPanel() { 15 | super(new BorderLayout()); 16 | } 17 | JPanel addPanel(JPanel p, String loc) { 18 | add(p, (loc != null) ? loc : BorderLayout.CENTER); 19 | return p; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/MyPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import java.awt.FlowLayout; 5 | import java.awt.LayoutManager; 6 | 7 | import javax.swing.BorderFactory; 8 | import javax.swing.Box; 9 | import javax.swing.BoxLayout; 10 | import javax.swing.JPanel; 11 | 12 | /** 13 | * GUI helper class allowing new components to be quickly added to the File Analyzer in a consistent fashion 14 | * @author TBrady 15 | * 16 | */ 17 | class MyPanel extends JPanel { 18 | private static final long serialVersionUID = 1L; 19 | Box main; 20 | MyPanel(LayoutManager layout) { 21 | super(layout); 22 | main = new Box(BoxLayout.Y_AXIS); 23 | add(main); 24 | } 25 | 26 | MyPanel() { 27 | this(new FlowLayout()); 28 | } 29 | 30 | 31 | JPanel addPanel() { 32 | return addPanel((String)null); 33 | } 34 | 35 | JPanel addPanel(String title) { 36 | return addPanel(title, null); 37 | } 38 | JPanel addPanel(String title, String loc) { 39 | JPanel p = new JPanel(new FlowLayout(FlowLayout.LEFT)); 40 | if (title != null) { 41 | p.setBorder(BorderFactory.createTitledBorder(title)); 42 | } 43 | return addPanel(p, loc); 44 | } 45 | 46 | JPanel addBorderPanel(String title) { 47 | JPanel p = new JPanel(new BorderLayout()); 48 | if (title != null) { 49 | p.setBorder(BorderFactory.createTitledBorder(title)); 50 | } 51 | return addPanel(p, (String)null); 52 | } 53 | 54 | JPanel addPanel(JPanel p, String loc) { 55 | if (loc == null) { 56 | main.add(p); 57 | } else { 58 | main.add(p, loc); 59 | } 60 | return p; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/MyProgress.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | /** 4 | * Helper class that reports on File Analyzer progress (on the Progress Tab) 5 | * @author TBrady 6 | * 7 | */ 8 | class MyProgress { 9 | int dircount; 10 | int lastdircount; 11 | int rptgap = 1; 12 | int rptgapmult = 5; 13 | int dirfound = 0; 14 | boolean processing; 15 | GuiFileTraversal gft; 16 | DirectoryTable dt; 17 | 18 | public MyProgress(GuiFileTraversal gft) { 19 | this.gft = gft; 20 | dt = gft.gftSW.dt; 21 | } 22 | 23 | public void resetDirCount() { 24 | dircount = 0; 25 | lastdircount = 0; 26 | rptgap = 1; 27 | } 28 | 29 | public void increment() { 30 | dircount++; 31 | testDirCount(false,processing ? " directories processed" : " directories found"); 32 | } 33 | 34 | public void testDirCount(boolean b, String note) { 35 | if (processing) { 36 | gft.gftSW.publish(""); 37 | } else if (b || (dircount >= lastdircount + rptgap)) { 38 | gft.gftSW.publish(dircount + " dirs "+note); 39 | lastdircount = dircount; 40 | if (dircount > rptgap*rptgapmult*2) { 41 | rptgap *= rptgapmult; 42 | gft.gftSW.publish("... Reporting every "+ rptgap); 43 | } 44 | } 45 | } 46 | 47 | public void complete(boolean processed) { 48 | testDirCount(true, processed ? " total directories processed" : " total directories found"); 49 | if (!processed){ 50 | dt.progressPanel.progress.setMaximum(dircount); 51 | dirfound = dircount; 52 | resetDirCount(); 53 | processing = !processed; 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/gui/MyTableModel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.util.Date; 6 | 7 | import javax.swing.JLabel; 8 | import javax.swing.JTable; 9 | import javax.swing.table.DefaultTableCellRenderer; 10 | import javax.swing.table.DefaultTableModel; 11 | import javax.swing.table.JTableHeader; 12 | import javax.swing.table.TableCellRenderer; 13 | import javax.swing.table.TableColumn; 14 | import javax.swing.table.TableColumnModel; 15 | 16 | /** 17 | * Table model for the Details Tab 18 | * @author TBrady 19 | * 20 | */ 21 | public class MyTableModel extends DefaultTableModel { 22 | private static final long serialVersionUID = 1L; 23 | 24 | public MyTableModel() { 25 | } 26 | 27 | public int getColumnCount() { 28 | return 6; 29 | } 30 | 31 | public Class getColumnClass(int col) { 32 | if (col == 4) { 33 | return Date.class; 34 | } else if (col == 3) { 35 | return Long.class; 36 | } else if (col == 5) { 37 | return Object.class; 38 | } else { 39 | return String.class; 40 | } 41 | } 42 | 43 | public void setColumns(JTable jt) { 44 | JTableHeader jth = jt.getTableHeader(); 45 | jth.setReorderingAllowed(true); 46 | TableColumnModel tcm = jt.getColumnModel(); 47 | TableColumn tc; 48 | Object[][] details = { 49 | {"Directory",300}, 50 | {"Filename",200}, 51 | {"Type",50}, 52 | {"Size",100}, 53 | {"Mod Date",100}, 54 | {"Other",500} 55 | }; 56 | for(int i=0; i 2 | 3 |

This package contains the Graphical User Interface elements of the File Analyzer.

4 |

Note: In order to provide a responsive user experience, long-running actions to be performed within a Java GUI should occur in a worker thread that periodically updates its status to the GUI application thread. Several functions that are straightforward in the command line mode of the File Analyzer, have been decomposed into worker thread and GUI thread actions.

5 | 6 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/importer/DefaultImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | import gov.nara.nwts.ftapp.ftprop.FTProp; 6 | import gov.nara.nwts.ftapp.ftprop.InitializationStatus; 7 | 8 | import java.io.File; 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | /** 14 | * Abstract base class for importer behaviors. 15 | * @author TBrady 16 | * 17 | */ 18 | public abstract class DefaultImporter implements Importer { 19 | protected FTDriver dt; 20 | protected ArrayListftprops; 21 | public DefaultImporter(FTDriver dt) { 22 | this.dt = dt; 23 | ftprops = new ArrayList(); 24 | } 25 | 26 | public abstract String toString(); 27 | public abstract ActionResult importFile(File selectedFile) throws IOException; 28 | public boolean allowForceKey() { 29 | return false; 30 | } 31 | public String getShortNameNormalized() { 32 | return getShortName().replaceAll("[\\s&]",""); 33 | } 34 | public List getPropertyList() { 35 | return ftprops; 36 | } 37 | 38 | public Object getProperty(String name) { 39 | return getProperty(name, null); 40 | } 41 | public Object getProperty(String name, Object def) { 42 | for(FTProp ftprop: ftprops) { 43 | if (ftprop.getName().equals(name)) { 44 | return ftprop.getValue(); 45 | } 46 | } 47 | return def; 48 | } 49 | public void setProperty(String name, String s) { 50 | for(FTProp ftprop: ftprops) { 51 | if (ftprop.getName().equals(name)) { 52 | ftprop.setValue(ftprop.validate(s)); 53 | return; 54 | } 55 | } 56 | } 57 | public InitializationStatus initValidate(File refFile) { 58 | InitializationStatus iStat = new InitializationStatus(); 59 | for(FTProp prop: ftprops) { 60 | iStat.addMessage(prop.initValidation(refFile)); 61 | } 62 | return iStat; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/importer/Importer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.ftprop.FTProp; 5 | import gov.nara.nwts.ftapp.ftprop.InitializationStatus; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.util.List; 10 | 11 | /** 12 | * Contract that Importers will fullfill. 13 | * @author TBrady 14 | * 15 | */ 16 | public interface Importer { 17 | public InitializationStatus initValidate(File refFile); 18 | 19 | public ActionResult importFile(File selectedFile) throws IOException; 20 | public String getDescription(); 21 | public boolean allowForceKey(); 22 | public String getShortName(); 23 | public String getShortNameNormalized(); 24 | 25 | public List getPropertyList(); 26 | public Object getProperty(String name); 27 | public void setProperty(String name, String str); 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/importer/ImporterRegistry.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filetest.CounterValidation; 5 | 6 | import java.util.Vector; 7 | 8 | /** 9 | * Activates the Importers that will be presented on the Import tab. 10 | * @author TBrady 11 | * 12 | */ 13 | public class ImporterRegistry extends Vector { 14 | 15 | private static final long serialVersionUID = 1L; 16 | 17 | public ImporterRegistry(FTDriver dt) { 18 | add(new DelimitedFileImporter(dt)); 19 | add(new Parser(dt)); 20 | add(new MultiParser(dt)); 21 | add(new CountKey(dt)); 22 | add(new CounterValidation(dt)); 23 | } 24 | 25 | public void removeImporter(Class c) { 26 | for(Importer ft: this) { 27 | if (c.isInstance(ft)) { 28 | this.remove(ft); 29 | break; 30 | } 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/importer/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

This package contains custom file import rules that can be invoked from the File Analyzer.

4 |

The primary reason for providing import functionality is to support the match/merge capabilities of the File Analyzer GUI.

5 |

As new import rules are created, the File Analyzer could also serve as an ingest tool.

6 | 7 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/CustomPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Abstract class handling an individual name validation check that requires custom coding beyond a regular expression match. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class CustomPattern extends NameValidationPattern { 13 | 14 | public CustomPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public CustomPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.NEXT); 19 | } 20 | public CustomPattern(Pattern pattern, boolean checkPath){ 21 | super(pattern, checkPath, RenameStatus.NEXT); 22 | } 23 | 24 | public File getNewFile(File f, Matcher m) { 25 | return null; 26 | } 27 | public String getMessage(File f, Matcher m) { 28 | return ""; 29 | } 30 | public abstract RenameDetails report(File f, Matcher m); 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/InvalidManualPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#INVALID_MANUAL} if true indicating that a file cannot be programmatically renamed. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class InvalidManualPattern extends NameValidationPattern { 13 | 14 | public InvalidManualPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public InvalidManualPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.INVALID_MANUAL); 19 | } 20 | 21 | public File getNewFile(File f, Matcher m) { 22 | return null; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/NameValidationPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Abstract class containing a regular expression pattern that will be used to perform file name validation 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class NameValidationPattern { 13 | Pattern pattern; 14 | RenameStatus status; 15 | boolean checkPath; 16 | 17 | public NameValidationPattern(Pattern pattern, boolean checkPath, RenameStatus status){ 18 | this.pattern = pattern; 19 | this.status = status; 20 | this.checkPath = checkPath; 21 | } 22 | 23 | public RenameDetails checkFile(File f) { 24 | String s = checkPath ? f.getAbsolutePath() : f.getName(); 25 | Matcher m = pattern.matcher(s); 26 | if (m.matches()) { 27 | return report(f, m); 28 | } 29 | return new RenameDetails(RenameStatus.NEXT, null, ""); 30 | } 31 | 32 | public RenameDetails report(File f, Matcher m) { 33 | return new RenameDetails(status, getNewFile(f, m), getMessage(f, m)); 34 | } 35 | 36 | public abstract File getNewFile(File f, Matcher m); 37 | 38 | public abstract String getMessage(File f, Matcher m); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/RenameDetails.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | 5 | public class RenameDetails { 6 | public RenameStatus status; 7 | File newFile; 8 | public String note; 9 | public RenameDetails(RenameStatus status, File newFile){ 10 | this(status,newFile,null); 11 | } 12 | public RenameDetails(RenameStatus status){ 13 | this(status,null,null); 14 | } 15 | public RenameDetails(RenameStatus status, File newFile, String note){ 16 | this.status = status; 17 | this.newFile = newFile; 18 | this.note = note; 19 | } 20 | public RenamePassFail getPassFail() {return status.getPassFail();} 21 | public String getMessage() { 22 | StringBuffer buf = new StringBuffer(); 23 | if (note!=null) { 24 | buf.append(note); 25 | buf.append(" "); 26 | } 27 | buf.append(status.getMessage()); 28 | return buf.toString(); 29 | } 30 | public String getRenameStatus() {return status.toString();} 31 | public File getFile() {return newFile;} 32 | public String getDetailNote(File root){ 33 | if (newFile == null) { 34 | return (note==null)?"":note; 35 | } 36 | String s = newFile.getAbsolutePath(); 37 | String r = root.getAbsolutePath(); 38 | String m = s; 39 | if (s.startsWith(r)) { 40 | m = s.substring(r.length()); 41 | } 42 | return m; 43 | } 44 | public String toString() { 45 | return status.toString() +": " + ((note==null)?"":note); 46 | } 47 | 48 | } 49 | 50 | 51 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/RenamePassFail.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | /** 4 | * Enumeration indicating a Pass or Failure of a test. This enumeration is useful beyond file name validation and could be suitable to move to a higher level pacakage. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum RenamePassFail {PASS,FAIL}; 9 | 10 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/RenameStatus.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | /** 4 | * Enumeration indicating all of the possible results of a file name/directory name validation test. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum RenameStatus { 9 | SKIP("Skip", RenamePassFail.PASS), 10 | NEXT("Test inconclusive", RenamePassFail.PASS), 11 | VALID("Original filename is valid, rename not required", RenamePassFail.PASS), 12 | INVALID_MANUAL("Original filename must be manually renamed",RenamePassFail.FAIL), 13 | PARSE_ERR("Original filename could not be parsed",RenamePassFail.FAIL), 14 | NEW_NAME_INVALID("File not renamed, new name is invalid",RenamePassFail.FAIL), 15 | RENAMABLE("New filename is valid (rename not performed)", RenamePassFail.PASS), 16 | RENAMED("Renamed",RenamePassFail.PASS), 17 | RENAME_FILE_EXISTS("Rename failed, file already exists", RenamePassFail.FAIL), 18 | RENAME_FAILURE("Rename failed", RenamePassFail.FAIL), 19 | DIRECTORY("Directory: Not Tested",RenamePassFail.PASS), 20 | DIRECTORY_VALID("Valid Directory",RenamePassFail.PASS), 21 | DIRECTORY_EMPTY("Empty Directory",RenamePassFail.FAIL), 22 | DIRECTORY_CHILD_INVALID("Directory contains Invalid File",RenamePassFail.FAIL), 23 | DIRECTORY_NAME_INVALID("Directory name is Invalid",RenamePassFail.FAIL), 24 | DIRECTORY_SEQUENCE_ERROR("Sequence Error in Directory",RenamePassFail.FAIL), 25 | DIRECTORY_INCOMPLETE("Incomplete Directory",RenamePassFail.FAIL); 26 | String message; 27 | RenamePassFail passfail; 28 | RenameStatus(String message, RenamePassFail passfail) { 29 | this.message = message; 30 | this.passfail = passfail; 31 | } 32 | public RenamePassFail getPassFail() {return passfail;} 33 | public String getMessage() {return message;} 34 | }; 35 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/RenameablePattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#RENAMABLE} if true indicating that the file could be automatically renamed. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class RenameablePattern extends NameValidationPattern { 13 | 14 | public RenameablePattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public RenameablePattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.RENAMABLE); 19 | } 20 | public RenameDetails report(File f, Matcher m) { 21 | File nf = getNewFile(f, m); 22 | if (nf == null) { 23 | return new RenameDetails(RenameStatus.NEW_NAME_INVALID, nf, getMessage(f, m)); 24 | } 25 | return new RenameDetails(status, nf, getMessage(f, m)); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/ValidPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#VALID} if true. 9 | * @author TBrady 10 | * 11 | */ 12 | public class ValidPattern extends NameValidationPattern { 13 | 14 | public ValidPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public ValidPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.VALID); 19 | } 20 | 21 | public File getNewFile(File f, Matcher m) { 22 | return null; 23 | } 24 | public String getMessage(File f, Matcher m) { 25 | return ""; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/nameValidation/package.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Georgetown-University-Libraries/File-Analyzer/e9fb1a3d5d5c7f12b0d579c9e70af43c8b26011a/core/src/main/gov/nara/nwts/ftapp/nameValidation/package.html -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

This package contains the main entry points for the File Analyzer as well as the workflow components that are common to both the GUI and the Command line versions of the File Analyzer. 4 |

The contents of this package are intended to be of generic use to other institutions that might use the File Analyzer. The contents of this package should not require any libraries other than the standard Java SE 1.7 or higher.

5 | 6 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/CRUDStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 6 | import gov.nara.nwts.ftapp.crud.CRUD; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Stats object that reports on database ingest actions. 12 | * @author TBrady 13 | * 14 | */ 15 | class CRUDStats extends Stats { 16 | 17 | public static enum CRUDStatsItems implements StatsItemEnum { 18 | Path(StatsItem.makeStringStatsItem("Path",450)), 19 | PassFail(StatsItem.makeEnumStatsItem(RenamePassFail.class, "Pass/Fail").setWidth(50)), 20 | Status(StatsItem.makeEnumStatsItem(CRUD.class, "Status").setWidth(150)); 21 | 22 | StatsItem si; 23 | CRUDStatsItems(StatsItem si) {this.si=si;} 24 | public StatsItem si() {return si;} 25 | } 26 | 27 | public static enum Generator implements StatsGenerator { 28 | INSTANCE; 29 | public CRUDStats create(String key) {return new CRUDStats(key);} 30 | } 31 | public static StatsItemConfig details = StatsItemConfig.create(CRUDStatsItems.class); 32 | 33 | public CRUDStats(String key) { 34 | super(CRUDStats.details, key); 35 | } 36 | 37 | public Object compute(File f, FileTest fileTest) { 38 | Object ret = fileTest.fileTest(f); 39 | return ret; 40 | } 41 | 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/ChecksumStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.YN; 4 | import gov.nara.nwts.ftapp.filetest.FileTest; 5 | import gov.nara.nwts.ftapp.filetest.NameChecksum; 6 | 7 | import java.io.File; 8 | 9 | /** 10 | * Identify items by checksum value, identify duplicate items. 11 | * @author TBrady 12 | * 13 | */ 14 | public class ChecksumStats extends Stats { 15 | public static enum DUP {Unique, FirstFound, Duplicate;} 16 | public static enum ChecksumStatsItems implements StatsItemEnum { 17 | Key(StatsItem.makeStringStatsItem("Key", 400)), 18 | Data(StatsItem.makeStatsItem(Object.class, "Data", 300).setInitVal("")), 19 | IsDuplicate(StatsItem.makeEnumStatsItem(YN.class, "Is Duplicate").setInitVal(YN.N)), 20 | DuplicateStat(StatsItem.makeEnumStatsItem(DUP.class, "Duplicate Stat").setInitVal(DUP.Unique)), 21 | MatchCount(StatsItem.makeIntStatsItem("Num of Matches").setInitVal(1)); 22 | 23 | StatsItem si; 24 | ChecksumStatsItems(StatsItem si) {this.si=si;} 25 | public StatsItem si() {return si;} 26 | } 27 | public static enum Generator implements StatsGenerator { 28 | INSTANCE; 29 | public ChecksumStats create(String key) {return new ChecksumStats(key);} 30 | } 31 | public static StatsItemConfig details = StatsItemConfig.create(ChecksumStatsItems.class); 32 | private ChecksumStats(String key) { 33 | super(ChecksumStats.details, key); 34 | } 35 | 36 | public Object compute(File f, FileTest fileTest) { 37 | Object o = fileTest.fileTest(f); 38 | setVal(ChecksumStatsItems.Data, o); 39 | 40 | if (fileTest instanceof NameChecksum) { 41 | if (o != null) { 42 | ((NameChecksum)fileTest).setChecksumKey(o.toString(), this); 43 | } 44 | } 45 | return o; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/CountStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Status objects that counts items by key. 9 | * @author TBrady 10 | * 11 | */ 12 | public class CountStats extends Stats { 13 | 14 | public static enum CountStatsItems implements StatsItemEnum { 15 | Type(StatsItem.makeStringStatsItem("Type")), 16 | Count(StatsItem.makeLongStatsItem("Count")); 17 | 18 | StatsItem si; 19 | CountStatsItems(StatsItem si) {this.si=si;} 20 | public StatsItem si() {return si;} 21 | } 22 | public static enum Generator implements StatsGenerator { 23 | INSTANCE; 24 | public CountStats create(String key) {return new CountStats(key);} 25 | } 26 | 27 | public static StatsItemConfig details = StatsItemConfig.create(CountStatsItems.class); 28 | 29 | public CountStats(String key) { 30 | super(CountStats.details, key); 31 | } 32 | 33 | public Object compute(File f, FileTest fileTest) { 34 | sumVal(CountStatsItems.Count, 1); 35 | return fileTest.fileTest(f); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/DirStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Statistics showing accumulated file counts within a directory structure. 9 | * @author TBrady 10 | * 11 | */ 12 | public class DirStats extends Stats { 13 | public static enum DirStatsItems implements StatsItemEnum { 14 | Dir(StatsItem.makeStringStatsItem("Dir", 100)), 15 | Count(StatsItem.makeLongStatsItem("Count")), 16 | CumulativeCount(StatsItem.makeLongStatsItem("Cumulative Count")); 17 | 18 | StatsItem si; 19 | DirStatsItems(StatsItem si) {this.si=si;} 20 | public StatsItem si() {return si;} 21 | } 22 | 23 | public static enum Generator implements StatsGenerator { 24 | INSTANCE; 25 | public DirStats create(String key) {return new DirStats(key);} 26 | } 27 | public static StatsItemConfig details = StatsItemConfig.create(DirStatsItems.class); 28 | 29 | private DirStats(String key) { 30 | super(DirStats.details, key); 31 | } 32 | 33 | public Object compute(File f, FileTest fileTest) { 34 | File root = fileTest.getRoot(); 35 | for(File ftest = f.getParentFile(); ftest!=null; ftest = ftest.getParentFile()){ 36 | DirStats stats = (DirStats)fileTest.getStats(fileTest.getKey(f,ftest)); 37 | stats.accumulate(f, fileTest, ftest); 38 | if (ftest.equals(root)){ 39 | break; 40 | } 41 | } 42 | return fileTest.fileTest(f); 43 | } 44 | 45 | public void accumulate(File f, FileTest fileTest, File parentdir) { 46 | if (f.getParentFile().equals(parentdir)){ 47 | sumVal(DirStatsItems.Count, 1); 48 | } 49 | sumVal(DirStatsItems.CumulativeCount, 1); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/FileCountStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Stats object showing file counts and file sizes. 9 | * @author TBrady 10 | * 11 | */ 12 | public class FileCountStats extends Stats { 13 | private static enum FileCountStatsItems implements StatsItemEnum { 14 | Type(StatsItem.makeStringStatsItem("Type")), 15 | Count(StatsItem.makeLongStatsItem("Count")), 16 | Size(StatsItem.makeLongStatsItem("Size")); 17 | 18 | StatsItem si; 19 | FileCountStatsItems(StatsItem si) {this.si=si;} 20 | public StatsItem si() {return si;} 21 | } 22 | 23 | public static enum Generator implements StatsGenerator { 24 | INSTANCE; 25 | public FileCountStats create(String key) {return new FileCountStats(key);} 26 | } 27 | public static StatsItemConfig details = StatsItemConfig.create(FileCountStatsItems.class); 28 | 29 | private FileCountStats(String key) { 30 | super(FileCountStats.details, key); 31 | } 32 | 33 | public Object compute(File f, FileTest fileTest) { 34 | Object ret = super.compute(f, fileTest); 35 | sumVal(FileCountStatsItems.Count, 1); 36 | sumVal(FileCountStatsItems.Size, f.length()); 37 | return ret; 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/NameStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Stats object showing file name and file size. 9 | * @author TBrady 10 | * 11 | */ 12 | public class NameStats extends Stats { 13 | private static enum NameStatsItems implements StatsItemEnum { 14 | Name(StatsItem.makeStringStatsItem("Name")), 15 | Count(StatsItem.makeLongStatsItem("Count")), 16 | Size(StatsItem.makeLongStatsItem("Size")); 17 | 18 | StatsItem si; 19 | NameStatsItems(StatsItem si) {this.si=si;} 20 | public StatsItem si() {return si;} 21 | } 22 | public static enum Generator implements StatsGenerator { 23 | INSTANCE; 24 | public NameStats create(String key) {return new NameStats(key);} 25 | } 26 | public static StatsItemConfig details = StatsItemConfig.create(NameStatsItems.class); 27 | 28 | private NameStats(String key) { 29 | super(NameStats.details, key); 30 | } 31 | 32 | public Object compute(File f, FileTest fileTest) { 33 | sumVal(NameStatsItems.Count, 1); 34 | sumVal(NameStatsItems.Size, f.length()); 35 | return fileTest.fileTest(f); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/NameValidationStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | import gov.nara.nwts.ftapp.nameValidation.RenameDetails; 5 | import gov.nara.nwts.ftapp.nameValidation.RenameStatus; 6 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Stats object displaying the results of a filename test. 12 | * @linkplain gov.nara.nwts.ftapp.filetest.NameValidationTest} contains the base logic that makes use of this Stats object. 13 | * @author TBrady 14 | * 15 | */ 16 | public class NameValidationStats extends Stats { 17 | private static enum NameValidationStatsItems implements StatsItemEnum { 18 | Path(StatsItem.makeStringStatsItem("Path", 450)), 19 | PassFail(StatsItem.makeEnumStatsItem(RenamePassFail.class, "Pass/Fail").setWidth(50)), 20 | Status(StatsItem.makeEnumStatsItem(RenameStatus.class,"Status").setWidth(150)), 21 | Message(StatsItem.makeStringStatsItem("Message", 250)), 22 | RecommendedPath(StatsItem.makeStringStatsItem("Path", 450)); 23 | 24 | StatsItem si; 25 | NameValidationStatsItems(StatsItem si) {this.si=si;} 26 | public StatsItem si() {return si;} 27 | } 28 | 29 | public static enum Generator implements StatsGenerator { 30 | INSTANCE; 31 | public Stats create(String key) {return new NameValidationStats(key);} 32 | } 33 | public static StatsItemConfig details = StatsItemConfig.create(NameValidationStatsItems.class); 34 | 35 | private NameValidationStats(String key) { 36 | super(NameValidationStats.details, key); 37 | } 38 | 39 | public Object compute(File f, FileTest fileTest) { 40 | Object ret = fileTest.fileTest(f); 41 | if (ret instanceof RenameDetails) { 42 | RenameDetails rdet = (RenameDetails)ret; 43 | setVal(NameValidationStatsItems.PassFail, rdet.getPassFail()); 44 | setVal(NameValidationStatsItems.Status, rdet.getRenameStatus()); 45 | setVal(NameValidationStatsItems.Message, rdet.getMessage()); 46 | setVal(NameValidationStatsItems.RecommendedPath, rdet.getDetailNote(fileTest.getRoot())); 47 | } 48 | return ret; 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/RandomStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | import java.util.Random; 5 | 6 | import gov.nara.nwts.ftapp.YN; 7 | import gov.nara.nwts.ftapp.filetest.FileTest; 8 | 9 | /** 10 | * Generate a random value for each file in a set. 11 | * @author TBrady 12 | * 13 | */ 14 | public class RandomStats extends Stats { 15 | 16 | public static enum RandomStatsItems implements StatsItemEnum { 17 | Path(StatsItem.makeStringStatsItem("Path", 450)), 18 | Selected(StatsItem.makeEnumStatsItem(YN.class,"Selected").setInitVal(YN.N).setExport(false)); 19 | 20 | StatsItem si; 21 | RandomStatsItems(StatsItem si) {this.si=si;} 22 | public StatsItem si() {return si;} 23 | } 24 | 25 | public static enum Generator implements StatsGenerator { 26 | INSTANCE; 27 | public Stats create(String key) {return new RandomStats(key);} 28 | } 29 | 30 | public static StatsItemConfig details = StatsItemConfig.create(RandomStatsItems.class); 31 | long randomVal; 32 | Random random; 33 | 34 | private RandomStats(String key) { 35 | super(RandomStats.details, key); 36 | random = new Random(); 37 | } 38 | 39 | public Object compute(File f, FileTest fileTest) { 40 | randomVal = random.nextLong(); 41 | if (fileTest instanceof Randomizer) { 42 | Randomizer r = (Randomizer)fileTest; 43 | while(r.getTreeSet().get(randomVal) != null) { 44 | randomVal = random.nextLong(); 45 | } 46 | r.getTreeSet().put(randomVal, key); 47 | } 48 | return randomVal; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/Randomizer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.util.TreeMap; 4 | 5 | /** 6 | * Helper class for generating a random sample. 7 | * @author TBrady 8 | * 9 | */ 10 | public interface Randomizer { 11 | public TreeMap getTreeSet(); 12 | } 13 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/StatsGenerator.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | public interface StatsGenerator { 4 | //Stats create(StatsItemConfig config, String key); 5 | Stats create(String key); 6 | } 7 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/StatsItemConfig.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.Iterator; 6 | import java.util.TreeMap; 7 | import java.util.TreeSet; 8 | 9 | public class StatsItemConfig implements Iterable { 10 | ArrayList columns; 11 | HashMap columnMap; 12 | 13 | public static & StatsItemEnum> StatsItemConfig create(Class eclass){ 14 | return new StatsItemConfig(eclass); 15 | } 16 | 17 | public & StatsItemEnum> StatsItemConfig(Class eclass) { 18 | this(); 19 | for(StatsItemEnum eitem : eclass.getEnumConstants()) { 20 | addStatsItem(eitem, eitem.si()); 21 | } 22 | } 23 | 24 | public StatsItemConfig() { 25 | columns = new ArrayList(); 26 | columnMap = new HashMap(); 27 | } 28 | 29 | public void addStatsItem(Object key, StatsItem si) { 30 | si.setIndex(columns.size()-1); 31 | columns.add(si); 32 | columnMap.put(key, si); 33 | } 34 | 35 | public int size() {return columns.size();} 36 | 37 | public StatsItem get(int i) { 38 | return columns.get(i); 39 | } 40 | 41 | public StatsItem getByKey(Object key) { 42 | return columnMap.get(key); 43 | } 44 | 45 | public boolean[] getExportArray() { 46 | boolean[] exarr = new boolean[size()]; 47 | int count = 0; 48 | for(StatsItem si: columns) { 49 | exarr[count++] = si.export; 50 | } 51 | return exarr; 52 | } 53 | 54 | public Iterator iterator() { 55 | return columns.iterator(); 56 | } 57 | 58 | public void createFilters(TreeMap data) { 59 | for(StatsItem si: columns) { 60 | if (si.getFilter()) { 61 | TreeSet vals = new TreeSet(); 62 | for(Stats stat: data.values()) { 63 | Object obj = stat.getKeyVal(si, null); 64 | if (obj == null) continue; 65 | vals.add(obj); 66 | } 67 | si.values = vals.toArray(); 68 | } 69 | } 70 | 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/StatsItemEnum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | public interface StatsItemEnum { 4 | public StatsItem si(); 5 | public int ordinal(); 6 | } 7 | -------------------------------------------------------------------------------- /core/src/main/gov/nara/nwts/ftapp/stats/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

The results of each {@link gov.nara.nwts.ftapp.filetest.FileTest} can be highly customized to meet a specific business need.

4 | 5 | 6 | -------------------------------------------------------------------------------- /core/src/main/overview.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Georgetown-University-Libraries/File-Analyzer/e9fb1a3d5d5c7f12b0d579c9e70af43c8b26011a/core/src/main/overview.html -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/BAG_TYPE.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | public enum BAG_TYPE { 4 | DIRECTORY, 5 | TAR, 6 | ZIP 7 | } 8 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/DemoBatchAnalyzer.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.DemoActionRegistry; 4 | import edu.georgetown.library.fileAnalyzer.importer.DemoImporterRegistry; 5 | 6 | import gov.nara.nwts.ftapp.BatchAnalyzer; 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 10 | /** 11 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 12 | * @author TBrady 13 | * 14 | */ 15 | public class DemoBatchAnalyzer extends BatchAnalyzer { 16 | 17 | public DemoBatchAnalyzer() { 18 | super(); 19 | } 20 | 21 | public ActionRegistry getActionRegistry(FTDriver ft) { 22 | return new DemoActionRegistry(ft, true); 23 | } 24 | 25 | protected ImporterRegistry getImporterRegistry(FTDriver ft) { 26 | return new DemoImporterRegistry(ft); 27 | } 28 | public static void main(String[] args) { 29 | DemoBatchAnalyzer ba = new DemoBatchAnalyzer(); 30 | ba.run(args); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/DemoBatchImporter.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.DemoActionRegistry; 4 | import edu.georgetown.library.fileAnalyzer.importer.DemoImporterRegistry; 5 | 6 | import gov.nara.nwts.ftapp.BatchImporter; 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 10 | /** 11 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 12 | * @author TBrady 13 | * 14 | */ 15 | public class DemoBatchImporter extends BatchImporter { 16 | 17 | public DemoBatchImporter() { 18 | super(); 19 | } 20 | 21 | public ActionRegistry getActionRegistry(FTDriver ft) { 22 | return new DemoActionRegistry(ft, true); 23 | } 24 | 25 | public ImporterRegistry getImporterRegistry(FTDriver ft) { 26 | return new DemoImporterRegistry(ft); 27 | } 28 | public static void main(String[] args) { 29 | DemoBatchImporter ba = new DemoBatchImporter(); 30 | ba.run(args); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/DemoFileAnalyzer.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import java.io.File; 4 | 5 | import edu.georgetown.library.fileAnalyzer.filetest.DemoActionRegistry; 6 | import edu.georgetown.library.fileAnalyzer.importer.DemoImporterRegistry; 7 | 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 10 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 11 | /** 12 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 13 | * @author TBrady 14 | * 15 | */ 16 | public class DemoFileAnalyzer extends DirectoryTable { 17 | 18 | public DemoFileAnalyzer(File f, boolean modifyAllowed) { 19 | super(f, modifyAllowed); 20 | this.title = "My Demo File Analyzer"; 21 | this.message = "Illustrates extensions to the file analzyer."; 22 | this.refreshTitle(); 23 | 24 | } 25 | 26 | protected ActionRegistry getActionRegistry() { 27 | return new DemoActionRegistry(this, modifyAllowed); 28 | } 29 | 30 | protected ImporterRegistry getImporterRegistry() { 31 | return new DemoImporterRegistry(this); 32 | } 33 | public static void main(String[] args) { 34 | if (args.length > 0) 35 | new DemoFileAnalyzer(new File(args[0]), false); 36 | else 37 | new DemoFileAnalyzer(null, false); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/dateValidation/DateValidationPattern.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.dateValidation; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | 6 | abstract public class DateValidationPattern { 7 | Pattern criteria; 8 | 9 | public DateValidationPattern(String criteriaStr) { 10 | criteria = Pattern.compile(criteriaStr); 11 | } 12 | 13 | public DateValidationResult test(String s) { 14 | if (s == null) return DateValidationResult.untestable(); 15 | s = s.trim(); 16 | if (s.equals("")) return DateValidationResult.missing(); 17 | Matcher m = criteria.matcher(s); 18 | if (!m.matches()) return DateValidationResult.untestable(); 19 | return makeResult(s, m); 20 | } 21 | 22 | abstract public DateValidationResult makeResult(String s, Matcher m); 23 | } 24 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/dateValidation/DateValidationResult.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.dateValidation; 2 | 3 | import gov.nara.nwts.ftapp.YN; 4 | 5 | import java.util.Date; 6 | 7 | public class DateValidationResult { 8 | public Date date; 9 | public String result; 10 | public DateValidationStatus status; 11 | 12 | public DateValidationResult(Date date, String result, DateValidationStatus status) { 13 | this.date = date; 14 | this.result = result; 15 | this.status = status; 16 | } 17 | 18 | public boolean doNext() { 19 | return (this.status == DateValidationStatus.UNTESTABLE); 20 | } 21 | 22 | public static DateValidationResult untestable() { 23 | return new DateValidationResult(null, "", DateValidationStatus.UNTESTABLE); 24 | } 25 | 26 | public static DateValidationResult missing() { 27 | return new DateValidationResult(null, "", DateValidationStatus.MISSING); 28 | } 29 | public static DateValidationResult invalid() { 30 | return new DateValidationResult(null, "", DateValidationStatus.INVALID); 31 | } 32 | public static DateValidationResult valid(Date d, String s) { 33 | return new DateValidationResult(d, s, DateValidationStatus.VALID); 34 | } 35 | public static DateValidationResult simplifiable(Date d, String s) { 36 | return new DateValidationResult(d, s, DateValidationStatus.SIMPLIFIABLE); 37 | } 38 | public static DateValidationResult parseable(Date d, String s) { 39 | return new DateValidationResult(d, s, DateValidationStatus.PARSEABLE); 40 | } 41 | public static DateValidationResult parseable(String s) { 42 | return new DateValidationResult(null, s, DateValidationStatus.PARSEABLE); 43 | } 44 | public static DateValidationResult valid(String s) { 45 | return new DateValidationResult(null, s, DateValidationStatus.VALID); 46 | } 47 | public YN exists() { 48 | return (status.exists()) ? YN.Y : YN.N; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/dateValidation/DateValidationStatus.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.dateValidation; 2 | 3 | public enum DateValidationStatus { 4 | VALID(true), SIMPLIFIABLE(true), PARSEABLE(false), INVALID(false), MISSING(false), UNTESTABLE(false); 5 | 6 | boolean valid; 7 | DateValidationStatus(boolean valid){ 8 | this.valid = valid; 9 | } 10 | 11 | public boolean isValid() { 12 | return valid; 13 | } 14 | 15 | public static DateValidationStatus overall(DateValidationStatus a, DateValidationStatus b){ 16 | return (a.ordinal() > b.ordinal()) ? a : b; 17 | } 18 | 19 | public static DateValidationStatus overall(DateValidationStatus[] dstats){ 20 | DateValidationStatus ret = DateValidationStatus.VALID; 21 | for(DateValidationStatus dvs: dstats) { 22 | ret = (ret.ordinal() > dvs.ordinal()) ? ret : dvs; 23 | } 24 | 25 | return ret; 26 | } 27 | 28 | public static DateValidationStatus worst(DateValidationStatus a, DateValidationStatus b){ 29 | return (a.ordinal() > b.ordinal()) ? a : b; 30 | } 31 | 32 | public static DateValidationStatus best(DateValidationStatus a, DateValidationStatus b){ 33 | return (a.ordinal() < b.ordinal()) ? a : b; 34 | } 35 | 36 | public boolean exists() { 37 | if ((this == DateValidationStatus.MISSING) || (this == DateValidationStatus.UNTESTABLE)) return false; 38 | return true; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/dateValidation/DateValidator.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.dateValidation; 2 | 3 | import java.util.Vector; 4 | 5 | public class DateValidator { 6 | Vector patterns; 7 | String defaultResultFormatStr; 8 | 9 | public DateValidator(String defaultResultFormatStr) { 10 | patterns = new Vector(); 11 | this.defaultResultFormatStr = defaultResultFormatStr; 12 | } 13 | 14 | public void addValidationPattern(String pattern, String dateFormatString) { 15 | addValidationPattern(pattern, dateFormatString, defaultResultFormatStr); 16 | } 17 | 18 | public void addValidationPattern(DateValidationPattern dvp) { 19 | patterns.add(dvp); 20 | } 21 | 22 | public void addValidationPattern(String pattern, String dateFormatString, String resultFormatStr) { 23 | patterns.add(new DefaultDateValidationPattern(pattern, dateFormatString, resultFormatStr)); 24 | } 25 | 26 | public DateValidationResult test(String s) { 27 | for(DateValidationPattern dvp: patterns) { 28 | DateValidationResult dvr = dvp.test(s); 29 | if (!dvr.doNext()) { 30 | return dvr; 31 | } 32 | } 33 | return DateValidationResult.invalid(); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/dateValidation/DefaultDateValidationPattern.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.dateValidation; 2 | 3 | import java.text.DateFormat; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | import java.util.regex.Matcher; 8 | 9 | public class DefaultDateValidationPattern extends DateValidationPattern { 10 | DateFormat parseFormat; 11 | DateFormat returnFormat; 12 | 13 | public DefaultDateValidationPattern(String criteriaStr, String parseFormatStr, String returnFormatStr) { 14 | super(criteriaStr); 15 | parseFormat = new SimpleDateFormat(parseFormatStr); 16 | parseFormat.setLenient(false); 17 | returnFormat = new SimpleDateFormat(returnFormatStr); 18 | } 19 | 20 | public DateValidationResult makeResult(String s, Matcher m) { 21 | try { 22 | Date d = parseFormat.parse(s); 23 | String ret = returnFormat.format(d); 24 | Date d2 = returnFormat.parse(ret); 25 | if (!d.equals(d2)) { 26 | return DateValidationResult.simplifiable(d, ret); 27 | } 28 | if (!s.equals(ret)){ 29 | return DateValidationResult.parseable(d, ret); 30 | } 31 | return DateValidationResult.valid(d, ret); 32 | } catch (ParseException e) { 33 | return DateValidationResult.invalid(); 34 | } 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/AIPDirToAPT.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | 8 | import edu.georgetown.library.fileAnalyzer.util.AIPDirToAPTHelper; 9 | import edu.georgetown.library.fileAnalyzer.util.AIPToAPTHelper; 10 | 11 | 12 | /** 13 | * Extract all metadata fields from a TIF or JPG using categorized tag defintions. 14 | * @author TBrady 15 | * 16 | */ 17 | class AIPDirToAPT extends AIPToAPT { 18 | public AIPDirToAPT(FTDriver dt) { 19 | super(dt); 20 | } 21 | 22 | public String toString() { 23 | return "Package AIP Dir for APT"; 24 | } 25 | public String getShortName(){return "AIP Dir->APT";} 26 | 27 | public String getDescription() { 28 | return "This rule will package a DSpace AIP Directory for APTrust"; 29 | } 30 | 31 | @Override public boolean isTestDirectory() { 32 | return true; 33 | } 34 | 35 | @Override public boolean isTestable(File f) { 36 | return f.equals(getRoot()); 37 | } 38 | 39 | @Override public boolean isTestFiles() { 40 | return false; 41 | } 42 | 43 | @Override public boolean processRoot() { 44 | return true; 45 | } 46 | 47 | @Override 48 | public AIPToAPTHelper getAIPToAPTHelper() throws IOException { 49 | return new AIPDirToAPTHelper(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/AIPZipToAPT.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filter.ZipFilter; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | import org.apache.commons.io.FileUtils; 10 | 11 | import edu.georgetown.library.fileAnalyzer.util.AIPToAPTHelper; 12 | import edu.georgetown.library.fileAnalyzer.util.AIPZipToAPTHelper; 13 | 14 | /** 15 | * Extract all metadata fields from a TIF or JPG using categorized tag defintions. 16 | * @author TBrady 17 | * 18 | */ 19 | class AIPZipToAPT extends AIPToAPT { 20 | 21 | public AIPZipToAPT(FTDriver dt) { 22 | super(dt); 23 | } 24 | 25 | public String getKey(File f) { 26 | return f.getName(); 27 | } 28 | public String toString() { 29 | return "Package AIP Zip for APT"; 30 | } 31 | public String getShortName(){return "AIP Zip->APT";} 32 | 33 | public String getDescription() { 34 | return "This rule will package a DSpace AIP Zip package for APTrust"; 35 | } 36 | 37 | @Override public boolean isTestDirectory() { 38 | return false; 39 | } 40 | @Override public boolean processRoot() { 41 | return false; 42 | } 43 | 44 | @Override public boolean isTestFiles() { 45 | return true; 46 | } 47 | 48 | @Override public boolean isTestable(File f) { 49 | return f.getName().toLowerCase().endsWith(".zip"); 50 | } 51 | 52 | public void initFilters() { 53 | filters.add(new ZipFilter()); 54 | } 55 | 56 | private File outdir; 57 | 58 | @Override public void cleanup(int count) { 59 | if (outdir != null) { 60 | try { 61 | FileUtils.deleteDirectory(outdir); 62 | } catch (IOException e) { 63 | e.printStackTrace(); 64 | } 65 | } 66 | } 67 | 68 | @Override 69 | public AIPToAPTHelper getAIPToAPTHelper() throws IOException { 70 | outdir = AIPZipToAPTHelper.createTempDir(); 71 | return new AIPZipToAPTHelper(outdir); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/CounterValidationXls.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import edu.georgetown.library.fileAnalyzer.importer.ExcelReader; 4 | import gov.nara.nwts.ftapp.counter.CounterStat; 5 | import gov.nara.nwts.ftapp.FTDriver; 6 | import gov.nara.nwts.ftapp.filetest.CounterValidation; 7 | import gov.nara.nwts.ftapp.filter.CSVFilter; 8 | import gov.nara.nwts.ftapp.filter.CounterFilterXls; 9 | import gov.nara.nwts.ftapp.filter.ExcelFilter; 10 | import gov.nara.nwts.ftapp.filter.TxtFilter; 11 | import gov.nara.nwts.ftapp.importer.DelimitedFileReader; 12 | import gov.nara.nwts.ftapp.stats.Stats; 13 | 14 | import java.io.File; 15 | import java.io.IOException; 16 | import java.util.Vector; 17 | 18 | import org.apache.poi.openxml4j.exceptions.InvalidFormatException; 19 | 20 | /** 21 | * Extract all metadata fields from a TIF or JPG using categorized tag defintions. 22 | * @author TBrady 23 | * 24 | */ 25 | public class CounterValidationXls extends CounterValidation { 26 | public CounterValidationXls(FTDriver dt) { 27 | super(dt); 28 | } 29 | @Override public Vector> getDataFromFile(File f, Stats s) { 30 | String ext = getExt(f); 31 | String sep = getSeparator(f, ext); 32 | try { 33 | Vector> data = new Vector>(); 34 | if (ext.equals("CSV") || ext.endsWith("TXT")) { 35 | data = DelimitedFileReader.parseFile(f, sep); 36 | } else { 37 | data = ExcelReader.readExcel(f); 38 | } 39 | return data; 40 | } catch (InvalidFormatException e) { 41 | s.setVal(CounterStatsItems.Stat, CounterStat.UNSUPPORTED_FILE); 42 | s.setVal(CounterStatsItems.Message, e.toString()); 43 | } catch (IOException e) { 44 | s.setVal(CounterStatsItems.Stat, CounterStat.UNSUPPORTED_FILE); 45 | s.setVal(CounterStatsItems.Message, e.toString()); 46 | } catch (org.apache.poi.hssf.OldExcelFormatException e) { 47 | s.setVal(CounterStatsItems.Stat, CounterStat.UNSUPPORTED_FILE); 48 | s.setVal(CounterStatsItems.Message, e.toString()); 49 | } catch (Exception e) { 50 | e.printStackTrace(); 51 | s.setVal(CounterStatsItems.Stat, CounterStat.UNSUPPORTED_FILE); 52 | s.setVal(CounterStatsItems.Message, e.toString()); 53 | } 54 | return null; 55 | } 56 | 57 | public String toString() { 58 | return "Counter Compliance"; 59 | } 60 | public void initFilters() { 61 | filters.add(new CounterFilterXls()); 62 | filters.add(new CSVFilter()); 63 | filters.add(new ExcelFilter()); 64 | filters.add(new TxtFilter()); 65 | } 66 | 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/DemoActionRegistry.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.CreateIIIFManifest; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | import gov.nara.nwts.ftapp.filetest.CounterValidation; 6 | 7 | /** 8 | * Initialize the File Analzyer with code relying on external libraries 9 | * 10 | */ 11 | public class DemoActionRegistry extends DSpaceActionRegistry { 12 | 13 | private static final long serialVersionUID = 1L; 14 | 15 | public DemoActionRegistry(FTDriver dt, boolean modifyAllowed) { 16 | super(dt, modifyAllowed); 17 | add(new PageCount(dt)); 18 | 19 | //The following rule intentionally inactive 20 | //add(new SwapDelimitedColumns(dt)); 21 | add(new ImageProperties(dt)); 22 | add(new YearbookNameValidationTest(dt)); 23 | add(new CreateBag(dt)); 24 | add(new CreateAPTrustBag(dt)); 25 | add(new AIPZipToAPT(dt)); 26 | add(new AIPDirToAPT(dt)); 27 | add(new VerifyBag(dt)); 28 | add(new VerifyBagZip(dt)); 29 | add(new VerifyBagTar(dt)); 30 | add(new VerifyAPTrustBagTar(dt)); 31 | removeFT(CounterValidation.class); 32 | add(new CounterValidationXls(dt)); 33 | add(new DemoFileTest(dt)); 34 | add(new MarcItemInventory(dt)); 35 | add(new CreateIIIFManifest(dt)); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/VerifyBagTar.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filter.TarFilter; 5 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | 11 | import org.apache.commons.io.FileUtils; 12 | 13 | import edu.georgetown.library.fileAnalyzer.stats.BagStatsItems; 14 | import edu.georgetown.library.fileAnalyzer.util.TarUtil; 15 | 16 | /** 17 | * Extract all metadata fields from a TIF or JPG using categorized tag defintions. 18 | * @author TBrady 19 | * 20 | */ 21 | class VerifyBagTar extends VerifyBag { 22 | public static StatsItemConfig details = StatsItemConfig.create(BagStatsItems.class); 23 | 24 | public VerifyBagTar(FTDriver dt) { 25 | super(dt); 26 | } 27 | 28 | public String toString() { 29 | return "Verify Bag - Tar"; 30 | } 31 | public String getShortName(){return "Ver Bag Tar";} 32 | 33 | public String getDescription() { 34 | return "This rule will validate the contents of a bag tar file"; 35 | } 36 | 37 | @Override public boolean isTestDirectory() { 38 | return false; 39 | } 40 | @Override public boolean processRoot() { 41 | return false; 42 | } 43 | 44 | @Override public boolean isTestFiles() { 45 | return true; 46 | } 47 | 48 | @Override public boolean isTestable(File f) { 49 | return f.getName().toLowerCase().endsWith(".tar"); 50 | } 51 | 52 | @Override public void initFilters() { 53 | filters.add(new TarFilter()); 54 | } 55 | 56 | @Override public File prepareFile(File f) throws IOException { 57 | File untarred = TarUtil.untar(f); 58 | if (untarred.listFiles().length == 1) { 59 | return untarred.listFiles()[0]; 60 | } 61 | return untarred; 62 | } 63 | 64 | @Override public void cleanupPreparedFile(File f) throws IOException { 65 | if (f == null) return; 66 | if (!f.isDirectory()) return; 67 | FileUtils.deleteDirectory(f); 68 | Files.deleteIfExists(f.toPath()); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/VerifyBagZip.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filter.ZipFilter; 5 | import gov.nara.nwts.ftapp.stats.StatsItemConfig; 6 | 7 | import java.io.File; 8 | 9 | import edu.georgetown.library.fileAnalyzer.stats.BagStatsItems; 10 | 11 | /** 12 | * Extract all metadata fields from a TIF or JPG using categorized tag defintions. 13 | * @author TBrady 14 | * 15 | */ 16 | class VerifyBagZip extends VerifyBag { 17 | public static StatsItemConfig details = StatsItemConfig.create(BagStatsItems.class); 18 | 19 | public VerifyBagZip(FTDriver dt) { 20 | super(dt); 21 | } 22 | 23 | public String toString() { 24 | return "Verify Bag - Zip"; 25 | } 26 | public String getShortName(){return "Ver Bag Zip";} 27 | 28 | public String getDescription() { 29 | return "This rule will validate the contents of a bag zip file"; 30 | } 31 | 32 | @Override public boolean isTestDirectory() { 33 | return false; 34 | } 35 | @Override public boolean processRoot() { 36 | return false; 37 | } 38 | 39 | @Override public boolean isTestFiles() { 40 | return true; 41 | } 42 | 43 | @Override public boolean isTestable(File f) { 44 | return f.getName().toLowerCase().endsWith(".zip"); 45 | } 46 | 47 | public void initFilters() { 48 | filters.add(new ZipFilter()); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/YearbookNameValidationTest.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | 4 | import java.io.File; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | 8 | import gov.nara.nwts.ftapp.FTDriver; 9 | import gov.nara.nwts.ftapp.filetest.NameValidationTest; 10 | import gov.nara.nwts.ftapp.nameValidation.CustomPattern; 11 | import gov.nara.nwts.ftapp.nameValidation.DirAnalysis; 12 | import gov.nara.nwts.ftapp.nameValidation.RenameDetails; 13 | import gov.nara.nwts.ftapp.nameValidation.RenameStatus; 14 | import gov.nara.nwts.ftapp.nameValidation.ValidPattern; 15 | /** 16 | * Filename validation rule to ensure that filenames are lowercase. 17 | * @author TBrady 18 | * 19 | */ 20 | class YearbookNameValidationTest extends NameValidationTest { 21 | 22 | Pattern dirPatternFolder = Pattern.compile("^\\d\\d\\d\\d*$"); 23 | Pattern dirPatternImage = Pattern.compile("gt_yearbooks_\\d\\d\\d\\d_(\\d\\d\\d).tif"); 24 | Pattern dirPatternIgnore = Pattern.compile("^(Thumbs.db|.*_st.pdf)$"); 25 | 26 | public YearbookNameValidationTest(FTDriver dt) { 27 | super(dt, new ValidPattern("^gt_yearbooks_\\d\\d\\d\\d_(\\d\\d\\d.tif|st.pdf)*$", false),null, "Yearbook","Yearbook"); 28 | 29 | testPatterns.add(new CustomPattern("^Thumbs.db$", false) { 30 | public RenameDetails report(File f, Matcher m) { 31 | return new RenameDetails(RenameStatus.SKIP); 32 | } 33 | 34 | }); 35 | 36 | dirTestPatterns.add(new CustomPattern(dirPatternFolder, false) { 37 | public RenameDetails report(File f, Matcher m) { 38 | RenameDetails det = DirAnalysis.analyze(f, dirPatternImage, 1, dirPatternIgnore, false, false); 39 | return det; 40 | } 41 | 42 | }); 43 | dirTestPatterns.add(new CustomPattern("^.*$", false) { 44 | public RenameDetails report(File f, Matcher m) { 45 | return new RenameDetails(RenameStatus.SKIP); 46 | } 47 | 48 | }); 49 | } 50 | 51 | public String getDescription() { 52 | return "This test will check conformance with the Georgetown Digitization Yearbook project." 53 | + getNameValidationDisclaimer(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/CollItemFolders.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | import java.util.regex.Pattern; 6 | 7 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFLookupEnum; 8 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFStandardProp; 9 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFType; 10 | 11 | public class CollItemFolders extends DefaultManifestProjectTranslate { 12 | public static Pattern pImage = Pattern.compile(".*\\.(tiff?|jpg|jpeg|jp2)$"); 13 | public static FilenameFilter filter = new FilenameFilter(){ 14 | @Override 15 | public boolean accept(File arg0, String arg1) { 16 | return pImage.matcher(arg1.toLowerCase()).matches(); 17 | } 18 | }; 19 | @Override 20 | public File getCollComponentRootAncestor(File root, File f) { 21 | return f; 22 | } 23 | 24 | @Override 25 | public String getCollComponentRootLabel(File root, File f, MetadataInputFile curMeta) { 26 | String s = curMeta.getValue(IIIFLookupEnum.Title.getLookup(), f.getParentFile().getName()); 27 | return this.translate(IIIFType.typeCollection, IIIFStandardProp.label, s); 28 | } 29 | 30 | @Override 31 | public String getCollManifestLabel(File root, File f, MetadataInputFile curMeta) { 32 | String s = curMeta.getValue(IIIFLookupEnum.Title.getLookup(), f.getParentFile().getName()); 33 | return this.translate(IIIFType.typeCollection, IIIFStandardProp.label, s); 34 | } 35 | 36 | @Override 37 | public RangePath getPrimaryRangePath(IIIFManifest manifest, String key, File f, MetadataInputFile itemMeta) { 38 | return RangePath.makeEmptyRangePath(manifest); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/CreateDateProjectTranslate.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.util.TreeMap; 5 | 6 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFLookupEnum; 7 | 8 | public class CreateDateProjectTranslate extends DefaultManifestProjectTranslate { 9 | TreeMap decadeRanges = new TreeMap<>(); 10 | RangePath decadesRangeRoot; 11 | 12 | @Override 13 | public void initProjectRanges(IIIFManifest manifest, File root, RangePath top) { 14 | decadesRangeRoot = top; 15 | } 16 | 17 | @Override 18 | public String getSequenceValue(String key, MetadataInputFile itemMeta) { 19 | return itemMeta.getValue(IIIFLookupEnum.DateCreated.getLookup(), IIIFManifest.EMPTY) + "_" + key; 20 | } 21 | @Override public String getSubtitle() {return "By Creation Date";} 22 | 23 | @Override 24 | public RangePath getPrimaryRangePath(IIIFManifest manifest, String key, File f, MetadataInputFile itemMeta) { 25 | String decade = getDecade(itemMeta.getValue(IIIFLookupEnum.DateCreated.getLookup(), IIIFManifest.EMPTY)); 26 | RangePath dr = decadeRanges.get(decade); 27 | if (dr == null) { 28 | dr = RangePath.makeRangePath(manifest, decade, decade); 29 | decadeRanges.put(decade, dr); 30 | } 31 | decadesRangeRoot.addChildRange(dr); 32 | return dr; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/DefaultInput.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.MetadataInputFileBuilder.InputFileType; 8 | 9 | abstract class DefaultInput implements MetadataInputFile { 10 | InputFileType fileType; 11 | @Override 12 | public String getValue(IIIFLookup key, String def){ 13 | return def; 14 | } 15 | @Override 16 | public String getValue(List keyList, String def, String sep){ 17 | StringBuilder sb = new StringBuilder(); 18 | for(IIIFLookup key: keyList) { 19 | String s = getValue(key, def); 20 | if (s.isEmpty()) { 21 | continue; 22 | } 23 | if (sb.length() > 0) { 24 | sb.append(sep); 25 | } 26 | sb.append(s); 27 | } 28 | return sb.toString(); 29 | } 30 | @Override 31 | public File getFile() { 32 | return null; 33 | } 34 | @Override 35 | public InputFileType getInputFileType() { 36 | return fileType; 37 | } 38 | @Override 39 | public void setCurrentKey(String s) { 40 | //no action except for CSV 41 | } 42 | @Override 43 | public List getInitRanges(IIIFManifest manifest, RangePath parent, ManifestProjectTranslate manifestTranslate) { 44 | return new ArrayList(); 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/DefaultInputFile.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | 5 | abstract class DefaultInputFile extends DefaultInput { 6 | File file; 7 | @Override 8 | public File getFile() { 9 | return file; 10 | } 11 | DefaultInputFile(File file) { 12 | this.file = file; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/DefaultManifestProjectTranslateEnum.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | 4 | public enum DefaultManifestProjectTranslateEnum implements ManifestProjectTranslateEnum { 5 | Default, 6 | ByCreationDate { 7 | public ManifestProjectTranslate getTranslator() { 8 | return new CreateDateProjectTranslate(); 9 | } 10 | }, 11 | ByFolderName { 12 | public ManifestProjectTranslate getTranslator() { 13 | return new FileSystemProjectTranslate(); 14 | } 15 | }, 16 | ByItemFolderName { 17 | public ManifestProjectTranslate getTranslator() { 18 | return new ItemFolderProjectTranslate(); 19 | } 20 | }, 21 | EADFolderMap { 22 | public ManifestProjectTranslate getTranslator() { 23 | return new EADFolderTranslate(); 24 | } 25 | }, 26 | EADFolderMapSubjectsOnly { 27 | public ManifestProjectTranslate getTranslator() { 28 | return new EADFolderTranslateSubjectsOnly(); 29 | } 30 | }, 31 | CollItemFolders { 32 | public ManifestProjectTranslate getTranslator() { 33 | return new CollItemFolders(); 34 | } 35 | } 36 | ; 37 | 38 | @Override 39 | public ManifestProjectTranslate getTranslator() { 40 | return new DefaultManifestProjectTranslate(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/EADFolderTranslateSubjectsOnly.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | public class EADFolderTranslateSubjectsOnly extends EADFolderTranslate { 4 | @Override 5 | public boolean showFolderRanges() { 6 | return false; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/FileSystemProjectTranslate.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | 5 | public class FileSystemProjectTranslate extends FolderProjectTranslate { 6 | RangePath containers; 7 | 8 | @Override 9 | public void initProjectRanges(IIIFManifest manifest, File root, RangePath top) { 10 | super.initProjectRanges(manifest, root, top); 11 | containers = RangePath.makeRangePath(manifest, "ZZContainers", "Containers"); 12 | if (showFolderRanges()) { 13 | top.addChildRange(containers); 14 | } 15 | dirPaths.put(root.getAbsolutePath(), containers); 16 | } 17 | 18 | @Override 19 | public RangePath makeRangePath(IIIFManifest manifest, File f) { 20 | RangePath rp = RangePath.makeRangePath(manifest, getRelPath(f), rangeTranslate(f.getName())); 21 | RangePath lastrp = rp; 22 | for(File parent = f.getParentFile(); parent != null; parent = parent.getParentFile()) { 23 | if (dirPaths.containsKey(parent.getAbsolutePath())) { 24 | RangePath parrp = dirPaths.get(parent.getAbsolutePath()); 25 | parrp.addChildRange(lastrp); 26 | lastrp.setParent(parrp); 27 | break; 28 | } 29 | 30 | RangePath parrp = RangePath.makeRangePath(manifest, getRelPath(parent), rangeTranslate(parent.getName())); 31 | parrp.addChildRange(lastrp); 32 | lastrp.setParent(parrp); 33 | dirPaths.put(parent.getAbsolutePath(), parrp); 34 | lastrp = parrp; 35 | } 36 | return rp; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/FolderProjectTranslate.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.util.TreeMap; 5 | 6 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFLookupEnum; 7 | 8 | public abstract class FolderProjectTranslate extends DefaultManifestProjectTranslate { 9 | File root; 10 | TreeMap dirPaths = new TreeMap<>(); 11 | RangePath top; 12 | 13 | @Override 14 | public void initProjectRanges(IIIFManifest manifest, File root, RangePath top) { 15 | this.root = root; 16 | this.top = top; 17 | } 18 | @Override 19 | public boolean showFolderRanges() { 20 | return true; 21 | } 22 | 23 | public String getRelPath(File f) { 24 | return f.getAbsolutePath().substring(root.getAbsolutePath().length()).replaceAll("[\\\\\\/]", "_"); 25 | } 26 | 27 | public abstract RangePath makeRangePath(IIIFManifest manifest, File f); 28 | 29 | @Override 30 | public RangePath getPrimaryRangePath(IIIFManifest manifest, String key, File f, MetadataInputFile itemMeta) { 31 | if (dirPaths.containsKey(f.getAbsolutePath())) { 32 | return dirPaths.get(f.getAbsolutePath()); 33 | } 34 | RangePath rp = makeRangePath(manifest, f); 35 | if (!manifest.isCollectionManifest() && manifest.getManifestProjectTranslate().isOneItemPerRange()) { 36 | rp.setDisplayPath(itemMeta.getValue(IIIFLookupEnum.Title.getLookup(), rp.displayPath)); 37 | } 38 | dirPaths.put(f.getAbsolutePath(), rp); 39 | return rp; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/IIIFProp.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | public interface IIIFProp { 4 | public String getLabel(); 5 | public String getDefault(); 6 | public boolean isMetadata(); 7 | } 8 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/InputFileException.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | public class InputFileException extends Exception { 4 | private static final long serialVersionUID = -8244295017254682738L; 5 | 6 | InputFileException(String s) { 7 | super(s); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/ItemFolderProjectTranslate.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | 5 | public class ItemFolderProjectTranslate extends FolderProjectTranslate { 6 | @Override 7 | public RangePath makeRangePath(IIIFManifest manifest, File f) { 8 | RangePath rp = RangePath.makeRangePath(manifest, getRelPath(f), rangeTranslate(f.getName())); 9 | top.addChildRange(rp); 10 | return rp; 11 | } 12 | @Override 13 | public boolean isOneItemPerRange() { 14 | return true; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/ManifestDimensions.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | import org.apache.tika.exception.TikaException; 7 | import org.apache.tika.metadata.Metadata; 8 | import org.apache.tika.metadata.TIFF; 9 | import org.apache.tika.parser.image.ImageMetadataExtractor; 10 | import org.xml.sax.SAXException; 11 | 12 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.DefaultDimensions; 13 | 14 | public class ManifestDimensions { 15 | int height; 16 | int width; 17 | ManifestDimensions(int height, int width) { 18 | this.height = height; 19 | this.width = width; 20 | } 21 | ManifestDimensions(File f) throws IOException, SAXException, TikaException { 22 | Metadata metadata = new Metadata(); 23 | ImageMetadataExtractor ime = new ImageMetadataExtractor(metadata); 24 | 25 | this.height = DefaultDimensions.PORTRAIT.dimensions.height; 26 | this.width = DefaultDimensions.PORTRAIT.dimensions.width; 27 | if (f.getName().toLowerCase().endsWith("tif") || f.getName().toLowerCase().endsWith("tiff")) { 28 | ime.parseTiff(f); 29 | } else if (f.getName().toLowerCase().endsWith("jpg") || f.getName().toLowerCase().endsWith("jpeg")) { 30 | ime.parseJpeg(f); 31 | } 32 | int h = metadata.getInt(TIFF.IMAGE_LENGTH); 33 | int w = metadata.getInt(TIFF.IMAGE_WIDTH); 34 | if (h == 0 || w == 0) { 35 | return; 36 | } 37 | if (h >= w) { 38 | this.width = (w * 1000) / h; 39 | this.height = 1000; 40 | } else { 41 | this.height = (h * 1000) / w; 42 | this.width = 1000; 43 | } 44 | } 45 | int height() { 46 | return height; 47 | } 48 | int width() { 49 | return width; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/ManifestProjectTranslate.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | 5 | import javax.xml.xpath.XPath; 6 | 7 | import org.w3c.dom.Node; 8 | 9 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFProp; 10 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.IIIFEnums.IIIFType; 11 | 12 | public interface ManifestProjectTranslate { 13 | public String getSubtitle(); 14 | public String getSequenceValue(String key, MetadataInputFile itemMeta); 15 | public boolean includeItem(MetadataInputFile itemMeta); 16 | public String translate(IIIFType type, IIIFProp key, String val); 17 | public String rangeTranslate(String val); 18 | public void registerEADRange(XPath xp, Node n, RangePath rangePath); 19 | public RangePath getPrimaryRangePath(IIIFManifest manifest, String key, File f, MetadataInputFile itemMeta); 20 | public boolean processInitRanges(); 21 | public boolean showFolderRanges(); 22 | public void initProjectRanges(IIIFManifest manifest, File root, RangePath top); 23 | public boolean isOneItemPerRange(); 24 | 25 | public File getCollComponentRootAncestor(File root, File f); 26 | public String getCollComponentRootLabel(File root, File f, MetadataInputFile curMeta); 27 | public String getCollManifestLabel(File root, File f, MetadataInputFile curMeta); 28 | } 29 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/ManifestProjectTranslateEnum.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | public interface ManifestProjectTranslateEnum { 4 | public String name(); 5 | public ManifestProjectTranslate getTranslator(); 6 | } 7 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/MetadataInputFile.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.util.List; 5 | 6 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.MetadataInputFileBuilder.InputFileType; 7 | 8 | public interface MetadataInputFile { 9 | public File getFile(); 10 | public String getValue(IIIFLookup key, String def); 11 | public String getValue(List keyList, String def, String sep); 12 | public InputFileType getInputFileType(); 13 | public void setCurrentKey(String key); 14 | public List getInitRanges(IIIFManifest manifest, RangePath parent, ManifestProjectTranslate manifestTranslate); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/PropertyFile.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest.iiif; 2 | 3 | import java.io.File; 4 | import java.io.FileReader; 5 | import java.util.Properties; 6 | 7 | import edu.georgetown.library.fileAnalyzer.filetest.iiif.MetadataInputFileBuilder.InputFileType; 8 | 9 | public class PropertyFile extends DefaultInputFile { 10 | Properties prop = new Properties(); 11 | PropertyFile(File file) throws InputFileException { 12 | super(file); 13 | fileType = InputFileType.Property; 14 | try { 15 | prop.load(new FileReader(file)); 16 | } catch (Exception e) { 17 | throw new InputFileException("Property Parsing Error "+e.getMessage()); 18 | } 19 | } 20 | 21 | @Override 22 | public String getValue(IIIFLookup key, String def) { 23 | String propkey = key.getFileTypeKey(fileType); 24 | return prop.getProperty(propkey, def); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/filetest/iiif/README.md: -------------------------------------------------------------------------------- 1 | ## Input Parameters 2 | - Manifest Property File: [manifestGenerate.prop](manifestGenerate.prop) 3 | - ManifestProjectTranslate [enum class](ManifestProjectTranslateEnum.java) that generates a class that implements [ManifestProjectTranslate interface](ManifestProjectTranslate.java) 4 | 5 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/importer/DemoImporterRegistry.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.importer; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.CounterValidationXls; 4 | import edu.georgetown.library.fileAnalyzer.filetest.MarcItemInventory; 5 | import gov.nara.nwts.ftapp.FTDriver; 6 | import gov.nara.nwts.ftapp.filetest.CounterValidation; 7 | 8 | 9 | /** 10 | * Activates the Importers that will be presented on the Import tab. 11 | * @author TBrady 12 | * 13 | */ 14 | public class DemoImporterRegistry extends DSpaceImporterRegistry { 15 | 16 | private static final long serialVersionUID = 1L; 17 | 18 | public DemoImporterRegistry(FTDriver dt) { 19 | super(dt); 20 | // add(new MarcValidator(dt)); 21 | 22 | removeImporter(CounterValidation.class); 23 | add(new CounterValidationXls(dt)); 24 | add(new DemoImporter(dt)); 25 | add(new MarcRecValidator(dt)); 26 | add(new OutsourcedMarcRecValidator(dt)); 27 | add(new EncodingCheck(dt)); 28 | add(new MarcInventory(dt)); 29 | add(new MarcItemInventory(dt)); 30 | add(new MarcSerializer(dt)); 31 | add(new OAIImporter(dt)); 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/stats/BagStatsItems.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.stats; 2 | 3 | import edu.georgetown.library.fileAnalyzer.util.APTrustHelper.STAT; 4 | import gov.nara.nwts.ftapp.stats.StatsItem; 5 | import gov.nara.nwts.ftapp.stats.StatsItemEnum; 6 | 7 | public enum BagStatsItems implements StatsItemEnum { 8 | Key(StatsItem.makeStringStatsItem("Source", 300)), 9 | Bag(StatsItem.makeStringStatsItem("Bag", 300)), 10 | Stat(StatsItem.makeEnumStatsItem(STAT.class, "Bag Status")), 11 | Count(StatsItem.makeIntStatsItem("Item Count")), 12 | Message(StatsItem.makeStringStatsItem("Message", 400)), 13 | ; 14 | StatsItem si; 15 | BagStatsItems(StatsItem si) {this.si=si;} 16 | public StatsItem si() {return si;} 17 | } 18 | -------------------------------------------------------------------------------- /demo/src/main/edu/georgetown/library/fileAnalyzer/stats/DetailedBagStatsItems.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.stats; 2 | 3 | import edu.georgetown.library.fileAnalyzer.util.APTrustHelper.STAT; 4 | import gov.nara.nwts.ftapp.stats.StatsItem; 5 | import gov.nara.nwts.ftapp.stats.StatsItemEnum; 6 | 7 | public enum DetailedBagStatsItems implements StatsItemEnum { 8 | Key(StatsItem.makeStringStatsItem("Bag Path", 200)), 9 | Stat(StatsItem.makeEnumStatsItem(STAT.class, "Bag Status")), 10 | Count(StatsItem.makeIntStatsItem("Item Count")), 11 | BagSourceOrg(StatsItem.makeStringStatsItem("Source Org",150)), 12 | BagSenderDesc(StatsItem.makeStringStatsItem("Sender Desc",150)), 13 | BagSenderId(StatsItem.makeStringStatsItem("Sender Id",150)), 14 | BagCount(StatsItem.makeStringStatsItem("Bag Count",150)), 15 | BagTotal(StatsItem.makeStringStatsItem("Bag Total",150)), 16 | Message(StatsItem.makeStringStatsItem("Message",400)), 17 | ; 18 | StatsItem si; 19 | DetailedBagStatsItems(StatsItem si) {this.si=si;} 20 | public StatsItem si() {return si;} 21 | } -------------------------------------------------------------------------------- /doc/NARA File Analyzer and Metadata Harvester.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Georgetown-University-Libraries/File-Analyzer/e9fb1a3d5d5c7f12b0d579c9e70af43c8b26011a/doc/NARA File Analyzer and Metadata Harvester.doc -------------------------------------------------------------------------------- /doc/TestPlan.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

File Analyzer Test Plan

4 |

File Tests

5 |
    6 |
  • 7 |
8 |

Filter Tests

9 |
    10 |
  • 11 |
12 |

Property Tests

13 |
    14 |
  • 15 |
16 |

Importer Tests

17 |
    18 |
  • 19 |
20 |

GUI Tests

21 |

Criteria Panel

22 |
    23 |
  • 24 |
25 |

Import Panel

26 |
    27 |
  • 28 |
29 |

Details Panel

30 |
    31 |
  • 32 |
33 |

Results Panel

34 |
    35 |
  • 36 |
37 |

Merge Panel

38 |
    39 |
  • 40 |
41 |

Batch Processing

42 |
    43 |
  • 44 |
45 |

Command Line Tests

46 |
    47 |
  • 48 |
49 | 50 | 51 | -------------------------------------------------------------------------------- /doc/TestPlan.txt: -------------------------------------------------------------------------------- 1 | File Tests 2 | 3 | Filter Tests 4 | 5 | Property Tests 6 | 7 | Import Tests -------------------------------------------------------------------------------- /dspace/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/DSpaceBatchAnalyzer.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.DSpaceActionRegistry; 4 | import edu.georgetown.library.fileAnalyzer.importer.DSpaceImporterRegistry; 5 | 6 | import gov.nara.nwts.ftapp.BatchAnalyzer; 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 10 | /** 11 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 12 | * @author TBrady 13 | * 14 | */ 15 | public class DSpaceBatchAnalyzer extends BatchAnalyzer { 16 | 17 | public DSpaceBatchAnalyzer() { 18 | super(); 19 | } 20 | 21 | public ActionRegistry getActionRegistry(FTDriver ft) { 22 | return new DSpaceActionRegistry(ft, true); 23 | } 24 | 25 | protected ImporterRegistry getImporterRegistry(FTDriver ft) { 26 | return new DSpaceImporterRegistry(ft); 27 | } 28 | public static void main(String[] args) { 29 | DSpaceBatchAnalyzer ba = new DSpaceBatchAnalyzer(); 30 | ba.run(args); 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/DSpaceBatchImporter.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import edu.georgetown.library.fileAnalyzer.filetest.DSpaceActionRegistry; 4 | import edu.georgetown.library.fileAnalyzer.importer.DSpaceImporterRegistry; 5 | 6 | import gov.nara.nwts.ftapp.BatchImporter; 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 10 | /** 11 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 12 | * @author TBrady 13 | * 14 | */ 15 | public class DSpaceBatchImporter extends BatchImporter { 16 | 17 | public DSpaceBatchImporter() { 18 | super(); 19 | } 20 | 21 | public ActionRegistry getActionRegistry(FTDriver ft) { 22 | return new DSpaceActionRegistry(ft, true); 23 | } 24 | 25 | public ImporterRegistry getImporterRegistry(FTDriver ft) { 26 | return new DSpaceImporterRegistry(ft); 27 | } 28 | public static void main(String[] args) { 29 | DSpaceBatchImporter ba = new DSpaceBatchImporter(); 30 | ba.run(args); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/DSpaceFileAnalyzer.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer; 2 | 3 | import java.io.File; 4 | 5 | import edu.georgetown.library.fileAnalyzer.filetest.DSpaceActionRegistry; 6 | import edu.georgetown.library.fileAnalyzer.importer.DSpaceImporterRegistry; 7 | 8 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 9 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 10 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 11 | /** 12 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 13 | * @author TBrady 14 | * 15 | */ 16 | public class DSpaceFileAnalyzer extends DirectoryTable { 17 | 18 | public DSpaceFileAnalyzer(File f, boolean modifyAllowed) { 19 | super(f, modifyAllowed); 20 | } 21 | 22 | @Override protected ActionRegistry getActionRegistry() { 23 | return new DSpaceActionRegistry(this, modifyAllowed); 24 | } 25 | 26 | @Override protected ImporterRegistry getImporterRegistry() { 27 | return new DSpaceImporterRegistry(this); 28 | } 29 | public static void main(String[] args) { 30 | if (args.length > 0) 31 | new DSpaceFileAnalyzer(new File(args[0]), false); 32 | else 33 | new DSpaceFileAnalyzer(null, false); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/filetest/DSpaceActionRegistry.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 5 | 6 | /** 7 | * Initialize the File Analzyer with generic image processing rules (but not NARA specific business rules) 8 | * @author TBrady 9 | * 10 | */ 11 | public class DSpaceActionRegistry extends ActionRegistry { 12 | 13 | private static final long serialVersionUID = 1L; 14 | 15 | public DSpaceActionRegistry(FTDriver dt, boolean modifyAllowed) { 16 | super(dt, modifyAllowed); 17 | add(new IngestInventory(dt)); 18 | add(new IngestValidate(dt)); 19 | add(new DSpaceDAT(dt)); 20 | add(new ProquestToIngest(dt)); 21 | add(new ProquestQC(dt)); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/filter/GUImageFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.filter; 2 | 3 | import gov.nara.nwts.ftapp.filter.DefaultFileTestFilter; 4 | 5 | /** 6 | * Filter for TIF or JPG files 7 | * @author TBrady 8 | * 9 | */ 10 | public class GUImageFileTestFilter extends DefaultFileTestFilter { 11 | 12 | public String getSuffix() { 13 | return ".*\\.(tif|jpg|jp2)$"; 14 | } 15 | public boolean isReSuffix() { 16 | return true; 17 | } 18 | public String getName(){return "Image";} 19 | 20 | } 21 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/importer/DSpaceImporterRegistry.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 5 | 6 | 7 | /** 8 | * Activates the Importers that will be presented on the Import tab. 9 | * @author TBrady 10 | * 11 | */ 12 | public class DSpaceImporterRegistry extends ImporterRegistry { 13 | 14 | private static final long serialVersionUID = 1L; 15 | 16 | public DSpaceImporterRegistry(FTDriver dt) { 17 | super(dt); 18 | add(new IngestFolderCreate(dt)); 19 | add(new ItemUpdateFolderCreateAddBitstream(dt)); 20 | add(new ItemUpdateFolderCreateDeleteBitstream(dt)); 21 | add(new CSVBatcher(dt)); 22 | add(new DSpaceMetadata2Marc(dt)); 23 | add(new EAD2DC(dt)); 24 | add(new EAD2DAO(dt)); 25 | add(new ReadMetadataRegistry(dt)); 26 | } 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/importer/Field.java: -------------------------------------------------------------------------------- 1 | /** 2 | * The contents of this file are subject to the license and copyright 3 | * detailed in the LICENSE and NOTICE files at the root of the source 4 | * tree and available online at 5 | * 6 | * http://www.dspace.org/license/ 7 | */ 8 | package edu.georgetown.library.fileAnalyzer.importer; 9 | 10 | /** 11 | * Convert metadata field registry to JSON* 12 | * @author twb27 13 | */ 14 | public class Field { 15 | public static String makeName(String prefix, String element, String qualifier) { 16 | StringBuilder sb = new StringBuilder(prefix); 17 | sb.append("."); 18 | sb.append(element); 19 | if (qualifier != null) { 20 | sb.append("."); 21 | sb.append(qualifier); 22 | } 23 | return sb.toString(); 24 | } 25 | 26 | private String name; 27 | private String element; 28 | private String qualifier; 29 | private String description; 30 | 31 | public String name() {return name;} 32 | public String element() {return element;} 33 | public String qualifier() {return qualifier;} 34 | public String description() {return description;} 35 | 36 | public void setName(String name) {this.name = name;} 37 | public void setElement(String element) {this.name = element;} 38 | public void setQualifier(String qualifier) {this.name = qualifier;} 39 | public void setDescription(String description) {this.name = description;} 40 | 41 | public Field() { 42 | } 43 | public Field(String name, String element, String qualifier, String description) { 44 | this.name = name; 45 | this.element = element; 46 | this.qualifier = qualifier; 47 | this.description = description; 48 | } 49 | 50 | public String toString() { 51 | return String.format("name:%s,element:%s,qualifier:%s,description:%s", name, element, qualifier, description == null ? null : description.replaceAll("\\s+", " ")); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/importer/ItemUpdateFolderCreateDeleteBitstream.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.importer; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.Vector; 6 | 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.stats.Stats; 9 | 10 | /** 11 | * @author TBrady 12 | * 13 | */ 14 | public class ItemUpdateFolderCreateDeleteBitstream extends ItemUpdateFolderCreate { 15 | public ItemUpdateFolderCreateDeleteBitstream(FTDriver dt) { 16 | super(dt); 17 | } 18 | 19 | @Override 20 | public String getSubtask() { 21 | return "Delete Bitstreams"; 22 | } 23 | 24 | @Override 25 | public String getCmdOpt() { 26 | return "-D"; 27 | } 28 | 29 | @Override 30 | public String getColDesc() { 31 | return "\t2) BitstreamId - required, the integer (or uuid in DSpace6) associated with the bitstream to delete\n"; 32 | } 33 | 34 | @Override 35 | public int getColCount(){ 36 | return FIXED.values().length; 37 | }; 38 | public static enum FIXED { 39 | HANDLE(0), BITID(1); 40 | int index; 41 | FIXED(int i) {index = i;} 42 | } 43 | 44 | public void createItem(Stats stats, File selectedFile, Vector cols) { 45 | String handle = cols.get(FIXED.HANDLE.index); 46 | String bitid = cols.get(FIXED.BITID.index); 47 | 48 | File dir = makeItemDir(stats, handle); 49 | try { 50 | writeDeleteContents(dir, bitid); 51 | } catch (IOException e1) { 52 | stats.setVal(ItemUpdateStatsItems.Status, status.FAIL); 53 | stats.setVal(ItemUpdateStatsItems.Message, e1.getMessage()); 54 | return; 55 | } 56 | stats.setVal(ItemUpdateStatsItems.Status, status.PASS); 57 | 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/importer/MetadataRegPropFile.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.importer; 2 | 3 | import java.io.File; 4 | import java.io.FileReader; 5 | import java.io.IOException; 6 | import java.util.HashMap; 7 | 8 | import com.google.gson.Gson; 9 | 10 | import gov.nara.nwts.ftapp.FTDriver; 11 | import gov.nara.nwts.ftapp.ftprop.FTPropFile; 12 | import gov.nara.nwts.ftapp.ftprop.InitializationStatus; 13 | 14 | public class MetadataRegPropFile extends FTPropFile { 15 | 16 | public static final String P_METAREG = "metadata-reg"; 17 | HashMap fields = new HashMap(); 18 | public MetadataRegPropFile(FTDriver ft) { 19 | super(ft, "MetadataRegPropFile", P_METAREG, P_METAREG, "Metadata Registry Export File (JSON format), Optional", ""); 20 | } 21 | 22 | @Override public InitializationStatus initValidation(File refFile) { 23 | InitializationStatus iStat = new InitializationStatus(); 24 | if (getFile() != null) { 25 | try(FileReader fr = new FileReader(getFile())){ 26 | Schema[] data = new Gson().fromJson(fr, Schema[].class); 27 | fields = Schema.getFields(data); 28 | if (fields.size() == 0) { 29 | iStat.addFailMessage("No fields defined in the Metadata Registry"); 30 | } else { 31 | //iStat.addMessage(data.length + " schemas in registry. " + fields.size() + " fields in the Metadata Registry."); 32 | } 33 | } catch (IOException|RuntimeException e) { 34 | iStat.addFailMessage("Cannot Parse the Metadata Registry File"); 35 | iStat.addFailMessage(e.getMessage()); 36 | } 37 | } 38 | return iStat; 39 | } 40 | 41 | public boolean isFieldInRegistry(String name) { 42 | if (fields.isEmpty()) return true; 43 | return fields.containsKey(name); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/importer/Schema.java: -------------------------------------------------------------------------------- 1 | /** 2 | * The contents of this file are subject to the license and copyright 3 | * detailed in the LICENSE and NOTICE files at the root of the source 4 | * tree and available online at 5 | * 6 | * http://www.dspace.org/license/ 7 | */ 8 | package edu.georgetown.library.fileAnalyzer.importer; 9 | 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | 14 | /** 15 | * Convert metadata schema registry to JSON 16 | * @author twb27 17 | */ 18 | public class Schema { 19 | private String prefix; 20 | private String namespace; 21 | private List fields; 22 | 23 | public String prefix() {return prefix;} 24 | public String namespace() {return namespace;} 25 | public List getFields() {return fields;} 26 | 27 | public void setPrefix(String prefix) {this.prefix = prefix;} 28 | public void setNamespace(String namespace) {this.namespace = namespace;} 29 | public void setFields(List fields) {this.fields = fields;} 30 | 31 | public Schema() { 32 | } 33 | public Schema(String prefix, String namespace) { 34 | this.prefix = prefix; 35 | this.namespace = namespace; 36 | this.fields = new ArrayList(); 37 | } 38 | public void addField(String name, String element, String qualifier, String description) { 39 | fields.add(new Field(name, element, qualifier, description)); 40 | } 41 | 42 | public String toString() { 43 | StringBuilder sb = new StringBuilder(); 44 | sb.append(String.format("prefix:%s,namespace:%s", prefix, namespace)); 45 | for(Field f: fields) { 46 | sb.append(String.format("%n\t%s",f)); 47 | } 48 | return sb.toString(); 49 | } 50 | 51 | static public HashMap getFields(Schema[] schemas) { 52 | HashMap fields = new HashMap(); 53 | if (schemas != null) { 54 | for(Schema schema: schemas) { 55 | if (schema.getFields() == null) continue; 56 | for(Field field: schema.getFields()) { 57 | fields.put(field.name(), field); 58 | } 59 | } 60 | } 61 | return fields; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /dspace/src/main/edu/georgetown/library/fileAnalyzer/proquestXsl/GUProquestURIResolver.java: -------------------------------------------------------------------------------- 1 | package edu.georgetown.library.fileAnalyzer.proquestXsl; 2 | 3 | import java.io.InputStream; 4 | 5 | import javax.xml.transform.Source; 6 | import javax.xml.transform.TransformerException; 7 | import javax.xml.transform.URIResolver; 8 | import javax.xml.transform.stream.StreamSource; 9 | 10 | public class GUProquestURIResolver implements URIResolver { 11 | public Source resolve(String href, String base) throws TransformerException { 12 | InputStream is = GUProquestURIResolver.class.getClassLoader().getResourceAsStream("edu/georgetown/library/fileAnalyzer/proquestXsl/" + href); 13 | if (is == null) is = GUProquestURIResolver.class.getClassLoader().getResourceAsStream("resources/edu/georgetown/library/fileAnalyzer/proquestXsl/" + href); 14 | return new StreamSource(is); 15 | } 16 | public static GUProquestURIResolver INSTANCE = new GUProquestURIResolver(); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /dspace/src/main/resources/edu/georgetown/library/fileAnalyzer/proquestXsl/common.xsl: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /dspace/src/main/resources/edu/georgetown/library/fileAnalyzer/proquestXsl/proquest2ingest-local.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | local 6 | embargo 7 | terms 8 | custom-date 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 6-months 25 | 1-year 26 | 2-years 27 | forever 28 | custom 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | edu.georgetown.library 6 | FileAnalyzer 7 | 2.0 8 | pom 9 | 10 | http://maven.apache.org 11 | 12 | 13 | UTF-8 14 | 15 | 16 | 17 | 18 | 19 | src/main/resources 20 | 21 | 22 | 23 | 24 | 25 | maven-assembly-plugin 26 | 2.4 27 | 28 | 29 | 30 | 31 | 32 | 33 | core-utils 34 | core 35 | dspace 36 | bag-utils 37 | demo 38 | 39 | 40 | --------------------------------------------------------------------------------