├── doc.zip ├── src ├── overview.html └── gov │ └── nara │ └── nwts │ └── ftapp │ ├── filetest │ ├── package.html │ ├── NameMD5Checksum.java │ ├── NameSha1Checksum.java │ ├── LowercaseTest.java │ ├── NameSha256Checksum.java │ ├── CountByType.java │ ├── NameMatch.java │ ├── BaseNameMatch.java │ ├── ByMD5Checksum.java │ ├── DirMatch.java │ ├── DirTypeNameMatch.java │ ├── ListDirectories.java │ ├── ActionRegistry.java │ ├── FileTest.java │ ├── NameChecksum.java │ ├── RandomFileTest.java │ ├── DefaultFileTest.java │ └── NameValidationTest.java │ ├── nameValidation │ ├── package.html │ ├── RenamePassFail.java │ ├── ValidPattern.java │ ├── InvalidManualPattern.java │ ├── RenameablePattern.java │ ├── CustomPattern.java │ ├── NameValidationPattern.java │ ├── RenameDetails.java │ ├── RenameStatus.java │ └── DirAnalysis.java │ ├── YN.java │ ├── crud │ ├── package.html │ ├── CRUD.java │ └── CRUDDetails.java │ ├── stats │ ├── Randomizer.java │ ├── NameStats.java │ ├── CountStats.java │ ├── Stats.java │ ├── DataStats.java │ ├── NumericStats.java │ ├── FileCountStats.java │ ├── package.html │ ├── RandomStats.java │ ├── CRUDStats.java │ ├── CountAppendStats.java │ ├── DirStats.java │ ├── NameValidationStats.java │ └── DirTypeStats.java │ ├── filter │ ├── TxtFilter.java │ ├── XmlFilter.java │ ├── package.html │ ├── ExcelFilter.java │ ├── CSVFilter.java │ ├── ODSFilter.java │ ├── ImageFileTestFilter.java │ ├── TiffJpegFileTestFilter.java │ ├── AVFileTestFilter.java │ ├── OdsCsvFilter.java │ ├── JpegFileTestFilter.java │ ├── TiffFileTestFilter.java │ ├── FileTestFilter.java │ └── DefaultFileTestFilter.java │ ├── ftprop │ ├── package.html │ ├── FTProp.java │ ├── FTPropString.java │ ├── DefaultFTProp.java │ └── FTPropEnum.java │ ├── importer │ ├── package.html │ ├── Importer.java │ ├── ImporterRegistry.java │ ├── CsvFileImporter.java │ ├── TabSepFileImporter.java │ ├── SemicolonFileImporter.java │ ├── DefaultImporter.java │ ├── FileListImporter.java │ ├── Parser.java │ └── DelimitedFileImporter.java │ ├── package.html │ ├── gui │ ├── package.html │ ├── MyBorderPanel.java │ ├── DirSelect.java │ ├── DirSelectChooser.java │ ├── MyProgress.java │ ├── MyPanel.java │ ├── FileCatalog.java │ ├── FileSelect.java │ ├── GuiFileTraversalSW.java │ ├── FilterPanel.java │ ├── FileSelectChooser.java │ ├── MyTableModel.java │ ├── SummaryPanel.java │ ├── GuiFileTraversal.java │ ├── DetailsPanel.java │ ├── TableSaver.java │ ├── ImportPanel.java │ ├── StatsTable.java │ ├── ProgressPanel.java │ └── CriteriaPanel.java │ ├── Timer.java │ ├── FileAnalyzer.java │ ├── FileAnalyzerMod.java │ ├── MyDirectoryFilter.java │ ├── ResultFilter.java │ ├── ActionResult.java │ ├── MyFilenameFilter.java │ ├── BatchImporter.java │ └── FileTraversal.java ├── bin └── fileAnalyzer.jar ├── samples └── ImageAnalyzer │ ├── src │ └── gov │ │ └── nara │ │ └── nwts │ │ └── ftappImg │ │ ├── tif │ │ ├── package.html │ │ └── TifExtractor.java │ │ ├── jpeg │ │ ├── package.html │ │ └── JpegExtractor.java │ │ ├── tags │ │ ├── package.html │ │ ├── Extractor.java │ │ ├── DefaultExtractor.java │ │ └── XMPExtractor.java │ │ ├── filetest │ │ ├── package.html │ │ ├── tmp0000.html │ │ ├── ImageActionRegistry.java │ │ ├── CountJpeg.java │ │ ├── CountTiff.java │ │ └── GenericImageProperties.java │ │ ├── stats │ │ ├── package.html │ │ ├── GenericImageStats.java │ │ ├── ImageStats.java │ │ └── JpegStats.java │ │ ├── package.html │ │ └── ImageFileAnalyzer.java │ └── README ├── doc └── NARA File Analyzer and Metadata Harvester.doc ├── javadoc.css ├── codeinventory.json ├── LICENSE.md └── README.md /doc.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/doc.zip -------------------------------------------------------------------------------- /src/overview.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/src/overview.html -------------------------------------------------------------------------------- /bin/fileAnalyzer.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/bin/fileAnalyzer.jar -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/tif/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Helper classes for parsing TIF metadata. 4 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/jpeg/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Helper classes for parsing jpeg metadata. 4 | 5 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/package.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/src/gov/nara/nwts/ftapp/filetest/package.html -------------------------------------------------------------------------------- /doc/NARA File Analyzer and Metadata Harvester.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/doc/NARA File Analyzer and Metadata Harvester.doc -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/tags/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Helper classes for reporting on metadata embedded in images. 4 | 5 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/package.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/usnationalarchives/File-Analyzer/HEAD/src/gov/nara/nwts/ftapp/nameValidation/package.html -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/filetest/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | File Test that perform generic image metadata extraction and reporting. 4 | {@link gov.nara.nwts.ftapp.filetest} 5 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/filetest/tmp0000.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | File Test that perform generic image metadata extraction and reporting. 4 | {@link gov.nara.nwts.ftapp.filetest} 5 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/stats/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | File Test Stats objects supporting File Tests that report on embedded image metadata. 4 | {@link gov.nara.nwts.ftapp.stats} 5 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/YN.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | /** 4 | * Simple enumeration when reporting statistics 5 | * @author TBrady 6 | * 7 | */ 8 | public enum YN implements Comparable { 9 | Y, 10 | N; 11 | } 12 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/crud/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | The Create, Replace, Update, Delete package is intended to contain helper classes when using the File Analyzer as a database ingest tool. This package has not yet been fully developed. 4 | 5 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/Randomizer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.util.TreeMap; 4 | 5 | /** 6 | * Helper class for generating a random sample. 7 | * @author TBrady 8 | * 9 | */ 10 | public interface Randomizer { 11 | public TreeMap getTreeSet(); 12 | } 13 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/README: -------------------------------------------------------------------------------- 1 | This sample code will extend the File Analyzer and introspect embedded metadata. 2 | 3 | This code requires the following additional libraries in order to execute. 4 | 5 | Java Advanced Imaging 1.1.3 jai_core.jar 6 | Java Advanced Imaging 1.1.3 jai_codec.jar 7 | Apache Tika 0.9 tika-app-0.9.jar 8 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/TxtFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TxtFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".txt"; 11 | } 12 | public String getName(){return "TXT";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/XmlFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for XML files 5 | * @author TBrady 6 | * 7 | */ 8 | public class XmlFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".xml"; 11 | } 12 | public String getName(){return "XML";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | A File Test Filter is a string-based or regular-expression based file name matcher that will determine files that will be considered for analysis by a FileTest. The File Analyzer user interface will allow a user to further refine the filter criteria for more specific analysis. 4 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/ExcelFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for excel files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ExcelFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".xls"; 11 | } 12 | public String getName(){return "Excel";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /javadoc.css: -------------------------------------------------------------------------------- 1 | table.bordered { 2 | border-collapse: collapse; 3 | } 4 | table.bordered td, table.bordered th { 5 | border: solid thin black; 6 | padding: 2px; 7 | } 8 | 9 | table.overview td, table.overview th { 10 | width: 140px; 11 | } 12 | 13 | tr.custom { 14 | background-color: #EEFFFF; 15 | } 16 | 17 | tr.img { 18 | background-color: #FFEEFF; 19 | } -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/crud/CRUD.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.crud; 2 | 3 | /** 4 | *Categorizes the resulting status of a database action. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum CRUD implements Comparable { 9 | NA, 10 | Created, 11 | Replaced, 12 | Updated, 13 | Deleted, 14 | Skipped, 15 | NotFound, 16 | AlreadyExists; 17 | } 18 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/CSVFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for comma-separated text files 5 | * @author TBrady 6 | * 7 | */ 8 | public class CSVFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".csv"; 11 | } 12 | public String getName(){return "CSV";} 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/RenamePassFail.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | /** 4 | * Enumeration indicating a Pass or Failure of a test. This enumeration is useful beyond file name validation and could be suitable to move to a higher level pacakage. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum RenamePassFail {PASS,FAIL}; 9 | 10 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ftprop/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | A File Test Property is an optional parameter that will alter the behavior of a File Test. Where possible, File Test Properties should be avoided in order to simplify the user experience. As new File Test Properties are created, care must be taken to ensure that they will behave in both the GUI mode and the Command Line mode. 4 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

This package contains custom file import rules that can be invoked from the File Analyzer.

4 |

The primary reason for providing import functionality is to support the match/merge capabilities of the File Analyzer GUI.

5 |

As new import rules are created, the File Analyzer could also serve as an ingest tool.

6 | 7 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/ODSFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for Open Office Spreadsheet files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ODSFilter extends DefaultFileTestFilter { 9 | public String getSuffix() { 10 | return ".ods"; 11 | } 12 | public String getName(){return "Open Office Spreadsheet";} 13 | 14 | public String getShortName(){return "ODS";} 15 | } 16 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | This package contains File Analyzer objects of general use beyond NARA but which are dependent on 3rd party libraries that extract metadata from image files. 4 |

Dependencies

5 |
    6 |
  • Java Advanced Imaging 1.1.3 jai_core.jar
    • 7 |
  • Java Advanced Imaging 1.1.3 jai_codec.jar
    • 8 |
  • Apache Tika 0.9 tika-app-0.9.jar
    • 9 |
10 | 11 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/ImageFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for images files 5 | * @author TBrady 6 | * 7 | */ 8 | public class ImageFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(tif|gif|jpg|png|bmp)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "Images";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/TiffJpegFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for TIF or JPG files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TiffJpegFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(tif|jpg)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "Tif Jpeg";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/AVFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for AV files 5 | * @author TBrady 6 | * 7 | */ 8 | public class AVFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*(\\.wav|\\.mov|\\.mp3|\\.avi|\\.dpx|\\.mxf)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "AV Files";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/OdsCsvFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for open office spreadsheet files OR Comma separated files 5 | * @author TBrady 6 | * 7 | */ 8 | public class OdsCsvFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".*\\.(ods|csv)$"; 12 | } 13 | public boolean isReSuffix() { 14 | return true; 15 | } 16 | public String getName(){return "ODS/CSV";} 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/JpegFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for JPG files 5 | * @author TBrady 6 | * 7 | */ 8 | public class JpegFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".jpg"; 12 | } 13 | public String getPrefix() { 14 | return "^[^\\.].*"; 15 | } 16 | public boolean isRePrefix() { 17 | return true; 18 | } 19 | public String getName(){return "Jpg";} 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

This package contains the main entry points for the File Analyzer as well as the workflow components that are common to both the GUI and the Command line versions of the File Analyzer. 4 |

The contents of this package are intended to be of generic use to other institutions that might use the File Analyzer. The contents of this package should not require any libraries other than the standard Java SE 1.6 or higher.

5 | 6 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/TiffFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Filter for TIF files 5 | * @author TBrady 6 | * 7 | */ 8 | public class TiffFileTestFilter extends DefaultFileTestFilter { 9 | 10 | public String getSuffix() { 11 | return ".tif"; 12 | } 13 | public String getPrefix() { 14 | return "^[^\\.].*"; 15 | } 16 | public boolean isRePrefix() { 17 | return true; 18 | } 19 | public String getName(){return "Tiffs";} 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

This package contains the Graphical User Interface elements of the File Analyzer.

4 |

Note: In order to provide a responsive user experience, long-running actions to be performed within a Java GUI should occur in a worker thread that periodically updates its status to the GUI application thread. Several functions that are straightforward in the command line mode of the File Analyzer, have been decomposed into worker thread and GUI thread actions.

5 | 6 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/tags/Extractor.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.tags; 2 | 3 | import gov.nara.nwts.ftappImg.tags.ImageTags.DUP; 4 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGS; 5 | 6 | /** 7 | * Contract for default behavior for an image metadata extractor 8 | * @author TBrady 9 | * 10 | */ 11 | public interface Extractor { 12 | public void close(); 13 | public String getString(DUP tags); 14 | public String getString(DUP tags, boolean b); 15 | public String getString(TAGS tags); 16 | } 17 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/Timer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.util.Date; 4 | /** 5 | * Helper class to report on the duration of a File Analyzer action. 6 | * @author TBrady 7 | * 8 | */ 9 | public class Timer { 10 | Date start; 11 | Date end; 12 | 13 | public Timer() { 14 | start = new Date(); 15 | } 16 | 17 | public void end() { 18 | end = new Date(); 19 | } 20 | 21 | public double getDuration() { 22 | if (end == null) 23 | end(); 24 | return (end.getTime() - start.getTime())*.001; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/MyBorderPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | 5 | import javax.swing.JPanel; 6 | 7 | /** 8 | * GUI helper class to add a panel with a border layout 9 | * @author TBrady 10 | * 11 | */ 12 | class MyBorderPanel extends MyPanel { 13 | private static final long serialVersionUID = 1L; 14 | MyBorderPanel() { 15 | super(new BorderLayout()); 16 | } 17 | JPanel addPanel(JPanel p, String loc) { 18 | add(p, (loc != null) ? loc : BorderLayout.CENTER); 19 | return p; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/Importer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | 8 | /** 9 | * Contract that Importers will fullfill. 10 | * @author TBrady 11 | * 12 | */ 13 | public interface Importer { 14 | public ActionResult importFile(File selectedFile) throws IOException; 15 | public String getDescription(); 16 | public boolean allowForceKey(); 17 | public String getShortName(); 18 | public String getShortNameFormatted(); 19 | public String getShortNameNormalized(); 20 | } 21 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/FileAnalyzer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Driver for the GUI version of the file analyzer with file modification rules disabled. 9 | * This application was originally created by Terry Brady in NARA's Digitization Services Branch. 10 | * @author TBrady 11 | * 12 | */ 13 | public class FileAnalyzer { 14 | 15 | public static void main(String[] args) { 16 | if (args.length > 0) 17 | new DirectoryTable(new File(args[0]),false); 18 | else 19 | new DirectoryTable(null,false); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/FileAnalyzerMod.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Driver for the GUI version of the file analyzer with file modification rules enabled. 9 | * This application was originally created by Terry Brady in NARA's Digitization Services Branch. 10 | * @author TBrady 11 | * 12 | */ 13 | public class FileAnalyzerMod { 14 | 15 | public static void main(String[] args) { 16 | if (args.length > 0) 17 | new DirectoryTable(new File(args[0]),true); 18 | else 19 | new DirectoryTable(null,true); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/DirSelect.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import javax.swing.JFileChooser; 4 | import javax.swing.JFrame; 5 | import javax.swing.JTextField; 6 | 7 | /** 8 | * Displays a directory selection dialog; results will be saved to a specified text field. 9 | * @author TBrady 10 | * 11 | */ 12 | class DirSelect extends FileSelect { 13 | private static final long serialVersionUID = 1L; 14 | 15 | public void configureChooser() { 16 | jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); 17 | } 18 | DirSelect(JFrame parent, JTextField result, String title) { 19 | super(parent, result, title); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/ImporterRegistry.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.util.Vector; 6 | 7 | /** 8 | * Activates the Importers that will be presented on the Import tab. 9 | * @author TBrady 10 | * 11 | */ 12 | public class ImporterRegistry extends Vector { 13 | 14 | private static final long serialVersionUID = 1L; 15 | 16 | public ImporterRegistry(FTDriver dt) { 17 | add(new TabSepFileImporter(dt)); 18 | add(new CsvFileImporter(dt)); 19 | add(new SemicolonFileImporter(dt)); 20 | add(new FileListImporter(dt)); 21 | add(new Parser(dt)); 22 | } 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/MyDirectoryFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | 6 | /** 7 | * Filter for identifying directories when traversing a directory tree 8 | * @author TBrady 9 | * 10 | */ 11 | public class MyDirectoryFilter implements FilenameFilter { 12 | boolean ignorePeriods; 13 | public MyDirectoryFilter(boolean ignorePeriods){ 14 | this.ignorePeriods = ignorePeriods; 15 | } 16 | public boolean accept(File dir, String filename) { 17 | 18 | if (ignorePeriods) { 19 | if (filename.contains(".")) return false; 20 | } 21 | return (new File(dir,filename)).isDirectory(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/crud/CRUDDetails.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.crud; 2 | 3 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 4 | /** 5 | * Stores the results of a database action. 6 | * @author TBrady 7 | * 8 | */ 9 | public class CRUDDetails { 10 | CRUD action; 11 | RenamePassFail status; 12 | int id; 13 | String note; 14 | public CRUDDetails(CRUD action, RenamePassFail status, int id, String note){ 15 | this.action = action; 16 | this.status = status; 17 | this.id = id; 18 | this.note = note; 19 | } 20 | public String toString() { 21 | return action.toString()+" "+ status.toString() +". ID=" + id+". "+((note==null)?"":note); 22 | } 23 | } 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/NameStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Stats object showing file name and file size. 9 | * @author TBrady 10 | * 11 | */ 12 | public class NameStats extends Stats { 13 | public static Object[][] details = { 14 | {String.class,"Name",250}, 15 | {Long.class,"Size",100}, 16 | }; 17 | 18 | 19 | public NameStats(String key) { 20 | super(key); 21 | vals.add(new Long(0)); 22 | } 23 | 24 | public Object compute(File f, FileTest fileTest) { 25 | Long size = f.length(); 26 | vals.set(0, size.longValue()+1); 27 | return fileTest.fileTest(f); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/FileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Contract that a File Test Filter must support 5 | * @author TBrady 6 | * 7 | */ 8 | public interface FileTestFilter { 9 | public String getName(); 10 | 11 | public String getPrefix(); 12 | public String getContains(); 13 | public String getSuffix(); 14 | public String getExclusion(); 15 | 16 | public boolean isRePrefix(); 17 | public boolean isReContains(); 18 | public boolean isReSuffix(); 19 | public boolean isReExclusion(); 20 | 21 | public String getShortName(); 22 | public String getShortNameFormatted(); 23 | public String getShortNameNormalized(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/CountStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Status objects that counts items by key. 9 | * @author TBrady 10 | * 11 | */ 12 | public class CountStats extends Stats { 13 | public static Object[][] details = { 14 | {String.class,"Type",100}, 15 | {Long.class,"Count",100}, 16 | }; 17 | 18 | 19 | public CountStats(String key) { 20 | super(key); 21 | vals.add(new Long(0)); 22 | } 23 | 24 | public Object compute(File f, FileTest fileTest) { 25 | Long count = (Long)vals.get(0); 26 | vals.set(0, count.longValue()+1); 27 | return fileTest.fileTest(f); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ftprop/FTProp.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import javax.swing.JComponent; 4 | 5 | /** 6 | * Contract that a File Test Property must satisfy in both the GUI and Command Line form of the File Analyzer 7 | * @author TBrady 8 | * 9 | */ 10 | public interface FTProp { 11 | public String getName(); 12 | public String describe(); 13 | public JComponent getEditor(); 14 | public Object getDefault(); 15 | public Object validate(String s); 16 | public Object getValue(); 17 | public void setValue(Object obj); 18 | 19 | public String getShortName(); 20 | public String getShortNameFormatted(); 21 | public String getShortNameNormalized(); 22 | public String describeFormatted(); 23 | } 24 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/Stats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | import java.util.Vector; 5 | 6 | import gov.nara.nwts.ftapp.filetest.FileTest; 7 | 8 | /** 9 | * Base class for Stats objects containing a key and a variable lenght list of values. 10 | * @author TBrady 11 | * 12 | */ 13 | public class Stats { 14 | public static Object[][] details = { 15 | {String.class,"Type",100}, 16 | }; 17 | 18 | public Vector vals; 19 | public String key; 20 | 21 | public Stats(String key) { 22 | this.key = key; 23 | vals = new Vector(); 24 | } 25 | 26 | public Object compute(File f, FileTest fileTest) { 27 | Object o = fileTest.fileTest(f); 28 | return o; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/filetest/ImageActionRegistry.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 5 | 6 | /** 7 | * Initialize the File Analzyer with generic image processing rules (but not NARA specific business rules) 8 | * @author TBrady 9 | * 10 | */ 11 | public class ImageActionRegistry extends ActionRegistry { 12 | 13 | private static final long serialVersionUID = 1L; 14 | 15 | public ImageActionRegistry(FTDriver dt, boolean modifyAllowed) { 16 | super(dt, modifyAllowed); 17 | //add(new CountTiff(dt)); 18 | //add(new CountJpeg(dt)); 19 | add(new GenericImageProperties(dt)); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/CsvFileImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | /** 6 | * Importer for comma delimited files 7 | * @author TBrady 8 | * 9 | */ 10 | public class CsvFileImporter extends DelimitedFileImporter { 11 | 12 | public CsvFileImporter(FTDriver dt) { 13 | super(dt); 14 | } 15 | 16 | public String getSeparator() { 17 | return ","; 18 | } 19 | 20 | public String toString() { 21 | return "Import Comma-Separated File"; 22 | } 23 | public String getDescription() { 24 | return "This rule will import a comma separated file and use the first column as a unique key"; 25 | } 26 | public String getShortName() { 27 | return "CSV"; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/TabSepFileImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | /** 6 | * Importer for tab delimited files 7 | * @author TBrady 8 | * 9 | */ 10 | public class TabSepFileImporter extends DelimitedFileImporter { 11 | 12 | public TabSepFileImporter(FTDriver dt) { 13 | super(dt); 14 | } 15 | 16 | public String getSeparator() { 17 | return "\t"; 18 | } 19 | public String toString() { 20 | return "Import Tab-Separated File"; 21 | } 22 | public String getDescription() { 23 | return "This rule will import a tab separated file and use the first column as a unique key"; 24 | } 25 | public String getShortName() { 26 | return "TAB"; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /codeinventory.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "NARA File Analyzer and Metadata Harvester", 3 | "description": "Software to walk a directory tree and perform a \"File Test\" on each file that is encountered. The application framework allows new File Tests to be quickly developed and deployed into the application. The results of each File Test are compiled into a table that summarizes the results of the analysis.", 4 | "license": "https://github.com/usnationalarchives/File-Analyzer/commit/7c2aefc27dec2ab9c2f49d644c4b55cbe67657ab", 5 | "openSourceProject": 1, 6 | "governmentWideReuseProject": 1, 7 | "tags": [ 8 | "NARA", 9 | "FileTest", 10 | ""Java", 11 | Metadata" 12 | ], 13 | "contact": { 14 | "email": "leslie.johnston@nara.gov" 15 | } 16 | } -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/SemicolonFileImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | /** 6 | * Importer for semicolon delimited files 7 | * @author TBrady 8 | * 9 | */ 10 | public class SemicolonFileImporter extends DelimitedFileImporter { 11 | 12 | public SemicolonFileImporter(FTDriver dt) { 13 | super(dt); 14 | } 15 | 16 | public String getSeparator() { 17 | return ";"; 18 | } 19 | 20 | public String toString() { 21 | return "Import Semicolon-Separated File"; 22 | } 23 | public String getDescription() { 24 | return "This rule will import a semicolon separated file and use the first column as a unique key"; 25 | } 26 | public String getShortName() { 27 | return "SEMI"; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/DataStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Stats object that generically displays a key and data value. When using this Stats object, the assumption is that the FileTest will customize the column headers. 9 | * @author TBrady 10 | * 11 | */ 12 | public class DataStats extends Stats { 13 | public static Object[][] details = { 14 | {String.class,"Key",200}, 15 | {Object.class,"Data",300}, 16 | }; 17 | 18 | 19 | public DataStats(String key) { 20 | super(key); 21 | vals.add(""); 22 | } 23 | 24 | public Object compute(File f, FileTest fileTest) { 25 | Object o = fileTest.fileTest(f); 26 | vals.set(0, o); 27 | return o; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/NumericStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Stats object showing a key and a numeric value. 9 | * This was used in one of the Census rules, it may not have much additional use. 10 | * @author TBrady 11 | * 12 | */ 13 | public class NumericStats extends Stats { 14 | static Object[][] details = { 15 | {String.class,"Key",100}, 16 | {Long.class,"Computed Count",100}, 17 | }; 18 | 19 | 20 | public NumericStats(String key) { 21 | super(key); 22 | vals.add(new Long(0)); 23 | } 24 | 25 | public Object compute(File f, FileTest fileTest) { 26 | Object count = fileTest.fileTest(f); 27 | vals.set(0, count); 28 | return count; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/FileCountStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Stats object showing file counts and file sizes. 9 | * @author TBrady 10 | * 11 | */ 12 | public class FileCountStats extends CountStats { 13 | public static Object[][] details = { 14 | {String.class,"Type",100}, 15 | {Long.class,"Count",100}, 16 | {Long.class,"Size",100}, 17 | }; 18 | 19 | 20 | public FileCountStats(String key) { 21 | super(key); 22 | vals.add(new Long(0)); 23 | } 24 | 25 | public Object compute(File f, FileTest fileTest) { 26 | Object ret = super.compute(f, fileTest); 27 | Long bytes = (Long)vals.get(1); 28 | vals.set(1, bytes.longValue()+f.length()); 29 | return ret; 30 | } 31 | 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/package.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

The results of each {@link gov.nara.nwts.ftapp.filetest.FileTest} can be highly customized to meet a specific business need.

4 |

A companion to each Stats object is a Stats description object implemented as Object[][]. The first dimension correlates to a column of output that will be displayed for each Stats object. The second dimension is used as follows:

5 |
    6 |
  • Class of the object that will be displayed. The class will most commonly be a String, Int, Long, Date or Enum.
    • 7 |
  • Column header to display
    • 8 |
  • Width of the column
    • 9 |
  • Filter values (in the GUI, a drop down will be created if present)
    • 10 |
11 |

Flag to indicate whether or not to save the column by default when exporting results.

12 | 13 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/DirSelectChooser.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.event.ActionEvent; 4 | import java.util.prefs.Preferences; 5 | 6 | import javax.swing.JFrame; 7 | 8 | /** 9 | * Widget that will display a chosen directory and will open a separate window allowing a directory to be selected. 10 | * @author TBrady 11 | * 12 | */ 13 | class DirSelectChooser extends FileSelectChooser { 14 | private static final long serialVersionUID = 1L; 15 | 16 | DirSelectChooser(JFrame parent, String title, String def) { 17 | this(parent, title, null, null, def); 18 | } 19 | DirSelectChooser(JFrame parent, String title, Preferences p, String key, String def) { 20 | super(parent, title, p, key, def); 21 | } 22 | 23 | public void actionPerformed(ActionEvent arg0) { 24 | new DirSelect(parent, tf, title); 25 | } 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/ValidPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#VALID} if true. 9 | * @author TBrady 10 | * 11 | */ 12 | public class ValidPattern extends NameValidationPattern { 13 | 14 | public ValidPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public ValidPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.VALID); 19 | } 20 | 21 | public File getNewFile(File f, Matcher m) { 22 | return null; 23 | } 24 | public String getMessage(File f, Matcher m) { 25 | return ""; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/RandomStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | import java.util.Random; 5 | 6 | import gov.nara.nwts.ftapp.filetest.FileTest; 7 | 8 | /** 9 | * Generate a random value for each file in a set. 10 | * @author TBrady 11 | * 12 | */ 13 | public class RandomStats extends Stats { 14 | 15 | long randomVal; 16 | Random random; 17 | 18 | public RandomStats(String key) { 19 | super(key); 20 | random = new Random(); 21 | } 22 | 23 | public Object compute(File f, FileTest fileTest) { 24 | randomVal = random.nextLong(); 25 | if (fileTest instanceof Randomizer) { 26 | Randomizer r = (Randomizer)fileTest; 27 | while(r.getTreeSet().get(randomVal) != null) { 28 | randomVal = random.nextLong(); 29 | } 30 | r.getTreeSet().put(randomVal, key); 31 | } 32 | return randomVal; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/CRUDStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 6 | import gov.nara.nwts.ftapp.crud.CRUD; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Stats object that reports on database ingest actions. 12 | * @author TBrady 13 | * 14 | */ 15 | public class CRUDStats extends Stats { 16 | 17 | public static Object[][] details = { 18 | {String.class,"Path",450}, 19 | {String.class,"Pass/Fail",50,RenamePassFail.values()}, 20 | {String.class,"Status",150, CRUD.values()}, 21 | }; 22 | 23 | 24 | public CRUDStats(String key) { 25 | super(key); 26 | vals.add(""); 27 | vals.add(""); 28 | } 29 | 30 | public Object compute(File f, FileTest fileTest) { 31 | Object ret = fileTest.fileTest(f); 32 | return ret; 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/InvalidManualPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#INVALID_MANUAL} if true indicating that a file cannot be programmatically renamed. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class InvalidManualPattern extends NameValidationPattern { 13 | 14 | public InvalidManualPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public InvalidManualPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.INVALID_MANUAL); 19 | } 20 | 21 | public File getNewFile(File f, Matcher m) { 22 | return null; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/NameMD5Checksum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.security.MessageDigest; 6 | import java.security.NoSuchAlgorithmException; 7 | 8 | /** 9 | * Generate MD5 checksums for a set of files. 10 | * @author TBrady 11 | * 12 | */ 13 | class NameMD5Checksum extends NameChecksum { 14 | 15 | public NameMD5Checksum(FTDriver dt) { 16 | super(dt); 17 | } 18 | 19 | public String toString() { 20 | return "Get MD5 Checksum By Name"; 21 | } 22 | public String getShortName(){return "MD5 ";} 23 | 24 | public MessageDigest getMessageDigest() throws NoSuchAlgorithmException { 25 | return MessageDigest.getInstance("MD5"); 26 | } 27 | 28 | public String getDescription() { 29 | return "This test reports the MD5 checksum for a given filename.\nNote, the checksum will be overwritten if the file is found more than once."; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/NameSha1Checksum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import java.security.MessageDigest; 5 | import java.security.NoSuchAlgorithmException; 6 | 7 | /** 8 | * Generate SHA1 checksums for a set of files. 9 | * @author TBrady 10 | * 11 | */ 12 | class NameSha1Checksum extends NameChecksum { 13 | 14 | public NameSha1Checksum(FTDriver dt) { 15 | super(dt); 16 | } 17 | 18 | public String toString() { 19 | return "Get SHA1 Checksum By Name"; 20 | } 21 | 22 | public String getShortName(){return "SHA1";} 23 | 24 | public MessageDigest getMessageDigest() throws NoSuchAlgorithmException { 25 | return MessageDigest.getInstance("SHA1"); 26 | } 27 | 28 | public String getDescription() { 29 | return "This test reports the SHA1 checksum for a given filename.\nNote, the checksum will be overwritten if the file is found more than once."; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/LowercaseTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | 6 | import gov.nara.nwts.ftapp.FTDriver; 7 | import gov.nara.nwts.ftapp.nameValidation.RenameablePattern; 8 | import gov.nara.nwts.ftapp.nameValidation.ValidPattern; 9 | /** 10 | * Filename validation rule to ensure that filenames are lowercase. 11 | * @author TBrady 12 | * 13 | */ 14 | class LowercaseTest extends NameValidationTest { 15 | 16 | public LowercaseTest(FTDriver dt, FileTest nextTest) { 17 | super(dt, new ValidPattern("^[^A-Z]*$", false),nextTest, "Lowercase","Lowercase"); 18 | testPatterns.add(new RenameablePattern(".*", false){ 19 | public String getMessage(File f, Matcher m) { 20 | return ""; 21 | } 22 | 23 | public File getNewFile(File f, Matcher m) { 24 | return new File(f.getParentFile(), f.getName().toLowerCase()); 25 | } 26 | 27 | }); 28 | } 29 | 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/NameSha256Checksum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import java.security.MessageDigest; 5 | import java.security.NoSuchAlgorithmException; 6 | 7 | /** 8 | * Generate SHA256 checksums for a set of files. 9 | * @author TBrady 10 | * 11 | */ 12 | class NameSha256Checksum extends NameChecksum { 13 | 14 | public NameSha256Checksum(FTDriver dt) { 15 | super(dt); 16 | } 17 | 18 | public String toString() { 19 | return "Get SHA-256 Checksum By Name"; 20 | } 21 | 22 | public String getShortName(){return "SHA-256";} 23 | 24 | public MessageDigest getMessageDigest() throws NoSuchAlgorithmException { 25 | return MessageDigest.getInstance("SHA-256"); 26 | } 27 | 28 | public String getDescription() { 29 | return "This test reports the SHA1 checksum for a given filename.\nNote, the checksum will be overwritten if the file is found more than once."; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/CountAppendStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Stats class that accumulates counts as well as accumulates notes into a text buffer as processing continues. 9 | * When using a checksum algorithm to find duplicate values, this routine will provide details about instances in which more than one file share the same checksum. 10 | * @author TBrady 11 | * 12 | */ 13 | public class CountAppendStats extends CountStats { 14 | public static Object[][] details = { 15 | {String.class,"Type",100}, 16 | {Long.class,"Count",100}, 17 | {String.class,"Details",2000}, 18 | }; 19 | 20 | 21 | public CountAppendStats(String key) { 22 | super(key); 23 | vals.add(""); 24 | } 25 | 26 | public Object compute(File f, FileTest fileTest) { 27 | vals.set(1, vals.get(1) + f.getAbsolutePath()+"; "); 28 | return super.compute(f, fileTest); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/DefaultImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | /** 10 | * Abstract base class for importer behaviors. 11 | * @author TBrady 12 | * 13 | */ 14 | public abstract class DefaultImporter implements Importer { 15 | protected FTDriver dt; 16 | public DefaultImporter(FTDriver dt) { 17 | this.dt = dt; 18 | } 19 | 20 | public abstract String toString(); 21 | public abstract ActionResult importFile(File selectedFile) throws IOException; 22 | public boolean allowForceKey() { 23 | return false; 24 | } 25 | public String getShortNameNormalized() { 26 | return getShortName().replaceAll("[\\s&]",""); 27 | } 28 | public String getShortNameFormatted() { 29 | StringBuffer buf = new StringBuffer(); 30 | buf.append(getShortNameNormalized()); 31 | buf.append(" "); 32 | return buf.substring(0,20); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ResultFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.util.Vector; 4 | 5 | /** 6 | * Container for the various filters that may be applied to a set of results allowing results to be filtered by one or more columns. 7 | * @author TBrady 8 | * 9 | */ 10 | public class ResultFilter { 11 | class ResultFilterItem { 12 | int col; 13 | String val; 14 | 15 | ResultFilterItem(int col, String val) { 16 | this.col = col; 17 | this.val = val; 18 | } 19 | } 20 | Vector resfilters; 21 | public ResultFilter() { 22 | resfilters = new Vector(); 23 | } 24 | 25 | public void add(int col, String val) { 26 | resfilters.add(new ResultFilterItem(col, val)); 27 | } 28 | 29 | public boolean evaluate(Vector obj) { 30 | for(ResultFilterItem rfi: resfilters){ 31 | if (rfi.col < obj.size()) { 32 | Object o = obj.get(rfi.col); 33 | if (o == null) o = ""; 34 | if (!rfi.val.equals(o.toString())){ 35 | return false; 36 | } 37 | } 38 | } 39 | return true; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/RenameablePattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * A name validation pattern which will return {@link RenameStatus#RENAMABLE} if true indicating that the file could be automatically renamed. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class RenameablePattern extends NameValidationPattern { 13 | 14 | public RenameablePattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public RenameablePattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.RENAMABLE); 19 | } 20 | public RenameDetails report(File f, Matcher m) { 21 | File nf = getNewFile(f, m); 22 | if (nf == null) { 23 | return new RenameDetails(RenameStatus.NEW_NAME_INVALID, nf, getMessage(f, m)); 24 | } 25 | return new RenameDetails(status, nf, getMessage(f, m)); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/ImageFileAnalyzer.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.ActionRegistry; 6 | import gov.nara.nwts.ftapp.gui.DirectoryTable; 7 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 8 | import gov.nara.nwts.ftappImg.filetest.ImageActionRegistry; 9 | /** 10 | * Driver for the File Analyzer GUI loading image-specific rules but not NARA specific rules. 11 | * @author TBrady 12 | * 13 | */ 14 | public class ImageFileAnalyzer extends DirectoryTable { 15 | 16 | public ImageFileAnalyzer(File f, boolean modifyAllowed) { 17 | super(f, modifyAllowed); 18 | } 19 | 20 | protected ActionRegistry getActionRegistry() { 21 | return new ImageActionRegistry(this, modifyAllowed); 22 | } 23 | 24 | protected ImporterRegistry getImporterRegistry() { 25 | return new ImporterRegistry(this); 26 | } 27 | public static void main(String[] args) { 28 | if (args.length > 0) 29 | new ImageFileAnalyzer(new File(args[0]), false); 30 | else 31 | new ImageFileAnalyzer(null, false); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/CustomPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Abstract class handling an individual name validation check that requires custom coding beyond a regular expression match. 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class CustomPattern extends NameValidationPattern { 13 | 14 | public CustomPattern(String pattern, boolean checkPath){ 15 | this(pattern, checkPath, 0); 16 | } 17 | public CustomPattern(String pattern, boolean checkPath, int pattFlags){ 18 | super(Pattern.compile(pattern, pattFlags), checkPath, RenameStatus.NEXT); 19 | } 20 | public CustomPattern(Pattern pattern, boolean checkPath){ 21 | super(pattern, checkPath, RenameStatus.NEXT); 22 | } 23 | 24 | public File getNewFile(File f, Matcher m) { 25 | return null; 26 | } 27 | public String getMessage(File f, Matcher m) { 28 | return ""; 29 | } 30 | public abstract RenameDetails report(File f, Matcher m); 31 | } 32 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/CountByType.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.FileCountStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | /** 9 | * Count items by file extension; this is the most basic and easy to understand File Analyzer rule. 10 | * @author TBrady 11 | * 12 | */ 13 | class CountByType extends DefaultFileTest { 14 | 15 | public CountByType(FTDriver dt) { 16 | super(dt); 17 | } 18 | 19 | public Object fileTest(File f) { 20 | return null; 21 | } 22 | 23 | public String toString() { 24 | return "Count Files By Type"; 25 | } 26 | 27 | public Stats createStats(String key){ 28 | return new FileCountStats(key); 29 | } 30 | public Object[][] getStatsDetails() { 31 | return FileCountStats.details; 32 | } 33 | public String getShortName(){return "By Type";} 34 | public void initFilters() { 35 | initAllFilters(); 36 | } 37 | 38 | public String getDescription() { 39 | return "This test counts the number of files found by file extension."; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/NameMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.NameStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | /** 9 | * Match files by file name (with file extension); this can be used to verify the completeness of a copy operation. 10 | * @author TBrady 11 | * 12 | */ 13 | class NameMatch extends DefaultFileTest { 14 | 15 | public NameMatch(FTDriver dt) { 16 | super(dt); 17 | } 18 | 19 | public String toString() { 20 | return "Match By Name"; 21 | } 22 | public String getKey(File f) { 23 | return f.getName(); 24 | } 25 | 26 | public String getShortName(){return "Name";} 27 | 28 | public Object fileTest(File f) { 29 | return null; 30 | } 31 | public Stats createStats(String key){ 32 | return new NameStats(key); 33 | } 34 | public Object[][] getStatsDetails() { 35 | return NameStats.details; 36 | } 37 | public void initFilters() { 38 | initAllFilters(); 39 | } 40 | 41 | public String getDescription() { 42 | return "This test reports on file size by name regardless of the directory in a file name is found."; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filter/DefaultFileTestFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filter; 2 | 3 | /** 4 | * Default filter which accepts all files 5 | * @author TBrady 6 | * 7 | */ 8 | public class DefaultFileTestFilter implements FileTestFilter { 9 | 10 | public String getName() { 11 | return "All Files"; 12 | } 13 | public String getContains() { 14 | return ""; 15 | } 16 | public String getPrefix() { 17 | return ""; 18 | } 19 | 20 | public String getExclusion() { 21 | return ""; 22 | } 23 | 24 | public String getSuffix() { 25 | return ""; 26 | } 27 | public boolean isRePrefix() { 28 | return false; 29 | } 30 | public boolean isReContains() { 31 | return false; 32 | } 33 | public boolean isReSuffix() { 34 | return false; 35 | } 36 | public boolean isReExclusion() { 37 | return false; 38 | } 39 | 40 | public String getShortName() { 41 | return getName(); 42 | } 43 | 44 | public String getShortNameNormalized() { 45 | return getShortName().replaceAll("[\\s&]",""); 46 | } 47 | public String getShortNameFormatted() { 48 | StringBuffer buf = new StringBuffer(); 49 | buf.append(getShortNameNormalized()); 50 | buf.append(" "); 51 | return buf.substring(0,20); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/tags/DefaultExtractor.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.tags; 2 | 3 | import java.util.HashMap; 4 | 5 | import gov.nara.nwts.ftappImg.tags.ImageTags.DUP; 6 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGS; 7 | 8 | /** 9 | * Abstract base class for image extractors 10 | * @author TBrady 11 | * 12 | */ 13 | public abstract class DefaultExtractor implements Extractor { 14 | 15 | public abstract void close(); 16 | public String getString(DUP dup) { 17 | return getString(dup, false); 18 | } 19 | public String getString(DUP dup, boolean getFirst) { 20 | HashMap tagvals = new HashMap(); 21 | HashMap vals = new HashMap(); 22 | for(TAGS ctag: dup.duptags) { 23 | String s = getString(ctag); 24 | if (s == null) continue; 25 | s = s.trim(); 26 | if (s.equals("")) continue; 27 | if (getFirst) return s; 28 | tagvals.put(ctag,s); 29 | Integer i = vals.get(s); 30 | if (i == null) { 31 | vals.put(s, 1); 32 | } else { 33 | vals.put(s, i+1); 34 | } 35 | } 36 | if (vals.size() == 0) return ""; 37 | String first = vals.keySet().iterator().next(); 38 | if (vals.size() == 1) { 39 | return first; 40 | } 41 | return "["+vals.size()+"] " + first; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/NameValidationPattern.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Abstract class containing a regular expression pattern that will be used to perform file name validation 9 | * @author TBrady 10 | * 11 | */ 12 | public abstract class NameValidationPattern { 13 | Pattern pattern; 14 | RenameStatus status; 15 | boolean checkPath; 16 | 17 | public NameValidationPattern(Pattern pattern, boolean checkPath, RenameStatus status){ 18 | this.pattern = pattern; 19 | this.status = status; 20 | this.checkPath = checkPath; 21 | } 22 | 23 | public RenameDetails checkFile(File f) { 24 | String s = checkPath ? f.getAbsolutePath() : f.getName(); 25 | Matcher m = pattern.matcher(s); 26 | if (m.matches()) { 27 | return report(f, m); 28 | } 29 | return new RenameDetails(RenameStatus.NEXT, null, ""); 30 | } 31 | 32 | public RenameDetails report(File f, Matcher m) { 33 | return new RenameDetails(status, getNewFile(f, m), getMessage(f, m)); 34 | } 35 | 36 | public abstract File getNewFile(File f, Matcher m); 37 | 38 | public abstract String getMessage(File f, Matcher m); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/DirStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Statistics showing accumulated file counts within a directory structure. 9 | * @author TBrady 10 | * 11 | */ 12 | public class DirStats extends Stats { 13 | public static Object[][] details = { 14 | {String.class,"Key",300}, 15 | {Long.class,"Count",100}, 16 | {Long.class,"Cumulative Count",100}, 17 | }; 18 | 19 | 20 | public DirStats(String key) { 21 | super(key); 22 | vals.add(new Long(0)); 23 | vals.add(new Long(0)); 24 | } 25 | 26 | public Object compute(File f, FileTest fileTest) { 27 | File root = fileTest.getRoot(); 28 | for(File ftest = f.getParentFile(); ftest!=null; ftest = ftest.getParentFile()){ 29 | DirStats stats = (DirStats)fileTest.getStats(fileTest.getKey(f,ftest)); 30 | stats.accumulate(f, fileTest, ftest); 31 | if (ftest.equals(root)){ 32 | break; 33 | } 34 | } 35 | return fileTest.fileTest(f); 36 | } 37 | 38 | public void accumulate(File f, FileTest fileTest, File parentdir) { 39 | Long count = (Long)vals.get(0); 40 | if (f.getParentFile().equals(parentdir)){ 41 | vals.set(0, count.longValue()+1); 42 | } 43 | count = (Long)vals.get(1); 44 | vals.set(1, count.longValue()+1); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/BaseNameMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.NameStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | /** 9 | * Match files based on the base file name (without a file extension). 10 | * @author TBrady 11 | * 12 | */ 13 | class BaseNameMatch extends DefaultFileTest { 14 | 15 | public BaseNameMatch(FTDriver dt) { 16 | super(dt); 17 | } 18 | 19 | public String toString() { 20 | return "Match By Base Name"; 21 | } 22 | public String getKey(File f) { 23 | String s = f.getName(); 24 | String[] sa = s.split("\\."); 25 | if (sa.length > 1) s = s.substring(0,s.length()-sa[sa.length-1].length()-1); 26 | return s; 27 | } 28 | 29 | public String getShortName(){return "Base Name";} 30 | 31 | public Object fileTest(File f) { 32 | return null; 33 | } 34 | public Stats createStats(String key){ 35 | return new NameStats(key); 36 | } 37 | public Object[][] getStatsDetails() { 38 | return NameStats.details; 39 | } 40 | public void initFilters() { 41 | initAllFilters(); 42 | } 43 | 44 | public String getDescription() { 45 | return "This test reports on file size by base name (no extension) regardless of the directory in a file name is found."; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/filetest/CountJpeg.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filetest.DefaultFileTest; 5 | import gov.nara.nwts.ftapp.filter.JpegFileTestFilter; 6 | import gov.nara.nwts.ftappImg.stats.JpegStats; 7 | import gov.nara.nwts.ftapp.stats.Stats; 8 | 9 | import java.io.File; 10 | 11 | /** 12 | * Extract key metadata from a JPG file; this implementation has been superceeded by better implementations 13 | * @author TBrady 14 | * 15 | */ 16 | class CountJpeg extends DefaultFileTest { 17 | 18 | public CountJpeg(FTDriver dt) { 19 | super(dt); 20 | } 21 | 22 | public String toString() { 23 | return "Jpeg Properties"; 24 | } 25 | public String getKey(File f) { 26 | return f.getName(); 27 | } 28 | 29 | public String getShortName(){return "Jpeg";} 30 | 31 | public Object fileTest(File f) { 32 | return null; 33 | } 34 | public Stats createStats(String key){ 35 | return new JpegStats(key); 36 | } 37 | public Object[][] getStatsDetails() { 38 | return JpegStats.details; 39 | } 40 | 41 | public void initFilters() { 42 | filters.add(new JpegFileTestFilter()); 43 | } 44 | 45 | public String getDescription() { 46 | return "This test will extract key metadata items from each Jpeg file that is found"; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/filetest/CountTiff.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.filetest.DefaultFileTest; 5 | import gov.nara.nwts.ftapp.filter.TiffFileTestFilter; 6 | import gov.nara.nwts.ftappImg.stats.ImageStats; 7 | import gov.nara.nwts.ftapp.stats.Stats; 8 | 9 | import java.io.File; 10 | 11 | /** 12 | * Extract key metadata from a TIF file; this implementation has been superceeded by better implementations 13 | * @author TBrady 14 | * 15 | */ 16 | class CountTiff extends DefaultFileTest { 17 | 18 | public CountTiff(FTDriver dt) { 19 | super(dt); 20 | } 21 | 22 | public String toString() { 23 | return "Tif Properties"; 24 | } 25 | public String getKey(File f) { 26 | return f.getName(); 27 | } 28 | 29 | public String getShortName(){return "Tif";} 30 | 31 | public Object fileTest(File f) { 32 | return null; 33 | } 34 | public Stats createStats(String key){ 35 | return new ImageStats(key); 36 | } 37 | public Object[][] getStatsDetails() { 38 | return ImageStats.details; 39 | } 40 | 41 | public void initFilters() { 42 | filters.add(new TiffFileTestFilter()); 43 | } 44 | 45 | public String getDescription() { 46 | return "This test will extract key metadata items from each TIF file that is found"; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/ByMD5Checksum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.CountAppendStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | import java.security.MessageDigest; 9 | import java.security.NoSuchAlgorithmException; 10 | 11 | /** 12 | * List files by MD5 checksum; this can be used to identify potentially duplicate digital objects. 13 | * @author TBrady 14 | * 15 | */ 16 | class ByMD5Checksum extends NameChecksum { 17 | 18 | public ByMD5Checksum(FTDriver dt) { 19 | super(dt); 20 | } 21 | public String getKey(File f) { 22 | return getChecksum(f); 23 | } 24 | public Object fileTest(File f) { 25 | return f.getAbsolutePath(); 26 | } 27 | public Stats createStats(String key){ 28 | return new CountAppendStats(key); 29 | } 30 | public Object[][] getStatsDetails() { 31 | return CountAppendStats.details; 32 | } 33 | 34 | public String toString() { 35 | return "By MD5 Checksum"; 36 | } 37 | public String getShortName(){return "By MD5";} 38 | 39 | public MessageDigest getMessageDigest() throws NoSuchAlgorithmException { 40 | return MessageDigest.getInstance("MD5"); 41 | } 42 | 43 | public String getDescription() { 44 | return "Lists files by MD5 checksum.\nNote, the checksum will be overwritten if the file is found more than once."; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | As a work of the United States Government, this project is in the 2 | public domain within the United States. 3 | 4 | Additionally, we waive copyright and related rights in the work 5 | worldwide through the CC0 1.0 Universal public domain dedication. 6 | 7 | ## CC0 1.0 Universal Summary 8 | 9 | This is a human-readable summary of the 10 | [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). 11 | 12 | ### No Copyright 13 | 14 | The person who associated a work with this deed has dedicated the work to 15 | the public domain by waiving all of his or her rights to the work worldwide 16 | under copyright law, including all related and neighboring rights, to the 17 | extent allowed by law. 18 | 19 | You can copy, modify, distribute and perform the work, even for commercial 20 | purposes, all without asking permission. 21 | 22 | ### Other Information 23 | 24 | In no way are the patent or trademark rights of any person affected by CC0, 25 | nor are the rights that other persons may have in the work or in how the 26 | work is used, such as publicity or privacy rights. 27 | 28 | Unless expressly stated otherwise, the person who associated a work with 29 | this deed makes no warranties about the work, and disclaims liability for 30 | all uses of the work, to the fullest extent permitted by applicable law. 31 | When using or citing the work, you should not imply endorsement by the 32 | author or the affirmer. 33 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/RenameDetails.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | 5 | public class RenameDetails { 6 | public RenameStatus status; 7 | File newFile; 8 | public String note; 9 | public RenameDetails(RenameStatus status, File newFile){ 10 | this(status,newFile,null); 11 | } 12 | public RenameDetails(RenameStatus status){ 13 | this(status,null,null); 14 | } 15 | public RenameDetails(RenameStatus status, File newFile, String note){ 16 | this.status = status; 17 | this.newFile = newFile; 18 | this.note = note; 19 | } 20 | public RenamePassFail getPassFail() {return status.getPassFail();} 21 | public String getMessage() { 22 | StringBuffer buf = new StringBuffer(); 23 | if (note!=null) { 24 | buf.append(note); 25 | buf.append(" "); 26 | } 27 | buf.append(status.getMessage()); 28 | return buf.toString(); 29 | } 30 | public String getRenameStatus() {return status.toString();} 31 | public File getFile() {return newFile;} 32 | public String getDetailNote(File root){ 33 | if (newFile == null) { 34 | return (note==null)?"":note; 35 | } 36 | String s = newFile.getAbsolutePath(); 37 | String r = root.getAbsolutePath(); 38 | String m = s; 39 | if (s.startsWith(r)) { 40 | m = s.substring(r.length()); 41 | } 42 | return m; 43 | } 44 | public String toString() { 45 | return status.toString() +": " + ((note==null)?"":note); 46 | } 47 | 48 | } 49 | 50 | 51 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/DirMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.DirStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | 9 | /** 10 | * Create FileAnalyzer statistics by directory. 11 | * @author TBrady 12 | * 13 | */ 14 | class DirMatch extends DefaultFileTest { 15 | 16 | public DirMatch(FTDriver dt) { 17 | super(dt); 18 | } 19 | 20 | public String toString() { 21 | return "Match By Path"; 22 | } 23 | public String getKey(File f) { 24 | return getKey(f, f.getParentFile()); 25 | } 26 | 27 | public String getKey(File f, Object parentdir) { 28 | String key = ""; 29 | if (parentdir instanceof File) { 30 | key = ((File)parentdir).getAbsolutePath().substring(getRoot().getAbsolutePath().length()); 31 | } 32 | return key; 33 | } 34 | 35 | public String getShortName(){return "Path";} 36 | 37 | public Object fileTest(File f) { 38 | return null; 39 | } 40 | public Stats createStats(String key){ 41 | return new DirStats(key); 42 | } 43 | public Object[][] getStatsDetails() { 44 | return DirStats.details; 45 | } 46 | public void initFilters() { 47 | initAllFilters(); 48 | } 49 | 50 | public String getDescription() { 51 | return "This test counts the number of items found in a specific directory. This test will also compute cumulative totals found for each directory that is scanned."; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/NameValidationStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | import gov.nara.nwts.ftapp.nameValidation.RenameDetails; 5 | import gov.nara.nwts.ftapp.nameValidation.RenameStatus; 6 | import gov.nara.nwts.ftapp.nameValidation.RenamePassFail; 7 | 8 | import java.io.File; 9 | 10 | /** 11 | * Stats object displaying the results of a filename test. 12 | * @linkplain gov.nara.nwts.ftapp.filetest.NameValidationTest} contains the base logic that makes use of this Stats object. 13 | * @author TBrady 14 | * 15 | */ 16 | public class NameValidationStats extends Stats { 17 | 18 | public static Object[][] details = { 19 | {String.class,"Path",450}, 20 | {String.class,"Pass/Fail",50,RenamePassFail.values()}, 21 | {String.class,"Status",150,RenameStatus.values()}, 22 | {String.class,"Message",250}, 23 | {Object.class,"Recommended Path",450}, 24 | }; 25 | 26 | 27 | public NameValidationStats(String key) { 28 | super(key); 29 | vals.add(""); 30 | vals.add(""); 31 | vals.add(""); 32 | vals.add(""); 33 | } 34 | 35 | public Object compute(File f, FileTest fileTest) { 36 | Object ret = fileTest.fileTest(f); 37 | if (ret instanceof RenameDetails) { 38 | RenameDetails rdet = (RenameDetails)ret; 39 | vals.set(0, rdet.getPassFail()); 40 | vals.set(1, rdet.getRenameStatus()); 41 | vals.set(2, rdet.getMessage()); 42 | vals.set(3, rdet.getDetailNote(fileTest.getRoot())); 43 | } 44 | return ret; 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/MyProgress.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | /** 4 | * Helper class that reports on File Analyzer progress (on the Progress Tab) 5 | * @author TBrady 6 | * 7 | */ 8 | class MyProgress { 9 | int dircount; 10 | int lastdircount; 11 | int rptgap = 1; 12 | int rptgapmult = 5; 13 | int dirfound = 0; 14 | boolean processing; 15 | GuiFileTraversal gft; 16 | DirectoryTable dt; 17 | 18 | public MyProgress(GuiFileTraversal gft) { 19 | this.gft = gft; 20 | dt = gft.gftSW.dt; 21 | } 22 | 23 | public void resetDirCount() { 24 | dircount = 0; 25 | lastdircount = 0; 26 | rptgap = 1; 27 | } 28 | 29 | public void increment() { 30 | dircount++; 31 | testDirCount(false,processing ? " directories processed" : " directories found"); 32 | } 33 | 34 | public void testDirCount(boolean b, String note) { 35 | if (processing) { 36 | gft.gftSW.publish(""); 37 | } else if (b || (dircount >= lastdircount + rptgap)) { 38 | gft.gftSW.publish(dircount + " dirs "+note); 39 | lastdircount = dircount; 40 | if (dircount > rptgap*rptgapmult*2) { 41 | rptgap *= rptgapmult; 42 | gft.gftSW.publish("... Reporting every "+ rptgap); 43 | } 44 | } 45 | } 46 | 47 | public void complete(boolean processed) { 48 | testDirCount(true, processed ? " total directories processed" : " total directories found"); 49 | if (!processed){ 50 | dt.progressPanel.progress.setMaximum(dircount); 51 | dirfound = dircount; 52 | resetDirCount(); 53 | processing = !processed; 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/DirTypeNameMatch.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | import gov.nara.nwts.ftapp.stats.DirTypeStats; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.File; 8 | 9 | /** 10 | * Create FileAnalyzer statistics by directory and by FileType. 11 | * This routine is useful for understanding the contents of a large volume of material. 12 | * This routine was built as a sample rule for NWME. 13 | * @author TBrady 14 | * 15 | */ 16 | class DirTypeNameMatch extends DirMatch { 17 | 18 | public DirTypeNameMatch(FTDriver dt) { 19 | super(dt); 20 | } 21 | public String toString() { 22 | return "Count By Type and Dir"; 23 | } 24 | public String getKey(File f) { 25 | return getKey(f, f.getParentFile()); 26 | } 27 | 28 | public String getKey(File f, Object parentdir) { 29 | String key = getExt(f); 30 | if (parentdir instanceof File) { 31 | key = getExt(f)+": " + ((File)parentdir).getAbsolutePath().substring(getRoot().getAbsolutePath().length()); 32 | } 33 | return key; 34 | } 35 | 36 | public String getShortName(){return "Type&Dir";} 37 | public Stats createStats(String key){ 38 | return new DirTypeStats(key); 39 | } 40 | public Object[][] getStatsDetails() { 41 | return DirTypeStats.details; 42 | } 43 | public String getDescription() { 44 | return "This test counts the number of occurrences of a specific filetype within a directory. \nA cumulative total is counted for each parent directory."; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/ListDirectories.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.FTDriver; 6 | import gov.nara.nwts.ftapp.stats.Stats; 7 | 8 | /** 9 | * List the full path for a dirctory; this can be used as input for the FileAnalzyer batch capability. 10 | * @author TBrady 11 | * 12 | */ 13 | class ListDirectories extends DefaultFileTest { 14 | public static Object[][] details = { 15 | {String.class,"Folder",250}, 16 | {String.class,"Name",100}, 17 | }; 18 | 19 | public ListDirectories(FTDriver dt) { 20 | super(dt); 21 | } 22 | 23 | public String toString() { 24 | return "List Dir"; 25 | } 26 | 27 | public Object fileTest(File f) { 28 | return f.getName(); 29 | } 30 | 31 | public String getKey(File f) { 32 | String path = f.getAbsolutePath(); 33 | return path; 34 | } 35 | 36 | public Stats createStats(String key) { 37 | Stats stats = new Stats(key) { 38 | public Object compute(File f, FileTest fileTest) { 39 | vals.set(0, f.getName()); 40 | return f.getName(); 41 | } 42 | }; 43 | stats.vals.add(""); 44 | return stats; 45 | } 46 | 47 | public Object[][] getStatsDetails() { 48 | return details; 49 | } 50 | 51 | public String getShortName() { 52 | return "DIR"; 53 | } 54 | 55 | 56 | public String getDescription() { 57 | return "Generate a list of directories"; 58 | } 59 | 60 | public boolean isTestDirectory() { 61 | return true; 62 | } 63 | public boolean isTestFiles() { 64 | return false; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/MyPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import java.awt.FlowLayout; 5 | import java.awt.LayoutManager; 6 | 7 | import javax.swing.BorderFactory; 8 | import javax.swing.Box; 9 | import javax.swing.BoxLayout; 10 | import javax.swing.JPanel; 11 | 12 | /** 13 | * GUI helper class allowing new components to be quickly added to the File Analyzer in a consistent fashion 14 | * @author TBrady 15 | * 16 | */ 17 | class MyPanel extends JPanel { 18 | private static final long serialVersionUID = 1L; 19 | Box main; 20 | MyPanel(LayoutManager layout) { 21 | super(layout); 22 | main = new Box(BoxLayout.Y_AXIS); 23 | add(main); 24 | } 25 | 26 | MyPanel() { 27 | this(new FlowLayout()); 28 | } 29 | 30 | 31 | JPanel addPanel() { 32 | return addPanel((String)null); 33 | } 34 | 35 | JPanel addPanel(String title) { 36 | return addPanel(title, null); 37 | } 38 | JPanel addPanel(String title, String loc) { 39 | JPanel p = new JPanel(new FlowLayout(FlowLayout.LEFT)); 40 | if (title != null) { 41 | p.setBorder(BorderFactory.createTitledBorder(title)); 42 | } 43 | return addPanel(p, loc); 44 | } 45 | 46 | JPanel addBorderPanel(String title) { 47 | JPanel p = new JPanel(new BorderLayout()); 48 | if (title != null) { 49 | p.setBorder(BorderFactory.createTitledBorder(title)); 50 | } 51 | return addPanel(p, (String)null); 52 | } 53 | 54 | JPanel addPanel(JPanel p, String loc) { 55 | if (loc == null) { 56 | main.add(p); 57 | } else { 58 | main.add(p, loc); 59 | } 60 | return p; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ActionResult.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.stats.Stats; 4 | 5 | import java.io.File; 6 | import java.util.TreeMap; 7 | /** 8 | * This class contains details about the completion of a {@link gov.nara.nwts.ftapp.filetest.FileTest} or a {@link gov.nara.nwts.ftapp.importer.Importer} 9 | * @author TBrady 10 | * 11 | */ 12 | public class ActionResult { 13 | /** 14 | * Unique name to assign to the results of the action 15 | */ 16 | public String name; 17 | /** 18 | * Descriptive name of the action that was performed 19 | */ 20 | public String action; 21 | /** 22 | * Root directory or file on which the action was performed 23 | */ 24 | public File root; 25 | /** 26 | * Descriptive structure defining the components of the action results 27 | */ 28 | public Object[][] details; 29 | /** 30 | * Container of the stats generated by the action 31 | */ 32 | public TreeMaptypes; 33 | /** 34 | * Indicates whether or not the task ran to completion or if the task was terminated manually or terminated by the file count threshold 35 | */ 36 | public boolean completed; 37 | /** 38 | * Time in milliseconds that the action took to complete 39 | */ 40 | public double duration; 41 | 42 | public ActionResult(File root, String name, String action, Object[][] details, TreeMaptypes, boolean completed, double duration) { 43 | this.root = root; 44 | this.name = name; 45 | this.action = action; 46 | this.details = details; 47 | this.types = types; 48 | this.completed = completed; 49 | this.duration = duration; 50 | } 51 | } 52 | 53 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/FileCatalog.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import javax.swing.JFileChooser; 5 | import javax.swing.JFrame; 6 | import javax.swing.JLabel; 7 | import javax.swing.JPanel; 8 | 9 | /** 10 | * UI component allowing the selction of the root directory for a File Test. 11 | * Note: this pre-dates the creation of the {@link FileSelectChooser} and {@link DirSelectChooser} classes but accomplishes a similar function 12 | * @author TBrady 13 | * 14 | */ 15 | class FileCatalog extends JFrame { 16 | private static final long serialVersionUID = 1L; 17 | 18 | DirectoryTable dt; 19 | FileCatalog(DirectoryTable dt) { 20 | super("File Analyzer: Set Input Directory"); 21 | this.dt = dt; 22 | setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 23 | JPanel p = new JPanel(new BorderLayout()); 24 | add(p); 25 | JFileChooser jfc = new JFileChooser() { 26 | private static final long serialVersionUID = 1L; 27 | public void cancelSelection() { 28 | FileCatalog.this.dispose(); 29 | } 30 | public void approveSelection() { 31 | FileCatalog.this.setVisible(false); 32 | FileCatalog.this.dt.criteriaPanel.rootLabel.setText(getSelectedFile().getAbsolutePath()); 33 | FileCatalog.this.dt.setSelectedFile(); 34 | FileCatalog.this.dispose(); 35 | } 36 | }; 37 | if (dt.root != null){ 38 | jfc.setCurrentDirectory(dt.root); 39 | } 40 | jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); 41 | p.add(jfc, BorderLayout.CENTER); 42 | p.add(new JLabel("Please select the directory that you wish to catalog"),BorderLayout.NORTH); 43 | pack(); 44 | setVisible(true); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/RenameStatus.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | /** 4 | * Enumeration indicating all of the possible results of a file name/directory name validation test. 5 | * @author TBrady 6 | * 7 | */ 8 | public enum RenameStatus { 9 | NEXT("Test inconclusive", RenamePassFail.PASS), 10 | VALID("Original filename is valid, rename not required", RenamePassFail.PASS), 11 | INVALID_MANUAL("Original filename must be manually renamed",RenamePassFail.FAIL), 12 | PARSE_ERR("Original filename could not be parsed",RenamePassFail.FAIL), 13 | NEW_NAME_INVALID("File not renamed, new name is invalid",RenamePassFail.FAIL), 14 | RENAMABLE("New filename is valid (rename not performed)", RenamePassFail.PASS), 15 | RENAMED("Renamed",RenamePassFail.PASS), 16 | RENAME_FILE_EXISTS("Rename failed, file already exists", RenamePassFail.FAIL), 17 | RENAME_FAILURE("Rename failed", RenamePassFail.FAIL), 18 | DIRECTORY("Directory: Not Tested",RenamePassFail.PASS), 19 | DIRECTORY_VALID("Valid Directory",RenamePassFail.PASS), 20 | DIRECTORY_EMPTY("Empty Directory",RenamePassFail.FAIL), 21 | DIRECTORY_CHILD_INVALID("Directory contains Invalid File",RenamePassFail.FAIL), 22 | DIRECTORY_NAME_INVALID("Directory name is Invalid",RenamePassFail.FAIL), 23 | DIRECTORY_SEQUENCE_ERROR("Sequence Error in Directory",RenamePassFail.FAIL), 24 | DIRECTORY_INCOMPLETE("Incomplete Directory",RenamePassFail.FAIL); 25 | String message; 26 | RenamePassFail passfail; 27 | RenameStatus(String message, RenamePassFail passfail) { 28 | this.message = message; 29 | this.passfail = passfail; 30 | } 31 | public RenamePassFail getPassFail() {return passfail;} 32 | public String getMessage() {return message;} 33 | }; 34 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/stats/DirTypeStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.stats; 2 | 3 | import java.io.File; 4 | 5 | import gov.nara.nwts.ftapp.filetest.FileTest; 6 | 7 | /** 8 | * Status class showing accumulated counts by directory and by file type. 9 | * This is useful in helping users discover the contents of a large volume of data. 10 | * @author TBrady 11 | * 12 | */ 13 | public class DirTypeStats extends Stats { 14 | public static Object[][] details = { 15 | {String.class,"Key",100}, 16 | {String.class,"Type",80}, 17 | {String.class,"Path",300}, 18 | {Long.class,"Count",100}, 19 | {Long.class,"Cumulative Count",100}, 20 | }; 21 | 22 | 23 | public DirTypeStats(String key) { 24 | super(key); 25 | vals.add(""); 26 | vals.add(""); 27 | vals.add(new Long(0)); 28 | vals.add(new Long(0)); 29 | } 30 | 31 | public Object compute(File f, FileTest fileTest) { 32 | File root = fileTest.getRoot(); 33 | for(File ftest = f.getParentFile(); ftest!=null; ftest = ftest.getParentFile()){ 34 | DirTypeStats stats = (DirTypeStats)fileTest.getStats(fileTest.getKey(f,ftest)); 35 | stats.accumulate(f, fileTest, ftest); 36 | if (ftest.equals(root)){ 37 | break; 38 | } 39 | } 40 | return fileTest.fileTest(f); 41 | } 42 | 43 | public void accumulate(File f, FileTest fileTest, File parentdir) { 44 | Long count = (Long)vals.get(2); 45 | if (f.getParentFile().equals(parentdir)){ 46 | vals.set(2, count.longValue()+1); 47 | } 48 | count = (Long)vals.get(3); 49 | vals.set(3, count.longValue()+1); 50 | vals.set(1, (parentdir==null) ? "" : parentdir.getAbsolutePath().substring(fileTest.getRoot().getAbsolutePath().length())); 51 | vals.set(0, fileTest.getExt(f)); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/FileSelect.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.BorderLayout; 4 | import java.io.File; 5 | 6 | import javax.swing.JDialog; 7 | import javax.swing.JFileChooser; 8 | import javax.swing.JFrame; 9 | import javax.swing.JLabel; 10 | import javax.swing.JPanel; 11 | import javax.swing.JTextField; 12 | 13 | /** 14 | * Displays a File selection dialog; results will be saved to a specified text field. 15 | * @author TBrady 16 | * 17 | */ 18 | class FileSelect extends JDialog { 19 | private static final long serialVersionUID = 1L; 20 | JTextField result; 21 | JFileChooser jfc; 22 | 23 | 24 | public void configureChooser() { 25 | jfc.setFileSelectionMode(JFileChooser.FILES_ONLY); 26 | } 27 | 28 | FileSelect(JFrame parent, JTextField result, String title) { 29 | super(parent, title); 30 | this.setModal(true); 31 | this.result = result; 32 | setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE); 33 | JPanel p = new JPanel(new BorderLayout()); 34 | add(p); 35 | jfc = new JFileChooser() { 36 | private static final long serialVersionUID = 1L; 37 | 38 | public void cancelSelection() { 39 | FileSelect.this.dispose(); 40 | } 41 | 42 | public void approveSelection() { 43 | FileSelect.this.setVisible(false); 44 | FileSelect.this.result.setText(this.getSelectedFile() 45 | .getAbsolutePath()); 46 | FileSelect.this.dispose(); 47 | } 48 | }; 49 | String root = FileSelect.this.result.getText(); 50 | if (root != null) { 51 | jfc.setCurrentDirectory(new File(root)); 52 | } 53 | p.add(jfc, BorderLayout.CENTER); 54 | p.add(new JLabel(title), BorderLayout.NORTH); 55 | configureChooser(); 56 | pack(); 57 | setVisible(true); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/ActionRegistry.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.util.Vector; 6 | /** 7 | * Activates the FileTest objects that will be made available to a user. 8 | * In the event of a load failure of an individual class, the runtime loading of those classes can be disabled in this method. 9 | * @author TBrady 10 | * 11 | */ 12 | public class ActionRegistry extends Vector { 13 | 14 | private static final long serialVersionUID = 1L; 15 | boolean modifyAllowed = true; 16 | 17 | public ActionRegistry(FTDriver dt, boolean modifyAllowed) { 18 | this.modifyAllowed = modifyAllowed; 19 | add(new CountByType(dt)); 20 | add(new ListDirectories(dt)); 21 | add(new NameMatch(dt)); 22 | add(new BaseNameMatch(dt)); 23 | add(new NameMD5Checksum(dt)); 24 | add(new NameSha1Checksum(dt)); 25 | add(new NameSha256Checksum(dt)); 26 | add(new ByMD5Checksum(dt)); 27 | add(new DirMatch(dt)); 28 | add(new DirTypeNameMatch(dt)); 29 | add(new RandomFileTest(dt)); 30 | FileTest next = new LowercaseTest(dt, null); 31 | add(next); 32 | if (modifyAllowed()) { 33 | add(new LowercaseTest(dt, next)); 34 | } 35 | } 36 | 37 | /** 38 | * By design, File Analyzer actions should not be destructive. 39 | * When implementing routines that will automatically rename items (vs validating file names), the registry can be configured to only load those actions for specific users. 40 | * Also, the base File Analyzer module can be initiated with or without modification enabled. 41 | * This check is only enforced if the FileTest honors this flag. 42 | * 43 | */ 44 | public boolean modifyAllowed() { 45 | return modifyAllowed; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/stats/GenericImageStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.stats; 2 | 3 | import gov.nara.nwts.ftapp.stats.Stats; 4 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGS; 5 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGLOC; 6 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGTYPE; 7 | import gov.nara.nwts.ftappImg.tags.ImageTags.NARAREQ; 8 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGCONTENT; 9 | import gov.nara.nwts.ftappImg.tags.ImageTags.DUP; 10 | 11 | /** 12 | * Stats object to report on metadata coming from either a TIF or a JPG using tag definitions 13 | * @author TBrady 14 | * 15 | */ 16 | public class GenericImageStats extends Stats { 17 | public static Object[][] details = { 18 | {String.class,"Key",60, null, false}, 19 | {String.class,"File",150}, 20 | {String.class,"Name",150, TAGS.values()}, 21 | {String.class,"Path",200}, 22 | {String.class,"Value",200}, 23 | {TAGTYPE.class,"Type",100,TAGTYPE.values()}, 24 | {TAGCONTENT.class,"Content",100,TAGCONTENT.values()}, 25 | {DUP.class,"Dup Info",100,DUP.values()}, 26 | {TAGLOC.class,"Loc",50,TAGLOC.values()}, 27 | {NARAREQ.class,"Nara Use",120,NARAREQ.values()}, 28 | }; 29 | 30 | public GenericImageStats(String key) { 31 | this(key,"","","","",TAGLOC.NA,TAGTYPE.TIFF_TAG,NARAREQ.UNDECIDED,TAGCONTENT.UNDECIDED, DUP.NA); 32 | } 33 | 34 | public GenericImageStats(String key, String file, String name, String path, String value, TAGLOC tiffloc, TAGTYPE tagtype, NARAREQ narareq, TAGCONTENT tagcontent, DUP dup) { 35 | super(key); 36 | vals.add(file); 37 | vals.add(name); 38 | vals.add(path); 39 | vals.add(value); 40 | vals.add(tagtype); 41 | vals.add(tagcontent); 42 | vals.add(dup); 43 | vals.add(tiffloc); 44 | vals.add(narareq); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/GuiFileTraversalSW.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.util.Iterator; 4 | import java.util.List; 5 | 6 | import javax.swing.SwingWorker; 7 | import javax.swing.table.DefaultTableModel; 8 | 9 | /** 10 | * Worker thread that will perform an actual file test and send periodic status updates to the GUI thread. 11 | * Note: this class is the primary reason for running Java 1.6 or higher. It handles GUI actions elegantly. 12 | * @author TBrady 13 | * 14 | */ 15 | class GuiFileTraversalSW extends SwingWorker { 16 | GuiFileTraversal traversal; 17 | DirectoryTable dt; 18 | 19 | public GuiFileTraversalSW(DirectoryTable dt, DefaultTableModel tm) { 20 | this.dt = dt; 21 | traversal = new GuiFileTraversal(this, tm); 22 | } 23 | 24 | public void publish(String s) { 25 | super.publish(s); 26 | } 27 | 28 | protected String doInBackground() throws Exception { 29 | try { 30 | publish("starting"); 31 | traversal.traverseFile(); 32 | publish("Complete"); 33 | return "complete.."; 34 | } catch (Throwable e) { 35 | e.printStackTrace(); 36 | this.cancel(true); 37 | return "err..."; 38 | } 39 | } 40 | 41 | protected void process(List messages) { 42 | if (traversal.myprogress.processing) { 43 | dt.progressPanel.progress.setValue(traversal.myprogress.dircount+1); 44 | } 45 | for(Iteratori=messages.iterator(); i.hasNext(); ){ 46 | String s = i.next(); 47 | if (s.equals("")) continue; 48 | dt.report(s); 49 | } 50 | 51 | } 52 | 53 | 54 | public void done() { 55 | if (isCancelled()) { 56 | publish("Processing cancelled"); 57 | } 58 | dt.report((traversal.myprogress.dircount+1) + " of " + traversal.myprogress.dirfound + " processed"); 59 | } 60 | 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/FileTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.List; 5 | import java.util.regex.Pattern; 6 | 7 | import gov.nara.nwts.ftapp.FTDriver; 8 | import gov.nara.nwts.ftapp.stats.Stats; 9 | import gov.nara.nwts.ftapp.filter.FileTestFilter; 10 | import gov.nara.nwts.ftapp.ftprop.FTProp; 11 | 12 | /** 13 | * Contract defining the behavior of a File Analyzer custom rule. 14 | * @author TBrady 15 | * 16 | */ 17 | public interface FileTest { 18 | public String toString(); 19 | public String getDescription(); 20 | public String getExt(File f); 21 | public String getKey(File f); 22 | public String getKey(File f, Object o); 23 | public boolean isTestable(File f); 24 | public Object fileTest(File f); 25 | public Stats getStats(File f); 26 | public Stats getStats(String key); 27 | 28 | public List getFilters(); 29 | 30 | public Stats createStats(String key); 31 | public Object[][] getStatsDetails(); 32 | public String getShortName(); 33 | public String getShortNameFormatted(); 34 | public String getShortNameNormalized(); 35 | 36 | public FileTestFilter getDefaultFilter(); 37 | 38 | public File getRoot(); 39 | 40 | void initFilters(); 41 | 42 | public boolean isTestFiles(); 43 | public boolean isTestDirectory(); 44 | public boolean processRoot(); 45 | public Pattern getDirectoryPattern(); 46 | 47 | public void refineResults(); 48 | public void init(); 49 | 50 | public void progress(int count); 51 | public FileTest resetOption(); 52 | public List getPropertyList(); 53 | public FTDriver getFTDriver(); 54 | public Object getProperty(String name); 55 | public void setProperty(String name, String str); 56 | } 57 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/tags/XMPExtractor.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.tags; 2 | 3 | import java.util.ArrayList; 4 | 5 | import com.adobe.xmp.XMPException; 6 | import com.adobe.xmp.XMPIterator; 7 | import com.adobe.xmp.XMPMeta; 8 | import com.adobe.xmp.properties.XMPProperty; 9 | import com.adobe.xmp.properties.XMPPropertyInfo; 10 | 11 | /** 12 | * Extractor for accessing data from XMP within a TIF or a JPG 13 | * @author TBrady 14 | * 15 | */ 16 | public class XMPExtractor { 17 | XMPMeta xmp; 18 | public static final String[] XMP_DESC = {"http://purl.org/dc/elements/1.1/","dc:description[1]"}; 19 | public static final String[] XMP_INSTR = {"http://ns.adobe.com/photoshop/1.0/","photoshop:Instructions"}; 20 | public static final String[] XMP_KEY = {"http://purl.org/dc/elements/1.1/","dc:subject[1]"}; 21 | public static final String[] XMP_ICC = {"http://ns.adobe.com/photoshop/1.0/","photoshop:ICCProfile"}; 22 | 23 | public XMPExtractor(XMPMeta xmp) { 24 | this.xmp = xmp; 25 | } 26 | public XMPExtractor() { 27 | } 28 | 29 | public String getXMP(String[] xmppath) { 30 | try { 31 | if (xmp==null) return ""; 32 | //System.err.println(xmp.dumpObject()); 33 | XMPProperty xp = xmp.getProperty(xmppath[0], xmppath[1]); 34 | if (xp==null) return ""; 35 | return xp.toString(); 36 | } catch (XMPException e) { 37 | e.printStackTrace(); 38 | return ""; 39 | } 40 | } 41 | public ArrayList getTags() { 42 | ArrayList list = new ArrayList(); 43 | if (xmp != null){ 44 | try { 45 | for(XMPIterator xi = xmp.iterator();xi.hasNext();) { 46 | XMPPropertyInfo o = (XMPPropertyInfo)xi.next(); 47 | list.add(o); 48 | } 49 | } catch (XMPException e) { 50 | e.printStackTrace(); 51 | } 52 | 53 | } 54 | return list; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ftprop/FTPropString.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import javax.swing.JComponent; 6 | import javax.swing.JTextField; 7 | import javax.swing.event.DocumentEvent; 8 | import javax.swing.event.DocumentListener; 9 | 10 | /** 11 | * File Test Property object for string values 12 | * @author TBrady 13 | * 14 | */ 15 | 16 | public class FTPropString extends DefaultFTProp { 17 | JTextField tf; 18 | public FTPropString(FileTest ft, String name, String shortname, String description, Object def) { 19 | super(ft, name, shortname, description, def); 20 | init(); 21 | tf = new JTextField(this.def.toString()); 22 | tf.getDocument().addDocumentListener(new DocumentListener(){ 23 | public void changedUpdate(DocumentEvent arg0) { 24 | if (FTPropString.this.ft.getFTDriver().hasPreferences()) { 25 | FTPropString.this.ft.getFTDriver().getPreferences().put(getPrefString(), tf.getText()); 26 | } 27 | } 28 | 29 | public void insertUpdate(DocumentEvent arg0) { 30 | if (FTPropString.this.ft.getFTDriver().hasPreferences()) { 31 | FTPropString.this.ft.getFTDriver().getPreferences().put(getPrefString(), tf.getText()); 32 | } 33 | } 34 | 35 | public void removeUpdate(DocumentEvent arg0) { 36 | if (FTPropString.this.ft.getFTDriver().hasPreferences()) { 37 | FTPropString.this.ft.getFTDriver().getPreferences().put(getPrefString(), tf.getText()); 38 | } 39 | } 40 | }); 41 | } 42 | 43 | public JComponent getEditor() { 44 | return tf; 45 | } 46 | 47 | public Object getValue() { 48 | return tf.getText(); 49 | } 50 | 51 | public void setValue(Object obj) { 52 | tf.setText(obj.toString()); 53 | } 54 | public Object validate(String s) { 55 | if (s == null) s = ""; 56 | return getValue(); 57 | } 58 | 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/FilterPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.filter.FileTestFilter; 4 | 5 | import javax.swing.JCheckBox; 6 | import javax.swing.JPanel; 7 | import javax.swing.JTextField; 8 | /** 9 | * Panel presenting the various filter values that can be overridden from the user interface 10 | * @author TBrady 11 | * 12 | */ 13 | class FilterPanel extends MyPanel { 14 | private static final long serialVersionUID = 1L; 15 | JTextField prefix; 16 | JTextField contains; 17 | JTextField suffix; 18 | JTextField exclusion; 19 | 20 | JCheckBox rePrefix; 21 | JCheckBox reContains; 22 | JCheckBox reSuffix; 23 | JCheckBox reExclusion; 24 | 25 | FilterPanel(DirectoryTable dt, FileTestFilter filter) { 26 | JPanel p = addPanel("Filename Prefix Filter"); 27 | prefix = new JTextField(filter.getPrefix(), 50); 28 | p.add(prefix); 29 | rePrefix = new JCheckBox(); 30 | rePrefix.setText("Regex"); 31 | rePrefix.setSelected(filter.isRePrefix()); 32 | p.add(rePrefix); 33 | 34 | p = addPanel("Filename Contains Filter"); 35 | contains = new JTextField(filter.getContains(), 50); 36 | p.add(contains); 37 | reContains = new JCheckBox(); 38 | reContains.setText("Regex"); 39 | reContains.setSelected(filter.isReContains()); 40 | p.add(reContains); 41 | 42 | p = addPanel("Filename Suffix Filter"); 43 | suffix = new JTextField(filter.getSuffix(), 50); 44 | p.add(suffix); 45 | reSuffix = new JCheckBox(); 46 | reSuffix.setText("Regex"); 47 | reSuffix.setSelected(filter.isReSuffix()); 48 | p.add(reSuffix); 49 | 50 | p = addPanel("Filename Exclusion Filter"); 51 | exclusion = new JTextField(filter.getExclusion(), 50); 52 | p.add(exclusion); 53 | reExclusion = new JCheckBox(); 54 | reExclusion.setText("Regex"); 55 | reExclusion.setSelected(filter.isReExclusion()); 56 | p.add(reExclusion); 57 | 58 | dt.criteriaPanel.filterTabs.add(filter.getName(), this); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/FileSelectChooser.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.awt.FlowLayout; 4 | import java.awt.event.ActionEvent; 5 | import java.awt.event.ActionListener; 6 | import java.util.prefs.Preferences; 7 | 8 | import javax.swing.JButton; 9 | import javax.swing.JFrame; 10 | import javax.swing.JPanel; 11 | import javax.swing.JTextField; 12 | import javax.swing.event.DocumentEvent; 13 | import javax.swing.event.DocumentListener; 14 | 15 | /** 16 | * Widget that will display a chosen file and will open a separate window allowing a file to be selected. 17 | * @author TBrady 18 | * 19 | */ 20 | class FileSelectChooser extends JPanel implements ActionListener, DocumentListener { 21 | private static final long serialVersionUID = 1L; 22 | private Preferences p; 23 | protected JFrame parent; 24 | public JTextField tf; 25 | private JButton button; 26 | protected String title; 27 | private String key; 28 | 29 | FileSelectChooser(JFrame parent, String title, String def) { 30 | this(parent, title, null, null, def); 31 | } 32 | FileSelectChooser(JFrame parent, String title, Preferences p, String key, String def) { 33 | this.p = p; 34 | this.parent = parent; 35 | this.title = title; 36 | this.key = key; 37 | setLayout(new FlowLayout(FlowLayout.LEFT)); 38 | String val = (p == null) ? def : p.get(key, def); 39 | tf = new JTextField(val, 40); 40 | tf.setEditable(false); 41 | add(tf); 42 | button = new JButton("..."); 43 | add(button); 44 | button.addActionListener(this); 45 | tf.getDocument().addDocumentListener(this); 46 | } 47 | 48 | public void change() { 49 | if (p != null) { 50 | p.put(key, tf.getText()); 51 | } 52 | } 53 | 54 | public void actionPerformed(ActionEvent arg0) { 55 | new FileSelect(parent, tf, title); 56 | } 57 | public void changedUpdate(DocumentEvent arg0) { 58 | change(); 59 | } 60 | public void insertUpdate(DocumentEvent arg0) { 61 | change(); 62 | } 63 | public void removeUpdate(DocumentEvent arg0) { 64 | change(); 65 | } 66 | 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ftprop/DefaultFTProp.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | /** 6 | * Abstract base class for File Test Properties 7 | * @author TBrady 8 | * 9 | */ 10 | 11 | public abstract class DefaultFTProp implements FTProp { 12 | String name; 13 | String shortname; 14 | String description; 15 | Object def; 16 | FileTest ft; 17 | 18 | public enum RUNMODE { 19 | TEST, 20 | PROD; 21 | } 22 | 23 | public DefaultFTProp(FileTest ft, String name, String shortname, String description, Object def) { 24 | this.name = name; 25 | this.shortname = shortname; 26 | this.description = description; 27 | this.ft = ft; 28 | this.def = def; 29 | } 30 | 31 | public void init() { 32 | if (ft.getFTDriver().hasPreferences()) { 33 | def = ft.getFTDriver().getPreferences().get(getPrefString(), def.toString()); 34 | } 35 | } 36 | public void init(Object[] vals) { 37 | if (ft.getFTDriver().hasPreferences()) { 38 | String s = ft.getFTDriver().getPreferences().get(getPrefString(), def.toString()); 39 | if (s == null) return; 40 | for(Object obj: vals) { 41 | if (s.equals(obj.toString())) { 42 | def = obj; 43 | return; 44 | } 45 | } 46 | } 47 | } 48 | 49 | public String getPrefString() { 50 | return ft.toString()+"--"+name; 51 | } 52 | 53 | public String describe() { 54 | return description; 55 | } 56 | public String describeFormatted() { 57 | return "\t\t\t"+description; 58 | } 59 | 60 | public Object getDefault() { 61 | return def; 62 | } 63 | 64 | public String getName() { 65 | return name; 66 | } 67 | public String getShortName() { 68 | return shortname; 69 | } 70 | 71 | public String getShortNameNormalized() { 72 | return getShortName().replaceAll("[\\s&]",""); 73 | } 74 | public String getShortNameFormatted() { 75 | StringBuffer buf = new StringBuffer(); 76 | buf.append(getShortNameNormalized()); 77 | buf.append(" "); 78 | return buf.substring(0,20); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/FileListImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | import gov.nara.nwts.ftapp.stats.Stats; 6 | 7 | import java.io.BufferedReader; 8 | import java.io.File; 9 | import java.io.FileReader; 10 | import java.io.IOException; 11 | import gov.nara.nwts.ftapp.Timer; 12 | import java.util.TreeMap; 13 | /** 14 | * This routine will import a list of paths from a file making the filename of each into a key. 15 | * On the 1940 Census project, several QC applications were written in which file paths were written to a text file in order to be imported into a thumbnail viewer. 16 | * This routine allowed those files to be used as input for other comparison processes. 17 | * This rule may or may not be useful for the future. 18 | * @author TBrady 19 | * 20 | */ 21 | public class FileListImporter extends DefaultImporter { 22 | public static Object[][] details = { 23 | {String.class,"Filename Import",200}, 24 | }; 25 | 26 | public FileListImporter(FTDriver dt) { 27 | super(dt); 28 | } 29 | 30 | public ActionResult importFile(File selectedFile) throws IOException { 31 | Timer timer = new Timer(); 32 | TreeMap types = new TreeMap(); 33 | FileReader fr = new FileReader(selectedFile); 34 | BufferedReader br = new BufferedReader(fr); 35 | for(String line=br.readLine(); line!=null; line=br.readLine()){ 36 | File f=new File(line); 37 | String s = f.getName(); 38 | Stats stats = new Stats(s); 39 | types.put(s,stats); 40 | } 41 | return new ActionResult(selectedFile, selectedFile.getName(), this.toString(), details, types, true, timer.getDuration()); 42 | } 43 | 44 | 45 | public String toString() { 46 | return "File List Importer"; 47 | } 48 | public String getDescription() { 49 | return "This rule is used to import a list of file paths from a file. The file name will be made the key for each record."; 50 | } 51 | public String getShortName() { 52 | return "LIST"; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/stats/ImageStats.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.stats; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | import gov.nara.nwts.ftapp.stats.Stats; 5 | import gov.nara.nwts.ftappImg.tags.XMPExtractor; 6 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGS; 7 | import gov.nara.nwts.ftappImg.tif.TifExtractor; 8 | 9 | import java.io.File; 10 | /** 11 | * Generic container for reporting on image properties; this implementation has been superceeded by better implementations 12 | * @author TBrady 13 | * 14 | */ 15 | public class ImageStats extends Stats { 16 | public static Object[][] details = { 17 | {String.class,"File",200}, 18 | {Integer.class,"Bits/Channel (258)",100}, 19 | {Integer.class,"Color Space (262)",100}, 20 | {String.class,"ICC Profile",120}, 21 | {String.class,"Description (270)",300}, 22 | {String.class,"Keywords (XMP)",200}, 23 | {String.class,"Instructions (XMP)",200}, 24 | {String.class,"Desc 1",150}, 25 | {String.class,"Desc 2",150}, 26 | {String.class,"Desc 3",150}, 27 | {String.class,"Desc 4",150}, 28 | }; 29 | 30 | public ImageStats(String key) { 31 | super(key); 32 | vals.add(new Integer(0)); 33 | vals.add(new Integer(0)); 34 | vals.add(""); 35 | vals.add(""); 36 | vals.add(""); 37 | vals.add(""); 38 | vals.add(""); 39 | vals.add(""); 40 | vals.add(""); 41 | vals.add(""); 42 | } 43 | 44 | public Object compute(File f, FileTest fileTest) { 45 | TifExtractor tiffext = new TifExtractor(f); 46 | vals.set(0,tiffext.getTiffInt(TAGS.TIFF_BITS_PER_CHANNEL)); 47 | vals.set(1,tiffext.getTiffInt(TAGS.TIFF_COLOR_SPACE)); 48 | vals.set(2,tiffext.getXMP(XMPExtractor.XMP_ICC)); 49 | String tfs = tiffext.getTiffString(TAGS.TIFF_DESCRIPTION); 50 | vals.set(3,tfs); 51 | vals.set(4, tiffext.getXMP(XMPExtractor.XMP_KEY)); 52 | vals.set(5, tiffext.getXMP(XMPExtractor.XMP_INSTR)); 53 | 54 | String[] parts = tfs.split("(\\s\\s\\s+|\n)"); 55 | for(int i=0; (i < parts.length) && (i <4); i++){ 56 | vals.set(6+i,parts[i]); 57 | } 58 | tiffext.close(); 59 | return fileTest.fileTest(f); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/ftprop/FTPropEnum.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.ftprop; 2 | 3 | import java.awt.event.ItemEvent; 4 | import java.awt.event.ItemListener; 5 | 6 | import gov.nara.nwts.ftapp.filetest.FileTest; 7 | 8 | import javax.swing.JComboBox; 9 | import javax.swing.JComponent; 10 | 11 | /** 12 | * File Test Property object presenting enumerated values as a choice 13 | * @author TBrady 14 | * 15 | */ 16 | 17 | public class FTPropEnum extends DefaultFTProp { 18 | JComboBox combo; 19 | 20 | public FTPropEnum(FileTest ft, String name, String shortname, String description, Object[]vals, Object def) { 21 | super(ft, name, shortname, description, def); 22 | init(vals); 23 | combo = new JComboBox(); 24 | initCombo(vals); 25 | combo.addItemListener(new ItemListener(){ 26 | public void itemStateChanged(ItemEvent arg0) { 27 | Object obj = combo.getSelectedItem(); 28 | if (obj == null) return; 29 | if (FTPropEnum.this.ft.getFTDriver().hasPreferences()){ 30 | FTPropEnum.this.ft.getFTDriver().getPreferences().put(getPrefString(), combo.getSelectedItem().toString()); 31 | } 32 | } 33 | }); 34 | } 35 | 36 | public void initCombo(Object[] vals) { 37 | for(Object obj: vals) { 38 | combo.addItem(obj); 39 | } 40 | setValue(def); 41 | } 42 | 43 | public JComponent getEditor() { 44 | return combo; 45 | } 46 | 47 | public Object getValue() { 48 | return combo.getSelectedItem(); 49 | } 50 | 51 | public Object validate(String s) { 52 | for(int i=0; i getColumnClass(int col) { 32 | if (col == 4) { 33 | return Date.class; 34 | } else if (col == 3) { 35 | return Long.class; 36 | } else if (col == 5) { 37 | return Object.class; 38 | } else { 39 | return String.class; 40 | } 41 | } 42 | 43 | public void setColumns(JTable jt) { 44 | JTableHeader jth = jt.getTableHeader(); 45 | jth.setReorderingAllowed(true); 46 | TableColumnModel tcm = jt.getColumnModel(); 47 | TableColumn tc; 48 | Object[][] details = { 49 | {"Directory",300}, 50 | {"Filename",200}, 51 | {"Type",50}, 52 | {"Size",100}, 53 | {"Mod Date",100}, 54 | {"Other",500} 55 | }; 56 | for(int i=0; i types) { 75 | st = new StatsTable(details,types, parent); 76 | tp.removeAll(); 77 | tp.add(new JScrollPane(st.jt), BorderLayout.CENTER); 78 | filterPanel.removeAll(); 79 | for(Iteratori=st.filters.iterator();i.hasNext();) { 80 | JComboBox cb = i.next(); 81 | if (cb!=null) filterPanel.add(cb); 82 | } 83 | } 84 | 85 | public void setFilterNote(int x, int y) { 86 | fnote.setText("["+FTDriver.nf.format(x)+ " of " + FTDriver.nf.format(y) + " showing]"); 87 | } 88 | } 89 | 90 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/Parser.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | import gov.nara.nwts.ftapp.Timer; 6 | import gov.nara.nwts.ftapp.stats.Stats; 7 | 8 | import java.io.BufferedReader; 9 | import java.io.File; 10 | import java.io.FileNotFoundException; 11 | import java.io.FileReader; 12 | import java.io.IOException; 13 | import java.util.TreeMap; 14 | import java.util.regex.Matcher; 15 | import java.util.regex.Pattern; 16 | 17 | /** 18 | * Base class parser to ananlyze and ingest individual rows from a file using a regular expression pattern. 19 | * @author TBrady 20 | * 21 | */ 22 | public class Parser extends DefaultImporter { 23 | public enum status {PASS,FAIL} 24 | public static Object[][] details = { 25 | {String.class,"Row",60}, 26 | {status.class,"Pass/Fail",100,status.values()}, 27 | {String.class,"Data",300}, 28 | }; 29 | Pattern p; 30 | int cols; 31 | Object[][]mydetails; 32 | 33 | public Object[][] getDetails() { 34 | return details; 35 | } 36 | public Pattern getPattern() { 37 | return Pattern.compile("^(.*)$"); 38 | } 39 | 40 | public Parser(FTDriver dt) { 41 | super(dt); 42 | mydetails = getDetails(); 43 | cols = mydetails.length - 1; 44 | p = getPattern(); 45 | } 46 | 47 | public Matcher test(String line) { 48 | return p.matcher(line); 49 | } 50 | 51 | public Object getVal(Matcher m, int i) { 52 | if (m.groupCount()>= i) { 53 | return m.group(i).trim(); 54 | } 55 | return ""; 56 | } 57 | 58 | public Object getDefVal(Matcher m, int i, String line) { 59 | if (m.groupCount()== i) { 60 | return line; 61 | } 62 | return ""; 63 | } 64 | 65 | public void setVals(Matcher m, Stats stats, String line) { 66 | if (m.matches()) { 67 | stats.vals.add(status.PASS); 68 | for(int i=1;i<=cols;i++) { 69 | stats.vals.add(getVal(m,i)); 70 | } 71 | } else { 72 | stats.vals.add(status.FAIL); 73 | for(int i=1;i<=cols;i++) { 74 | stats.vals.add(getDefVal(m,i,line)); 75 | } 76 | } 77 | 78 | } 79 | 80 | public ActionResult importFile(File selectedFile) throws IOException { 81 | Timer timer = new Timer(); 82 | TreeMap types = new TreeMap(); 83 | try { 84 | BufferedReader br = new BufferedReader(new FileReader(selectedFile)); 85 | int i=1000000; 86 | for(String line=br.readLine(); line!=null; line=br.readLine()){ 87 | String key = ""+ (i++); 88 | Stats stats = new Stats(key); 89 | types.put(key, stats); 90 | Matcher m = test(line); 91 | setVals(m, stats, line); 92 | } 93 | br.close(); 94 | return new ActionResult(selectedFile, selectedFile.getName(), this.toString(), mydetails, types, true, timer.getDuration()); 95 | } catch (FileNotFoundException e) { 96 | e.printStackTrace(); 97 | } catch (IOException e) { 98 | e.printStackTrace(); 99 | } 100 | return new ActionResult(selectedFile, selectedFile.getName(), this.toString(), mydetails, types, false, timer.getDuration()); 101 | } 102 | 103 | public String toString() { 104 | return "Parser"; 105 | } 106 | public String getDescription() { 107 | return "This rule will parse each line of a file and add it to the results table."; 108 | } 109 | public String getShortName() { 110 | return "Parse"; 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/GuiFileTraversal.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import java.io.File; 4 | import java.util.Date; 5 | import java.util.Vector; 6 | 7 | import javax.swing.table.DefaultTableModel; 8 | 9 | import gov.nara.nwts.ftapp.FileTraversal; 10 | import gov.nara.nwts.ftapp.filetest.FileTest; 11 | import gov.nara.nwts.ftapp.stats.Stats; 12 | 13 | /** 14 | * As files are processed within the FileAnalyzer, this class manages status updates. 15 | * This class is used by {@link GuiFileTraversalSW} to provide a responsive user experience. 16 | * Since GuiFileTraversalSW must extend SwingWorker, this companion class was written so that it could override 17 | * the default behaviors in the {@link gov.nara.nwts.ftapp.FileTraversal} class. 18 | * 19 | * 20 | * @author TBrady 21 | * 22 | */ 23 | class GuiFileTraversal extends FileTraversal { 24 | DefaultTableModel tm; 25 | GuiFileTraversalSW gftSW; 26 | 27 | MyProgress myprogress; 28 | 29 | public void completeDirectoryScan() { 30 | myprogress.complete(false); 31 | } 32 | public void completeFileScan() { 33 | myprogress.complete(true); 34 | } 35 | public int getNumProcessed() { 36 | return tm.getRowCount(); 37 | } 38 | 39 | public GuiFileTraversal(GuiFileTraversalSW gftsw, DefaultTableModel tm) { 40 | super(gftsw.dt); 41 | gftSW = gftsw; 42 | this.tm = tm; 43 | myprogress = new MyProgress(this); 44 | } 45 | 46 | public void increment() { 47 | myprogress.increment(); 48 | } 49 | 50 | public void reportCancel() { 51 | gftSW.publish("Stopping: " +max + " items found."); 52 | } 53 | 54 | 55 | public void checkFile(File thefile, FileTest fileTest) { 56 | Vector v = new Vector(); 57 | String s = thefile.getParent(); 58 | int len = gftSW.dt.root.getPath().length(); 59 | if (s.length() > len) s = s.substring(len); 60 | v.add(s); 61 | String name = thefile.getName(); 62 | v.add(name); 63 | v.add(fileTest.getExt(thefile)); 64 | if (fileTest.isTestable(thefile)){ 65 | Stats mystats = fileTest.getStats(thefile); 66 | long size = thefile.length(); 67 | v.add(new Long(size)); 68 | v.add(new Date(thefile.lastModified())); 69 | Object o = null; 70 | if (mystats!=null){ 71 | try { 72 | o = mystats.compute(thefile, fileTest); 73 | } catch (Exception e) { 74 | e.printStackTrace(); 75 | } 76 | } 77 | v.add((o == null) ? "" : o); 78 | tm.addRow(v); 79 | } 80 | } 81 | public void checkDirFile(File thefile, FileTest fileTest) { 82 | Vector v = new Vector(); 83 | if (thefile.equals(gftSW.dt.root)){ 84 | v.add("root"); 85 | } else { 86 | v.add(thefile.getParent().substring(gftSW.dt.root.getPath().length())); 87 | } 88 | String name = thefile.getName(); 89 | v.add(name); 90 | v.add(""); 91 | if (fileTest.isTestable(thefile)){ 92 | Stats mystats = fileTest.getStats(thefile); 93 | v.add(new Long(0)); 94 | v.add(new Date(thefile.lastModified())); 95 | Object o = mystats.compute(thefile, fileTest); 96 | v.add((o == null) ? "" : o); 97 | tm.addRow(v); 98 | } 99 | } 100 | public void clear() { 101 | tm.setNumRows(0); 102 | myprogress.resetDirCount(); 103 | } 104 | 105 | public boolean traverseFile() { 106 | return traverseFile(fileTest, max); 107 | } 108 | public void reportDuration(double duration) { 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /samples/ImageAnalyzer/src/gov/nara/nwts/ftappImg/jpeg/JpegExtractor.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftappImg.jpeg; 2 | 3 | import gov.nara.nwts.ftappImg.tags.XMPExtractor; 4 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGLOC; 5 | import gov.nara.nwts.ftappImg.tags.ImageTags.TAGS; 6 | 7 | import java.io.BufferedInputStream; 8 | import java.io.ByteArrayOutputStream; 9 | import java.io.File; 10 | import java.io.FileInputStream; 11 | import java.io.FileNotFoundException; 12 | import java.io.IOException; 13 | import java.util.ArrayList; 14 | 15 | 16 | import org.apache.tika.exception.TikaException; 17 | import org.apache.tika.metadata.Metadata; 18 | import org.apache.tika.parser.ParseContext; 19 | import org.apache.tika.parser.jpeg.JpegParser; 20 | import org.apache.tika.parser.image.xmp.XMPPacketScanner; 21 | import org.xml.sax.ContentHandler; 22 | import org.xml.sax.SAXException; 23 | import org.xml.sax.helpers.DefaultHandler; 24 | 25 | import com.adobe.xmp.XMPException; 26 | import com.adobe.xmp.XMPMetaFactory; 27 | import gov.nara.nwts.ftappImg.tags.DefaultExtractor; 28 | 29 | /** 30 | * Helper class for traversing JPG metadata 31 | * @author TBrady 32 | * 33 | */ 34 | public class JpegExtractor extends DefaultExtractor{ 35 | Metadata m; 36 | JpegParser jp; 37 | public XMPExtractor xmpex; 38 | 39 | public JpegExtractor(File f) { 40 | 41 | ContentHandler ch = new DefaultHandler(); 42 | 43 | m = new Metadata(); 44 | ParseContext pc = new ParseContext(); 45 | jp = new JpegParser(); 46 | XMPPacketScanner xps = new XMPPacketScanner(); 47 | 48 | try { 49 | jp.parse(new BufferedInputStream(new FileInputStream(f)), ch, m, pc); 50 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 51 | xps.parse(new FileInputStream(f), baos); 52 | if (baos.size() > 0){ 53 | try { 54 | xmpex = new XMPExtractor(XMPMetaFactory.parseFromBuffer(baos.toByteArray())); 55 | } catch (XMPException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } catch (FileNotFoundException e1) { 60 | e1.printStackTrace(); 61 | } catch (IOException e1) { 62 | e1.printStackTrace(); 63 | } catch (SAXException e1) { 64 | e1.printStackTrace(); 65 | } catch (TikaException e1) { 66 | e1.printStackTrace(); 67 | } 68 | 69 | } 70 | 71 | public TAGS getTagDef(String tag) { 72 | 73 | for(TAGS t: TAGS.values()) { 74 | if (t.tiffloc == TAGLOC.JPG && tag.equals(t.path)) { 75 | return t; 76 | } 77 | } 78 | return TAGS.UNDEFINED; 79 | } 80 | public ArrayList getTags() { 81 | ArrayList list = new ArrayList(); 82 | for(String s: m.names()) { 83 | list.add(s); 84 | } 85 | if (xmpex!=null){ 86 | list.addAll(xmpex.getTags()); 87 | } 88 | return list; 89 | } 90 | 91 | public int getInt(String name, int def) { 92 | try { 93 | return Integer.parseInt(getAttribute(name)); 94 | } catch (NumberFormatException e){ 95 | return def; 96 | } 97 | } 98 | 99 | public float getFloat(String name, int def) { 100 | try { 101 | return Float.parseFloat(getAttribute(name)); 102 | } catch (NumberFormatException e){ 103 | return def; 104 | } 105 | } 106 | 107 | public void close() { 108 | } 109 | 110 | public String getAttribute(String name) { 111 | return m.get(name); 112 | } 113 | 114 | public String getString(TAGS tags) { 115 | return getAttribute(tags.path); 116 | } 117 | } 118 | 119 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/MyFilenameFilter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Filter used to identify files for analysis honoring the values supplied within the filter options 9 | * @author TBrady 10 | * 11 | */ 12 | public class MyFilenameFilter implements FilenameFilter { 13 | boolean ignorePeriods; 14 | String prefix; 15 | String contains; 16 | String suffix; 17 | String exclusion; 18 | 19 | boolean isRePrefix; 20 | boolean isReContains; 21 | boolean isReSuffix; 22 | boolean isReExclusion; 23 | 24 | boolean isTestFiles; 25 | 26 | Pattern rePrefix; 27 | Pattern reContains; 28 | Pattern reSuffix; 29 | Pattern reExclusion; 30 | 31 | public MyFilenameFilter( 32 | boolean ignorePeriods, 33 | String prefix, 34 | boolean isRePrefix, 35 | String contains, 36 | boolean isReContains, 37 | String suffix, 38 | boolean isReSuffix, 39 | String exclusion, 40 | boolean isReExclusion, 41 | boolean isTestFiles 42 | ){ 43 | this.ignorePeriods = ignorePeriods; 44 | this.prefix = prefix; 45 | this.contains = contains; 46 | this.suffix = suffix; 47 | this.exclusion = exclusion; 48 | 49 | this.isRePrefix = isRePrefix; 50 | this.isReContains = isReContains; 51 | this.isReSuffix = isReSuffix; 52 | this.isReExclusion = isReExclusion; 53 | 54 | this.isTestFiles = isTestFiles; 55 | 56 | if (isRePrefix) 57 | rePrefix = Pattern.compile(prefix, Pattern.CASE_INSENSITIVE); 58 | if (isReContains) 59 | reContains = Pattern.compile(contains, Pattern.CASE_INSENSITIVE); 60 | if (isReSuffix) 61 | reSuffix = Pattern.compile(suffix, Pattern.CASE_INSENSITIVE); 62 | if (isReExclusion) 63 | reExclusion = Pattern.compile(exclusion, Pattern.CASE_INSENSITIVE); 64 | } 65 | public boolean accept(File dir, String filename) { 66 | 67 | if (ignorePeriods) { 68 | if (!filename.contains(".")){ 69 | if ((new File(dir,filename)).isDirectory()){ 70 | return true; 71 | } 72 | } 73 | }else { 74 | if ((new File(dir,filename)).isDirectory()){ 75 | return true; 76 | } 77 | } 78 | 79 | if (!isTestFiles) return false; 80 | 81 | if (prefix.length() > 0){ 82 | if (isRePrefix) { 83 | if (!rePrefix.matcher(filename).matches()) 84 | return false; 85 | } else if (!filename.toLowerCase().startsWith(prefix.toLowerCase())){ 86 | return false; 87 | } 88 | } 89 | if (contains.length() > 0){ 90 | if (isReContains) { 91 | if (!reContains.matcher(filename).matches()) 92 | return false; 93 | } else if (!filename.toLowerCase().contains(contains.toLowerCase())){ 94 | return false; 95 | } 96 | } 97 | if (suffix.length() > 0){ 98 | if (isReSuffix) { 99 | if (!reSuffix.matcher(filename).matches()) 100 | return false; 101 | } else if (!filename.toLowerCase().endsWith(suffix.toLowerCase())){ 102 | return false; 103 | } 104 | } 105 | if (exclusion.length() > 0){ 106 | if (isReExclusion) { 107 | if (reExclusion.matcher(filename).matches()) 108 | return false; 109 | } else if (filename.toLowerCase().contains(exclusion.toLowerCase())){ 110 | return false; 111 | } 112 | } 113 | return true; 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NARA File Analyzer and Metadata Harvester 2 | == 3 | 4 | The NARA File Analyzer Tool walks a directory tree and performs a "File Test" 5 | on each file that is encountered. The application framework allows new File 6 | Tests to be quickly developed and deployed into the application. The results 7 | of each File Test are compiled into a table that summarizes the results of the 8 | analysis. 9 | 10 | A File Test is a simple set of actions that are performed upon a single file 11 | such as filename validation, file size statistical analysis, checksum 12 | calculation, file type extraction. Depending on the action, the content of 13 | the file may or may not be read. Each File Test is configured with filters 14 | that determine which files will be processed by the File Test (i.e. only image 15 | files). 16 | 17 | Each File Test will generate a table of results. The number of columns and 18 | the definition of the columns will vary from test to test. For example, a 19 | file type analysis will report the file extension and the number of files 20 | discovered with that extension. The checksum file tests will report the name 21 | of a file and the checksum string associated with that file. 22 | 23 | The File Analyzer tool can be run as a GUI in which the results are 24 | displayed in a table. The File Analyzer can also be run in batch mode. In 25 | batch mode, the results will be written to a tab-separated file. The GUI 26 | version of the application allows the results of multiple executions to be 27 | merged. The merged information can be filtered to display matching values 28 | and mismatched values. 29 | 30 | ## Benefits 31 | 32 | The NARA File Analyzer automates a number of simple tasks that would be 33 | tedious to perform either manually or with other COTS applications. 34 | 35 | The match/merge capabilities have provided a very powerful and simple 36 | mechanism to ensure quality control checks on large numbers of files. 37 | 38 | Note: NARA has also deployed a customized version of this application 39 | that performs file tests that implement custom business rules such as 40 | file name validation and metadata introspection. 41 | 42 | ## Public Domain 43 | 44 | This project is in the public domain within the United States, and 45 | copyright and related rights in the work worldwide are waived through 46 | the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). 47 | 48 | All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest 49 | 50 | For more information, see [license](https://github.com/usnationalarchives/File-Analyzer/blob/master/LICENSE.md). 51 | 52 | ## Privacy 53 | 54 | All comments, messages, pull requests, and other submissions received through official NARA pages including this GitHub page may be subject to archiving requirements. See the [Privacy Statement](http://www.archives.gov/global-pages/privacy.html) for more information. 55 | 56 | ## Contributions 57 | 58 | We welcome contributions. If you would like to contribute to the project you can do so by forking the repository and submitting your changes in a pull request. You can submit issues using [GitHub Issues](https://github.com/usnationalarchives/File-Analyzer/issues). 59 | 60 | ## Deployment 61 | 62 | The basic File Analyzer application is deployed as a self-extracting 63 | jar file. The application requires Java SE 1.6 or higher to be present 64 | on the user's workstation. The application can be launched by double 65 | clicking the jar file. 66 | 67 | If additional runtime memory is needed when running the application, a 68 | simple windows bat file can be created to launch the application with 69 | a larger memory allocation. 70 | 71 | java -mx1000m -jar fileAnalyzer.jar 72 | 73 | Additional documentation can be found in the `doc/NARA File Analyzer and Metadata Harvester.doc` document. 74 | 75 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/nameValidation/DirAnalysis.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.nameValidation; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | import java.util.TreeMap; 7 | import java.util.regex.Matcher; 8 | import java.util.regex.Pattern; 9 | 10 | /** 11 | * Special file name validation logic that applies to directories. 12 | * When naming conventions exist for files, there is usually a different rule governing the containing directories. 13 | * This class also has a facility to ensure unique and sequential sequencing of individual file names within a directory 14 | * @author TBrady 15 | * 16 | */ 17 | public class DirAnalysis{ 18 | 19 | public static RenameDetails analyze(File dir, Pattern p, int group, Pattern pIgnore){ 20 | return analyze(dir,p,group,pIgnore,true); 21 | } 22 | public static RenameDetails analyze(File dir, Pattern p, int group, Pattern pIgnore, boolean rptIgnore){ 23 | ArrayList names = new ArrayList(); 24 | for(String s: dir.list()){ 25 | names.add(s); 26 | } 27 | return analyze(names, dir,p,group,pIgnore,rptIgnore); 28 | } 29 | public static RenameDetails recursiveAnalyze(File dir, Pattern p, int group, Pattern pIgnore, boolean rptIgnore){ 30 | ArrayList names = new ArrayList(); 31 | gatherFileNames(names, dir); 32 | return analyze(names, dir,p,group,pIgnore,rptIgnore); 33 | } 34 | 35 | public static void gatherFileNames(Listnames, File f) { 36 | if (f.isDirectory()) { 37 | for(File cf: f.listFiles()){ 38 | gatherFileNames(names, cf); 39 | } 40 | } else { 41 | names.add(f.getName()); 42 | } 43 | } 44 | 45 | public static RenameDetails analyze(List names, File dir, Pattern p, int group, Pattern pIgnore, boolean rptIgnore){ 46 | TreeMap seq = new TreeMap(); 47 | StringBuffer buf = new StringBuffer(); 48 | buf.append(names.size()); 49 | buf.append(" Files: "); 50 | 51 | boolean invalid = false; 52 | 53 | if (names.size() == 0) { 54 | return new RenameDetails(RenameStatus.DIRECTORY_EMPTY, null, null); 55 | } 56 | 57 | if (p == null) { 58 | return new RenameDetails(RenameStatus.DIRECTORY, null, buf.toString()); 59 | } 60 | 61 | for(String name: names){ 62 | Matcher m = p.matcher(name); 63 | if (m.matches()) { 64 | if (group < 0) continue; 65 | if (group > m.groupCount()) continue; 66 | String s = m.group(group); 67 | try { 68 | int cur = Integer.parseInt(s); 69 | Integer x = seq.get(cur); 70 | if (x == null) x = 0; 71 | //handle dup? 72 | seq.put(cur,x++); 73 | } catch(NumberFormatException e) { 74 | invalid=true; 75 | buf.append(" ["+name+"]"); 76 | } 77 | } else if (pIgnore.matcher(name).matches()){ 78 | if (rptIgnore) buf.append(" {Skip: "+name+"}"); 79 | } else { 80 | buf.append(" ["+name+"]"); 81 | invalid=true; 82 | } 83 | } 84 | 85 | if (invalid) { 86 | return new RenameDetails(RenameStatus.DIRECTORY_CHILD_INVALID, null, buf.toString()); 87 | } 88 | 89 | if (seq.size() == 0) { 90 | return new RenameDetails(RenameStatus.DIRECTORY_INCOMPLETE, null, buf.toString()); 91 | } 92 | boolean sequenceValid = true; 93 | int last = -1; 94 | int last_rangestart = -1; 95 | for(int cur: seq.keySet()){ 96 | if (last == -1) { 97 | buf.append(cur); 98 | last_rangestart = cur; 99 | }else if (cur != last+1) { 100 | if (last_rangestart != last) { 101 | buf.append(" - " + last); 102 | } 103 | buf.append(", " + cur); 104 | last_rangestart = cur; 105 | sequenceValid = false; 106 | } 107 | last = cur; 108 | } 109 | if (last != last_rangestart) { 110 | buf.append(" - " + last); 111 | } 112 | 113 | if (sequenceValid) { 114 | return new RenameDetails(RenameStatus.DIRECTORY_VALID, null, buf.toString()); 115 | } 116 | return new RenameDetails(RenameStatus.DIRECTORY_SEQUENCE_ERROR, null, buf.toString()); 117 | } 118 | 119 | } 120 | 121 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/DetailsPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.gui.MyTableModel; 4 | 5 | import java.awt.BorderLayout; 6 | import java.awt.Dimension; 7 | import java.awt.event.ActionEvent; 8 | import java.awt.event.ActionListener; 9 | import java.util.ArrayList; 10 | 11 | import javax.swing.BorderFactory; 12 | import javax.swing.JButton; 13 | import javax.swing.JLabel; 14 | import javax.swing.JPanel; 15 | import javax.swing.JScrollPane; 16 | import javax.swing.JTable; 17 | import javax.swing.JTextField; 18 | import javax.swing.RowFilter; 19 | import javax.swing.table.TableModel; 20 | import javax.swing.table.TableRowSorter; 21 | 22 | /** 23 | * User interface tab displaying the files that have been processed by a FileTest 24 | * @author TBrady 25 | * 26 | */ 27 | class DetailsPanel extends MyBorderPanel { 28 | private static final long serialVersionUID = 1L; 29 | JTable jt; 30 | MyTableModel tm; 31 | JTextField jtfRoot; 32 | TableRowSorter sorter; 33 | DirectoryTable parent; 34 | 35 | DetailsPanel(DirectoryTable dt) { 36 | parent = dt; 37 | JPanel ph = addPanel("", BorderLayout.NORTH); 38 | ph.setLayout(new BorderLayout()); 39 | jtfRoot = new JTextField("",50); 40 | jtfRoot.setEditable(false); 41 | ph.add(jtfRoot, BorderLayout.NORTH); 42 | 43 | JPanel p = addBorderPanel("Details"); 44 | tm = new MyTableModel(); 45 | jt = new JTable(tm); 46 | jt.setAutoResizeMode(JTable.AUTO_RESIZE_OFF); 47 | sorter = new TableRowSorter(tm); 48 | jt.setRowSorter(sorter); 49 | tm.setColumns(jt); 50 | jt.setPreferredScrollableViewportSize(new Dimension(600,400)); 51 | p.add(new JScrollPane(jt),BorderLayout.CENTER); 52 | ph.add(new DetailsFilter(), BorderLayout.SOUTH); 53 | 54 | p = addPanel((String)null, BorderLayout.SOUTH); 55 | dt.countLabel = new JLabel(); 56 | p.add(dt.countLabel); 57 | JButton save = new JButton("Export Table"); 58 | save.addActionListener( 59 | new ActionListener() { 60 | public void actionPerformed(ActionEvent arg0) { 61 | new TableSaver(parent,tm,jt,"Details"); 62 | } 63 | } 64 | ); 65 | p.add(save); 66 | } 67 | class DetailsFilter extends JPanel { 68 | private static final long serialVersionUID = 1L; 69 | ArrayList tfs; 70 | 71 | DetailsFilter() { 72 | tfs = new ArrayList(); 73 | createField("Dir",15); 74 | createField("File",10); 75 | createField("Type",5); 76 | createField("Other",15); 77 | JButton bf = new JButton("Filter"); 78 | add(bf); 79 | bf.addActionListener(new ActionListener(){ 80 | public void actionPerformed(ActionEvent arg0) { 81 | setFilter(); 82 | } 83 | }); 84 | JButton bc = new JButton("Clear"); 85 | bc.addActionListener(new ActionListener(){ 86 | public void actionPerformed(ActionEvent arg0) { 87 | for(JTextField tf: tfs) { 88 | tf.setText(""); 89 | } 90 | setFilter(); 91 | } 92 | }); 93 | add(bc); 94 | } 95 | 96 | JTextField createField(String name, int len) { 97 | JTextField tf = new JTextField(len); 98 | tf.setBorder(BorderFactory.createTitledBorder(name)); 99 | add(tf); 100 | tfs.add(tf); 101 | return tf; 102 | } 103 | 104 | void setFilter() { 105 | DetailsPanel.this.sorter.setRowFilter( 106 | new RowFilter(){ 107 | public boolean include( 108 | javax.swing.RowFilter.Entry row) { 109 | boolean b = true; 110 | b = b && compare(row.getStringValue(0), tfs.get(0).getText()); 111 | b = b && compare(row.getStringValue(1), tfs.get(1).getText()); 112 | b = b && compare(row.getStringValue(2), tfs.get(2).getText()); 113 | b = b && compare(row.getStringValue(5), tfs.get(3).getText()); 114 | return b; 115 | } 116 | public boolean compare(String val, String filter) { 117 | if (filter==null) return true; 118 | if (filter.trim().equals("")) return true; 119 | if (val == null) return false; 120 | return (val.trim().contains(filter.trim())); 121 | } 122 | } 123 | ); 124 | } 125 | 126 | 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/BatchImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import gov.nara.nwts.ftapp.importer.Importer; 4 | import gov.nara.nwts.ftapp.importer.ImporterRegistry; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | /** 10 | * Driver for the command line version of the File Analyzer (for performing file imports) 11 | * This application was originally created by Terry Brady in NARA's Digitization Services Branch. 12 | * @author TBrady 13 | * 14 | */ 15 | public class BatchImporter { 16 | FTDriver dt; 17 | File infile; 18 | File outdir; 19 | String outfile; 20 | Importer impsel; 21 | ImporterRegistry ar; 22 | boolean overwrite = true; 23 | 24 | public ImporterRegistry getImporterRegistry(FTDriver dt) { 25 | return new ImporterRegistry(dt); 26 | } 27 | public BatchImporter() { 28 | dt = new FTDriver(infile); 29 | ar = getImporterRegistry(dt); 30 | outdir = new File(System.getProperty("user.dir")); 31 | outfile = ""; 32 | } 33 | 34 | public String getArg(String[] args, int i) { 35 | if (i >= args.length) reportError("Missing argument for "+args[args.length-1]); 36 | String s = args[i]; 37 | if ((s.length() > 2) && (s.startsWith("\"")) && (s.endsWith("\""))) { 38 | return s.substring(1,s.length()-2); 39 | } 40 | return s; 41 | } 42 | 43 | public void parse(String[] args) { 44 | String ftname = null; 45 | for(int i=0; i keySet; 22 | public static final String LAQL="AQL"; 23 | 24 | public static Object[][] details = { 25 | {String.class,"Path",450}, 26 | {YN.class,"Selected",60, YN.values(),false}, 27 | }; 28 | 29 | public RandomFileTest(FTDriver dt) { 30 | super(dt); 31 | keySet = new TreeMap(); 32 | ftprops.add(new FTPropEnum(this, LAQL,"AQL","Set the acceptible quality level",AQL.values(),AQL.AQLp04)); 33 | } 34 | 35 | public String toString() { 36 | return "Random Sampling Mil 105E"; 37 | } 38 | public String getKey(File f) { 39 | return f.getAbsolutePath(); 40 | } 41 | public Stats createStats(String key){ 42 | Stats stats = new RandomStats(key); 43 | stats.vals.add(YN.N); 44 | return stats; 45 | } 46 | 47 | public String getShortName(){return "Random";} 48 | 49 | public Object[][] getStatsDetails() { 50 | return details; 51 | } 52 | 53 | public void initFilters() { 54 | initAllFilters(); 55 | } 56 | 57 | public String getDescription() { 58 | return "This test will return a list of files in random order for QC processing"; 59 | } 60 | 61 | public Object fileTest(File f) { 62 | return null; 63 | } 64 | 65 | public TreeMap getTreeSet() { 66 | return keySet; 67 | } 68 | 69 | public void refineResults() { 70 | int count=0; 71 | AQL aql = (AQL)getProperty(LAQL); 72 | int batchSize = dt.types.size(); 73 | int sampleSize = aql.getSampleSize(batchSize); 74 | 75 | for(Iteratori=keySet.values().iterator(); i.hasNext(); ){ 76 | Stats stats = dt.types.get(i.next()); 77 | stats.vals.set(0, (count< sampleSize) ? YN.Y : YN.N); 78 | count++; 79 | } 80 | keySet.clear(); 81 | } 82 | 83 | /** 84 | * Acceptable quality levels as defined by Mil 105E. 85 | * Five AQL's have been coded as an illustrative example. 86 | * @author TBrady 87 | * 88 | */ 89 | enum AQL { 90 | AQLp04("AQL 0.04%", -1,-1,-1,-1,-1,-1,-1,315,315,315,315,315,490,715,715), 91 | AQL1("AQL 1.00%",-1,13,13,13,13,13,20,29,34,42,50,60,74,90,102), 92 | AQL2p5("AQL 2.50%",5,5,5,5,7,11,13,16,19,23,29,35,40,40,40), 93 | AQL4("AQL 4.00%",3,3,3,5,6,7,10,11,15,18,22,29,29,29,29), 94 | AQL6p5("AQL 6.50%",2,2,3,5,5,6,7,9,11,13,15,15,15,15,15); 95 | 96 | String name; 97 | public String toString() { 98 | return name; 99 | } 100 | class Sample { 101 | int batchSize; 102 | int sampleSize; 103 | Sample(int batchSize, int sampleSize) { 104 | this.batchSize = batchSize; 105 | this.sampleSize = sampleSize; 106 | } 107 | } 108 | ArrayList sampleSizes; 109 | 110 | AQL(String name,int v8, int v15, int v25,int v50, int v90,int v150, int v280, int v500, int v1200, int v3200, int v10000, int v35000, int v150000, int v500000, int vmax) { 111 | this.name = name; 112 | sampleSizes = new ArrayList(); 113 | sampleSizes.add(new Sample(8,v8)); 114 | sampleSizes.add(new Sample(15,v15)); 115 | sampleSizes.add(new Sample(25,v25)); 116 | sampleSizes.add(new Sample(50,v50)); 117 | sampleSizes.add(new Sample(90,v90)); 118 | sampleSizes.add(new Sample(150,v150)); 119 | sampleSizes.add(new Sample(280,v280)); 120 | sampleSizes.add(new Sample(500,v500)); 121 | sampleSizes.add(new Sample(1200,v1200)); 122 | sampleSizes.add(new Sample(3200,v3200)); 123 | sampleSizes.add(new Sample(10000,v10000)); 124 | sampleSizes.add(new Sample(35000,v35000)); 125 | sampleSizes.add(new Sample(150000,v150000)); 126 | sampleSizes.add(new Sample(500000,v500000)); 127 | sampleSizes.add(new Sample(-1,vmax)); 128 | }; 129 | 130 | int getSampleSize(int batchSize) { 131 | for(Iteratori=sampleSizes.iterator(); i.hasNext();){ 132 | Sample s = i.next(); 133 | if (s.batchSize >= batchSize) { 134 | return (s.sampleSize == -1) ? batchSize : s.sampleSize; 135 | } 136 | if (s.batchSize == -1) { 137 | return s.sampleSize; 138 | } 139 | } 140 | return batchSize; 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/importer/DelimitedFileImporter.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.importer; 2 | 3 | import gov.nara.nwts.ftapp.ActionResult; 4 | import gov.nara.nwts.ftapp.FTDriver; 5 | import gov.nara.nwts.ftapp.Timer; 6 | import gov.nara.nwts.ftapp.stats.Stats; 7 | 8 | import java.io.BufferedReader; 9 | import java.io.File; 10 | import java.io.FileReader; 11 | import java.io.IOException; 12 | import java.util.TreeMap; 13 | import java.util.Vector; 14 | 15 | /** 16 | * Abstract class handling the import of a character-delimited text file allowing for individual values to be wrapped by quotation marks. 17 | * @author TBrady 18 | * 19 | */ 20 | public abstract class DelimitedFileImporter extends DefaultImporter { 21 | public DelimitedFileImporter(FTDriver dt) { 22 | super(dt); 23 | } 24 | boolean forceKey; 25 | int rowKey = 1000000; 26 | 27 | public abstract String getSeparator(); 28 | protected static String getNextString(String in, String sep) { 29 | return getNextString(in,sep,0); 30 | } 31 | protected static String getNextString(String in, String sep, int start) { 32 | int pos = -1; 33 | if (in.startsWith("\"")) { 34 | int qpos = in.indexOf("\"", (start==0) ? 1 : start); 35 | int qqpos = in.indexOf("\"\"", (start==0) ? 1 : start); 36 | if ((qpos==qqpos)&&(qqpos >= 0)) { 37 | return getNextString(in,sep,qqpos+2); 38 | } 39 | if (qpos == in.length()) { 40 | return in; 41 | } 42 | if (qpos == -1) { 43 | qpos = 0; 44 | } 45 | pos = in.indexOf(sep,qpos+1); 46 | } else { 47 | pos = in.indexOf(sep, 0); 48 | } 49 | if (pos == -1) return in; 50 | return in.substring(0,pos); 51 | } 52 | 53 | public static Vector parseLine(String line, String sep){ 54 | String pline = line; 55 | Vector cols = new Vector(); 56 | while(pline!=null){ 57 | pline = pline.trim(); 58 | String tpline = getNextString(pline,sep); 59 | cols.add(normalize(tpline)); 60 | if (pline.length() == tpline.length()) break; 61 | pline = pline.substring(tpline.length()+1); 62 | } 63 | return cols; 64 | } 65 | 66 | public static Vector> parseFile(File f, String sep) throws IOException{ 67 | return parseFile(f,sep,false); 68 | } 69 | public static Vector> parseFile(File f, String sep,boolean skipFirstLine) throws IOException{ 70 | BufferedReader br = new BufferedReader(new FileReader(f)); 71 | if (skipFirstLine) br.readLine(); 72 | Vector> rows = new Vector>(); 73 | for(String line=br.readLine(); line!=null; line=br.readLine()){ 74 | rows.add(parseLine(line, sep)); 75 | } 76 | br.close(); 77 | return rows; 78 | } 79 | 80 | public ActionResult importFile(File selectedFile) throws IOException { 81 | Timer timer = new Timer(); 82 | forceKey = dt.getImporterForceKey(); 83 | BufferedReader br = new BufferedReader(new FileReader(selectedFile)); 84 | int colcount = 0; 85 | TreeMap types = new TreeMap(); 86 | for(String line=br.readLine(); line!=null; line=br.readLine()){ 87 | Vector cols = parseLine(line, getSeparator()); 88 | colcount = Math.max(colcount,cols.size()); 89 | String key = cols.get(0); 90 | if (forceKey) { 91 | key = "" + (rowKey++); 92 | } 93 | Stats stats = new Stats(key); 94 | if (forceKey) { 95 | stats.vals.add(cols.get(0)); 96 | } 97 | for(int i=1; i checks; 35 | 36 | DirectoryTable parent; 37 | TableSaver(DirectoryTable parent, DefaultTableModel tm, JTable jt, String fname) { 38 | this(parent, tm, jt, fname, new ArrayList()); 39 | } 40 | TableSaver(DirectoryTable parent, DefaultTableModel tm, JTable jt, String fname, ListnoExport) { 41 | super(parent.frame, "Export Table: "+fname); 42 | this.tm = tm; 43 | this.parent = parent; 44 | this.jt = jt; 45 | setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); 46 | setModal(true); 47 | setLayout(new BorderLayout()); 48 | JFileChooser jfc = new JFileChooser(){ 49 | private static final long serialVersionUID = 1L; 50 | public String getApproveButtonText() { 51 | return "Export"; 52 | } 53 | public void cancelSelection() { 54 | TableSaver.this.dispose(); 55 | } 56 | public void approveSelection() { 57 | File f = getSelectedFile(); 58 | if (f.exists()) { 59 | if (JOptionPane.showConfirmDialog(this, "File Exists. Do you wish to replace it?","File Exists", JOptionPane.YES_NO_OPTION) != JOptionPane.YES_OPTION) { 60 | return; 61 | } 62 | } 63 | try { 64 | save(getSelectedFile()); 65 | TableSaver.this.parent.preferences.put("importexport",getSelectedFile().getParentFile().getAbsolutePath()); 66 | //throw new IOException("testing..."); 67 | } catch (IOException e) { 68 | e.printStackTrace(); 69 | JOptionPane.showMessageDialog(this, e.getMessage(), "Error", JOptionPane.ERROR_MESSAGE); 70 | } 71 | TableSaver.this.dispose(); 72 | } 73 | }; 74 | String defpath = parent.preferences.get("importexport", ""); 75 | jfc.setSelectedFile(new File(fname +".txt")); 76 | if (!defpath.equals("")){ 77 | File p = new File(defpath); 78 | if (p.exists()) { 79 | jfc.setSelectedFile(new File(p,fname +".txt")); 80 | } 81 | } 82 | add(jfc, BorderLayout.NORTH); 83 | colPanel = new JPanel(new GridLayout(0,1)); 84 | add(new JScrollPane(colPanel), BorderLayout.CENTER); 85 | checks = new ArrayList(); 86 | for(int c=0; c rs = jt.getRowSorter(); 116 | for(int r=0; r0) buf.append(","); 106 | buf.append(l); 107 | } 108 | return buf.toString(); 109 | } else if (tf.getType() == TIFFField.TIFF_UNDEFINED) { 110 | return "Undef: "; 111 | } else if (tf.getType() == TIFFField.TIFF_SBYTE) { 112 | return "Undef: "; 113 | } 114 | return tf.getAsString(0); 115 | } catch (Exception e) { 116 | e.printStackTrace(); 117 | return "**" + tf.getType(); 118 | } 119 | } 120 | 121 | 122 | public String getTagName(TIFFField tf) { 123 | if (tiffdir == null) return ""; 124 | int tag = tf.getTag(); 125 | TAGS tags = getTagDef(tf); 126 | if (tags == TAGS.UNDEFINED) { 127 | return "Tiff Tag " + tag; 128 | } 129 | return tags.name(); 130 | } 131 | public TAGS getTagDef(TIFFField tf) { 132 | if (tiffdir == null) return TAGS.UNDEFINED; 133 | int tag = tf.getTag(); 134 | 135 | for(TAGS t: TAGS.values()) { 136 | if (t.tiffloc == TAGLOC.TAG && tag == t.tag) { 137 | return t; 138 | } 139 | } 140 | return TAGS.UNDEFINED; 141 | } 142 | 143 | public ArrayList getTags() { 144 | ArrayList list = new ArrayList(); 145 | if (tiffdir != null){ 146 | for(TIFFField tf: tiffdir.getFields()) { 147 | list.add(tf); 148 | } 149 | } 150 | if (xmpex != null){ 151 | list.addAll(xmpex.getTags()); 152 | } 153 | return list; 154 | } 155 | 156 | 157 | public String getXMP(String[] xmppath) { 158 | return xmpex.getXMP(xmppath); 159 | } 160 | 161 | public void close() { 162 | try { 163 | if (fss != null) fss.close(); 164 | } catch (IOException e) { 165 | e.printStackTrace(); 166 | } 167 | } 168 | 169 | 170 | } 171 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/DefaultFileTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | import java.util.StringTokenizer; 7 | import java.util.regex.Pattern; 8 | 9 | import gov.nara.nwts.ftapp.FTDriver; 10 | import gov.nara.nwts.ftapp.filter.AVFileTestFilter; 11 | import gov.nara.nwts.ftapp.filter.DefaultFileTestFilter; 12 | import gov.nara.nwts.ftapp.filter.FileTestFilter; 13 | import gov.nara.nwts.ftapp.filter.ImageFileTestFilter; 14 | import gov.nara.nwts.ftapp.filter.JpegFileTestFilter; 15 | import gov.nara.nwts.ftapp.filter.TiffFileTestFilter; 16 | import gov.nara.nwts.ftapp.ftprop.FTProp; 17 | import gov.nara.nwts.ftapp.stats.Stats; 18 | import gov.nara.nwts.ftapp.stats.CountStats; 19 | 20 | /** 21 | * Abstract implementation of the FileTest interface; FileTest objects will derive these behaviors unless explicitly overridden. 22 | * @author TBrady 23 | * 24 | */ 25 | public abstract class DefaultFileTest implements FileTest { 26 | protected FTDriver dt; 27 | protected ArrayListfilters; 28 | protected ArrayListftprops; 29 | 30 | public File getRoot() { 31 | return dt.getRoot(); 32 | } 33 | 34 | public DefaultFileTest(FTDriver dt) { 35 | this.dt = dt; 36 | filters = new ArrayList(); 37 | ftprops = new ArrayList(); 38 | initFilters(); 39 | } 40 | 41 | public void initFilters() { 42 | filters.add(new DefaultFileTestFilter()); 43 | } 44 | 45 | public FileTest resetOption() { 46 | return null; 47 | } 48 | public void initAllFilters() { 49 | filters.add(new DefaultFileTestFilter()); 50 | filters.add(new AVFileTestFilter()); 51 | filters.add(new ImageFileTestFilter()); 52 | filters.add(new TiffFileTestFilter()); 53 | filters.add(new JpegFileTestFilter()); 54 | } 55 | 56 | public FileTestFilter getDefaultFilter() { 57 | if (filters.size() > 0) 58 | return filters.get(0); 59 | return null; 60 | } 61 | 62 | public List getFilters() { 63 | return filters; 64 | } 65 | 66 | public String getExt(File f) { 67 | String ext = ""; 68 | StringTokenizer st = new StringTokenizer(f.getName(), "."); 69 | while(st.hasMoreElements()) { 70 | ext = st.nextElement().toString().toUpperCase(); 71 | } 72 | return ext; 73 | } 74 | 75 | public String getKey(File f) { 76 | return getExt(f); 77 | } 78 | public String getKey(File f, Object o) { 79 | return getKey(f); 80 | } 81 | 82 | public String getShortNameNormalized() { 83 | return getShortName().replaceAll("[\\s&]",""); 84 | } 85 | public String getShortNameFormatted() { 86 | StringBuffer buf = new StringBuffer(); 87 | buf.append(getShortNameNormalized()); 88 | buf.append(" "); 89 | return buf.substring(0,20); 90 | } 91 | public Stats getStats(File f) { 92 | return getStats(getKey(f)); 93 | } 94 | 95 | public Stats getStats(String key) { 96 | Stats mystats = dt.types.get(key); 97 | if (mystats == null) { 98 | mystats = createStats(key); 99 | } 100 | dt.types.put(key, mystats); 101 | 102 | return mystats; 103 | } 104 | 105 | public boolean isTestable(File f) { 106 | return true; 107 | } 108 | 109 | public Stats createStats(String key){ 110 | return new CountStats(key); 111 | } 112 | public Object[][] getStatsDetails() { 113 | return CountStats.details; 114 | } 115 | 116 | public boolean isTestDirectory() { 117 | return false; 118 | } 119 | public boolean processRoot() { 120 | return false; 121 | } 122 | public boolean isTestFiles() { 123 | return true; 124 | } 125 | public Pattern getDirectoryPattern() { 126 | return null; 127 | } 128 | public void init() { 129 | } 130 | public void refineResults() { 131 | } 132 | 133 | public void progress(int count) { 134 | } 135 | 136 | public void showCount(int count) { 137 | if (count>0) System.out.println("Progress Count: "+count+" files processed."); 138 | } 139 | 140 | /* Some tasks such as checksum validation had errors due to too many open file handles although the files were out of scope. 141 | * Explicitly calling GC to force cleanup.*/ 142 | public void cleanup(int count) { 143 | if (count>0) System.out.println("Progress Count: "+count+" files processed. Start Cleanup."); 144 | System.gc(); 145 | System.runFinalization(); 146 | if (count>0) System.out.println("Cleanup complete."); 147 | System.out.flush(); 148 | } 149 | public List getPropertyList() { 150 | return ftprops; 151 | } 152 | 153 | public FTDriver getFTDriver() { 154 | return dt; 155 | } 156 | 157 | public Object getProperty(String name) { 158 | return getProperty(name, null); 159 | } 160 | public Object getProperty(String name, Object def) { 161 | for(FTProp ftprop: ftprops) { 162 | if (ftprop.getName().equals(name)) { 163 | return ftprop.getValue(); 164 | } 165 | } 166 | return def; 167 | } 168 | public void setProperty(String name, String s) { 169 | for(FTProp ftprop: ftprops) { 170 | if (ftprop.getName().equals(name)) { 171 | ftprop.setValue(ftprop.validate(s)); 172 | return; 173 | } 174 | } 175 | } 176 | 177 | } 178 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/ImportPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.importer.Importer; 4 | import gov.nara.nwts.ftapp.stats.Stats; 5 | 6 | import java.awt.BorderLayout; 7 | import java.awt.FlowLayout; 8 | import java.awt.GridLayout; 9 | import java.awt.event.ActionEvent; 10 | import java.awt.event.ActionListener; 11 | import java.io.File; 12 | import java.io.IOException; 13 | import java.text.NumberFormat; 14 | import java.util.TreeMap; 15 | 16 | import javax.swing.BorderFactory; 17 | import javax.swing.JButton; 18 | import javax.swing.JCheckBox; 19 | import javax.swing.JComboBox; 20 | import javax.swing.JFormattedTextField; 21 | import javax.swing.JLabel; 22 | import javax.swing.JOptionPane; 23 | import javax.swing.JPanel; 24 | import javax.swing.JTextArea; 25 | import javax.swing.JTextField; 26 | 27 | /** 28 | * User interface component presenting file import options and auto-sequencing options. 29 | * @author TBrady 30 | * 31 | */ 32 | class ImportPanel extends MyPanel { 33 | private static final long serialVersionUID = 1L; 34 | JTextField prefix; 35 | JTextField suffix; 36 | JFormattedTextField start; 37 | JFormattedTextField end; 38 | JComboBox pad; 39 | JComboBox importers; 40 | JTextArea importerDesc; 41 | JCheckBox forceKey; 42 | DirectoryTable parent; 43 | FileSelectChooser fsc; 44 | 45 | void updateDesc() { 46 | Importer i = (Importer)importers.getSelectedItem(); 47 | importerDesc.setText(i.getDescription()); 48 | forceKey.setEnabled(i.allowForceKey()); 49 | } 50 | 51 | ImportPanel(DirectoryTable dt) { 52 | parent = dt; 53 | JPanel p = addBorderPanel("Contents to Import for Analysis"); 54 | JPanel tp = new JPanel(new BorderLayout()); 55 | p.add(tp, BorderLayout.NORTH); 56 | fsc = new FileSelectChooser(parent.frame, "Select file to Import", parent.preferences, "IMPORT", ""); 57 | fsc.setBorder(BorderFactory.createTitledBorder("File to Import")); 58 | tp.add(fsc, BorderLayout.NORTH); 59 | JPanel p1 = new JPanel(new FlowLayout(FlowLayout.RIGHT)); 60 | p.add(p1, BorderLayout.SOUTH); 61 | forceKey = new JCheckBox("Force Unique Keys"); 62 | p1.add(forceKey); 63 | JButton jb = new JButton("Import File"); 64 | jb.addActionListener( 65 | new ActionListener(){ 66 | public void actionPerformed(ActionEvent arg0) { 67 | File f = new File(fsc.tf.getText()); 68 | if (f.exists()) { 69 | Importer imp = (Importer)parent.importPanel.importers.getSelectedItem(); 70 | try { 71 | parent.importFile(imp, f); 72 | } catch (IOException e) { 73 | JOptionPane.showMessageDialog(parent.frame, e.getMessage()+": "+f.getAbsolutePath()); 74 | } 75 | } 76 | } 77 | } 78 | ); 79 | p1.add(jb); 80 | p1 = new JPanel(); 81 | p1.add(new JLabel("File Import Action")); 82 | importers = new JComboBox(parent.importerRegistry); 83 | importers.addActionListener(new ActionListener(){ 84 | public void actionPerformed(ActionEvent arg0) { 85 | updateDesc(); 86 | } 87 | }); 88 | p1.add(importers); 89 | tp.add(p1, BorderLayout.SOUTH); 90 | importerDesc = new JTextArea(5,60); 91 | importerDesc.setLineWrap(true); 92 | importerDesc.setWrapStyleWord(true); 93 | importerDesc.setBackground(this.getBackground()); 94 | importerDesc.setEditable(false); 95 | p.add(importerDesc); 96 | updateDesc(); 97 | 98 | p = addPanel("Auto-Number"); 99 | p.setLayout(new GridLayout(0,2)); 100 | NumberFormat nf = NumberFormat.getIntegerInstance(); 101 | nf.setGroupingUsed(false); 102 | p.add(new JLabel("Prefix")); 103 | prefix = new JTextField(25); 104 | p.add(prefix); 105 | p.add(new JLabel("Start Number")); 106 | start = new JFormattedTextField(nf); 107 | start.setColumns(8); 108 | p.add(start); 109 | p.add(new JLabel("End Number")); 110 | end = new JFormattedTextField(nf); 111 | end.setColumns(8); 112 | p.add(end); 113 | p.add(new JLabel()); 114 | Object[] objs = {"No Padding",2,3,4,5,6,7,8}; 115 | pad = new JComboBox(objs); 116 | p.add(pad); 117 | p.add(new JLabel("Suffix")); 118 | suffix = new JTextField(25); 119 | p.add(suffix); 120 | p.add(new JLabel("")); 121 | jb = new JButton("Auto-Generate Keys for Analysis"); 122 | p.add(jb); 123 | jb.addActionListener(new ActionListener(){ 124 | Object[][] details = { 125 | {String.class,"Key",100}, 126 | {Integer.class,"Count",10} 127 | }; 128 | public void actionPerformed(ActionEvent arg0) { 129 | TreeMap types = new TreeMap(); 130 | Long startval = (Long)start.getValue(); 131 | Long endval = (Long)end.getValue(); 132 | 133 | if ((startval == null)||(endval==null)) return; 134 | 135 | for(long i=startval.intValue(); i<=endval.intValue(); i++){ 136 | NumberFormat nf = NumberFormat.getIntegerInstance(); 137 | nf.setGroupingUsed(false); 138 | if (pad.getSelectedItem() instanceof Integer) { 139 | nf.setMinimumIntegerDigits((Integer)pad.getSelectedItem()); 140 | } 141 | String key = prefix.getText() + nf.format(i) + suffix.getText(); 142 | Stats stats = new Stats(key); 143 | stats.vals.add(1); 144 | types.put(key, stats); 145 | } 146 | parent.showSummary("Generated "+(++parent.summaryCount), details, types, true); 147 | } 148 | }); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/FileTraversal.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp; 2 | 3 | import java.io.File; 4 | import java.io.FilenameFilter; 5 | import java.util.regex.Pattern; 6 | 7 | 8 | import gov.nara.nwts.ftapp.FTDriver; 9 | import gov.nara.nwts.ftapp.filetest.FileTest; 10 | import gov.nara.nwts.ftapp.stats.Stats; 11 | 12 | /** 13 | * Base class for handling file traversal logic. 14 | * This is used by the command line version of the application. 15 | * Some behaviors are further enhanced in the GUI application @linkplain gov.nara.nwts.ftapp.gui.GuiFileTraversal. 16 | * @author TBrady 17 | * 18 | */ 19 | public class FileTraversal { 20 | protected FTDriver driver; 21 | protected FilenameFilter fileFilter; 22 | protected FilenameFilter dirnameFilter; 23 | 24 | public FileTest fileTest; 25 | protected int max; 26 | protected int numProcessed = 0; 27 | protected boolean cancelled = false; 28 | 29 | public int getNumProcessed() { 30 | return numProcessed; 31 | } 32 | 33 | public FileTraversal(FTDriver dt) { 34 | this.driver = dt; 35 | } 36 | 37 | public boolean isCancelled() { 38 | return cancelled; 39 | } 40 | 41 | public void increment() { 42 | } 43 | 44 | public void setTraversal(FileTest fileTest, int max) { 45 | this.fileTest = fileTest; 46 | this.max = max; 47 | } 48 | 49 | public void reportCancel() { 50 | System.err.println("Stopping: " +max + " items found."); 51 | } 52 | public boolean traverse(File f, FileTest fileTest, int max) { 53 | if (f==null) return false; 54 | File[] files = f.listFiles(fileFilter); 55 | if (files == null) return true; 56 | if (fileTest.processRoot() && fileTest.isTestDirectory()) { 57 | boolean test = true; 58 | Pattern p = fileTest.getDirectoryPattern(); 59 | if (p != null){ 60 | test = p.matcher(f.getAbsolutePath()).matches(); 61 | } 62 | if (test) { 63 | checkDirFile(f, fileTest); 64 | } 65 | 66 | } 67 | for(int i=0; i= max) { 71 | return false; 72 | } 73 | traverse(files[i], fileTest, max); 74 | if (fileTest.isTestDirectory()) { 75 | boolean test = true; 76 | Pattern p = fileTest.getDirectoryPattern(); 77 | if (p != null){ 78 | test = p.matcher(files[i].getAbsolutePath()).matches(); 79 | } 80 | if (test) { 81 | checkDirFile(files[i], fileTest); 82 | } 83 | } 84 | increment(); 85 | } else { 86 | if (isCancelled()) return false; 87 | File thefile = files[i]; 88 | checkFile(thefile, fileTest); 89 | fileTest.progress(getNumProcessed()); 90 | numProcessed++; 91 | if (getNumProcessed() >= max) { 92 | reportCancel(); 93 | return false; 94 | } 95 | } 96 | } 97 | return true; 98 | } 99 | 100 | public void checkFile(File thefile, FileTest fileTest) { 101 | if (fileTest.isTestable(thefile)){ 102 | Stats mystats = fileTest.getStats(thefile); 103 | if (mystats!=null){ 104 | mystats.compute(thefile, fileTest); 105 | } 106 | } 107 | } 108 | public void checkDirFile(File thefile, FileTest fileTest) { 109 | if (fileTest.isTestable(thefile)){ 110 | Stats mystats = fileTest.getStats(thefile); 111 | mystats.compute(thefile, fileTest); 112 | } 113 | } 114 | 115 | 116 | public void countDirectories(File f) { 117 | if (f==null) return; 118 | File[] files = f.listFiles(dirnameFilter); 119 | 120 | increment(); 121 | if (files == null) return; 122 | for(int i=0; i filters; 37 | TableRowSorter sorter; 38 | ArrayListnoExport; 39 | DirectoryTable dt; 40 | 41 | class MyStatsTableModel extends DefaultTableModel { 42 | 43 | private static final long serialVersionUID = 1L; 44 | 45 | public int getColumnCount() { 46 | return details.length; 47 | } 48 | 49 | public Class getColumnClass(int col) { 50 | return (Class) details[col][0]; 51 | } 52 | 53 | }; 54 | 55 | StatsTable(Object[][] details, TreeMap mystats, DirectoryTable dt) { 56 | patt = Pattern.compile("\\s\\s+"); 57 | tm = new MyStatsTableModel(); 58 | this.details = details; 59 | this.dt = dt; 60 | noExport = new ArrayList(); 61 | filters = new Vector(); 62 | for (Iterator i = mystats.keySet().iterator(); i.hasNext();) { 63 | String s = i.next(); 64 | Vector v = new Vector(); 65 | v.add(s); 66 | for (Iterator j = mystats.get(s).vals.iterator(); j 67 | .hasNext();) { 68 | v.add(j.next()); 69 | } 70 | tm.addRow(v); 71 | } 72 | jt = new JTable(tm); 73 | 74 | sorter = new TableRowSorter(tm); 75 | jt.setRowSorter(sorter); 76 | jt.setAutoResizeMode(JTable.AUTO_RESIZE_OFF); 77 | 78 | setColumns(); 79 | 80 | jt.setPreferredScrollableViewportSize(new Dimension(500, 400)); 81 | for (Object[] o : details) { 82 | if (o.length > 3) { 83 | if (o[3] != null) { 84 | StringBuffer buf = new StringBuffer((String) o[1]); 85 | for (Enum e : (Enum[]) o[3]) { 86 | buf.append(";"); 87 | buf.append(e.toString()); 88 | } 89 | JComboBox cb = new JComboBox(buf.toString().split(";")); 90 | cb.addActionListener(new ActionListener() { 91 | public void actionPerformed(ActionEvent arg0) { 92 | RowFilter rf = new RowFilter() { 93 | public boolean include( 94 | Entry arg0) { 95 | for (int i = 0; i < arg0.getValueCount(); i++) { 96 | Object o = arg0.getValue(i); 97 | if (o == null) 98 | continue; 99 | JComboBox cb = filters.get(i); 100 | if (cb == null) 101 | continue; 102 | if (cb.getSelectedIndex() == 0) 103 | continue; 104 | if (!cb.getSelectedItem().equals(o)) { 105 | if (!cb.getSelectedItem() 106 | .toString().equals( 107 | o.toString())) { 108 | return false; 109 | } 110 | } 111 | } 112 | return true; 113 | } 114 | }; 115 | StatsTable.this.sorter.setRowFilter(rf); 116 | StatsTable.this.dt.summaryPanel.setFilterNote(jt.getRowCount(), tm.getRowCount()); 117 | } 118 | }); 119 | filters.add(cb); 120 | } else { 121 | filters.add(null); 122 | } 123 | } else { 124 | filters.add(null); 125 | } 126 | if (o.length > 4) { 127 | if (o[4]!=null) { 128 | if (!(Boolean)o[4]) { 129 | noExport.add((String)o[1]); 130 | } 131 | } 132 | } 133 | } 134 | } 135 | 136 | void setColumns() { 137 | JTableHeader jth = jt.getTableHeader(); 138 | jth.setReorderingAllowed(true); 139 | TableColumnModel tcm = jt.getColumnModel(); 140 | TableColumn tc; 141 | int width = 0; 142 | for (int i = 0; i < details.length; i++) { 143 | tc = tcm.getColumn(i); 144 | int cw = (Integer) details[i][2]; 145 | width += cw; 146 | tc.setPreferredWidth(cw); 147 | tc.setHeaderValue(details[i][1]); 148 | 149 | if (i != 0) { 150 | tc.setCellRenderer(new DefaultTableCellRenderer() { 151 | private static final long serialVersionUID = 1L; 152 | 153 | public void setValue(Object value) { 154 | if (value == null) 155 | return; 156 | if (value instanceof Long) { 157 | long v = (Long) value; 158 | setText(DirectoryTable.nf.format(v)); 159 | setHorizontalAlignment(JLabel.RIGHT); 160 | } else if (value instanceof Integer) { 161 | int v = (Integer) value; 162 | setText(DirectoryTable.nf.format(v)); 163 | setHorizontalAlignment(JLabel.RIGHT); 164 | } else { 165 | String s = value.toString(); 166 | if (s.contains("\n")) { 167 | s = "" + s + ""; 168 | s = patt.matcher(s).replaceAll("
"); 169 | s = s.replaceAll("\n", "
"); 170 | this.setToolTipText(s); 171 | }else if (s.contains("
")) { 172 | s = "" + s + ""; 173 | this.setToolTipText(s); 174 | }else { 175 | this.setToolTipText(null); 176 | } 177 | setText(value.toString()); 178 | } 179 | } 180 | 181 | }); 182 | 183 | } 184 | } 185 | } 186 | 187 | } 188 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/ProgressPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.FTDriver; 4 | 5 | import java.awt.BorderLayout; 6 | import java.awt.Color; 7 | import java.awt.Component; 8 | import java.awt.Dimension; 9 | import java.awt.event.ActionEvent; 10 | import java.awt.event.ActionListener; 11 | import java.io.File; 12 | import java.io.IOException; 13 | import java.util.Vector; 14 | 15 | import javax.swing.JButton; 16 | import javax.swing.JOptionPane; 17 | import javax.swing.JPanel; 18 | import javax.swing.JProgressBar; 19 | import javax.swing.JScrollPane; 20 | import javax.swing.JTable; 21 | import javax.swing.JTextArea; 22 | import javax.swing.JLabel; 23 | import javax.swing.JTextField; 24 | import javax.swing.JToggleButton; 25 | import javax.swing.table.DefaultTableCellRenderer; 26 | import javax.swing.table.DefaultTableModel; 27 | 28 | /** 29 | * User interface component displaying File Test progress; the batch processing options are also visible (if enabled) on this tab. 30 | * @author TBrady 31 | * 32 | */ 33 | class ProgressPanel extends MyPanel { 34 | private static final long serialVersionUID = 1L; 35 | JTextArea status; 36 | JProgressBar progress; 37 | JButton cancel; 38 | DirectoryTable parent; 39 | JTable jt; 40 | DefaultTableModel tm; 41 | String[] cols = {"Name","File Test","Root","Complete","Num Items","Duration","Output File"}; 42 | FileSelectChooser batch; 43 | JToggleButton pause; 44 | JButton doBatch; 45 | JButton clearBatch; 46 | JTextField batchCount; 47 | JPanel batchp; 48 | 49 | ProgressPanel(DirectoryTable dt) { 50 | parent = dt; 51 | JPanel p = addPanel("Status"); 52 | status = new JTextArea(10,60); 53 | status.setEditable(false); 54 | p.add(new JScrollPane(status)); 55 | p = addBorderPanel("File Processing Status"); 56 | progress = new JProgressBar(); 57 | setProgressColor(true); 58 | progress.setBackground(Color.LIGHT_GRAY); 59 | p.add(progress, BorderLayout.NORTH); 60 | p = addPanel(); 61 | cancel = new JButton("Cancel"); 62 | cancel.addActionListener( 63 | new ActionListener(){ 64 | public void actionPerformed(ActionEvent arg0) { 65 | if (parent.fileTraversal!=null){ 66 | parent.fileTraversal.cancel(true); 67 | } 68 | } 69 | 70 | } 71 | ); 72 | cancel.setEnabled(false); 73 | p.add(cancel); 74 | p = addBorderPanel("Test Log"); 75 | tm = new DefaultTableModel(cols,0); 76 | jt = new JTable(tm) { 77 | private static final long serialVersionUID = 1L; 78 | 79 | public boolean isCellEditable(int row, int col) { 80 | return false; 81 | } 82 | }; 83 | for(int i=0; isaveFile",""); 123 | pp1.add(batch); 124 | doBatch = new JButton("Process Batch"); 125 | pp2.add(doBatch); 126 | doBatch.addActionListener(new ActionListener(){ 127 | public void actionPerformed(ActionEvent arg0) { 128 | File f = new File(batch.tf.getText()); 129 | if (f.exists()) { 130 | try { 131 | parent.loadBatch(f); 132 | parent.batchStart(); 133 | } catch (IOException e) { 134 | JOptionPane.showMessageDialog(parent.frame, e.getMessage()); 135 | } 136 | } 137 | 138 | } 139 | }); 140 | pause = new JToggleButton("Pause"); 141 | pause.setEnabled(false); 142 | pause.addActionListener( 143 | new ActionListener(){ 144 | public void actionPerformed(ActionEvent arg0) { 145 | if (pause.isSelected()) { 146 | parent.pauseBatch(); 147 | } else { 148 | parent.batchStart(); 149 | } 150 | } 151 | } 152 | ); 153 | pp2.add(pause); 154 | clearBatch = new JButton("Clear Batch"); 155 | clearBatch.setEnabled(false); 156 | clearBatch.addActionListener( 157 | new ActionListener(){ 158 | public void actionPerformed(ActionEvent arg0) { 159 | parent.clearBatch(); 160 | pause.setSelected(false); 161 | parent.logBatchSize(); 162 | } 163 | } 164 | ); 165 | pp2.add(clearBatch); 166 | batchCount = new JTextField(6); 167 | batchCount.setEditable(false); 168 | batchCount.setBackground(getBackground()); 169 | pp2.add(batchCount); 170 | } 171 | 172 | void logTest(String name,String action, File root, boolean status, int numitems, double duration, File file){ 173 | Vectorv=new Vector(); 174 | v.add(name); 175 | v.add(action); 176 | v.add(root); 177 | v.add(status); 178 | v.add(numitems); 179 | v.add(FTDriver.ndurf.format(duration)+" secs"); 180 | v.add(file); 181 | tm.addRow(v); 182 | } 183 | 184 | void setProgressColor(boolean success) { 185 | progress.setForeground(success ? Color.GREEN : Color.RED); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/gui/CriteriaPanel.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.gui; 2 | 3 | import gov.nara.nwts.ftapp.filetest.FileTest; 4 | 5 | import java.awt.BorderLayout; 6 | import java.awt.FlowLayout; 7 | import java.awt.Font; 8 | import java.awt.event.ActionEvent; 9 | import java.awt.event.ActionListener; 10 | import java.awt.event.FocusEvent; 11 | import java.awt.event.FocusListener; 12 | import java.io.File; 13 | import java.util.ArrayList; 14 | import java.util.Collections; 15 | 16 | import javax.swing.JButton; 17 | import javax.swing.JCheckBox; 18 | import javax.swing.JComboBox; 19 | import javax.swing.JLabel; 20 | import javax.swing.JOptionPane; 21 | import javax.swing.JPanel; 22 | import javax.swing.JScrollPane; 23 | import javax.swing.JTabbedPane; 24 | import javax.swing.JTextArea; 25 | import javax.swing.JTextField; 26 | 27 | /** 28 | * User interface component defining a File Test to be performed 29 | * @author TBrady 30 | * 31 | */ 32 | class CriteriaPanel extends MyPanel { 33 | public static final int[] LIMITS = {10000,20000,40000,100000,500000,1000000,2000000,5000000,100,200,500,1000,2000,5000}; 34 | private static final long serialVersionUID = 1L; 35 | JTextField rootLabel; 36 | JButton analyze; 37 | 38 | JTabbedPane propFilter; 39 | JPanel propPanel; 40 | JTabbedPane filterTabs; 41 | JComboBox actions; 42 | JComboBox limit; 43 | JCheckBox ignorePeriods; 44 | JTextArea description; 45 | JCheckBox autoSave; 46 | DirSelectChooser fsc; 47 | JTextField fctf; 48 | DirectoryTable parent; 49 | 50 | CriteriaPanel(DirectoryTable dt) { 51 | this.parent = dt; 52 | JPanel p = addPanel("Root Directory to Scan."); 53 | rootLabel = new JTextField(50); 54 | rootLabel.addFocusListener(new FocusListener(){ 55 | public void focusGained(FocusEvent arg0) { 56 | } 57 | public void focusLost(FocusEvent arg0) { 58 | parent.setSelectedFile(); 59 | }}); 60 | p.add(rootLabel); 61 | JButton jb = new JButton("..."); 62 | jb.addActionListener( 63 | new ActionListener(){ 64 | public void actionPerformed(ActionEvent arg0) { 65 | new FileCatalog(parent); 66 | } 67 | } 68 | ); 69 | p.add(jb); 70 | jb = new JButton("Recent"); 71 | p.add(jb); 72 | jb.addActionListener( 73 | new ActionListener(){ 74 | @SuppressWarnings("unchecked") 75 | public void actionPerformed(ActionEvent arg0) { 76 | ArrayList recentrev = (ArrayList)parent.recent.clone(); 77 | Collections.reverse(recentrev); 78 | File o = (File)JOptionPane.showInputDialog( 79 | parent.frame, 80 | "Select a recently scanned directory", 81 | "Recent Directories", 82 | JOptionPane.INFORMATION_MESSAGE, 83 | null, 84 | recentrev.toArray(), 85 | null 86 | ); 87 | if (o != null) { 88 | parent.criteriaPanel.rootLabel.setText(o.getAbsolutePath()); 89 | parent.setSelectedFile(); 90 | } 91 | } 92 | } 93 | ); 94 | 95 | p = addPanel("Directory Identification"); 96 | ignorePeriods = new JCheckBox(); 97 | ignorePeriods.setSelected(true); 98 | ignorePeriods.setText("Assume directory names do not contain periods (faster)"); 99 | p.add(ignorePeriods); 100 | 101 | p = addBorderPanel("Auto-save Output directory"); 102 | fsc = new DirSelectChooser(parent.frame, "Output dir for results", parent.preferences, "outdir", ""); 103 | p.add(fsc, BorderLayout.CENTER); 104 | JPanel pp = new JPanel(new FlowLayout(FlowLayout.LEFT)); 105 | p.add(pp, BorderLayout.SOUTH); 106 | autoSave = new JCheckBox("Auto-save Results"); 107 | autoSave.setSelected(parent.preferences.getBoolean("autoSave", false)); 108 | autoSave.addActionListener(new ActionListener(){ 109 | public void actionPerformed(ActionEvent arg0) { 110 | parent.preferences.putBoolean("autoSave", autoSave.isSelected()); 111 | fctf.setEnabled(autoSave.isSelected()); 112 | } 113 | }); 114 | autoSave.setToolTipText("The system has the ability to auto-save the results of long running processes.\nIf not selected, you must explicitly export results."); 115 | pp.add(autoSave); 116 | fctf = new JTextField(10); 117 | fctf.setToolTipText("Base filename (WITHOUT extension) to assign to output file. If blank, a generated filename will be used."); 118 | fctf.setEnabled(autoSave.isSelected()); 119 | pp.add(fctf); 120 | 121 | p = addPanel("Action to perform on Files"); 122 | actions = new JComboBox(parent.actionRegistry); 123 | 124 | String action = parent.preferences.get("action", ""); 125 | for(int i=0; i list = jpegext.getTags(); 62 | for(Object obj: list){ 63 | if (obj instanceof String) { 64 | addObject(doJpgObject(filekey, jpegext, (String)obj)); 65 | } else if (obj instanceof XMPPropertyInfo) { 66 | addObject(doXMPObject(filekey, jpegext.xmpex, (XMPPropertyInfo)obj)); 67 | } 68 | } 69 | dt.types.remove(filekey); 70 | jpegext.close(); 71 | return null; 72 | } 73 | public Object doTiff(File f) { 74 | String filekey = getKey(f); 75 | TifExtractor tiffext = new TifExtractor(f); 76 | ArrayList list = tiffext.getTags(); 77 | for(Object obj: list){ 78 | if (obj instanceof TIFFField) { 79 | addObject(doTifObject(filekey, tiffext, (TIFFField)obj)); 80 | } else if (obj instanceof XMPPropertyInfo) { 81 | addObject(doXMPObject(filekey, tiffext.xmpex, (XMPPropertyInfo)obj)); 82 | } 83 | } 84 | dt.types.remove(filekey); 85 | tiffext.close(); 86 | return null; 87 | } 88 | 89 | public GenericImageStats doJpgObject(String filekey, JpegExtractor jpegext, String tag) { 90 | String val = jpegext.getAttribute(tag); 91 | TAGLOC tiffloc = TAGLOC.JPG; 92 | TAGTYPE tagtype = TAGTYPE.JPG; 93 | NARAREQ narareq = NARAREQ.UNDECIDED; 94 | TAGCONTENT tagcontent = TAGCONTENT.UNDECIDED; 95 | DUP dupfield = DUP.NA; 96 | String name = tag; 97 | 98 | TAGS tagdef = jpegext.getTagDef(tag); 99 | if (tagdef != TAGS.UNDEFINED) { 100 | tag = tagdef.name(); 101 | tiffloc = tagdef.tiffloc; 102 | tagtype = tagdef.tagtype; 103 | narareq = tagdef.narareq; 104 | tagcontent = tagdef.tagcontent; 105 | dupfield = tagdef.dup; 106 | } 107 | 108 | return new GenericImageStats(""+counter,filekey,tag,name, val,tiffloc,tagtype,narareq,tagcontent, dupfield); 109 | } 110 | public GenericImageStats doTifObject(String filekey, TifExtractor tiffext, TIFFField tf) { 111 | String tag = ""; 112 | TAGLOC tiffloc = TAGLOC.TAG; 113 | TAGTYPE tagtype = TAGTYPE.TIFF_TAG; 114 | NARAREQ narareq = NARAREQ.UNDECIDED; 115 | TAGCONTENT tagcontent = TAGCONTENT.UNDECIDED; 116 | DUP dupfield = DUP.NA; 117 | String path =""; 118 | 119 | TAGS tagdef = tiffext.getTagDef(tf); 120 | if (tagdef == TAGS.UNDEFINED) { 121 | tag=tiffext.getTagName(tf); 122 | } else { 123 | tag = tagdef.name(); 124 | path = "Tag: "+tagdef.tag; 125 | tiffloc = tagdef.tiffloc; 126 | tagtype = tagdef.tagtype; 127 | narareq = tagdef.narareq; 128 | tagcontent = tagdef.tagcontent; 129 | dupfield = tagdef.dup; 130 | } 131 | String val = tiffext.getTiffObject(tf).toString(); 132 | return new GenericImageStats(""+counter,filekey,tag,path,val,tiffloc,tagtype,narareq,tagcontent, dupfield); 133 | } 134 | public GenericImageStats doXMPObject(String filekey, XMPExtractor xmpex, XMPPropertyInfo xpi) { 135 | String tag = ""; 136 | String val = ""; 137 | 138 | TAGLOC tiffloc = TAGLOC.XMP; 139 | TAGTYPE tagtype = TAGTYPE.OTHER_XMP; 140 | NARAREQ narareq = NARAREQ.UNDECIDED; 141 | TAGCONTENT tagcontent = TAGCONTENT.UNDECIDED; 142 | DUP dupfield = DUP.NA; 143 | 144 | String[] path = new String[2]; 145 | path[0] = xpi.getNamespace(); 146 | path[1] = xpi.getPath(); 147 | tag=xpi.getPath(); 148 | if (tag == null) { 149 | return null; 150 | } else if (tag.endsWith("xml:lang")) { 151 | return null; 152 | } else if (tag.equals("")) { 153 | return null; 154 | } else { 155 | for(TAGS t: TAGS.values()){ 156 | if ((t.tiffloc == TAGLOC.XMP) && tag.equals(t.path)) { 157 | tag = t.name(); 158 | tagtype = t.tagtype; 159 | narareq = t.narareq; 160 | tagcontent = t.tagcontent; 161 | dupfield = t.dup; 162 | break; 163 | } 164 | } 165 | 166 | if (tagcontent == TAGCONTENT.CONTAINER) 167 | return null; 168 | 169 | if (tagtype == TAGTYPE.OTHER_XMP) { 170 | tag=xpi.getPath()+" --> "+xpi.getNamespace(); 171 | } 172 | 173 | val = xmpex.getXMP(path); 174 | return new GenericImageStats(""+counter,filekey,tag,path[1],val,tiffloc,tagtype,narareq,tagcontent, dupfield); 175 | } 176 | } 177 | 178 | public void addObject(GenericImageStats gis) { 179 | if (gis == null) return; 180 | dt.types.put(gis.key, gis); 181 | counter++; 182 | } 183 | 184 | public Stats createStats(String key){ 185 | return new GenericImageStats(key); 186 | } 187 | public Object[][] getStatsDetails() { 188 | return GenericImageStats.details; 189 | } 190 | 191 | public void initFilters() { 192 | filters.add(new TiffJpegFileTestFilter()); 193 | filters.add(new TiffFileTestFilter()); 194 | filters.add(new JpegFileTestFilter()); 195 | } 196 | 197 | public String getDescription() { 198 | return ""; 199 | } 200 | 201 | } 202 | -------------------------------------------------------------------------------- /src/gov/nara/nwts/ftapp/filetest/NameValidationTest.java: -------------------------------------------------------------------------------- 1 | package gov.nara.nwts.ftapp.filetest; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | 6 | import gov.nara.nwts.ftapp.FTDriver; 7 | import gov.nara.nwts.ftapp.filetest.DefaultFileTest; 8 | import gov.nara.nwts.ftapp.nameValidation.CustomPattern; 9 | import gov.nara.nwts.ftapp.nameValidation.NameValidationPattern; 10 | import gov.nara.nwts.ftapp.nameValidation.RenameDetails; 11 | import gov.nara.nwts.ftapp.nameValidation.RenameStatus; 12 | import gov.nara.nwts.ftapp.nameValidation.ValidPattern; 13 | import gov.nara.nwts.ftapp.stats.NameValidationStats; 14 | import gov.nara.nwts.ftapp.stats.Stats; 15 | /** 16 | * Abstract class defining the core functionality of a filename validation rule. 17 | * 18 | *

Sample Usage applying a complex set of patterns:

19 | * 
 20 |  * class CustomFilenameTest extends NameValidationTest {
 21 |  * public static String fold = "(\\d\\d\\d\\d[A-Z]?)";
 22 |  * public static String fold2 = "\\\\" + fold + "\\\\";
 23 |  * public static String box = "\\d\\d\\d\\d?[A-Z]?";
 24 |  * public static String box2 = "^.*\\\\Box(" + box + ")";
 25 |  * public static String box3 = "^(.*)\\\\Box(\\d\\d\\d\\d?[A-Za-z]?)";
 26 |  * public static String suff = "(001|002)_(MA|AC).tif$";
 27 |  * public static String seqsuff = "(\\d\\d\\d)_" + suff;
 28 |  * 
 29 |  * Pattern dirPatternFolder;
 30 |  * Pattern dirPatternImage;
 31 |  * Pattern dirPatternIgnore;
 32 |  * Pattern dirPatternIgnoreR;
 33 |  * 
 34 |  * public CustomFilenameTest(FTDriver dt, FileTest next) {
 35 |  * 		super(dt, new ValidPattern(box2 + fold2 + "Box\\1_" + seqsuff, true),
 36 |  * 			next, "Forest Service Filename", "RG95 ");
 37 |  * 		seqs = new TreeMap();
 38 |  * 		dirseqs = new TreeMap();
 39 |  * 
 40 |  * 		dirPatternFolder = Pattern.compile("^(\\d\\d\\d\\d)[A-Z]?$");
 41 |  * 		dirPatternImage = Pattern.compile("^Box"+box+"_" + seqsuff);
 42 |  * 		dirPatternIgnore = Pattern.compile("^(\\..*|Thumbs.db)$");
 43 |  * 		dirPatternIgnoreR = Pattern
 44 |  * 				.compile("^(\\..*|Thumbs.db|\\d\\d\\d\\d[A-Z]?)$");
 45 |  * 
 46 |  * 		testPatterns.add(new InvalidManualPattern(box2 + fold2
 47 |  * 				+ "Box(\\d\\d\\d[A-Z]?)_" + seqsuff, true) {
 48 |  * 			public String getMessage(File f, Matcher m) {
 49 |  * 				return "Box number mismatch in filename: [" + m.group(1) + "/"
 50 |  * 						+ m.group(3) + "]";
 51 |  * 			}
 52 |  * 		});
 53 |  * 		testPatterns.add(new InvalidManualPattern("^.*\\\\(Box" + box
 54 |  * 				+ "[^\\\\]+)" + fold2 + "Box(" + box + ")" + seqsuff, true) {
 55 |  * 			public String getMessage(File f, Matcher m) {
 56 |  * 				return "Box folder name is invalid: [" + m.group(1) + "]";
 57 |  * 			}
 58 |  * 		});
 59 |  * 		testPatterns.add(new InvalidManualPattern("(" + box3 + "\\\\"
 60 |  * 				+ "\\d{6,6}_.*_(\\d\\d\\d))_(MA|AC).tif$", true) {
 61 |  * 			public String getMessage(File f, Matcher m) {
 62 |  * 				return "No FOLDER folder present";
 63 |  * 			}
 64 |  * 		});
 65 |  * 		testPatterns.add(new RenameablePattern("^.*\\\\\\s*(Box" + box
 66 |  * 				+ ")[\\-_](\\d\\d\\d)[\\-_](001|002)[\\-_]*(MA|AC)\\s*.tif$",
 67 |  * 				true) {
 68 |  * 			public String getMessage(File f, Matcher m) {
 69 |  * 				return "Separators replaced";
 70 |  * 			} 
 71 |  * 
 72 |  * 			public File getNewFile(File f, Matcher m) {
 73 |  * 				String newname = m.group(1) + "_" + m.group(2) + "_"
 74 |  * 						+ m.group(3) + "_" + m.group(4) + ".tif";
 75 |  * 				return new File(f.getParentFile(), newname);
 76 |  * 			}
 77 |  * 		});
 78 |  * 		testPatterns.add(new RenameablePattern(box2 + fold2
 79 |  * 				+ "(Box\\1)_\\d{5,5}_" + seqsuff, true) {
 80 |  * 			public String getMessage(File f, Matcher m) {
 81 |  * 				return "Subject code removed";
 82 |  * 			} 
 83 |  * 
 84 |  * 			public File getNewFile(File f, Matcher m) {
 85 |  * 				String newname = m.group(3) + "_" + m.group(4) + "_"
 86 |  * 						+ m.group(5) + "_" + m.group(6) + ".tif";
 87 |  * 				return new File(f.getParentFile(), newname);
 88 |  * 			}
 89 |  * 		});
 90 |  * 		testPatterns.add(new RenameablePattern(box3 + fold2 + "Box\\d{1,3}_"
 91 |  * 				+ seqsuff, true) {
 92 |  * 			public String getMessage(File f, Matcher m) {
 93 |  * 				return "Extraneous identifiers removed";
 94 |  * 			}
 95 |  * 
 96 |  * 			public File getNewFile(File f, Matcher m) {
 97 |  * 				String mbox = "Box" + m.group(2).toUpperCase();
 98 |  * 				String newname = m.group(1) + "\\" + mbox + "\\" + m.group(3)
 99 |  * 						+ "\\" + mbox + "_" + m.group(4) + "_" + m.group(5)
100 |  * 						+ "_" + m.group(6) + ".tif";
101 |  * 				return new File(CustomFilenameTest.this.dt.root, newname);
102 |  * 			}
103 |  * 		});
104 |  * 		testPatterns.add(new RenameablePattern("("+box3 + fold2 + "\\d{6,6}[a-zA-Z]?_.*_)(\\d\\d\\d)_(MA|AC).tif$", true) {
105 |  * 			String message = "Sequence number generated";
106 |  * 			public String getMessage(File f, Matcher m) {
107 |  * 				return message;
108 |  * 			}
109 |  * 
110 |  * 			
111 |  * 			public File getNewFile(File f, Matcher m) {
112 |  * 			   ... custom logic to introspect the file system and determine the next sequence number to assign.
113 |  * 			}
114 |  * 		});
115 |  * 
116 |  * 
117 |  * 		dirTestPatterns.add(new CustomPattern("^Box\\d\\d\\d[A-Z]?$", false) {
118 |  * 			public RenameDetails report(File f, Matcher m) {
119 |  * 				RenameDetails det = DirAnalysis.analyze(f, dirPatternFolder, 1,
120 |  * 						dirPatternIgnore, false);
121 |  * 				if (det.status == RenameStatus.DIRECTORY_VALID) {
122 |  * 					det = DirAnalysis.recursiveAnalyze(f, dirPatternImage, 1,
123 |  * 							dirPatternIgnoreR, false);
124 |  * 				}
125 |  * 				return det;
126 |  * 			}
127 |  * 		});
128 |  * 		dirTestPatterns.add(new CustomPattern(dirPatternFolder, false) {
129 |  * 			public RenameDetails report(File f, Matcher m) {
130 |  * 				return DirAnalysis.analyze(f, dirPatternImage, 1,
131 |  * 						dirPatternIgnore, false);
132 |  * 			}
133 |  * 		});
134 |  * 	}
135 |  *
136 | * @author TBrady 137 | * 138 | */ 139 | abstract public class NameValidationTest extends DefaultFileTest { 140 | 141 | int numRename = 0; 142 | String name; 143 | String shortname; 144 | protected ArrayList testPatterns; 145 | protected ArrayList dirTestPatterns; 146 | boolean renameParent = false; 147 | ValidPattern valPatt; 148 | FileTest nextTest; 149 | protected boolean allowRename = false; 150 | 151 | public NameValidationTest(FTDriver dt, ValidPattern valPatt, FileTest nextTest, String name, String shortname) { 152 | super(dt); 153 | this.nextTest = nextTest; 154 | this.allowRename = (nextTest!=null); 155 | this.name = name; 156 | this.shortname = shortname; 157 | this.valPatt = valPatt; 158 | testPatterns = new ArrayList(); 159 | testPatterns.add(valPatt); 160 | dirTestPatterns = new ArrayList(); 161 | } 162 | 163 | public FileTest resetOption() { 164 | return nextTest; 165 | } 166 | 167 | public String toString() { 168 | return name + (allowRename ? " Rename" : " Test"); 169 | } 170 | 171 | public Object dirTest(File f) { 172 | return new RenameDetails(RenameStatus.DIRECTORY, null, ""); 173 | } 174 | 175 | public Object fileTest(File f) { 176 | if (f.isDirectory()) { 177 | for(NameValidationPattern nvp: dirTestPatterns) { 178 | RenameDetails det = nvp.checkFile(f); 179 | if (det.status == RenameStatus.NEXT) continue; 180 | return det; 181 | } 182 | return new RenameDetails(RenameStatus.DIRECTORY, null, null); 183 | } 184 | for(NameValidationPattern nvp: testPatterns) { 185 | RenameDetails det = nvp.checkFile(f); 186 | if (det.status == RenameStatus.NEXT) continue; 187 | File newFile = det.getFile(); 188 | if (newFile != null) { 189 | if (valPatt.checkFile(newFile).status != RenameStatus.VALID) { 190 | det.status = RenameStatus.NEW_NAME_INVALID; 191 | return det; 192 | } 193 | if (f.equals(newFile)) { 194 | } else if (newFile.exists()) { 195 | det.status = RenameStatus.RENAME_FILE_EXISTS; 196 | return det; 197 | } 198 | if (allowRename) { 199 | return renameFile(f, det); 200 | } 201 | } 202 | return det; 203 | } 204 | return new RenameDetails(RenameStatus.PARSE_ERR, null, ""); 205 | } 206 | 207 | public String getKey(File f) { 208 | String path = f.getAbsolutePath(); 209 | return path; 210 | } 211 | 212 | public Stats createStats(String key) { 213 | return new NameValidationStats(key); 214 | } 215 | 216 | public Object[][] getStatsDetails() { 217 | return NameValidationStats.details; 218 | } 219 | 220 | public String getShortName() { 221 | return shortname + (allowRename ? " Rename" : ""); 222 | } 223 | 224 | public String getDescription() { 225 | return "This test will check filename case." 226 | + ((allowRename) ? "\nFILES WILL BE RENAMED IF POSSIBLE." : ""); 227 | } 228 | 229 | public boolean isTestFiles() { 230 | return true; 231 | } 232 | public boolean isTestDirectory() { 233 | return true; 234 | } 235 | 236 | public RenameDetails renameFile(File f, RenameDetails det) { 237 | File newFile = det.getFile(); 238 | if (!newFile.getParentFile().exists()) { 239 | if (!newFile.getParentFile().mkdirs()) { 240 | det.status = RenameStatus.RENAME_FAILURE; 241 | return det; 242 | } 243 | } 244 | File pf = f.getParentFile(); 245 | if (!f.renameTo(newFile)) { 246 | det.status = RenameStatus.RENAME_FAILURE; 247 | return det; 248 | } 249 | if (renameParent) { 250 | if (pf.list().length == 0) { 251 | File gpf = pf.getParentFile(); 252 | File newpf = new File(gpf, "__" + pf.getName()); 253 | if (!pf.renameTo(newpf)){ 254 | System.err.println("Could not rename" + pf.getAbsolutePath()); 255 | } 256 | } 257 | } 258 | det.status = RenameStatus.RENAMED; 259 | return det; 260 | 261 | } 262 | } 263 | --------------------------------------------------------------------------------