├── demo.sh ├── pixelCheck-im.sh ├── README.md ├── jp2topdf-grok.sh ├── mastertoaccess-grok.sh ├── mastertoaccess-opj.sh ├── tifftojp2-grok.sh ├── jp2totiff.sh ├── doc └── grok-installation.md ├── mastertoaccess-kdu.sh ├── tifftojp2-lossy5-kdu.sh └── tifftojp2-kdu.sh /demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Demo script: 4 | # - Convert several directory trees from JP2 to TIFF 5 | # - From these TIFFs do a roundtrip conversion back to JP2 6 | # - Verify if JP2s from roundtrip conversion meet KB requirements 7 | # using jprofile (https://github.com/KBNLresearch/jprofile) 8 | # - power off the machine when finished 9 | 10 | jp2totiff=~/ownCloud/jp2totiff/jp2totiff.sh 11 | tifftojp2=~/ownCloud/jp2totiff/tifftojp2.sh 12 | 13 | # Convert to TIFF 14 | $jp2totiff /media/johan/Elements/MMUBWA03_000000001_1_01 MMUBWA03_000000001_1_01-TIFF 15 | $jp2totiff /media/johan/Elements/MMKB18A_000000001_1_04 MMKB18A_000000001_1_04-TIFF 16 | 17 | # Roundtrip conversion to JP2 18 | $tifftojp2 MMUBWA03_000000001_1_01-TIFF MMUBWA03_000000001_1_01-JP2-roundtrip 19 | $tifftojp2 MMKB18A_000000001_1_04-TIFF MMKB18A_000000001_1_04-JP2-roundtrip 20 | 21 | # Analyse JP2s 22 | jprofile -p kb_300Colour_2014.xml MMUBWA03_000000001_1_01-JP2-roundtrip MMUBWA03_000000001_1_01 23 | jprofile -p kb_300Colour_2014.xml MMKB18A_000000001_1_04-JP2-roundtrip MMKB18A_000000001_1_04 24 | 25 | # Power off the machine 26 | poweroff 27 | -------------------------------------------------------------------------------- /pixelCheck-im.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Performs pixel-wise check between TIFF images in two directory trees 4 | # Requires ImageMagick 5 | #set -euo pipefail 6 | 7 | set -uo pipefail 8 | 9 | if [ "$#" -ne 3 ] ; then 10 | echo "Usage: pixelCheck-im.sh dir1 dir2 dirOut" >&2 11 | exit 1 12 | fi 13 | 14 | # Input and output directories 15 | dir1="$1" 16 | dir2="$2" 17 | dirOut="$3" 18 | 19 | if ! [ -d "$dir1" ] ; then 20 | echo "input directory 1 does not exist" >&2 21 | exit 1 22 | fi 23 | 24 | if ! [ -d "$dir2" ] ; then 25 | echo "input directory 2 does not exist" >&2 26 | exit 1 27 | fi 28 | 29 | if ! [ -d "$dirOut" ] ; then 30 | mkdir "$dirOut" 31 | fi 32 | 33 | # Output file 34 | fileOut=$dirOut/pixelCheck.csv 35 | 36 | # Remove output file if it exists already (writing done in append mode!) 37 | if [ -f $fileOut ] ; then 38 | rm $fileOut 39 | fi 40 | 41 | # Write header to output file 42 | echo "file1","file2","ae","psnr" >> $fileOut 43 | 44 | # Iterate over all images in dir1, and do pixel check 45 | # with corresponding image in dir2 46 | 47 | while IFS= read -d $'\0' -r file ; do 48 | 49 | file1="$file" 50 | file2Name=$(basename "$file") 51 | 52 | # Input path 53 | path1=$(dirname "$file1") 54 | 55 | # Input path, relative to dir1 56 | inPathRel=$(realpath --relative-to=$dir1 $path1) 57 | 58 | # Full path to corresponding file in dir2 59 | file2="$dir2/$inPathRel/$file2Name" 60 | 61 | ae=$(compare -quiet -metric AE "$file1""[0]" "$file2""[0]" null: 2>&1) 62 | psnr=$(compare -quiet -metric psnr "$file1""[0]" "$file2""[0]" null: 2>&1) 63 | 64 | echo "$file1","$file2","$ae","$psnr" >> $fileOut 65 | 66 | done < <(find $dir1 -type f -regex '.*\.\(tiff\|TIFF\|tif\|TIF\)' -print0) 67 | 68 | 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Contents of this repo 2 | 3 | Various scripts for converting to and from JP2 (JPEG 2000 Part 1). 4 | 5 | ### jp2totiff.sh 6 | 7 | Takes a directory tree with JP2 images, and converts them to TIFF. 8 | 9 | Usage: 10 | 11 | ``` 12 | jp2totiff.sh dirIn dirOut 13 | ``` 14 | 15 | The directory structure of *dirIn* is cloned to *dirOut*. 16 | 17 | Requires: 18 | 19 | - Kakadu (kdu_expand) 20 | - ExifTool 21 | - realpath 22 | 23 | ### tifftojp2.sh 24 | 25 | Takes a directory tree with TIFF images, and converts them to JP2. 26 | 27 | Usage: 28 | 29 | ``` 30 | tifftojp2.sh dirIn dirOut 31 | ``` 32 | 33 | The directory structure of *dirIn* is cloned to *dirOut*. 34 | 35 | Requires: 36 | 37 | - Kakadu (kdu_expand) 38 | - ExifTool 39 | - realpath 40 | 41 | ### mastertoaccess-grok.sh 42 | 43 | Takes a directory with (lossless master) JP2 images, and converts them to lossy JP2 according to KB specifications using Grok. 44 | 45 | Usage: 46 | 47 | ``` 48 | mastertoaccess-grok.sh dirIn dirOut 49 | ``` 50 | 51 | Requires: 52 | 53 | - Grok (grk_decompress and grk_compress) 54 | - ExifTool 55 | - [jprofile](https://github.com/KBNLresearch/jprofile) 56 | 57 | ### mastertoaccess-kdu.sh 58 | 59 | Takes a directory with (lossless master) JP2 images, and converts them to lossy JP2 according to KB specifications using Kakadu. 60 | 61 | Usage: 62 | 63 | ``` 64 | mastertoaccess-kdu.sh dirIn dirOut 65 | ``` 66 | 67 | Requires: 68 | 69 | - Kakadu (kdu_expand and kdu_compress) 70 | - ExifTool 71 | - sed 72 | - [jprofile](https://github.com/KBNLresearch/jprofile) 73 | 74 | ### mastertoaccess-opj.sh 75 | 76 | Takes a directory with (lossless master) JP2 images, and converts them to lossy JP2 according to KB specifications using OpenJPEG. 77 | 78 | Usage: 79 | 80 | ``` 81 | mastertoaccess-grok.sh dirIn dirOut 82 | ``` 83 | 84 | Requires: 85 | 86 | - OpenJPEG (opj_decompress and opj_compress) 87 | - [jprofile](https://github.com/KBNLresearch/jprofile) 88 | 89 | ### jp2topdf-grok.sh 90 | 91 | Takes a directory with (lossless master) JP2 images, and converts them to single PDF files using different JPEG quality levels. 92 | 93 | Usage: 94 | 95 | ``` 96 | jp2topdf-grok.sh prefix dirIn dirOut 97 | ``` 98 | 99 | Requires: 100 | 101 | - Grok (grk_decompress) 102 | - ImageMagick 103 | -------------------------------------------------------------------------------- /jp2topdf-grok.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory with JP2 lossless master images to PDF with 4 | # various JPEG quality levels via TIFF 5 | # Requires: 6 | # - Grok 7 | # - ImageMagick 8 | 9 | if [ "$#" -ne 3 ] ; then 10 | echo "Usage: jp2totpdf-grok.sh prefix dirIn dirOut" >&2 11 | exit 1 12 | fi 13 | 14 | # Input and output directories 15 | prefix="$1" 16 | dirIn="$2" 17 | dirOut="$3" 18 | 19 | # PDF JPEG parameters 20 | density=300 21 | qualityLevels=(100 92 85 70 60 50) 22 | 23 | if ! [ -d "$dirIn" ] ; then 24 | echo "input directory does not exist" >&2 25 | exit 1 26 | fi 27 | 28 | if ! [ -d "$dirOut" ] ; then 29 | mkdir "$dirOut" 30 | fi 31 | 32 | # Output CSV file 33 | csvOut="$dirOut"/"$prefix"_qsize.csv 34 | 35 | if [ -f "$csvOut" ] ; then 36 | rm $csvOut 37 | fi 38 | 39 | # Log file (used too store Grok stdout, stderr) 40 | logFile=$dirOut/jp2topdf-grok.log 41 | 42 | # Grok status file (used to store Kakadu exit status) 43 | grokStatusFile=$dirOut/grokStatus.csv 44 | 45 | # Remove log and status files if they exist already (writing done in append mode!) 46 | if [ -f $logFile ] ; then 47 | rm $logFile 48 | fi 49 | 50 | if [ -f $grokStatusFile ] ; then 51 | rm $grokStatusFile 52 | fi 53 | 54 | # Iterate over all files in dirIn and convert JP2s 55 | # to TIFF 56 | 57 | while IFS= read -d $'\0' file ; do 58 | 59 | # File basename, extension removed 60 | bName=$(basename "$file" | cut -f 1 -d '.') 61 | 62 | # Output names 63 | outNameTIF=$bName.tif 64 | 65 | # Input path 66 | inPath=$(dirname "$file") 67 | 68 | # Full paths to output files 69 | tifOut="$dirOut/$outNameTIF" 70 | 71 | # First convert master JP2 to TIFF 72 | cmdDecompress="grk_decompress -i "$file" 73 | -o "$tifOut" 74 | -W "$logFile"" 75 | 76 | $cmdDecompress 77 | grokDecompressStatus=$? 78 | echo $tifOut,$grokDecompressStatus >> $grokStatusFile 79 | 80 | done < <(find $dirIn -type f -regex '.*\.\(jp2\|JP2\)' -print0) 81 | 82 | echo "q","fileSize" >> $csvOut 83 | 84 | # Create PDFs from TIFF images 85 | 86 | for q in "${qualityLevels[@]}"; do 87 | fileOut="$dirOut/$prefix"_"$q".pdf 88 | convert "$dirOut/*.{tif}" \ 89 | -compress jpeg \ 90 | -quality "$q" \ 91 | -density "$density" \ 92 | "$fileOut" 93 | fileSize=$(ls -l $fileOut | awk '{print $5}') 94 | echo "$q","$fileSize" >> $csvOut 95 | done 96 | 97 | # Remove TIFF images 98 | 99 | while IFS= read -d $'\0' file ; do 100 | rm $file 101 | done < <(find $dirOut -type f -regex '.*\.\(tif\|TIF\)' -print0) 102 | -------------------------------------------------------------------------------- /mastertoaccess-grok.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory with JP2 lossless master images to lossy access JP2s 4 | # Requires: 5 | # - Grok 6 | # - ExifTool 7 | # - jprofile 8 | 9 | if [ "$#" -ne 2 ] ; then 10 | echo "Usage: mastertoaccess-grok.sh dirIn dirOut" >&2 11 | exit 1 12 | fi 13 | 14 | # Input and output directories 15 | dirIn="$1" 16 | dirOut="$2" 17 | 18 | if ! [ -d "$dirIn" ] ; then 19 | echo "input directory does not exist" >&2 20 | exit 1 21 | fi 22 | 23 | if ! [ -d "$dirOut" ] ; then 24 | mkdir "$dirOut" 25 | fi 26 | 27 | dirAccess="$dirOut"/access 28 | 29 | if ! [ -d "$dirAccess" ] ; then 30 | mkdir "$dirAccess" 31 | fi 32 | 33 | # Log file (used too store Grok stdout, stderr) 34 | logFile=$dirOut/mastertoaccess-grok.log 35 | 36 | # Grok status file (used to store Kakadu exit status) 37 | grokStatusFile=$dirOut/grokStatus.csv 38 | 39 | # Remove log and status files if they exist already (writing done in append mode!) 40 | if [ -f $logFile ] ; then 41 | rm $logFile 42 | fi 43 | 44 | if [ -f $grokStatusFile ] ; then 45 | rm $grokStatusFile 46 | fi 47 | 48 | # Codestream comment string for access images 49 | cCommentAccess="KB_ACCESS_LOSSY_01/01/2015" 50 | 51 | # Iterate over all files in dirIn and convert JP2s 52 | # to TIFF, then convert those TIFFs to lossy JP2 53 | # according to KB access specs. 54 | 55 | while IFS= read -d $'\0' file ; do 56 | 57 | # File basename, extension removed 58 | bName=$(basename "$file" | cut -f 1 -d '.') 59 | 60 | # Output names 61 | outNameTIF=$bName.tif 62 | outNameJP2=$bName.jp2 63 | 64 | # Input path 65 | inPath=$(dirname "$file") 66 | 67 | # Full paths to output files 68 | tifOut="$dirAccess/$outNameTIF" 69 | jp2Out="$dirAccess/$outNameJP2" 70 | 71 | # First convert master JP2 to TIFF 72 | cmdDecompress="grk_decompress -i "$file" 73 | -o "$tifOut" 74 | -W "$logFile"" 75 | 76 | $cmdDecompress 77 | grokDecompressStatus=$? 78 | echo $tifOut,$grokDecompressStatus >> $grokStatusFile 79 | 80 | # Convert TIFF to lossy access JP2 according to KB specs 81 | # Note: XMP metadata are written to UUID box, whereas KB 82 | # specs prescribe XML box. Don't think this is a problem 83 | # for access images. 84 | cmdCompress="grk_compress -i "$tifOut" 85 | -o "$jp2Out" 86 | -I 87 | -n 6 88 | -p RPCL 89 | -t 1024,1024 90 | -b 64,64 91 | -c [256,256],[256,256],[128,128],[128,128],[128,128],[128,128] 92 | -r 2560,1280,640,320,160,80,40,20 93 | -S 94 | -E 95 | -M 32 96 | -C "$cCommentAccess" 97 | -W "$logFile"" 98 | 99 | $cmdCompress 100 | grokCompressStatus=$? 101 | echo $jp2Out,$grokCompressStatus >> $grokStatusFile 102 | 103 | # Remove TIFF file 104 | rm $tifOut 105 | 106 | done < <(find $dirIn -type f -regex '.*\.\(jp2\|JP2\)' -print0) 107 | 108 | # Run jprofile 109 | jprofile -p kb_300Colour_2014.xml $dirAccess jprofile-grok -------------------------------------------------------------------------------- /mastertoaccess-opj.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory with JP2 lossless master images to lossy access JP2s 4 | # Requires: 5 | # - OpenJPEG 6 | # - ExifTool 7 | # - jprofile 8 | 9 | if [ "$#" -ne 2 ] ; then 10 | echo "Usage: mastertoaccess-opj.sh dirIn dirOut" >&2 11 | exit 1 12 | fi 13 | 14 | # Input and output directories 15 | dirIn="$1" 16 | dirOut="$2" 17 | 18 | if ! [ -d "$dirIn" ] ; then 19 | echo "input directory does not exist" >&2 20 | exit 1 21 | fi 22 | 23 | if ! [ -d "$dirOut" ] ; then 24 | mkdir "$dirOut" 25 | fi 26 | 27 | dirAccess="$dirOut"/access 28 | 29 | if ! [ -d "$dirAccess" ] ; then 30 | mkdir "$dirAccess" 31 | fi 32 | 33 | # Location of OpenJPEG binaries 34 | opjPath=/Applications/openjpeg/bin 35 | 36 | # Log file (used too store opj stdout, stderr) 37 | logFile=$dirOut/mastertoaccess-opj.log 38 | 39 | # opj status file (used to store Kakadu exit status) 40 | opjStatusFile=$dirOut/opjStatus.csv 41 | 42 | # Remove log and status files if they exist already (writing done in append mode!) 43 | if [ -f $logFile ] ; then 44 | rm $logFile 45 | fi 46 | 47 | if [ -f $opjStatusFile ] ; then 48 | rm $opjStatusFile 49 | fi 50 | 51 | # Codestream comment string for access images 52 | cCommentAccess="KB_ACCESS_LOSSY_01/01/2015" 53 | 54 | # Iterate over all files in dirIn and convert JP2s 55 | # to TIFF, then convert those TIFFs to lossy JP2 56 | # according to KB access specs. 57 | 58 | while IFS= read -d $'\0' file ; do 59 | 60 | # File basename, extension removed 61 | bName=$(basename "$file" | cut -f 1 -d '.') 62 | 63 | # Output names 64 | outNameTIF=$bName.tif 65 | outNameJP2=$bName.jp2 66 | 67 | # Input path 68 | inPath=$(dirname "$file") 69 | 70 | # Full paths to output files 71 | tifOut="$dirAccess/$outNameTIF" 72 | jp2Out="$dirAccess/$outNameJP2" 73 | 74 | # First convert master JP2 to TIFF 75 | cmdDecompress="$opjPath/opj_decompress -threads ALL_CPUS 76 | -i "$file" 77 | -o "$tifOut"" 78 | 79 | $cmdDecompress 80 | opjDecompressStatus=$? 81 | echo $tifOut,$opjDecompressStatus >> $opjStatusFile 82 | 83 | # Convert TIFF to lossy access JP2 according to KB specs 84 | # Note: XMP metadata are written to UUID box, whereas KB 85 | # specs prescribe XML box. Don't think this is a problem 86 | # for access images. 87 | cmdCompress="$opjPath/opj_compress -threads ALL_CPUS 88 | -i "$tifOut" 89 | -o "$jp2Out" 90 | -I 91 | -n 6 92 | -p RPCL 93 | -t 1024,1024 94 | -b 64,64 95 | -c [256,256],[256,256],[128,128],[128,128],[128,128],[128,128] 96 | -r 2560,1280,640,320,160,80,40,20 97 | -S 98 | -E 99 | -M 32 100 | -C "$cCommentAccess"" 101 | 102 | $cmdCompress 103 | opjCompressStatus=$? 104 | echo $jp2Out,$opjCompressStatus >> $opjStatusFile 105 | 106 | # Remove TIFF file 107 | rm $tifOut 108 | 109 | done < <(find $dirIn -type f -regex '.*\.\(jp2\|JP2\)' -print0) 110 | 111 | # Run jprofile 112 | jprofile -p kb_300Colour_2014.xml $dirAccess jprofile-opj -------------------------------------------------------------------------------- /tifftojp2-grok.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory tree with TIFF images to JP2 4 | # Requires: 5 | # - Grok 6 | # - realpath 7 | 8 | if [ "$#" -ne 2 ] ; then 9 | echo "Usage: tifftojp2-grok.sh dirIn dirOut" >&2 10 | exit 1 11 | fi 12 | 13 | # Input and output directories 14 | dirIn="$1" 15 | dirOut="$2" 16 | 17 | if ! [ -d "$dirIn" ] ; then 18 | echo "input directory does not exist" >&2 19 | exit 1 20 | fi 21 | 22 | if ! [ -d "$dirOut" ] ; then 23 | mkdir "$dirOut" 24 | fi 25 | 26 | # Log file (used too store Grok stdout, stderr) 27 | logFile=$dirOut/tifftojp2-grok.log 28 | 29 | # Groku status file (used to store Grok exit status) 30 | grokStatusFile=$dirOut/grokStatus.csv 31 | 32 | # Checksum file 33 | checksumFile=$dirOut/checksums.md5 34 | 35 | # Remove log and checksum files if they exist already (writing done in append mode!) 36 | if [ -f $logFile ] ; then 37 | rm $logFile 38 | fi 39 | 40 | if [ -f $grokStatusFile ] ; then 41 | rm $grokStatusFile 42 | fi 43 | 44 | if [ -f $checksumFile ] ; then 45 | rm $checksumFile 46 | fi 47 | 48 | # Codestream comment strings for master and access images 49 | cCommentMaster="KB_MASTER_LOSSLESS_01/01/2015" 50 | cCommentAccess="KB_ACCESS_LOSSY_01/01/2015" 51 | 52 | # First clone the directory structure of dirIn to dirOut 53 | 54 | while IFS= read -d $'\0' -r directory ; do 55 | # Directory path, relative to dirIn 56 | dirInRel=$(realpath --relative-to=$dirIn $directory) 57 | 58 | # Absolute path to folder in output directory 59 | dirOutAbs=$dirOut/$dirInRel 60 | 61 | # Create folder in output directory 62 | mkdir -p $dirOutAbs 63 | done < <(find $dirIn -type d -print0) 64 | 65 | # Iterate over all files in dirIn and convert JP2s 66 | # to TIFF, writing result to corresponding folder in dirout 67 | 68 | while IFS= read -d $'\0' -r file ; do 69 | 70 | echo $file >> $logFile 71 | 72 | # File basename, extension removed 73 | bName=$(basename "$file" | cut -f 1 -d '.') 74 | 75 | # Output name 76 | outName=$bName.jp2 77 | 78 | # Input path 79 | inPath=$(dirname "$file") 80 | 81 | # Input path, relative to dirIn 82 | inPathRel=$(realpath --relative-to=$dirIn $inPath) 83 | 84 | # Full path to output file 85 | jp2Out="$dirOut/$inPathRel/$outName" 86 | 87 | # Convert TIFF to lossy access JP2 according to KB specs 88 | # Note: Grok writes XMP metadata UUID box, whereas KB 89 | # specs prescribe XML box! 90 | cmdCompress="grk_compress -i "$file" 91 | -o "$jp2Out" 92 | -n 6 93 | -p RPCL 94 | -t 1024,1024 95 | -b 64,64 96 | -c [256,256],[256,256],[128,128],[128,128],[128,128],[128,128] 97 | -r 2560,1280,640,320,160,80,40,20,10,5,1 98 | -S 99 | -E 100 | -M 32 101 | -C "$cCommentMaster" 102 | -W "$logFile"" 103 | 104 | echo "*** grok log: ***" >> $logFile 105 | 106 | $cmdCompress >>$logFile 2>&1 107 | grokCompressStatus=$? 108 | echo $jp2Out,$grokCompressStatus >> $grokStatusFile 109 | 110 | #md5sum "$jp2Out" >> $checksumFile 111 | 112 | echo "------" >> $logFile 113 | 114 | done < <(find $dirIn -type f -regex '.*\.\(tiff\|TIFF\|tif\|TIF\)' -print0) 115 | 116 | -------------------------------------------------------------------------------- /jp2totiff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory tree with JP2 images to TIFF 4 | # Requires: 5 | # - Kakadu (kdu_expand) 6 | # - ExifTool 7 | # - tiffinfo 8 | # - realpath 9 | 10 | 11 | if [ "$#" -ne 2 ] ; then 12 | echo "Usage: jp2totiff.sh dirIn dirOut" >&2 13 | exit 1 14 | fi 15 | 16 | # Input and output directories 17 | dirIn="$1" 18 | dirOut="$2" 19 | 20 | if ! [ -d "$dirIn" ] ; then 21 | echo "input directory does not exist" >&2 22 | exit 1 23 | fi 24 | 25 | if ! [ -d "$dirOut" ] ; then 26 | mkdir "$dirOut" 27 | fi 28 | 29 | # Location of Kakadu binaries 30 | kduPath=~/kakadu 31 | 32 | # Add Kakadu path to LD_LIBRARY_PATH 33 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$kduPath 34 | 35 | # Log file (used to store Kakadu and Exiftool stdout, stderr) 36 | logFile=$dirOut/jp2totiff.log 37 | 38 | # Kakadu status file (used to store Kakadu exit status) 39 | kakaduStatusFile=$dirOut/kduStatus.csv 40 | 41 | # Checksum file 42 | checksumFile=$dirOut/checksums.md5 43 | 44 | # Remove log and checksum files if they exist already (writing done in append mode!) 45 | if [ -f $logFile ] ; then 46 | rm $logFile 47 | fi 48 | 49 | if [ -f $kakaduStatusFile ] ; then 50 | rm $kakaduStatusFile 51 | fi 52 | 53 | if [ -f $checksumFile ] ; then 54 | rm $checksumFile 55 | fi 56 | 57 | # First clone the directory structure of dirIn to dirOut 58 | 59 | while IFS= read -d $'\0' -r directory ; do 60 | # Directory path, relative to dirIn 61 | dirInRel=$(realpath --relative-to=$dirIn $directory) 62 | 63 | # Absolute path to folder in output directory 64 | dirOutAbs=$dirOut/$dirInRel 65 | 66 | # Create folder in output directory 67 | mkdir -p $dirOutAbs 68 | done < <(find $dirIn -type d -print0) 69 | 70 | # Iterate over all files in dirIn and convert JP2s 71 | # to TIFF, writing result to corresponding folder in dirout 72 | 73 | while IFS= read -d $'\0' -r file ; do 74 | 75 | echo $file >> $logFile 76 | echo "*** Kakadu log: ***" >> $logFile 77 | 78 | # File basename, extension removed 79 | bName=$(basename "$file" | cut -f 1 -d '.') 80 | 81 | # Output name 82 | outName=$bName.tiff 83 | 84 | # Input path 85 | inPath=$(dirname "$file") 86 | 87 | # Input path, relative to dirIn 88 | inPathRel=$(realpath --relative-to=$dirIn $inPath) 89 | 90 | # Full path to output file 91 | tiffOut="$dirOut/$inPathRel/$outName" 92 | 93 | # Kakadu command line 94 | kduCmd="$kduPath/kdu_expand -i "$file" 95 | -o "$tiffOut"" 96 | 97 | # Convert to TIFF 98 | $kduCmd >>$logFile 2>&1 99 | kakaduStatus=$? 100 | echo $tiffOut,$kakaduStatus >> $kakaduStatusFile 101 | 102 | echo "*** Exiftool log: ***" >> $logFile 103 | 104 | # Remove XMP tags, except xmp-tiff ones 105 | # Skipping this step results in not well-formed XML 106 | # in round-trip JP2 generation with Kakadu (uuidbox) 107 | exiftool -overwrite_original -xmp:all= "-all:all>$logFile 2>&1 108 | 109 | # Compute MD5 checksum 110 | md5sum "$tiffOut" >> $checksumFile 111 | 112 | # Run TIFF through TIFFInfo, errors to log file 113 | echo "*** tiffinfo stderr: ***" >> $logFile 114 | tiffinfo -d "$tiffOut" 2>> $logFile >/dev/null 115 | 116 | echo "------" >> $logFile 117 | 118 | done < <(find $dirIn -type f -regex '.*\.\(jp2\|JP2\)' -print0) 119 | 120 | # Power off the machine 121 | #poweroff 122 | -------------------------------------------------------------------------------- /doc/grok-installation.md: -------------------------------------------------------------------------------- 1 | # Grok build process and installation 2 | 3 | This note documents the build process for Grok (including all third-party libraries that are needed to support various input formats andv ICC profiles). Partially based on [Bill Comstock's notes](https://wiki.harvard.edu/confluence/display/DigitalImaging/Installing+OpenJPEG+on+Windows+10%2C+Linux%2C+and+MacOS). 4 | 5 | ## Target OS 6 | 7 | Linux Mint 20.1 Ulyssa (based on Ubuntu Focal Fossa 20.04). 8 | 9 | ## Dev environment 10 | 11 | Install latest C and C++ compilers: 12 | 13 | ``` 14 | sudo apt install gcc-10 15 | sudo apt install g++-10 16 | ``` 17 | 18 | Configure update alternatives so that newly installed compilers are used: 19 | 20 | ``` 21 | sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 22 | ``` 23 | 24 | Uninstall current version of cmake (which is too old for Grok): 25 | 26 | ``` 27 | sudo apt purge --auto-remove cmake 28 | ``` 29 | 30 | Install newer version as per [here](https://askubuntu.com/a/1157132/1052776): 31 | 32 | ``` 33 | wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null 34 | sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' 35 | sudo apt update 36 | sudo apt install cmake 37 | ``` 38 | 39 | ## Install libraries needed by Grok 40 | 41 | ``` 42 | sudo apt-get install pkg-config libltdl-dev liblcms2-dev libtiff-dev libpng-dev libz-dev libzstd-dev libwebp-dev 43 | ``` 44 | 45 | ## Grok build process 46 | 47 | Download latest source distribution from: 48 | 49 | 50 | 51 | Then unzip. 52 | 53 | Note: earlier (9.7.3 and before) source distributions failed to compile because of [missing subdirectories in the "thirdparty" directory](https://github.com/GrokImageCompression/grok/issues/296). This is fixed from version 9.7.4 onwards. 54 | 55 | 56 | Go to Grok directory: 57 | 58 | ``` 59 | cd grok 60 | ``` 61 | 62 | Create build directory, the go there: 63 | 64 | ``` 65 | mkdir build 66 | cd build 67 | ``` 68 | 69 | If you want to list all the build options: 70 | 71 | ``` 72 | cmake .. -LA '{if(f)print} /-- Cache values/{f=1}' 73 | ``` 74 | 75 | From this huge list the following flags are important: 76 | 77 | - `CMAKE_BUILD_TYPE=Release` 78 | - `GROK_BUILD_THIRDPARTY:BOOL=ON` 79 | 80 | Configure build: 81 | 82 | ``` 83 | cmake -DCMAKE_BUILD_TYPE=Release -DGROK_BUILD_THIRDPARTY:BOOL=ON .. 84 | ``` 85 | For build we can specify number of logical cores. Check out using: 86 | 87 | ``` 88 | lscpu 89 | ``` 90 | 91 | Result: 92 | 93 | ``` 94 | Architecture: x86_64 95 | CPU op-mode(s): 32-bit, 64-bit 96 | Byte Order: Little Endian 97 | Address sizes: 39 bits physical, 48 bits virtual 98 | CPU(s): 4 99 | On-line CPU(s) list: 0-3 100 | Thread(s) per core: 1 101 | Core(s) per socket: 4 102 | Socket(s): 1 103 | ``` 104 | 105 | So in this case we have 4 physical cores, each running 1 thread (so 4 logical cores as well). So build using: 106 | 107 | ``` 108 | make -j4 109 | ``` 110 | 111 | Install: 112 | 113 | ``` 114 | sudo make install 115 | ``` 116 | 117 | Configure shared libraries: 118 | 119 | ``` 120 | sudo ldconfig 121 | ``` 122 | Clean: 123 | 124 | ``` 125 | sudo make clean 126 | ``` 127 | 128 | Done! 129 | -------------------------------------------------------------------------------- /mastertoaccess-kdu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory with JP2 lossless master images to lossy access JP2s 4 | # Requires: 5 | # - Kakadu 6 | # - ExifTool 7 | # -sed 8 | # - jprofile 9 | 10 | if [ "$#" -ne 2 ] ; then 11 | echo "Usage: mastertoaccess-kdu.sh dirIn dirOut" >&2 12 | exit 1 13 | fi 14 | 15 | # Input and output directories 16 | dirIn="$1" 17 | dirOut="$2" 18 | 19 | if ! [ -d "$dirIn" ] ; then 20 | echo "input directory does not exist" >&2 21 | exit 1 22 | fi 23 | 24 | if ! [ -d "$dirOut" ] ; then 25 | mkdir "$dirOut" 26 | fi 27 | 28 | dirAccess="$dirOut"/access 29 | 30 | if ! [ -d "$dirAccess" ] ; then 31 | mkdir "$dirAccess" 32 | fi 33 | 34 | # Location of Kakadu binaries 35 | kduPath=/Applications/kakadu 36 | 37 | # Add Kakadu path to LD_LIBRARY_PATH 38 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$kduPath 39 | 40 | # Log file (used too store kakadu stdout, stderr) 41 | logFile=$dirOut/mastertoaccess-kdu.log 42 | 43 | # Kakadu status file (used to store Kakadu exit status) 44 | kduStatusFile=$dirOut/kduStatus.csv 45 | 46 | # Remove log and status files if they exist already (writing done in append mode!) 47 | if [ -f $logFile ] ; then 48 | rm $logFile 49 | fi 50 | 51 | if [ -f $kduStatusFile ] ; then 52 | rm $kduStatusFile 53 | fi 54 | 55 | # Codestream comment string for access images 56 | cCommentAccess="KB_ACCESS_LOSSY_01/01/2015" 57 | 58 | # Iterate over all files in dirIn and convert JP2s 59 | # to TIFF, then convert those TIFFs to lossy JP2 60 | # according to KB access specs. 61 | 62 | while IFS= read -d $'\0' file ; do 63 | 64 | # File basename, extension removed 65 | bName=$(basename "$file" | cut -f 1 -d '.') 66 | 67 | # Output names 68 | outNameTIF=$bName.tif 69 | outNameJP2=$bName.jp2 70 | 71 | # Input path 72 | inPath=$(dirname "$file") 73 | 74 | # Full paths to output files 75 | tifOut="$dirAccess/$outNameTIF" 76 | jp2Out="$dirAccess/$outNameJP2" 77 | 78 | # First convert master JP2 to TIFF 79 | cmdDecompress="$kduPath/kdu_expand -i "$file" 80 | -o "$tifOut"" 81 | 82 | $cmdDecompress >>$logFile 2>&1 83 | kduDecompressStatus=$? 84 | echo $tifOut,$kduDecompressStatus >> $kduStatusFile 85 | 86 | # Get SamplesPerPixel value from output TIFF 87 | samplesPerPixel=$(exiftool -s -s -s -SamplesPerPixel "$tifOut") 88 | 89 | # Determine bitrate values, depending on samplesPerPixel value 90 | # Since bitrate = (BPP/CompressionRatio) 91 | if [ $samplesPerPixel -eq 3 ] ; then 92 | bitratesAccess="1.2,0.6,0.3,0.15,0.075,0.0375,0.01875,0.009375" 93 | fi 94 | 95 | if [ $samplesPerPixel -eq 1 ] ; then 96 | bitratesAccess="0.4,0.2,0.1,0.05,0.025,0.0125,0.00625,0.003125" 97 | fi 98 | 99 | # Name for temporary XMP sidecar file 100 | xmpName=$bName.xmp 101 | 102 | # Extract metadata from TIFF with Exiftool and write to XMP sidecar 103 | exiftool "$file" -o "$xmpName" >>$logFile 2>&1 104 | 105 | # Insert string "xml "at start of sidecar file so Kakadu knows to use XML box 106 | sed -i "1s/^/xml /" "$xmpName" 107 | 108 | # Convert TIFF to lossy access JP2 according to KB specs 109 | # Note: XMP metadata are written to UUID box, whereas KB 110 | # specs prescribe XML box. Don't think this is a problem 111 | # for access images. 112 | 113 | cmdCompress="$kduPath/kdu_compress -i "$tifOut" 114 | -o "$jp2Out" 115 | Creversible=no 116 | Clevels=5 117 | Corder=RPCL 118 | Stiles={1024,1024} 119 | Cblk={64,64} 120 | Cprecincts={256,256},{256,256},{128,128} 121 | Clayers=8 122 | -rate $bitratesAccess 123 | Cuse_sop=yes 124 | Cuse_eph=yes 125 | Cmodes=SEGMARK 126 | -jp2_box "$xmpName" 127 | -com "$cCommentAccess"" 128 | 129 | $cmdCompress >>$logFile 2>&1 130 | kduCompressStatus=$? 131 | echo $jp2Out,$kduCompressStatus >> $kduStatusFile 132 | 133 | # Remove TIFF file 134 | rm $tifOut 135 | 136 | # Remove XMP sidecar file 137 | rm $xmpName 138 | 139 | done < <(find $dirIn -type f -regex '.*\.\(jpf\|JPF\)' -print0) 140 | 141 | # Run jprofile 142 | jprofile -p kb_300Colour_2014.xml $dirAccess jprofile-kdu 143 | -------------------------------------------------------------------------------- /tifftojp2-lossy5-kdu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory tree with TIFF images to lossy JP2 with 5:1 compression ratio 4 | # Requires: 5 | # - Kakadu (kdu_expand) 6 | # - ExifTool 7 | # - realpath 8 | 9 | 10 | if [ "$#" -ne 2 ] ; then 11 | echo "Usage: tifftojp2-lossy-5-kdu.sh dirIn dirOut" >&2 12 | exit 1 13 | fi 14 | 15 | # Input and output directories 16 | dirIn="$1" 17 | dirOut="$2" 18 | 19 | if ! [ -d "$dirIn" ] ; then 20 | echo "input directory does not exist" >&2 21 | exit 1 22 | fi 23 | 24 | if ! [ -d "$dirOut" ] ; then 25 | mkdir "$dirOut" 26 | fi 27 | 28 | # Location of Kakadu binaries 29 | kduPath=~/kakadu 30 | 31 | # Add Kakadu path to LD_LIBRARY_PATH 32 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$kduPath 33 | 34 | # Log file (used too store Kakadu and Exiftool stdout, stderr) 35 | logFile=$dirOut/tifftojp2.log 36 | 37 | # Kakadu status file (used to store Kakadu exit status) 38 | kakaduStatusFile=$dirOut/kduStatus.csv 39 | 40 | # Checksum file 41 | checksumFile=$dirOut/checksums.md5 42 | 43 | # Remove log and checksum files if they exist already (writing done in append mode!) 44 | if [ -f $logFile ] ; then 45 | rm $logFile 46 | fi 47 | 48 | if [ -f $kakaduStatusFile ] ; then 49 | rm $kakaduStatusFile 50 | fi 51 | 52 | if [ -f $checksumFile ] ; then 53 | rm $checksumFile 54 | fi 55 | 56 | # Codestream comment strings for master and access images 57 | cCommentMaster="KB_MASTER_LOSSY_05/03/2024" 58 | 59 | # First clone the directory structure of dirIn to dirOut 60 | 61 | while IFS= read -d $'\0' -r directory ; do 62 | # Directory path, relative to dirIn 63 | dirInRel=$(realpath --relative-to=$dirIn $directory) 64 | 65 | # Absolute path to folder in output directory 66 | dirOutAbs=$dirOut/$dirInRel 67 | 68 | # Create folder in output directory 69 | mkdir -p $dirOutAbs 70 | done < <(find $dirIn -type d -print0) 71 | 72 | # Iterate over all files in dirIn and convert JP2s 73 | # to TIFF, writing result to corresponding folder in dirout 74 | 75 | while IFS= read -d $'\0' -r file ; do 76 | 77 | echo $file >> $logFile 78 | 79 | # File basename, extension removed 80 | bName=$(basename "$file" | cut -f 1 -d '.') 81 | 82 | # Output name 83 | outName=$bName.jp2 84 | 85 | # Input path 86 | inPath=$(dirname "$file") 87 | 88 | # Input path, relative to dirIn 89 | inPathRel=$(realpath --relative-to=$dirIn $inPath) 90 | 91 | # Full path to output file 92 | jp2Out="$dirOut/$inPathRel/$outName" 93 | 94 | # Name for temporary XMP sidecar file 95 | xmpName=$bName.xmp 96 | 97 | echo "*** Exiftool log: ***" >> $logFile 98 | 99 | # Extract metadata from TIFF with Exiftool and write to XMP sidecar 100 | exiftool "$file" -o "$xmpName" >>$logFile 2>&1 101 | 102 | # Insert string "xml "at start of sidecar file so Kakadu knows to use XML box 103 | sed -i "1s/^/xml /" "$xmpName" 104 | 105 | # Get SamplesPerPixel value 106 | samplesPerPixel=$(exiftool -s -s -s -SamplesPerPixel "$file") 107 | 108 | # Determine bitrate values, depending on samplesPerPixel value 109 | # Since bitrate = (BPP/CompressionRatio) 110 | if [ $samplesPerPixel -eq 3 ] ; then 111 | bitratesMaster="4.8,2.4,1.2,0.6,0.3,0.15,0.075,0.0375,0.01875,0.009375" 112 | fi 113 | 114 | if [ $samplesPerPixel -eq 1 ] ; then 115 | bitratesMaster="1.6,0.8,0.4,0.2,0.1,0.05,0.025,0.0125,0.00625,0.003125" 116 | fi 117 | 118 | # Construct Kakadu command lines (lossless master, lossy access copy) 119 | 120 | cmdlineMaster="$kduPath/kdu_compress -i "$file" 121 | -o "$jp2Out" 122 | Creversible=no 123 | Clevels=5 124 | Corder=RPCL 125 | Stiles={1024,1024} 126 | Cblk={64,64} 127 | Cprecincts={256,256},{256,256},{128,128} 128 | Clayers=10 129 | -rate $bitratesMaster 130 | Cuse_sop=yes 131 | Cuse_eph=yes 132 | Cmodes=SEGMARK 133 | -jp2_box "$xmpName" 134 | -com "$cCommentMaster"" 135 | 136 | echo "*** Kakadu log: ***" >> $logFile 137 | 138 | # Convert to JP2 139 | $cmdlineMaster >>$logFile 2>&1 140 | 141 | kakaduStatus=$? 142 | echo $jp2Out,$kakaduStatus >> $kakaduStatusFile 143 | 144 | # Remove XMP sidecar file 145 | rm $xmpName 146 | 147 | #md5sum "$jp2Out" >> $checksumFile 148 | 149 | echo "------" >> $logFile 150 | 151 | done < <(find $dirIn -type f -regex '.*\.\(tiff\|TIFF\|tif\|TIF\)' -print0) 152 | 153 | # Power off the machine 154 | # poweroff 155 | -------------------------------------------------------------------------------- /tifftojp2-kdu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Convert directory tree with TIFF images to JP2 4 | # Requires: 5 | # - Kakadu (kdu_expand) 6 | # - ExifTool 7 | # - realpath 8 | 9 | 10 | if [ "$#" -ne 2 ] ; then 11 | echo "Usage: tifftojp2.sh dirIn dirOut" >&2 12 | exit 1 13 | fi 14 | 15 | # Input and output directories 16 | dirIn="$1" 17 | dirOut="$2" 18 | 19 | if ! [ -d "$dirIn" ] ; then 20 | echo "input directory does not exist" >&2 21 | exit 1 22 | fi 23 | 24 | if ! [ -d "$dirOut" ] ; then 25 | mkdir "$dirOut" 26 | fi 27 | 28 | # Location of Kakadu binaries 29 | kduPath=~/kakadu 30 | 31 | # Add Kakadu path to LD_LIBRARY_PATH 32 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$kduPath 33 | 34 | # Log file (used too store Kakadu and Exiftool stdout, stderr) 35 | logFile=$dirOut/tifftojp2.log 36 | 37 | # Kakadu status file (used to store Kakadu exit status) 38 | kakaduStatusFile=$dirOut/kduStatus.csv 39 | 40 | # Checksum file 41 | checksumFile=$dirOut/checksums.md5 42 | 43 | # Remove log and checksum files if they exist already (writing done in append mode!) 44 | if [ -f $logFile ] ; then 45 | rm $logFile 46 | fi 47 | 48 | if [ -f $kakaduStatusFile ] ; then 49 | rm $kakaduStatusFile 50 | fi 51 | 52 | if [ -f $checksumFile ] ; then 53 | rm $checksumFile 54 | fi 55 | 56 | # Codestream comment strings for master and access images 57 | cCommentMaster="KB_MASTER_LOSSLESS_01/01/2015" 58 | cCommentAccess="KB_ACCESS_LOSSY_01/01/2015" 59 | 60 | # First clone the directory structure of dirIn to dirOut 61 | 62 | while IFS= read -d $'\0' -r directory ; do 63 | # Directory path, relative to dirIn 64 | dirInRel=$(realpath --relative-to=$dirIn $directory) 65 | 66 | # Absolute path to folder in output directory 67 | dirOutAbs=$dirOut/$dirInRel 68 | 69 | # Create folder in output directory 70 | mkdir -p $dirOutAbs 71 | done < <(find $dirIn -type d -print0) 72 | 73 | # Iterate over all files in dirIn and convert JP2s 74 | # to TIFF, writing result to corresponding folder in dirout 75 | 76 | while IFS= read -d $'\0' -r file ; do 77 | 78 | echo $file >> $logFile 79 | 80 | # File basename, extension removed 81 | bName=$(basename "$file" | cut -f 1 -d '.') 82 | 83 | # Output name 84 | outName=$bName.jp2 85 | 86 | # Input path 87 | inPath=$(dirname "$file") 88 | 89 | # Input path, relative to dirIn 90 | inPathRel=$(realpath --relative-to=$dirIn $inPath) 91 | 92 | # Full path to output file 93 | jp2Out="$dirOut/$inPathRel/$outName" 94 | 95 | # Name for temporary XMP sidecar file 96 | xmpName=$bName.xmp 97 | 98 | echo "*** Exiftool log: ***" >> $logFile 99 | 100 | # Extract metadata from TIFF with Exiftool and write to XMP sidecar 101 | exiftool "$file" -o "$xmpName" >>$logFile 2>&1 102 | 103 | # Insert string "xml "at start of sidecar file so Kakadu knows to use XML box 104 | sed -i "1s/^/xml /" "$xmpName" 105 | 106 | # Get SamplesPerPixel value 107 | samplesPerPixel=$(exiftool -s -s -s -SamplesPerPixel "$file") 108 | 109 | # Determine bitrate values, depending on samplesPerPixel value 110 | # Since bitrate = (BPP/CompressionRatio) 111 | if [ $samplesPerPixel -eq 3 ] ; then 112 | bitratesMaster="-,4.8,2.4,1.2,0.6,0.3,0.15,0.075,0.0375,0.01875,0.009375" 113 | bitratesAccess="1.2,0.6,0.3,0.15,0.075,0.0375,0.01875,0.009375" 114 | fi 115 | 116 | if [ $samplesPerPixel -eq 1 ] ; then 117 | bitratesMaster="-,1.6,0.8,0.4,0.2,0.1,0.05,0.025,0.0125,0.00625,0.003125" 118 | bitratesAccess="0.4,0.2,0.1,0.05,0.025,0.0125,0.00625,0.003125" 119 | fi 120 | 121 | # Construct Kakadu command lines (lossless master, lossy access copy) 122 | 123 | cmdlineMaster="$kduPath/kdu_compress -i "$file" 124 | -o "$jp2Out" 125 | Creversible=yes 126 | Clevels=5 127 | Corder=RPCL 128 | Stiles={1024,1024} 129 | Cblk={64,64} 130 | Cprecincts={256,256},{256,256},{128,128} 131 | Clayers=11 132 | -rate $bitratesMaster 133 | Cuse_sop=yes 134 | Cuse_eph=yes 135 | Cmodes=SEGMARK 136 | -jp2_box "$xmpName" 137 | -com "$cCommentMaster"" 138 | 139 | #cmdlineAccess="$kduPath/kdu_compress -i "$file" 140 | # -o "$outAccess" 141 | # Creversible=no 142 | # Clevels=5 143 | # Corder=RPCL 144 | # Stiles={1024,1024} 145 | # Cblk={64,64} 146 | # Cprecincts={256,256},{256,256},{128,128} 147 | # Clayers=8 148 | # -rate $bitratesAccess 149 | # Cuse_sop=yes 150 | # Cuse_eph=yes 151 | # Cmodes=SEGMARK 152 | # -jp2_box "$xmpName" 153 | # -com "$cCommentAccess"" 154 | 155 | echo "*** Kakadu log: ***" >> $logFile 156 | 157 | # Convert to JP2 158 | $cmdlineMaster >>$logFile 2>&1 159 | 160 | kakaduStatus=$? 161 | echo $jp2Out,$kakaduStatus >> $kakaduStatusFile 162 | 163 | # Remove XMP sidecar file 164 | rm $xmpName 165 | 166 | #md5sum "$jp2Out" >> $checksumFile 167 | 168 | echo "------" >> $logFile 169 | 170 | done < <(find $dirIn -type f -regex '.*\.\(tiff\|TIFF\|tif\|TIF\)' -print0) 171 | 172 | # Power off the machine 173 | # poweroff 174 | --------------------------------------------------------------------------------