├── .gitattribute ├── .github └── workflows │ └── release.yml ├── .gitignore ├── .gitmodules ├── .goreleaser.yml ├── .idea ├── .gitignore ├── gocfl.iml ├── modules.xml └── vcs.xml ├── LICENSE ├── README.md ├── build ├── build_decrypt.ps1 └── build_gocfl.ps1 ├── config ├── config.go ├── default.toml ├── embed.go ├── gocfl.toml ├── gocfl2.toml ├── gocfl_linux.toml ├── gocfl_sci.toml └── hek_gocfl.toml ├── data ├── defaultextensions │ ├── object │ │ ├── 0001-digest-algorithms │ │ │ └── config.json │ │ ├── 0011-direct-clean-path-layout │ │ │ └── config.json │ │ ├── NNNN-filesystem │ │ │ └── config.json │ │ ├── NNNN-gocfl-extension-manager │ │ │ └── config.json │ │ ├── NNNN-indexer │ │ │ └── config.json │ │ ├── NNNN-metafile │ │ │ ├── config.json │ │ │ └── gocfl-info-1.0.json │ │ ├── NNNN-thumbnail │ │ │ └── config.json │ │ ├── embed.go │ │ └── initial │ │ │ └── config.json │ └── storageroot │ │ ├── 0006-flat-omit-prefix-storage-layout │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ └── config.json │ │ ├── NNNN-direct-path-layout │ │ └── config.json │ │ ├── NNNN-gocfl-extension-manager │ │ └── config.json │ │ ├── embed.go │ │ └── initial │ │ └── config.json ├── displaydata │ ├── bootstrapdist │ │ ├── css │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.css.map │ │ └── js │ │ │ ├── bootstrap.bundle.min.js │ │ │ └── bootstrap.bundle.min.js.map │ ├── css │ │ ├── interface.css │ │ └── sidebar.css │ ├── embed.go │ ├── js │ │ ├── json-viewer.bundle.js │ │ ├── json-viewer.md │ │ ├── paged.js │ │ ├── paged.polyfill.js │ │ └── pagedjs.md │ └── templates │ │ ├── detail.gohtml │ │ ├── manifest.gohtml │ │ ├── object.gohtml │ │ ├── report.gohtml │ │ ├── storageroot.gohtml │ │ └── version.gohtml ├── fullextensions │ ├── object │ │ ├── 0001-digest-algorithms │ │ │ └── config.json │ │ ├── 0011-direct-clean-path-layout │ │ │ └── config.json │ │ ├── NNNN-content-subpath │ │ │ └── config.json │ │ ├── NNNN-filesystem │ │ │ └── config.json │ │ ├── NNNN-gocfl-extension-manager │ │ │ └── config.json │ │ ├── NNNN-indexer │ │ │ └── config.json │ │ ├── NNNN-metafile │ │ │ ├── config.json │ │ │ └── gocfl-info-1.0.json │ │ ├── NNNN-mets │ │ │ └── config.json │ │ ├── NNNN-migration │ │ │ └── config.json │ │ ├── NNNN-thumbnail │ │ │ └── config.json │ │ └── initial │ │ │ └── config.json │ └── storageroot │ │ ├── 0006-flat-omit-prefix-storage-layout │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ └── config.json │ │ ├── NNNN-direct-path-layout │ │ └── config.json │ │ ├── NNNN-gocfl-extension-manager │ │ └── config.json │ │ └── initial │ │ └── config.json ├── migration │ ├── pdfa.ps1 │ └── pdfa_def.ps ├── nomigrationextensions │ ├── object │ │ ├── 0001-digest-algorithms │ │ │ └── config.json │ │ ├── NNNN-content-subpath │ │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ │ └── config.json │ │ ├── NNNN-filesystem │ │ │ └── config.json │ │ ├── NNNN-indexer │ │ │ └── config.json │ │ ├── NNNN-thumbnail │ │ │ └── config.json │ │ └── initial │ │ │ └── config.json │ └── storageroot │ │ ├── 0006-flat-omit-prefix-storage-layout │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ └── config.json │ │ ├── NNNN-direct-path-layout │ │ └── config.json │ │ └── initial │ │ └── config.json ├── scicoreextensions │ ├── object │ │ ├── 0001-digest-algorithms │ │ │ └── config.json │ │ ├── NNNN-content-subpath │ │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ │ └── config.json │ │ ├── NNNN-filesystem │ │ │ └── config.json │ │ ├── NNNN-indexer │ │ │ └── config.json │ │ └── initial │ │ │ └── config.json │ └── storageroot │ │ ├── 0006-flat-omit-prefix-storage-layout │ │ └── config.json │ │ ├── NNNN-direct-clean-path-layout │ │ └── config.json │ │ ├── NNNN-direct-path-layout │ │ └── config.json │ │ └── initial │ │ └── config.json ├── scripts │ ├── pdf2thumb.ps1 │ └── video2thumb.ps1 └── specs │ ├── MARC21slim.xsd │ ├── embed.go │ ├── loc.gov_standards_premis_v3_premis-v3-0.xsd │ ├── mets.xsd │ ├── ocfl_1.1.md │ ├── premis.xsd │ └── xlink.xsd ├── decrypt └── main.go ├── docs ├── 0011-direct-clean-path-layout.md ├── NNNN-content-subpath.md ├── NNNN-direct-clean-path-layout.md ├── NNNN-filesystem.md ├── NNNN-gocfl-extension-manager.md ├── NNNN-indexer.md ├── NNNN-metafile.md ├── NNNN-mets.md ├── NNNN-migration.md ├── NNNN-thumbnail.md ├── add.md ├── bad-objects.txt ├── create.md ├── display.md ├── display_detail_migrated.png ├── display_detail_png.png ├── display_object_overview.png ├── display_start.png ├── display_version.png ├── embed.go ├── extract.md ├── extractmeta.md ├── good-objects.txt ├── init.md ├── initial.md ├── mets.xml ├── ocfl_spec_1.1.md ├── premis.xml ├── presentation.md ├── quickstart.md ├── stat.md ├── update.md ├── validate.md └── warn-objects.txt ├── fixtures └── new │ └── 1.1 │ └── warn-objects │ └── W003_empty_content_folder │ ├── 0=ocfl_object_1.1 │ ├── inventory.json │ ├── inventory.json.sha512 │ └── v1 │ ├── inventory.json │ └── inventory.json.sha512 ├── go.mod ├── go.sum ├── gocfl-info-1.0.json ├── gocfl ├── cmd │ ├── add.go │ ├── create.go │ ├── display.go │ ├── display │ │ └── server.go │ ├── errors.go │ ├── extract.go │ ├── extractmeta.go │ ├── helper.go │ ├── init.go │ ├── root.go │ ├── stat.go │ ├── update.go │ └── validate.go ├── main.go ├── main_imagick.go └── main_vips.go ├── internal ├── embed.go ├── errors.toml └── siegfried │ └── default.sig ├── justfile ├── pkg ├── dilcis │ ├── DILCISExtensionMETS.xsd │ ├── DILCISExtensionSIPMETS.xsd │ ├── ead3.xsd │ ├── ead3 │ │ └── ead3.go │ ├── ead3_undeprecated.xsd │ ├── mets.xsd │ ├── mets │ │ ├── DILCISExtensionMETS.go │ │ ├── DILCISExtensionSIPMETS.go │ │ ├── mets.go │ │ └── xlink.go │ ├── premis-v3-0.xsd │ ├── premis │ │ ├── premis.go │ │ ├── premis.go.sik │ │ └── premisFunc.go │ └── xlink.xsd ├── extension │ ├── 0001-digest-algorithms.go │ ├── 0002-flat-direct-storage-layout.go │ ├── 0002-flat-direct-storage-layout_test.go │ ├── 0003-hash-and-id-n-tuple-storage-layout.go │ ├── 0003-hash-and-id-n-tuple-storage-layout_test.go │ ├── 0004-hashed-n-tuple-storage-layout.go │ ├── 0004-hashed-n-tuple-storage-layout_test.go │ ├── 0006-flat-omit-prefix-storage-layout.go │ ├── 0006-flat-omit-prefix-storage-layout_test.go │ ├── 0006-flat-omit-prefix-storage-layout_test.go_test.go │ ├── 0007-n-tuple-omit-prefix-storage-layout.go │ ├── 0011-direct-clean-path-layout.go │ ├── 0011-direct-clean-path-layout_test.go │ ├── NNNN-content-subpath.go │ ├── NNNN-direct-clean-path-layout.go │ ├── NNNN-filesystem.go │ ├── NNNN-filesystem_darwin.go │ ├── NNNN-filesystem_unix.go │ ├── NNNN-filesystem_win.go │ ├── NNNN-gocfl-extension-manager.go │ ├── NNNN-indexer-logging-object.go │ ├── NNNN-indexer.go │ ├── NNNN-metafile.go │ ├── NNNN-mets.go │ ├── NNNN-migration.go │ ├── NNNN-pairtree-storage-layout.go │ ├── NNNN-pairtree-storage-layout_test.go │ ├── NNNN-thumbnail.go │ ├── NNNN-thumbnail_imagick.go │ ├── NNNN-thumbnail_native.go │ ├── NNNN-thumbnail_vips.go │ ├── config.go │ ├── initial.go │ └── pathdirect.go ├── ocfl │ ├── extension.go │ ├── extensionExternalParam.go │ ├── extensionFactory.go │ ├── extensionInitialDummy.go │ ├── extensionManager.go │ ├── fixity.go │ ├── fs.go │ ├── helper.go │ ├── inventory.go │ ├── inventory1_0.go │ ├── inventory1_1.go │ ├── inventorybase.go │ ├── inventorytypes.go │ ├── metadata.go │ ├── object.go │ ├── object1_0.go │ ├── object1_1.go │ ├── object2_0.go │ ├── objectbase.go │ ├── stat.go │ ├── storageroot.go │ ├── storageroot1_0.go │ ├── storageroot1_1.go │ ├── storageroot2_0.go │ ├── storagerootbase.go │ ├── validation.go │ ├── validationerror.go │ ├── validationerror1_0.go │ └── validationerror1_1.go └── subsystem │ ├── indexer │ ├── actions.go │ └── config.go │ ├── migration │ ├── helper.go │ └── migrate.go │ └── thumbnail │ ├── helper.go │ └── thumbnail.go └── version └── version.go /.gitattribute: -------------------------------------------------------------------------------- 1 | config/** linguist-vendored 2 | data/** linguist-vendored 3 | docs/** linguist-vendored 4 | fixtures/** linguist-vendored 5 | *.go linguist-language=Go 6 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | jobs: 7 | goreleaser: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: "checkout" 11 | uses: actions/checkout@v4 12 | with: 13 | fetch-depth: 0 14 | - name: Set up Go 15 | uses: actions/setup-go@v5 16 | with: 17 | go-version: 1.23.0 18 | - name: "run GoReleaser" 19 | uses: goreleaser/goreleaser-action@v5 20 | with: 21 | distribution: goreleaser 22 | version: '~> v2' 23 | args: release --clean -f .goreleaser.yml 24 | env: 25 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/gocfl_darwin_amd64 2 | /build/gocfl_darwin_arm64 3 | /build/gocfl_linux_amd64 4 | /build/gocfl_linux_arm64 5 | /build/gocfl_windows_amd64.exe 6 | /build/gocfl_windows_arm64.exe 7 | /config/hek_gocfl.toml 8 | /cmd/test/ 9 | go.work 10 | go.work.sum 11 | /data/temp/ 12 | dist/* 13 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "fixtures/fixtures"] 2 | path = fixtures/fixtures 3 | url = https://github.com/OCFL/fixtures.git 4 | branch = main 5 | [submodule "bootstrap"] 6 | path = data/displaydata/bootstrap 7 | url = https://github.com/twbs/bootstrap.git 8 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: gocfl 2 | version: 2 3 | builds: 4 | - main: ./gocfl/ 5 | binary: gocfl 6 | env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - linux 10 | - windows 11 | - darwin 12 | ignore: 13 | - goos: free bsd 14 | goarch: 386 15 | - goos: freebsd 16 | goarch: arm64 17 | - goos: windows 18 | goarch: arm64 19 | - goos: linux 20 | goarch: 386 21 | mod_timestamp: '{{ .CommitTimestamp }}' 22 | ldflags: 23 | -s 24 | -w 25 | -X main.appname={{.ProjectName}} 26 | -X main.builtBy=gocfl-goreleaser 27 | -X github.com/ocfl-archive/gocfl/v2/version.Version={{.Version}} 28 | -X github.com/ocfl-archive/gocfl/v2/version.Commit={{.Commit}} 29 | -X github.com/ocfl-archive/gocfl/v2/version.Date={{.CommitDate}} 30 | -X github.com/ocfl-archive/gocfl/v2/version.BuiltBy=goreleaser 31 | archives: 32 | - name_template: >- 33 | {{ .ProjectName }}_ 34 | {{ .Version }}_ 35 | {{- title .Os }}_ 36 | {{- if eq .Arch "amd64" }}x86_64 37 | {{- else if eq .Arch "386" }}i386 38 | {{- else if eq .Arch "arm64" }}arm64 39 | {{- else }}{{ .Arch }}{{ end }} 40 | format: tar.gz 41 | format_overrides: 42 | - goos: windows 43 | format: zip 44 | checksum: 45 | name_template: 'checksums.txt' 46 | snapshot: 47 | version_template: "{{ .Version }}-SNAPSHOT" 48 | changelog: 49 | sort: asc 50 | filters: 51 | exclude: 52 | - '^docs:' 53 | - '^test:' 54 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | # GitHub Copilot persisted chat sessions 10 | /copilot/chatSessions 11 | -------------------------------------------------------------------------------- /.idea/gocfl.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /build/build_decrypt.ps1: -------------------------------------------------------------------------------- 1 | $archs = "amd64","arm64" 2 | $oss = "windows","linux","darwin" 3 | 4 | foreach ($os in $oss) { 5 | $env:GOOS=$os 6 | foreach ($arch in $archs) { 7 | $env:GOARCH=$arch 8 | Write-Output "$os/$arch" 9 | if ($os -eq "windows") { 10 | Start-Process -FilePath "go.exe" -ArgumentList "build -o ./decrypt_$($os)_$($arch).exe ../decrypt" -Wait 11 | } else { 12 | Start-Process -FilePath "go.exe" -ArgumentList "build -o ./decrypt_$($os)_$($arch) ../decrypt" -Wait 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /build/build_gocfl.ps1: -------------------------------------------------------------------------------- 1 | $archs = "amd64","arm64" 2 | $oss = "windows","linux","darwin" 3 | 4 | foreach ($os in $oss) { 5 | $env:GOOS=$os 6 | foreach ($arch in $archs) { 7 | $env:GOARCH=$arch 8 | Write-Output "$os/$arch" 9 | if ($os -eq "windows") { 10 | Start-Process -FilePath "go.exe" -ArgumentList "build -o ./gocfl_$($os)_$($arch).exe ../gocfl" -Wait 11 | } else { 12 | Start-Process -FilePath "go.exe" -ArgumentList "build -o ./gocfl_$($os)_$($arch) ../gocfl" -Wait 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /config/default.toml: -------------------------------------------------------------------------------- 1 | [log] 2 | # "trace" 3 | # "debug" 4 | # "info" 5 | # "warn" 6 | # "error" 7 | # "fatal" 8 | # "panic" 9 | level = "info" 10 | 11 | # global config for all operations on S3 storages 12 | [S3] 13 | # --s3-endpoint 14 | #Endpoint="%%GOCFL_S3_ENDPOINT%%" 15 | # --s3-access-key-id 16 | #AccessKeyID="%%GOCFL_S3_ACCESS_KEY_ID%%" 17 | # --s3-secret-access-key 18 | #AccessKey="%%GOCFL_S3_ACCESS_KEY%%" 19 | # --s3-region 20 | #Region="%%GOCFL_S3_REGION%%" 21 | 22 | [AES] 23 | Enable=false 24 | KeepassFile="c:/temp/test.kdbx" 25 | KeepassEntry="keepass2://test.kdbx/Root/gocfl/test" 26 | KeepassKey="%%GOCFL_KEEPASS_KEY%%" 27 | 28 | [Init] 29 | # --ocfl-version 30 | OCFLVersion="1.1" 31 | # --default-storageroot-extensions 32 | #StorageRootExtensions="./data/defaultextensions/storageroot" 33 | # --digest 34 | Digest="sha512" 35 | 36 | [Add] 37 | # --message 38 | Message="initial commit" 39 | # --digest 40 | Digest="sha512" 41 | # --fixity 42 | Fixity=["sha256", "sha1", "md5"] 43 | # --default-object-extensions 44 | #ObjectExtensions="./data/fullextensions/object" 45 | 46 | DefaultArea="content" 47 | 48 | [Add.User] 49 | # --user-name 50 | Name="unknown user" 51 | # --user-address 52 | Address="https://github.com/ocfl-archive/gocfl" 53 | 54 | # 55 | # Extension parameter 56 | # 57 | [Extension] 58 | [Extension.NNNN-metafile] 59 | Source="" 60 | 61 | [Extension.NNNN-mets] 62 | # --ext-NNNN-mets-descriptive-metadata 63 | descriptive-metadata="other:metadata:info.json" 64 | # 65 | # Indexer 66 | # 67 | [Indexer] 68 | Enabled=true 69 | # Enable this, if there are problem detecting length of audio files with ffmpeg 70 | LocalCache=false 71 | 72 | [Indexer.Checksum] 73 | Enabled=true 74 | Digest=["sha512"] 75 | 76 | [Indexer.XML] 77 | Enabled=true 78 | [Indexer.XML.Format.document] 79 | Attributes.xmlns = "http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml" 80 | Type = "ocr" 81 | Subtype = "FineReader10" 82 | Mime = "application/xml" 83 | [Indexer.XML.Format."mets:mets"] 84 | Regexp = true 85 | Attributes."xmlns:mets" = "^https?://www.loc.gov/METS/?$" 86 | Type = "metadata" 87 | Subtype = "METS" 88 | Mime = "application/xml" 89 | 90 | [Indexer.Siegfried] 91 | Enabled=true 92 | #Signature = "/usr/share/siegfried/default.sig" 93 | #Signature = "file://C:/Users/micro/siegfried/default.sig" 94 | Signature = "internal:/siegfried/default.sig" 95 | 96 | # mapping of pronom id to mimetype if not found in siegfried 97 | [Indexer.Siegfried.MimeMap] 98 | "x-fmt/92" = "image/psd" 99 | "fmt/134" = "audio/mp3" 100 | "x-fmt/184" = "image/x-sun-raster" 101 | "fmt/202" = "image/x-nikon-nef" 102 | "fmt/211" = "image/x-photo-cd" 103 | "x-fmt/383" = "image/fits" 104 | "fmt/405" = "image/x-portable-anymap" 105 | "fmt/406" = "image/x-portable-graymap" 106 | "fmt/408" = "image/x-portable-pixmap" 107 | "fmt/436" = "image/x-adobe-dng" 108 | "fmt/437" = "image/x-adobe-dng" 109 | "fmt/592" = "image/x-canon-cr2" 110 | "fmt/642" = "image/x-raw-fuji" 111 | "fmt/662" = "image/x-raw-panasonic" 112 | "fmt/668" = "image/x-olympus-orf" 113 | "fmt/986" = "text/xmp" 114 | "fmt/1001" = "image/x-exr" 115 | "fmt/1040" = "image/vnd.ms-dds" 116 | "fmt/1781" = "image/x-pentax-pef" 117 | 118 | # relevance of mimetype for sorting 119 | # relevance < 100: rate down 120 | # relevance > 100: rate up 121 | # default = 100 122 | [Indexer.MimeRelevance.1] 123 | Regexp = "^application/octet-stream" 124 | Weight = 1 125 | [Indexer.MimeRelevance.2] 126 | Regexp = "^text/plain" 127 | Weight = 3 128 | [Indexer.MimeRelevance.3] 129 | Regexp = "^audio/mpeg" 130 | Weight = 6 131 | [Indexer.MimeRelevance.4] 132 | Regexp = "^video/mpeg" 133 | Weight = 5 134 | [Indexer.MimeRelevance.5] 135 | Regexp = "^application/vnd\\..+" 136 | Weight = 4 137 | [Indexer.MimeRelevance.6] 138 | Regexp = "^application/rtf" 139 | Weight = 4 140 | [Indexer.MimeRelevance.7] 141 | Regexp = "^application/.+" 142 | Weight = 2 143 | [Indexer.MimeRelevance.8] 144 | Regexp = "^text/.+" 145 | Weight = 4 146 | [Indexer.MimeRelevance.9] 147 | Regexp = "^audio/.+" 148 | Weight = 5 149 | [Indexer.MimeRelevance.10] 150 | Regexp = "^video/.+" 151 | Weight = 4 152 | [Indexer.MimeRelevance.11] 153 | Regexp = "^.+/x-.+" 154 | Weight = 80 155 | 156 | 157 | # 158 | # Thumbnail 159 | # 160 | [Thumbnail] 161 | Enable=true 162 | Background="none" 163 | 164 | # 165 | # Migration 166 | # 167 | [Migration] 168 | # --with-migration 169 | Enable=false 170 | -------------------------------------------------------------------------------- /config/embed.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import _ "embed" 4 | 5 | //go:embed default.toml 6 | var DefaultConfig []byte 7 | -------------------------------------------------------------------------------- /data/defaultextensions/object/0001-digest-algorithms/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0001-digest-algorithms" 3 | } 4 | -------------------------------------------------------------------------------- /data/defaultextensions/object/0011-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0011-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxPathSegmentLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": true 8 | } 9 | -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-filesystem/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-filesystem", 3 | "folders": "", 4 | "storageType": "extension", 5 | "storageName": "metadata", 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-gocfl-extension-manager/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "ObjectContentPath": [ 5 | "NNNN-direct-clean-path-layout", 6 | "NNNN-content-subpath" 7 | ], 8 | "ObjectChange": [ 9 | "NNNN-indexer", 10 | "NNNN-metafile" 11 | ] 12 | }, 13 | "exclusion": { 14 | "ObjectContentPath": [ 15 | [ 16 | "NNNN-direct-clean-path-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-indexer/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-indexer", 3 | "storageType": "extension", 4 | "storageName": "metadata", 5 | "actions": ["siegfried", "xml"], 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-metafile/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-metafile", 3 | "storageType": "extension", 4 | "storageName": "metadata", 5 | "name": "info.json", 6 | "schema": "gocfl-info-1.0.json", 7 | "schemaUrl": "https://raw.githubusercontent.com/ocfl-archive/gocfl/main/gocfl-info-1.0.json" 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-metafile/gocfl-info-1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "$ref": "#/$defs/Info", 4 | "$defs": { 5 | "Info": { 6 | "properties": { 7 | "signature": { 8 | "type": "string", 9 | "maxLength": 128, 10 | "minLength": 3, 11 | "pattern": "[a-zA-Z0-9/.-:-]+", 12 | "title": "archival signature", 13 | "description": "unique identifier within the archive system" 14 | }, 15 | "organisation_id": { 16 | "type": "string", 17 | "pattern": "[a-zA-Z0-9/.-:-]+", 18 | "title": "organisation identifier", 19 | "description": "id or abbreviation of organisation responsible for the object" 20 | }, 21 | "organisation": { 22 | "type": "string", 23 | "title": "organisation name", 24 | "description": "name of organisation responsible for the object" 25 | }, 26 | "organisation_address": { 27 | "type": "string", 28 | "title": "address", 29 | "description": "adress of person ingesting this archive (email" 30 | }, 31 | "collection_id": { 32 | "type": "string", 33 | "pattern": "[a-zA-Z0-9/._:-]+", 34 | "title": "collection identifier", 35 | "description": "id of collection the object belongs to" 36 | }, 37 | "collection": { 38 | "type": "string", 39 | "title": "collection name", 40 | "description": "name of collection the object belongs to" 41 | }, 42 | "sets": { 43 | "items": { 44 | "type": "string" 45 | }, 46 | "type": "array", 47 | "uniqueItems": true, 48 | "title": "sets", 49 | "description": "list of datasets object is belonging to" 50 | }, 51 | "identifiers": { 52 | "items": { 53 | "type": "string" 54 | }, 55 | "type": "array", 56 | "uniqueItems": true, 57 | "title": "identifiers", 58 | "description": "list of identifiers" 59 | }, 60 | "title": { 61 | "type": "string", 62 | "title": "title", 63 | "description": "title of object" 64 | }, 65 | "alternative_titles": { 66 | "items": { 67 | "type": "string" 68 | }, 69 | "type": "array", 70 | "uniqueItems": true, 71 | "title": "alternative titles", 72 | "description": "list of alternative titles of this object or parts of it" 73 | }, 74 | "description": { 75 | "type": "string" 76 | }, 77 | "keywords": { 78 | "items": { 79 | "type": "string" 80 | }, 81 | "type": "array", 82 | "uniqueItems": true 83 | }, 84 | "user": { 85 | "type": "string", 86 | "title": "user", 87 | "description": "name of person ingesting this object" 88 | }, 89 | "address": { 90 | "type": "string", 91 | "title": "address", 92 | "description": "adress of person ingesting this archive (email" 93 | }, 94 | "created": { 95 | "type": "string", 96 | "format": "date-time", 97 | "title": "creation date", 98 | "description": "date" 99 | }, 100 | "last_changed": { 101 | "type": "string", 102 | "format": "date-time", 103 | "title": "last changed", 104 | "description": "date" 105 | }, 106 | "deprecates": { 107 | "type": "string", 108 | "title": "deprecates", 109 | "description": "signature of object" 110 | }, 111 | "references": { 112 | "items": { 113 | "type": "string" 114 | }, 115 | "type": "array", 116 | "title": "references", 117 | "description": "list of signatures" 118 | }, 119 | "ingest_workflow": { 120 | "type": "string", 121 | "title": "ingest workflow", 122 | "description": "name of the workflow" 123 | }, 124 | "additional": { 125 | "type":["number","string","boolean","object","array", "null"], 126 | "title": "additional data", 127 | "description": "unstructured additional data" 128 | } 129 | }, 130 | "additionalProperties": false, 131 | "type": "object", 132 | "required": [ 133 | "signature", 134 | "organisation_id", 135 | "organisation", 136 | "organisation_address", 137 | "title", 138 | "user", 139 | "address", 140 | "created", 141 | "last_changed" 142 | ] 143 | } 144 | } 145 | } -------------------------------------------------------------------------------- /data/defaultextensions/object/NNNN-thumbnail/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storageType": "extension", 3 | "storageName": "metadata", 4 | "extensionName": "NNNN-thumbnail", 5 | "compress": "gzip", 6 | "width": 256, 7 | "height": 256, 8 | "singleDirectory": false 9 | } 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /data/defaultextensions/object/embed.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import "embed" 4 | 5 | //go:embed NNNN-filesystem/config.json 6 | //go:embed NNNN-indexer/config.json 7 | //go:embed 0011-direct-clean-path-layout/config.json 8 | //go:embed 0001-digest-algorithms/config.json 9 | //go:embed NNNN-thumbnail/config.json 10 | var DefaultObjectExtensionFS embed.FS 11 | -------------------------------------------------------------------------------- /data/defaultextensions/object/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "initial", 3 | "extension": "NNNN-gocfl-extension-manager" 4 | } 5 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/0006-flat-omit-prefix-storage-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0006-flat-omit-prefix-storage-layout", 3 | "delimiter": ":" 4 | } 5 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxPathSegmentLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": false 8 | } 9 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/NNNN-direct-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-path-layout" 3 | } 4 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/NNNN-gocfl-extension-manager/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "StorageRootPath": [ 5 | "NNNN-direct-clean-path-layout" 6 | ] 7 | }, 8 | "exclusion": { 9 | "StorageRootPath": [ 10 | [ 11 | "NNNN-direct-clean-path-layout", 12 | "0003-hash-and-id-n-tuple-storage-layout", 13 | "0004-hashed-n-tuple-storage-layout", 14 | "0002-flat-direct-storage-layout", 15 | "0006-flat-omit-prefix-storage-layout", 16 | "NNNN-pairtree-storage-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/embed.go: -------------------------------------------------------------------------------- 1 | package storageroot 2 | 3 | import "embed" 4 | 5 | // go:embed initial/*.json 6 | // 7 | //go:embed NNNN-direct-clean-path-layout 8 | var DefaultStorageRootExtensionFS embed.FS 9 | -------------------------------------------------------------------------------- /data/defaultextensions/storageroot/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "initial", 3 | "extension": "NNNN-gocfl-extension-manager" 4 | } 5 | -------------------------------------------------------------------------------- /data/displaydata/css/sidebar.css: -------------------------------------------------------------------------------- 1 | body { 2 | min-height: 100vh; 3 | min-height: -webkit-fill-available; 4 | } 5 | 6 | html { 7 | height: -webkit-fill-available; 8 | } 9 | 10 | main { 11 | display: flex; 12 | flex-wrap: nowrap; 13 | height: 100vh; 14 | height: -webkit-fill-available; 15 | max-height: 100vh; 16 | overflow-x: auto; 17 | overflow-y: hidden; 18 | } 19 | 20 | .b-example-divider { 21 | flex-shrink: 0; 22 | width: 1.5rem; 23 | height: 100vh; 24 | background-color: rgba(0, 0, 0, .1); 25 | border: solid rgba(0, 0, 0, .15); 26 | border-width: 1px 0; 27 | box-shadow: inset 0 .5em 1.5em rgba(0, 0, 0, .1), inset 0 .125em .5em rgba(0, 0, 0, .15); 28 | } 29 | 30 | .bi { 31 | vertical-align: -.125em; 32 | pointer-events: none; 33 | fill: currentColor; 34 | } 35 | 36 | .dropdown-toggle { outline: 0; } 37 | 38 | .nav-flush .nav-link { 39 | border-radius: 0; 40 | } 41 | 42 | .btn-toggle { 43 | display: inline-flex; 44 | align-items: center; 45 | padding: .25rem .5rem; 46 | font-weight: 600; 47 | color: rgba(0, 0, 0, .65); 48 | background-color: transparent; 49 | border: 0; 50 | } 51 | .btn-toggle:hover, 52 | .btn-toggle:focus { 53 | color: rgba(0, 0, 0, .85); 54 | background-color: #d2f4ea; 55 | } 56 | 57 | .btn-toggle::before { 58 | width: 1.25em; 59 | line-height: 0; 60 | content: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='rgba%280,0,0,.5%29' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M5 14l6-6-6-6'/%3e%3c/svg%3e"); 61 | transition: transform .35s ease; 62 | transform-origin: .5em 50%; 63 | } 64 | 65 | .btn-toggle[aria-expanded="true"] { 66 | color: rgba(0, 0, 0, .85); 67 | } 68 | .btn-toggle[aria-expanded="true"]::before { 69 | transform: rotate(90deg); 70 | } 71 | 72 | .btn-toggle-nav a { 73 | display: inline-flex; 74 | padding: .1875rem .5rem; 75 | margin-top: .125rem; 76 | margin-left: 1.25rem; 77 | text-decoration: none; 78 | } 79 | .btn-toggle-nav a:hover, 80 | .btn-toggle-nav a:focus { 81 | background-color: #d2f4ea; 82 | } 83 | 84 | .scrollarea { 85 | overflow-y: auto; 86 | } 87 | 88 | .fw-semibold { font-weight: 600; } 89 | .lh-tight { line-height: 1.25; } -------------------------------------------------------------------------------- /data/displaydata/embed.go: -------------------------------------------------------------------------------- 1 | package displaydata 2 | 3 | import ( 4 | "embed" 5 | ) 6 | 7 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-grid.min.css 8 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-grid.min.css.map 9 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-reboot.min.css 10 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-reboot.min.css.map 11 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-utilities.min.css 12 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap-utilities.min.css.map 13 | // go:embed bootstrap-5.3.0-alpha3-dist/js/bootstrap.min.js 14 | // go:embed bootstrap-5.3.0-alpha3-dist/js/bootstrap.min.js.map 15 | // 16 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap.min.css 17 | // go:embed bootstrap-5.3.0-alpha3-dist/css/bootstrap.min.css.map 18 | // go:embed bootstrap-5.3.0-alpha3-dist/js/bootstrap.bundle.min.js 19 | // go:embed bootstrap-5.3.0-alpha3-dist/js/bootstrap.bundle.min.js.map 20 | // 21 | // go:embed AdminLTE3.2/dist/css/adminlte.min.css 22 | // go:embed AdminLTE3.2/dist/css/adminlte.min.css.map 23 | // go:embed AdminLTE3.2/dist/js/adminlte.min.js 24 | // go:embed AdminLTE3.2/dist/js/adminlte.min.js.map 25 | // go:embed AdminLTE3.2/dist/img/* 26 | // go:embed AdminLTE3.2/plugins/fontawesome-free/css/all.min.css 27 | // go:embed AdminLTE3.2/plugins/fontawesome-free/webfonts/* 28 | // go:embed AdminLTE3.2/plugins/jquery/jquery.min.js 29 | // go:embed AdminLTE3.2/plugins/jquery/jquery.min.map 30 | // 31 | //go:embed bootstrapdist/css/bootstrap.min.css 32 | //go:embed bootstrapdist/css/bootstrap.min.css.map 33 | //go:embed bootstrapdist/js/bootstrap.bundle.min.js 34 | //go:embed bootstrapdist/js/bootstrap.bundle.min.js.map 35 | //go:embed css/sidebar.css 36 | //go:embed css/interface.css 37 | //go:embed js/json-viewer.bundle.js 38 | //go:embed js/paged.js 39 | //go:embed js/paged.polyfill.js 40 | var WebRoot embed.FS 41 | 42 | //go:embed templates/object.gohtml 43 | //go:embed templates/storageroot.gohtml 44 | //go:embed templates/manifest.gohtml 45 | //go:embed templates/version.gohtml 46 | //go:embed templates/detail.gohtml 47 | //go:embed templates/report.gohtml 48 | var TemplateRoot embed.FS 49 | -------------------------------------------------------------------------------- /data/fullextensions/object/0001-digest-algorithms/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0001-digest-algorithms" 3 | } 4 | 5 | 6 | -------------------------------------------------------------------------------- /data/fullextensions/object/0011-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0011-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxPathSegmentLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": true 8 | } 9 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-content-subpath/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-content-subpath", 3 | "subPath": { 4 | "content": { 5 | "path": "data", 6 | "description": "Payload of archival object" 7 | }, 8 | "metadata": { 9 | "path": "metadata", 10 | "description": "additional semantic metadata" 11 | }, 12 | "documentation": { 13 | "path": "documentation", 14 | "description": "documentation of the archival object" 15 | } 16 | } 17 | } 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-filesystem/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-filesystem", 3 | "folders": "", 4 | "storageType": "area", 5 | "storageName": "metadata", 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-gocfl-extension-manager/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "ObjectChange": [ 5 | "NNNN-indexer", 6 | "NNNN-migration" 7 | ], 8 | "ObjectContentPath": [ 9 | "NNNN-direct-clean-path-layout", 10 | "NNNN-content-subpath" 11 | ] 12 | }, 13 | "exclusion": { 14 | "ObjectContentPath": [ 15 | [ 16 | "NNNN-direct-clean-path-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-indexer/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-indexer", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "actions": ["siegfried", "ffprobe", "identify", "tika", "xml"], 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-metafile/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-metafile", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "name": "info.json", 6 | "schema": "gocfl-info-1.0.json", 7 | "schemaUrl": "https://raw.githubusercontent.com/ocfl-archive/gocfl/main/gocfl-info-1.0.json" 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-mets/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-mets", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "primaryDescriptiveMetadata": "json:metadata:info.json", 6 | "metsFile": "mets.xml", 7 | "premisFile": "premis.xml" 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-migration/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-migration", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "compress": "gzip", 6 | "shortName": true 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/fullextensions/object/NNNN-thumbnail/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storageType": "area", 3 | "storageName": "metadata", 4 | "extensionName": "NNNN-thumbnail", 5 | "compress": "gzip", 6 | "width": 256, 7 | "height": 256, 8 | "singleDirectory": true 9 | } 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /data/fullextensions/object/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "initial", 3 | "extension": "NNNN-gocfl-extension-manager" 4 | } 5 | -------------------------------------------------------------------------------- /data/fullextensions/storageroot/0006-flat-omit-prefix-storage-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0006-flat-omit-prefix-storage-layout", 3 | "delimiter": ":" 4 | } 5 | -------------------------------------------------------------------------------- /data/fullextensions/storageroot/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxPathSegmentLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": false 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/fullextensions/storageroot/NNNN-direct-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-path-layout" 3 | } 4 | -------------------------------------------------------------------------------- /data/fullextensions/storageroot/NNNN-gocfl-extension-manager/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "StorageRootPath": [ 5 | "NNNN-direct-clean-path-layout" 6 | ] 7 | }, 8 | "exclusion": { 9 | "StorageRootPath": [ 10 | [ 11 | "NNNN-direct-clean-path-layout", 12 | "0003-hash-and-id-n-tuple-storage-layout", 13 | "0004-hashed-n-tuple-storage-layout", 14 | "0002-flat-direct-storage-layout", 15 | "0006-flat-omit-prefix-storage-layout", 16 | "NNNN-pairtree-storage-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/fullextensions/storageroot/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "initial", 3 | "extension": "NNNN-gocfl-extension-manager" 4 | } 5 | -------------------------------------------------------------------------------- /data/migration/pdfa.ps1: -------------------------------------------------------------------------------- 1 | $temp = New-TemporaryFile 2 | $Input | Out-File $temp 3 | $tempFolder = $temp.DirectoryName -replace '\\', '/' 4 | $pdf = "{0}.pdf" -f $temp.FullName 5 | Rename-Item $temp $pdf 6 | $pdfSlash = $pdf -replace '\\', '/' 7 | Start-Process gswin64.exe -NoNewWindow -Wait -ArgumentList "-dBATCH","-dNODISPLAY","-dNOPAUSE","-dNOSAFER","-sDEVICE=pdfwrite","-dPDFA=2","-sColorConversionStrategy=RGB","-dPDFACompatibilityPolicy=1","--permit-file-read=$($tempFolder)","-sOutputFile=-","c:/daten/go/dev/gocfl/data/migration/pdfa_def.ps","$($pdfSlash)" 8 | #Remove-Item -Path $pdf 9 | -------------------------------------------------------------------------------- /data/migration/pdfa_def.ps: -------------------------------------------------------------------------------- 1 | %! 2 | % This is a sample prefix file for creating a PDF/A document. 3 | % Users should modify entries marked with "Customize". 4 | % This assumes an ICC profile resides in the file (srgb.icc), 5 | % in the current directory unless the user modifies the corresponding line below. 6 | 7 | % Define entries in the document Info dictionary : 8 | [ /Title (Title) % Customise 9 | /DOCINFO pdfmark 10 | 11 | % Define an ICC profile : 12 | /ICCProfile (c:/daten/go/dev/gocfl/data/migration/srgb.icc) % Customise 13 | def 14 | 15 | [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark 16 | 17 | %% This code attempts to set the /N (number of components) key for the ICC colour space. 18 | %% To do this it checks the ColorConversionStrategy or the device ProcessColorModel if 19 | %% ColorConversionStrategy is not set. 20 | %% This is not 100% reliable. A better solution is for the user to edit this and replace 21 | %% the code between the ---8<--- lines with a simple declaration like: 22 | %% /N 3 23 | %% where the value of N is the number of components from the profile defined in /ICCProfile above. 24 | %% 25 | [{icc_PDFA} 26 | << 27 | %% ----------8<--------------8<-------------8<--------------8<---------- 28 | systemdict /ColorConversionStrategy known { 29 | systemdict /ColorConversionStrategy get cvn dup /Gray eq { 30 | pop /N 1 false 31 | }{ 32 | dup /RGB eq { 33 | pop /N 3 false 34 | }{ 35 | /CMYK eq { 36 | /N 4 false 37 | }{ 38 | (ColorConversionStrategy not a device space, falling back to ProcessColorModel, output may not be valid PDF/A.)= 39 | true 40 | } ifelse 41 | } ifelse 42 | } ifelse 43 | } { 44 | (ColorConversionStrategy not set, falling back to ProcessColorModel, output may not be valid PDF/A.)= 45 | true 46 | } ifelse 47 | 48 | { 49 | currentpagedevice /ProcessColorModel get 50 | dup /DeviceGray eq { 51 | pop /N 1 52 | }{ 53 | dup /DeviceRGB eq { 54 | pop /N 3 55 | }{ 56 | dup /DeviceCMYK eq { 57 | pop /N 4 58 | } { 59 | (ProcessColorModel not a device space.)= 60 | /ProcessColorModel cvx /rangecheck signalerror 61 | } ifelse 62 | } ifelse 63 | } ifelse 64 | } if 65 | %% ----------8<--------------8<-------------8<--------------8<---------- 66 | 67 | >> /PUT pdfmark 68 | [{icc_PDFA} ICCProfile (r) file /PUT pdfmark 69 | 70 | % Define the output intent dictionary : 71 | 72 | [/_objdef {OutputIntent_PDFA} /type /dict /OBJ pdfmark 73 | [{OutputIntent_PDFA} << 74 | /Type /OutputIntent % Must be so (the standard requires). 75 | /S /GTS_PDFA1 % Must be so (the standard requires). 76 | /DestOutputProfile {icc_PDFA} % Must be so (see above). 77 | /OutputConditionIdentifier (sRGB) % Customize 78 | >> /PUT pdfmark 79 | [{Catalog} <> /PUT pdfmark -------------------------------------------------------------------------------- /data/nomigrationextensions/object/0001-digest-algorithms/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0001-digest-algorithms" 3 | } 4 | 5 | 6 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/NNNN-content-subpath/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-content-subpath", 3 | "subPath": { 4 | "content": { 5 | "path": "data", 6 | "description": "Payload of archival object" 7 | }, 8 | "metadata": { 9 | "path": "metadata", 10 | "description": "additional semantic metadata" 11 | } 12 | } 13 | } 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxFilenameLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": true 8 | } 9 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/NNNN-filesystem/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-filesystem", 3 | "folders": "", 4 | "storageType": "area", 5 | "storageName": "metadata", 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/NNNN-indexer/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-indexer", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "actions": ["siegfried", "ffprobe", "identify", "tika"], 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/NNNN-thumbnail/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-thumbnail", 3 | "compress": "gzip", 4 | "width": 256, 5 | "height": 256 6 | } 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /data/nomigrationextensions/object/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "ObjectChange": [ 5 | "NNNN-indexer", 6 | "NNNN-migration" 7 | ], 8 | "ObjectContentPath": [ 9 | "NNNN-direct-clean-path-layout", 10 | "NNNN-content-subpath" 11 | ] 12 | }, 13 | "exclusion": { 14 | "ObjectContentPath": [ 15 | [ 16 | "NNNN-direct-clean-path-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /data/nomigrationextensions/storageroot/0006-flat-omit-prefix-storage-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0006-flat-omit-prefix-storage-layout", 3 | "delimiter": ":" 4 | } 5 | -------------------------------------------------------------------------------- /data/nomigrationextensions/storageroot/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxFilenameLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": false 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/nomigrationextensions/storageroot/NNNN-direct-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-path-layout" 3 | } 4 | -------------------------------------------------------------------------------- /data/nomigrationextensions/storageroot/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "StorageRootPath": [ 5 | "NNNN-direct-clean-path-layout" 6 | ] 7 | }, 8 | "exclusion": { 9 | "StorageRootPath": [ 10 | [ 11 | "NNNN-direct-clean-path-layout", 12 | "0003-hash-and-id-n-tuple-storage-layout", 13 | "0004-hashed-n-tuple-storage-layout", 14 | "0002-flat-direct-storage-layout", 15 | "0006-flat-omit-prefix-storage-layout", 16 | "NNNN-pairtree-storage-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/0001-digest-algorithms/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0001-digest-algorithms" 3 | } 4 | 5 | 6 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/NNNN-content-subpath/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-content-subpath", 3 | "subPath": { 4 | "content": { 5 | "path": "data", 6 | "description": "Payload of archival object" 7 | }, 8 | "metadata": { 9 | "path": "metadata", 10 | "description": "additional semantic metadata" 11 | } 12 | } 13 | } 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxFilenameLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": true 8 | } 9 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/NNNN-filesystem/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-filesystem", 3 | "folders": ".keep", 4 | "storageType": "area", 5 | "storageName": "metadata", 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/NNNN-indexer/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-indexer", 3 | "storageType": "area", 4 | "storageName": "metadata", 5 | "actions": ["siegfried", "ffprobe", "identify", "tika"], 6 | "compress": "gzip" 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /data/scicoreextensions/object/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "ObjectChange": [ 5 | "NNNN-indexer", 6 | "NNNN-migration" 7 | ], 8 | "ObjectContentPath": [ 9 | "NNNN-direct-clean-path-layout", 10 | "NNNN-content-subpath" 11 | ] 12 | }, 13 | "exclusion": { 14 | "ObjectContentPath": [ 15 | [ 16 | "NNNN-direct-clean-path-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /data/scicoreextensions/storageroot/0006-flat-omit-prefix-storage-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "0006-flat-omit-prefix-storage-layout", 3 | "delimiter": ":" 4 | } 5 | -------------------------------------------------------------------------------- /data/scicoreextensions/storageroot/NNNN-direct-clean-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-clean-path-layout", 3 | "maxPathnameLen": 32000, 4 | "maxFilenameLen": 127, 5 | "replacementString": "_", 6 | "whitespaceReplacementString": " ", 7 | "utfEncode": false 8 | } 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /data/scicoreextensions/storageroot/NNNN-direct-path-layout/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-direct-path-layout" 3 | } 4 | -------------------------------------------------------------------------------- /data/scicoreextensions/storageroot/initial/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensionName": "NNNN-gocfl-extension-manager", 3 | "sort": { 4 | "StorageRootPath": [ 5 | "NNNN-direct-clean-path-layout" 6 | ] 7 | }, 8 | "exclusion": { 9 | "StorageRootPath": [ 10 | [ 11 | "NNNN-direct-clean-path-layout", 12 | "0003-hash-and-id-n-tuple-storage-layout", 13 | "0004-hashed-n-tuple-storage-layout", 14 | "0002-flat-direct-storage-layout", 15 | "0006-flat-omit-prefix-storage-layout", 16 | "NNNN-pairtree-storage-layout", 17 | "NNNN-direct-path-layout" 18 | ] 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/scripts/pdf2thumb.ps1: -------------------------------------------------------------------------------- 1 | Param ( 2 | [Parameter(Mandatory=$true, ValueFromPipeline=$false)] 3 | [string]$Source, 4 | 5 | [Parameter(Mandatory=$true, ValueFromPipeline=$false)] 6 | [string]$Destination, 7 | 8 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 9 | [string]$Background = "none", 10 | 11 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 12 | [int]$Width = 256, 13 | 14 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 15 | [int]$Height = 256 16 | ) 17 | 18 | $gsparams = "-dNOPAUSE -dBATCH -sDEVICE=png16m -dFirstPage=1 -dLastPage=1 -sOutputFile=$($Destination).png $($Source)" 19 | Start-Process -FilePath gswin64.exe -ArgumentList $gsparams -NoNewWindow -Wait 20 | 21 | $convertParams = "$($Destination).png -resize $($Width)x$($Height) -background $($Background) -gravity Center -extent $($Width)x$($Height) $($Destination)" 22 | Start-Process -FilePath convert.exe -ArgumentList $convertparams -NoNewWindow -Wait 23 | 24 | Remove-Item -Path "$($Destination).png" 25 | -------------------------------------------------------------------------------- /data/scripts/video2thumb.ps1: -------------------------------------------------------------------------------- 1 | Param ( 2 | [Parameter(Mandatory=$true, ValueFromPipeline=$false)] 3 | [string]$Source, 4 | 5 | [Parameter(Mandatory=$true, ValueFromPipeline=$false)] 6 | [string]$Destination, 7 | 8 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 9 | [string]$Background = "none", 10 | 11 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 12 | [int]$Width = 256, 13 | 14 | [Parameter(Mandatory=$false, ValueFromPipeline=$false)] 15 | [int]$Height = 256 16 | ) 17 | 18 | $ffmpegparams = "-ss 00:00:35 -i $($Source) -frames:v 1 $($Destination).png" 19 | Start-Process -FilePath ffmpeg.exe -ArgumentList $ffmpegparams -NoNewWindow -Wait 20 | 21 | $convertParams = "$($Destination).png -resize $($Width)x$($Height) -background $($Background) -gravity Center -extent $($Width)x$($Height) $($Destination)" 22 | Start-Process -FilePath convert.exe -ArgumentList $convertparams -NoNewWindow -Wait 23 | 24 | Remove-Item -Path "$($Destination).png" 25 | -------------------------------------------------------------------------------- /data/specs/embed.go: -------------------------------------------------------------------------------- 1 | package specs 2 | 3 | import ( 4 | _ "embed" 5 | ) 6 | 7 | //go:embed ocfl_1.1.md 8 | var OCFL1_1 []byte 9 | 10 | //go:embed mets.xsd 11 | var METSXSD []byte 12 | 13 | //go:embed xlink.xsd 14 | var XLinkXSD []byte 15 | 16 | //go:embed premis.xsd 17 | var PremisXSD []byte 18 | -------------------------------------------------------------------------------- /data/specs/xlink.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /decrypt/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "emperror.dev/emperror" 6 | "emperror.dev/errors" 7 | "encoding/json" 8 | "flag" 9 | "github.com/google/tink/go/keyset" 10 | "github.com/google/tink/go/streamingaead" 11 | "github.com/je4/utils/v2/pkg/encrypt" 12 | "github.com/je4/utils/v2/pkg/keepass2kms" 13 | "io" 14 | "os" 15 | "path/filepath" 16 | ) 17 | 18 | var inputFile = flag.String("input", "", "input file") 19 | var kdbxFile = flag.String("kdbx", "", "kdbx file") 20 | var kdbxSecret = flag.String("kdbx-secret", "", "kdbx secret") 21 | var kdbxKey = flag.String("kdbx-key", "", "kdbx key path") 22 | var outputFile = flag.String("output", "", "output file") 23 | 24 | func main() { 25 | flag.Parse() 26 | if inputFile == nil { 27 | panic("input file not set") 28 | } 29 | encFile := *inputFile 30 | keyfile := *inputFile + ".key.json" 31 | keyData, err := os.ReadFile(keyfile) 32 | if err != nil { 33 | emperror.Panic(errors.Errorf("cannot read key file '%s': %v", keyfile, err)) 34 | } 35 | kStruct := &encrypt.KeyStruct{} 36 | if err := json.Unmarshal(keyData, kStruct); err != nil { 37 | emperror.Panic(errors.Errorf("cannot unmarshal key file '%s': %v", keyfile, err)) 38 | } 39 | 40 | db, err := keepass2kms.LoadKeePassDBFromFile(*kdbxFile, *kdbxSecret) 41 | if err != nil { 42 | emperror.Panic(errors.Errorf("cannot load keepass2 database '%s': %v", *kdbxFile, err)) 43 | } 44 | client, err := keepass2kms.NewClient(db, filepath.Base(*kdbxFile)) 45 | if err != nil { 46 | emperror.Panic(errors.Errorf("cannot create keepass2 client: %v", err)) 47 | } 48 | // registry.RegisterKMSClient(client) 49 | 50 | aead, err := client.GetAEAD(*kdbxKey) 51 | if err != nil { 52 | emperror.Panic(errors.Errorf("cannot get aead '%s': %v", *kdbxKey, err)) 53 | } 54 | 55 | kh, err := keyset.Read(keyset.NewBinaryReader(bytes.NewBuffer(kStruct.EncryptedKey)), aead) 56 | if err != nil { 57 | emperror.Panic(errors.Errorf("cannot read keyset: %v", err)) 58 | } 59 | 60 | fp, err := os.Open(encFile) 61 | if err != nil { 62 | emperror.Panic(errors.Errorf("cannot open file '%s': %v", encFile, err)) 63 | } 64 | defer fp.Close() 65 | 66 | stream, err := streamingaead.New(kh) 67 | if err != nil { 68 | emperror.Panic(errors.Errorf("cannot create streamingaead: %v", err)) 69 | } 70 | dec, err := stream.NewDecryptingReader(fp, kStruct.Aad) 71 | if err != nil { 72 | emperror.Panic(errors.Errorf("cannot create decrypting reader: %v", err)) 73 | } 74 | 75 | var out io.Writer 76 | if outputFile != nil && *outputFile != "" { 77 | outFP, err := os.Create(*outputFile) 78 | if err != nil { 79 | emperror.Panic(errors.Errorf("cannot create output file '%s': %v", *outputFile, err)) 80 | } 81 | defer outFP.Close() 82 | out = outFP 83 | } else { 84 | out = os.Stdout 85 | } 86 | 87 | if _, err := io.Copy(out, dec); err != nil { 88 | emperror.Panic(errors.Errorf("cannot write to stdout: %v", err)) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /docs/NNNN-content-subpath.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: Content Subpath 2 | 3 | * **Extension Name:** NNNN-content-subpath 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | This object extension permits the creation of an additional path hierarchy within the content folder of an object version. In essence, the concept of an "area" encompasses both a folder name and a description. This allows for the alteration of subfolders while ensuring that the gocfl tools are able to identify the location of the content. 13 | It is imperative that one ˋareaˋ is designated as "content" to guarantee that the payload can be readily accessed by any ocfl tool. 14 | 15 | ### Usage Scenario 16 | 17 | This extra path layer lets you create subfolders for meta, data and log, for example. The data folder is where you'll find the payload for the archived object. You can use these three folders to organise content, metadata and logging. 18 | 19 | ## Parameters 20 | 21 | ### Summary 22 | 23 | * **Name:** `subPath` 24 | * **Description:** map of named `PathDescription`. The entry name is the `area`. 25 | * **Type:** map 26 | * **Default:** 27 | 28 | #### `PathDescription` 29 | 30 | * **Name:** `path` 31 | * **Description:** subpath in object content 32 | * **Type:** string 33 | * **Default:** 34 | 35 | * **Name:** `description` 36 | * **Description:** description of content belonging to this subfolder 37 | * **Type:** string 38 | * **Default:** 39 | 40 | ## Caveat 41 | 42 | There MUST exist an `area` called `content` since this is the default area for adding payload 43 | files. 44 | 45 | ## Procedure 46 | 47 | When adding a content file the subfolder will be automatically inserted into the content path of the 48 | manifest. Within the version `content` folder write a `readme.md` file containing the description of 49 | the folders. 50 | 51 | ## Examples 52 | 53 | ### Parameters 54 | 55 | It is not necessary to specify any parameters to use the default configuration. 56 | However, if you were to do so, it would look like the following: 57 | 58 | ```json 59 | { 60 | "extensionName": "NNNN-content-subpath", 61 | "subPath": { 62 | "content": { 63 | "path": "data", 64 | "description": "Payload of archival object" 65 | }, 66 | "metadata": { 67 | "path": "meta", 68 | "description": "additional semantic metadata" 69 | }, 70 | "index": { 71 | "path": "index", 72 | "description": "additional technical metadata" 73 | } 74 | } 75 | } 76 | ``` 77 | 78 | ### Result 79 | 80 | #### File Structure 81 | 82 | ``` 83 | \---content 84 | | README.md 85 | | 86 | +---data 87 | | | [...] 88 | | | 89 | | \---[...] 90 | +---meta 91 | | | [...] 92 | | 93 | \---index 94 | indexer_v1.jsonl 95 | ``` 96 | 97 | #### readme.md 98 | ```markdown 99 | ### Description of folders 100 | 101 | 102 | ##### data 103 | Payload of archival object 104 | 105 | ##### meta 106 | additional semantic metadata 107 | 108 | ##### index 109 | additional technical metadata 110 | ``` 111 | -------------------------------------------------------------------------------- /docs/NNNN-filesystem.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: Filesystem 2 | 3 | * **Extension Name:** NNNN-filesystem 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | This object extension integrates filesystem metadata during the ingest process. 13 | The filesystem metadata is stored as newline delimited JSON format. Furthermore, 14 | if `folders` is configured, the folder metadata is stored with the file whose name 15 | is configured in the `folders` parameter. This even keeps empty folders within the OCFL 16 | structure. 17 | 18 | ### Usage Scenario 19 | 20 | Since OCFL does not preserve filesystem metadata like creation date, 21 | access rights, etc. this extension is a way to preserve this information. 22 | 23 | 24 | ## Parameters 25 | 26 | ### Summary 27 | 28 | * **Name:** `storageType` 29 | * **Description:** Location Type where the technical metadata is stored. Possible values are 30 | `area`, `path` or `extension`. 31 | * **area:** within an `area` defined by [NNNN-content-subpath](NNNN-content-subpath.md) 32 | extension 33 | * **path:** directly within content folder 34 | * **extension:** within the extension subfolder 35 | * **Type:** string 36 | * **Default:** 37 | * 38 | * **Name:** `storageName` 39 | * **Description:** Location within the specified Type 40 | * **area:** area name 41 | * **path:** subfolder within content folder 42 | * **extension:** subfolder within extension folder 43 | * **Type:** string 44 | * **Default:** 45 | 46 | * **Name:** `folders` 47 | * **Description:** name of the file containing the folder metadata (i.e. ".keep"). Empty means no folder metadata. 48 | * **Type:** string 49 | * **Default:** 50 | 51 | * **Name:** `compress` 52 | * **Description:** Compression type for JSONL file 53 | * **none:** no compression 54 | * **gzip:** [gzip compression](https://en.wikipedia.org/wiki/Gzip) 55 | * **brotli:** [brotli compression](https://en.wikipedia.org/wiki/Brotli) 56 | * **Type:** string 57 | * **Default:** 58 | 59 | 60 | ## Procedure (tbd.) 61 | 62 | Every entry whithin the [OCFL Object Manifest](https://ocfl.io/1.1/spec/#manifest) 63 | is represented by a JSON line in a file called `filesystem_.jsonl[.gz|.br]`. 64 | Since this file is immutable, every version of the ocfl object gets its own indexer file. 65 | 66 | ## Examples 67 | 68 | JSON Entry for a file from Linux OS 69 | ```json 70 | { 71 | "path": "data/test.odt", 72 | "meta": { 73 | "aTime": "2023-05-03T11:52:02.6948384+02:00", 74 | "mTime": "2023-01-15T13:49:09.7643455+01:00", 75 | "cTime": "2023-01-15T13:52:22.9096886+01:00", 76 | "attr": "-rwxrwxrwx", 77 | "os": "linux", 78 | "sysStat": { 79 | "Dev": 72, 80 | "Ino": 23643898043722929, 81 | "Nlink": 1, 82 | "Mode": 33279, 83 | "Uid": 1000, 84 | "Gid": 1000, 85 | "X__pad0": 0, 86 | "Rdev": 0, 87 | "Size": 4456, 88 | "Blksize": 4096, 89 | "Blocks": 16, 90 | "Atim": { 91 | "Sec": 1683107522, 92 | "Nsec": 694838400 93 | }, 94 | "Mtim": { 95 | "Sec": 1673786949, 96 | "Nsec": 764345500 97 | }, 98 | "Ctim": { 99 | "Sec": 1673787142, 100 | "Nsec": 909688600 101 | }, 102 | "X__unused": [ 103 | 0, 104 | 0, 105 | 0 106 | ] 107 | }, 108 | "stateVersion": "v1" 109 | } 110 | } 111 | ``` 112 | 113 | JSON Entry for a file from Windows OS 114 | ```json 115 | { 116 | "path": "data/test.odt", 117 | "meta": { 118 | "aTime": "2023-05-05T10:16:16.634688+02:00", 119 | "mTime": "2023-01-15T13:49:09.7643455+01:00", 120 | "cTime": "2023-01-15T13:50:27.6733047+01:00", 121 | "attr": "Archive", 122 | "os": "windows", 123 | "sysStat": { 124 | "FileAttributes": 32, 125 | "CreationTime": { 126 | "LowDateTime": 4049774711, 127 | "HighDateTime": 31008991 128 | }, 129 | "LastAccessTime": { 130 | "LowDateTime": 3711819904, 131 | "HighDateTime": 31031081 132 | }, 133 | "LastWriteTime": { 134 | "LowDateTime": 3270685119, 135 | "HighDateTime": 31008991 136 | }, 137 | "FileSizeHigh": 0, 138 | "FileSizeLow": 4456 139 | }, 140 | "stateVersion": "v1" 141 | } 142 | } 143 | ``` 144 | 145 | 146 | ### Result 147 | 148 | -------------------------------------------------------------------------------- /docs/NNNN-metafile.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: Metafile 2 | 3 | * __Extension Name:__ NNNN-metafile 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | This object extension allows the import of one metadata file, which is 13 | validated against a json schema. 14 | 15 | ### Usage Scenario 16 | 17 | To allow OCFL Viewers the display of easy to find semantic metadata, this 18 | extension gives a schema, position and name of the metadata file. This can be used for external 19 | archive Managers to store semantic metadata. 20 | 21 | ## Parameters 22 | 23 | ### Summary 24 | 25 | * **Name:** `storageType` 26 | * **Description:** Location Type where the technical metadata is stored. Possible values are 27 | `area`, `path` or `extension`. 28 | * **area:** within an `area` defined by [NNNN-content-subpath](NNNN-content-subpath.md) 29 | extension 30 | * **path:** directly within content folder 31 | * **extension:** within the extension subfolder 32 | * **Type:** string 33 | * **Default:** 34 | * 35 | * **Name:** `storageName` 36 | * **Description:** Location within the specified Type 37 | * **area:** area name 38 | * **path:** subfolder within content folder 39 | * **extension:** subfolder within extension folder 40 | * **Type:** string 41 | * **Default:** 42 | 43 | * **Name:** `schemaUrl` 44 | * **Description:** url of the json metadata schema, to check the metafile content 45 | * **Type:** string 46 | * **Default:** 47 | 48 | * **Name:** `schema` 49 | * **Description:** local filename of schema, which contains the content of `schemaUrl` 50 | * **Type:** string 51 | * **Default:** 52 | 53 | * **Name:** `name` 54 | * **Description:** the name of the metadata file. Extension MUST be `.json` 55 | * **Type:** string 56 | * **Default:** `info.json` 57 | 58 | 59 | ## Procedure (tbd.) 60 | 61 | While adding or updating an OCFL object, a metadata file is added at the specified storage location with 62 | the specified name. Within this process, the file is validated against the given json schema. 63 | The schema file is stored next to the config.json file within the extension folder. 64 | 65 | ## Examples 66 | 67 | ### Parameters 68 | 69 | ```json 70 | { 71 | "extensionName": "NNNN-metafile", 72 | "storageType": "extension", 73 | "storageName": "metadata", 74 | "name": "info.json", 75 | "schema": "gocfl-info-1.0.json", 76 | "schemaUrl": "https://raw.githubusercontent.com/ocfl-archive/gocfl/main/gocfl-info-1.0.json" 77 | } 78 | ``` -------------------------------------------------------------------------------- /docs/NNNN-mets.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: METS 2 | 3 | * **Extension Name:** NNNN-mets 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Uses:** [NNNN-indexer](NNNN-indexer.md)(optional), [NNNN-metafile](NNNN-metafile.md)(optional), [NNNN-migration](NNNN-migration.md)(optional), [NNNN-content-subpath](NNNN-content-subpath.md)(optional) 8 | * **Obsoletes:** n/a 9 | * **Obsoleted by:** n/a 10 | 11 | ## Overview 12 | 13 | For enhancing compatibility with classic archive information package 14 | formats (i.e. https://dilcis.eu), this extension provides a way to 15 | integrate a METS and a Premis file for every version of 16 | the OCFL object based on the inventory. 17 | 18 | Technical metadata is provided by the [NNNN-indexer](NNNN-indexer.md) extension. 19 | The [NNNN-metafile](NNNN-metafile.md) extension is for the mandatory 20 | entry of the descriptive metadata section within the mets file. 21 | Migrations provided by [NNNN-migration](NNNN-migration.md) are referenced 22 | within the premis file. Using the [NNNN-content-subpath](NNNN-content-subpath.md) 23 | extension the METS and Premis files can be stored within the metadata folder. 24 | 25 | ### Usage Scenario 26 | 27 | In order to be compatible with classic archive information package formats, 28 | the OCFL object should contain a METS file for every version. 29 | This extension avoids the need to generate the METS file manually or to embed 30 | another aip format within the OCFL object. 31 | 32 | ## Parameters 33 | 34 | ### Summary 35 | 36 | * **Name:** `storageType` 37 | * **Description:** Location Type where the technical metadata is stored. Possible values are 38 | `area`, `path` or `extension`. 39 | * **area:** within an `area` defined by [NNNN-content-subpath](NNNN-content-subpath.md) 40 | extension 41 | * **path:** directly within content folder 42 | * **extension:** within the extension subfolder 43 | * **Type:** string 44 | * **Default:** 45 | 46 | * **Name:** `storageName` 47 | * **Description:** Location within the specified Type 48 | * **area:** area name 49 | * **path:** subfolder within content folder 50 | * **extension:** subfolder within extension folder 51 | * **Type:** string 52 | * **Default:** 53 | 54 | * **Name:** `primaryDescriptiveMetadata` 55 | * **Description:** File with primary descriptive metadata (mets:dmdSec) 56 | * **Format:** :: 57 | * **Type:** string 58 | * **Default:** `info:metadata:info.json` 59 | 60 | * **Name:** `metsFile` 61 | * **Description:** Name of the mets file 62 | * **Type:** string 63 | * **Default:** `mets.xml` 64 | 65 | * **Name:** `premisFile` 66 | * **Description:** Name of the premis file 67 | * **Type:** string 68 | * **Default:** `premis.xml` 69 | 70 | 71 | ## Caveat 72 | 73 | Make sure, that the extensions are used in the correct order. 74 | (`NNNN-mets` is normally at the end of the `UpdateObjectAfter` chain) 75 | 76 | 77 | ## Procedure (tbd.) 78 | 79 | This extension is used to place a METS and Premis file within the object at the given location. 80 | The source of these files depends on the implementation of this extension. 81 | 82 | ### Generation of METS and Premis files 83 | Both files are generated based on information available in the inventory and the metadata files. 84 | 85 | ### Insertion 86 | Both files already exists und are inserted into the object. 87 | 88 | -------------------------------------------------------------------------------- /docs/NNNN-migration.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: Migration 2 | 3 | * **Extension Name:** NNNN-migration 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | Preservation management requires a migration strategy. This extension provides a way to 13 | migrate old file formats to new ones. It needs the [NNNN-indexer](NNNN-indexer.md) extension to 14 | get Pronom IDs for the files. The migration is done by an external migration service. 15 | The migrated files are stored in a new version of the OCFL object. 16 | 17 | ### Usage Scenario 18 | 19 | If you have for example old PDF files, you can migrate them to PDF/A. This is a standard for 20 | archival PDF files. It is a good idea to migrate the files to PDF/A, because it is a more 21 | future-proof format. 22 | 23 | ## Parameters 24 | 25 | ### Summary 26 | 27 | * **Name:** `storageType` 28 | * **Description:** Location Type where the technical metadata is stored. Possible values are 29 | `area`, `path` or `extension`. 30 | * **area:** within an `area` defined by [NNNN-content-subpath](NNNN-content-subpath.md) 31 | extension 32 | * **path:** directly within content folder 33 | * **extension:** within the extension subfolder 34 | * **Type:** string 35 | * **Default:** 36 | 37 | * **Name:** `storageName` 38 | * **Description:** Location within the specified Type 39 | * **area:** area name 40 | * **path:** subfolder within content folder 41 | * **extension:** subfolder within extension folder 42 | * **Type:** string 43 | * **Default:** 44 | 45 | * **Name:** `compress` 46 | * **Description:** Compression type for JSONL file 47 | * **none:** no compression 48 | * **gzip:** [gzip compression](https://en.wikipedia.org/wiki/Gzip) 49 | * **brotli:** [brotli compression](https://en.wikipedia.org/wiki/Brotli) 50 | * **Type:** string 51 | * **Default:** 52 | 53 | 54 | ## Caveat 55 | 56 | The migration rules itself are not part of this extension. They are defined by the migration service. 57 | 58 | ## Procedure (tbd.) 59 | 60 | Every migrated entry whithin the [OCFL Object Manifest](https://ocfl.io/1.1/spec/#manifest) 61 | is represented by a JSON line in a file called `migration_.jsonl[.gz|.br]`. 62 | Since this file is immutable, every version of the ocfl object gets its own indexer file. 63 | 64 | ## Examples 65 | 66 | JSON Entry for a migrated pdf 67 | ```json 68 | { 69 | "path": "v2/content/data/=u007Eblä=u0020blubb=u005Bin=u005D/Modulhandbuch_MA_Gestaltung.pdf", 70 | "migration": { 71 | "source": "v1/content/data/=u007Eblä=u0020blubb=u005Bin=u005D/Modulhandbuch_MA_Gestaltung.pdf", 72 | "id": "PDFA#01" 73 | } 74 | } 75 | ``` 76 | 77 | ### Result 78 | 79 | -------------------------------------------------------------------------------- /docs/NNNN-thumbnail.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension NNNN: Thumbnail 2 | 3 | * **Extension Name:** NNNN-thumbnail 4 | * **Authors:** Jürgen Enge (Basel) 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | This object extension generates thumbnails of Media-Files 13 | 14 | ### Usage Scenario 15 | 16 | To generate reports or content sites for OCFL Objects, it is helpful to display thumbnails to get an overview of 17 | the content. 18 | 19 | ## Parameters 20 | 21 | ### Summary 22 | 23 | * **Name:** `storageType` 24 | * **Description:** Location Type where the technical metadata is stored. Possible values are 25 | `area`, `path` or `extension`. 26 | * **area:** within an `area` defined by [NNNN-content-subpath](NNNN-content-subpath.md) 27 | extension 28 | * **path:** directly within content folder 29 | * **extension:** within the extension subfolder 30 | * **Type:** string 31 | * **Default:** "extension" 32 | 33 | * **Name:** `storageName` 34 | * **Description:** Location within the specified Type 35 | * **area:** area name 36 | * **path:** subfolder within content folder 37 | * **extension:** subfolder within extension folder 38 | * **Type:** string 39 | * **Default:** "data" 40 | * 41 | * **Name:** `compress` 42 | * **Description:** Compression type for JSONL file 43 | * **none:** no compression 44 | * **gzip:** [gzip compression](https://en.wikipedia.org/wiki/Gzip) 45 | * **brotli:** [brotli compression](https://en.wikipedia.org/wiki/Brotli) 46 | * **Type:** string 47 | * **Default:** none 48 | * **Name:** `ext` 49 | * **Description:** Image Format (Extension) 50 | * **Type:** string 51 | * **Default:** png 52 | * **Name:** `width` 53 | * **Description:** Thumbnail width 54 | * **Type:** integer 55 | * **Default:** 256 56 | * **Name:** `height` 57 | * **Description:** Thumbnail height 58 | * **Type:** integer 59 | * **Default:** 256 60 | * **Name:** `singleDirectory` 61 | * **Description:** Write all thumbnails to a single directory. This is useful, if the number of thumbnails do not create problems with directory size. 62 | * **Type:** boolean 63 | * **Default:** false 64 | 65 | 66 | ## Procedure (tbd.) 67 | 68 | Every entry whithin the [OCFL Object Manifest](https://ocfl.io/1.1/spec/#manifest) 69 | is represented by a JSON line in a file called `thumbnail_.jsonl[.gz|.br]`. 70 | Since this file is immutable, every version of the ocfl object gets its own indexer file. 71 | Since thumbnails only add visual value to the OCFL Object, data and metadata is stored within the extension folder. 72 | 73 | ## Examples 74 | 75 | It is not necessary to specify any parameters to use the default configuration. 76 | However, if you were to do so, it would look like the following: 77 | 78 | ```json 79 | { 80 | "extensionName": "NNNN-thumbnail", 81 | "compress": "gzip", 82 | "ext": "png", 83 | "width": 256, 84 | "height": 256 85 | } 86 | ``` 87 | 88 | ### Result 89 | 90 | JSON Entry for a file with a "png" thumbnail based on process "Image#01". The file is identified by its digest/checksum. 91 | ```json 92 | { 93 | "ext": "png", 94 | "id": "Image#01", 95 | "checksum": "5cb8c60eb3c7641561df988493acdd0fbc6b6325ec396a6eaf6a9cbc329e1790b006d61b4465371c21a105b0fb5a77dff9a219ed57ead6cd074d6b8a6e2be896" 96 | } 97 | ``` 98 | The thumbnail itself resides in the file `extensions/NNNN-thumbnail/data/5/c/5cb8c60eb3c7641561df988493acdd0fbc6b6325ec396a6eaf6a9cbc329e1790b006d61b4465371c21a105b0fb5a77dff9a219ed57ead6cd074d6b8a6e2be896.png` 99 | 100 | -------------------------------------------------------------------------------- /docs/add.md: -------------------------------------------------------------------------------- 1 | # Add 2 | 3 | The `add` command add a new object to an OCFL Storage Root. 4 | Deduplication is disabled by default. 5 | If the object already exists, an error will occur. 6 | 7 | The [default extension configs](../data/defaultextensions/object) are used for that. 8 | 9 | ```text 10 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe add --help 11 | opens an existing ocfl structure and adds a new object. if an object with the given id already exists, an error is produced 12 | 13 | Usage: 14 | gocfl add [path to ocfl structure] [flags] 15 | 16 | Examples: 17 | gocfl add ./archive.zip /tmp/testdata -u 'Jane Doe' -a 'mailto:user@domain' -m 'initial add' -object-id 'id:abc123' 18 | 19 | Flags: 20 | --deduplicate force deduplication (slower) 21 | --default-object-extensions string folder with initial extension configurations for new OCFL objects 22 | -d, --digest string digest to use for ocfl checksum 23 | --ext-NNNN-metafile-source string url with metadata file. $ID will be replaced with object ID i.e. file:///c:/temp/$ID.json 24 | --ext-NNNN-mets-descriptive-metadata string reference to archived descriptive metadata (i.e. ead:metadata:ead.xml) 25 | -f, --fixity string comma separated list of digest algorithms for fixity 26 | -h, --help help for add 27 | -m, --message string message for new object version (required) 28 | --no-compress do not compress data in zip file 29 | -i, --object-id string object id to update (required) 30 | -a, --user-address string user address for new object version (required) 31 | -u, --user-name string user name for new object version (required) 32 | 33 | Global Flags: 34 | --config string config file (default is embedded) 35 | --log-file string log output file (default is console) 36 | --log-level string log level (CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG) 37 | --s3-access-key-id string Access Key ID for S3 Buckets 38 | --s3-endpoint string Endpoint for S3 Buckets 39 | --s3-region string Region for S3 Access 40 | --s3-secret-access-key string Secret Access Key for S3 Buckets 41 | ``` 42 | 43 | ## Examples 44 | 45 | All Examples refer to the same [config file](../config/gocfl.toml). 46 | 47 | # Storage Root on local filesystem 48 | 49 | ``` 50 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe add C:/temp/ocflroot C:/temp/ocfltest --config ./config/gocfl.toml -i "id:blah-blubb" 51 | Using config file: ./config/gocfl.toml 52 | opening 'C:/temp/ocflroot' 53 | 2023-01-08T13:37:00.814 cmd::doAdd [add.go:147] > INFO - opening 'C:/temp/ocflroot' 54 | 2023-01-08T13:37:00.817 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:2017-06-05_10-36-51_793.jpeg 55 | 2023-01-08T13:37:00.826 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:2017-07-25_20-15-18_980.jpeg 56 | 2023-01-08T13:37:00.831 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:DSC_0111.JPG 57 | 2023-01-08T13:37:00.837 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:Kopie von Kopie von bangbang 26_3_gemacht_V2.xlsx 58 | 2023-01-08T13:37:00.859 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:Kopie von bangbang 26_3_gemacht_V2.xlsx 59 | 2023-01-08T13:37:00.877 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:bangbang_0226_10078_naegelin_2015_prisoners_dilemma_model_plusminus_staged.mp4--web_master.mp4 60 | 2023-01-08T13:37:03.650 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:collage.png 61 | 2023-01-08T13:37:06.600 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:salon.json 62 | 2023-01-08T13:37:06.624 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/Modulhandbuch_MA_Gestaltung.pdf 63 | 2023-01-08T13:37:06.638 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/bangbang.csv 64 | 2023-01-08T13:37:06.652 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/bangbang26_3V2.csv 65 | 2023-01-08T13:37:06.667 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/bangbang_names.csv 66 | 2023-01-08T13:37:06.675 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/bla.pptx 67 | 2023-01-08T13:37:06.682 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/salon.json 68 | 2023-01-08T13:37:06.710 ocfl::(*ObjectBase).AddFile [objectbase.go:599] > INFO - adding file content:~blä blubb[in]/sizecalculation.xlsx 69 | 2023-01-08T13:37:06.716 ocfl::(*ObjectBase).Close [objectbase.go:450] > INFO - Closing object 'id:blah-blubb' 70 | 71 | no errors found 72 | 2023-01-08T13:37:06.721 cmd::doAdd.func1 [add.go:144] > INFO - Duration: 5.9077761s 73 | ``` 74 | 75 | -------------------------------------------------------------------------------- /docs/display.md: -------------------------------------------------------------------------------- 1 | # Display 2 | 3 | Displays content of OCFL object in a webbrowser 4 | 5 | This functionality has been implemented mainly for debugging purposes. 6 | It's not for large Storage Roots or large Objects (several hundreds of thousands elements) 7 | 8 | ```text 9 | show content of ocfl object in webbrowser 10 | 11 | Usage: 12 | gocfl display [path to ocfl structure] [flags] 13 | 14 | Aliases: 15 | display, viewer 16 | 17 | Examples: 18 | gocfl display ./archive.zip 19 | 20 | Flags: 21 | -a, --display-addr string address to listen on (default "localhost:8080") 22 | -e, --display-external-addr string external address to access the server (default "http://localhost:8080") 23 | -t, --display-templates string path to templates 24 | -c, --display-tls-cert string path to tls certificate 25 | -k, --display-tls-key string path to tls certificate key 26 | -h, --help help for display 27 | 28 | Global Flags: 29 | --config string config file (default is embedded) 30 | --log-file string log output file (default is console) 31 | --log-level string log level (CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG) 32 | --s3-access-key-id string Access Key ID for S3 Buckets 33 | --s3-endpoint string Endpoint for S3 Buckets 34 | --s3-region string Region for S3 Access 35 | --s3-secret-access-key string Secret Access Key for S3 Buckets 36 | ``` 37 | 38 | ## Examples 39 | 40 | ``` 41 | PS C:\daten\go\dev\gocfl\build> .\gocfl_windows_amd64.exe display c:/temp/test/ocfl_test.zip 42 | [GIN] 2023/05/19 - 15:13:53 | 200 | 773.1µs | 127.0.0.1 | GET "/" 43 | [GIN] 2023/05/19 - 15:13:53 | 200 | 53.3494ms | 127.0.0.1 | GET "/static/bootstrap-5.3.0-alpha3-dist/css/bootstrap.min.css" 44 | [GIN] 2023/05/19 - 15:13:53 | 200 | 0s | 127.0.0.1 | GET "/static/css/sidebar.css" 45 | [GIN] 2023/05/19 - 15:13:53 | 200 | 999.5µs | 127.0.0.1 | GET "/static/bootstrap-5.3.0-alpha3-dist/js/bootstrap.bundle.min.js" 46 | [GIN] 2023/05/19 - 15:13:53 | 404 | 0s | 127.0.0.1 | GET "/favicon.ico" 47 | ``` 48 | ### Storage Root 49 | 50 | ![Start](display_start.png) 51 | 52 | ### Object Overview 53 | ![Object Overview](display_object_overview.png) 54 | 55 | ### Version 56 | ![Object Version](display_version.png) 57 | 58 | ### File Detail 59 | ![File Detail](display_detail_png.png) 60 | 61 | ### File Detail of migrated file 62 | ![Migrated File](display_detail_migrated.png) -------------------------------------------------------------------------------- /docs/display_detail_migrated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/docs/display_detail_migrated.png -------------------------------------------------------------------------------- /docs/display_detail_png.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/docs/display_detail_png.png -------------------------------------------------------------------------------- /docs/display_object_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/docs/display_object_overview.png -------------------------------------------------------------------------------- /docs/display_start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/docs/display_start.png -------------------------------------------------------------------------------- /docs/display_version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/docs/display_version.png -------------------------------------------------------------------------------- /docs/embed.go: -------------------------------------------------------------------------------- 1 | package docs 2 | 3 | import ( 4 | "embed" 5 | ) 6 | 7 | //go:embed NNNN-*.md 0011-direct-clean-path-layout.md initial.md 8 | //go:embed ocfl_spec_1.1.md 9 | var ExtensionDocs embed.FS 10 | -------------------------------------------------------------------------------- /docs/good-objects.txt: -------------------------------------------------------------------------------- 1 | object folder 'minimal_content_dir_called_stuff' 2 | object folder 'minimal_mixed_digests' 3 | object folder 'minimal_no_content' 4 | object folder 'minimal_one_version_one_file' 5 | object folder 'minimal_uppercase_digests' 6 | object folder 'ocfl_object_all_fixity_digests' 7 | object folder 'spec-ex-full' 8 | object folder 'updates_all_actions' 9 | object folder 'updates_three_versions_one_file' 10 | 11 | no errors found 12 | -------------------------------------------------------------------------------- /docs/init.md: -------------------------------------------------------------------------------- 1 | # Init 2 | 3 | The `init` command initializes an OCFL Storage Root. 4 | [Default extension configs](../data/defaultextensions/storageroot) are used. 5 | 6 | ```text 7 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe init --help 8 | initializes an empty ocfl structure 9 | 10 | Usage: 11 | gocfl init [path to ocfl structure] [flags] 12 | 13 | Examples: 14 | gocfl init ./archive.zip 15 | 16 | Flags: 17 | --default-storageroot-extensions string folder with initial extension configurations for new OCFL Storage Root 18 | -d, --digest string digest to use for ocfl checksum 19 | -h, --help help for init 20 | --no-compress do not compress data in zip file 21 | --ocfl-version string ocfl version for new storage root 22 | 23 | Global Flags: 24 | --config string config file (default is embedded) 25 | --log-file string log output file (default is console) 26 | --log-level string log level (CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG) 27 | --s3-access-key-id string Access Key ID for S3 Buckets 28 | --s3-endpoint string Endpoint for S3 Buckets 29 | --s3-region string Region for S3 Access 30 | --s3-secret-access-key string Secret Access Key for S3 Buckets 31 | ``` 32 | 33 | ## Examples 34 | 35 | All Examples refer to the same [config file](../config/gocfl.toml) 36 | 37 | ### Storage Root on File System 38 | 39 | ``` 40 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe init c:/temp/ocflroot --config ./config/gocfl.toml 41 | Using config file: ./config/gocfl.toml 42 | 2023-01-08T13:27:31.770 cmd::doInit [init.go:104] > INFO - creating 'c:/temp/ocflroot' 43 | 44 | no errors found 45 | 2023-01-08T13:27:31.775 cmd::doInit.func1 [init.go:106] > INFO - Duration: 5.3941ms 46 | 47 | PS C:\Users\micro> Get-ChildItem /temp/ocflroot -recurse 48 | 49 | Directory: C:\temp\ocflroot 50 | 51 | Mode LastWriteTime Length Name 52 | ---- ------------- ------ ---- 53 | d---- 08.01.2023 13:27 extensions 54 | -a--- 08.01.2023 13:27 9 0=ocfl_1.1 55 | -a--- 08.01.2023 13:27 110 ocfl_layout.json 56 | 57 | Directory: C:\temp\ocflroot\extensions 58 | 59 | Mode LastWriteTime Length Name 60 | ---- ------------- ------ ---- 61 | d---- 08.01.2023 13:27 initial 62 | d---- 08.01.2023 13:27 NNNN-direct-clean-path-layout 63 | d---- 08.01.2023 13:27 NNNN-direct-path-layout 64 | 65 | Directory: C:\temp\ocflroot\extensions\initial 66 | 67 | Mode LastWriteTime Length Name 68 | ---- ------------- ------ ---- 69 | -a--- 08.01.2023 13:27 510 config.json 70 | 71 | Directory: C:\temp\ocflroot\extensions\NNNN-direct-clean-path-layout 72 | 73 | Mode LastWriteTime Length Name 74 | ---- ------------- ------ ---- 75 | -a--- 08.01.2023 13:27 298 config.json 76 | 77 | Directory: C:\temp\ocflroot\extensions\NNNN-direct-path-layout 78 | 79 | Mode LastWriteTime Length Name 80 | ---- ------------- ------ ---- 81 | -a--- 08.01.2023 13:27 50 config.json 82 | ``` 83 | 84 | ### Storage Root on ZIP File 85 | 86 | ``` 87 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe init c:/temp/ocfl_init.zip --config ./config/gocfl.toml 88 | Using config file: ./config/gocfl.toml 89 | 2023-01-08T13:25:58.771 cmd::doInit [init.go:104] > INFO - creating 'c:/temp/ocfl_init.zip' 90 | 91 | no errors found 92 | 2023-01-08T13:25:58.774 cmd::doInit.func1 [init.go:106] > INFO - Duration: 2.7303ms 93 | PS C:\daten\go\dev\gocfl> dir /temp/ocfl_init.* 94 | 95 | Directory: C:\temp 96 | 97 | Mode LastWriteTime Length Name 98 | ---- ------------- ------ ---- 99 | -a--- 08.01.2023 13:25 1648 ocfl_init.zip 100 | -a--- 08.01.2023 13:25 144 ocfl_init.zip.sha512 101 | ``` 102 | 103 | 104 | -------------------------------------------------------------------------------- /docs/initial.md: -------------------------------------------------------------------------------- 1 | # OCFL Community Extension `initial`: Initial Extension 2 | 3 | * **Extension Name:** `initial` 4 | * **Authors:** OCFL Editors 5 | * **Minimum OCFL Version:** 1.0 6 | * **OCFL Community Extensions Version:** 1.0 7 | * **Obsoletes:** n/a 8 | * **Obsoleted by:** n/a 9 | 10 | ## Overview 11 | 12 | This extension allows indication that the semantics of a particular extension takes precedence over all other extensions. It ensures that the special extension name `initial` is a registered extension name and thus that an extension directory `initial` is also valid in both objects and storage roots. 13 | 14 | An extension directory MAY contain an `initial` extension identified by the extension directory name `initial`. If it exists, the `initial` extension specifies another extension that MUST be applied before all other extensions in the directory. 15 | 16 | The extension configuration file indicates the functional extension to be applied first by specifying that extension's name in the `extension` parameter (not `initial`). This extension can be used to address otherwise undefined behaviors, such as: 17 | 18 | * Should extensions be applied in a specific order? 19 | * Is an extension deactivated, only applying to earlier versions of the object? 20 | * Does one extension depend on another? 21 | 22 | ## Parameter 23 | 24 | * **Name:** `extension` 25 | * **Description:** The name of the extension to be applied first 26 | * **Type:** string 27 | * **Constraints:** Must be a valid extension name 28 | * **Default:** Not applicable 29 | 30 | ## Example 31 | 32 | The following `config.json` configuration file indicates that the extension named `NNNN-functional-extension-name` should be applied first. 33 | 34 | ``` 35 | { 36 | "extensionName": "initial", 37 | "extension": "NNNN-functional-extension-name" 38 | } 39 | ``` 40 | 41 | ## Revision History 42 | 43 | | Date | Description | 44 | | ---- | ----------- | 45 | | 2024-09-19 | First published | -------------------------------------------------------------------------------- /docs/presentation.md: -------------------------------------------------------------------------------- 1 | # Default 2 | 3 | ## Create test dataset 4 | ```bash 5 | gocfl create c:/temp/ocfl/ocfl_test0.zip c:/temp/ocfl/sip/payload/ --object-id "id:abc_123" -m "Initial Commit" -u "Juergen Enge" -a "mailto:juergen@info-age.net" 6 | ``` 7 | 8 | ## Validate it 9 | ```bash 10 | gocfl validate c:/temp/ocfl/ocfl_test0.zip 11 | ``` 12 | 13 | ## Extract Metadata 14 | ```bash 15 | gocfl extractmeta c:/temp/ocfl/ocfl_test0.zip --output c:/temp/ocfl/ocfl_test0.json 16 | ``` 17 | 18 | ## Look inside 19 | ```bash 20 | gocfl display c:/temp/ocfl/ocfl_test0.zip 21 | ``` 22 | 23 | # Extended (Extension) Version 24 | 25 | ## Create test dataset 26 | ```bash 27 | cd /temp/ocfl 28 | gocfl create c:/temp/ocfl/ocfl_test1.zip c:/temp/ocfl/sip/payload metadata:c:/temp/ocfl/sip/meta --config c:/temp/ocfl/gocfl2.toml --ext-NNNN-metafile-source file://C:/temp/ocfl/sip/info.json --object-id "id:abc_123" -m "Initial Commit" -u "Juergen Enge" -a "mailto:juergen@info-age.net" 29 | ``` 30 | 31 | ## Validate it 32 | ```bash 33 | gocfl validate c:/temp/ocfl/ocfl_test1.zip --config c:/temp/ocfl/gocfl2.toml 34 | ``` 35 | 36 | ## Extract Metadata 37 | ```bash 38 | gocfl extractmeta c:/temp/ocfl/ocfl_test1.zip --output c:/temp/ocfl/ocfl_test1.json 39 | ``` 40 | 41 | ## Look inside 42 | ```bash 43 | gocfl display c:/temp/ocfl/ocfl_test1.zip --config c:/temp/ocfl/gocfl2.toml 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/validate.md: -------------------------------------------------------------------------------- 1 | # Validate 2 | 3 | Validates an OCFL Storage Root with one or all Objects. Validation is non-blocking which allows to 4 | get a list of multiple errors (which may be follow-ups of previous ones). 5 | 6 | ```text 7 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe validate --help 8 | validates an ocfl structure 9 | 10 | Usage: 11 | gocfl validate [path to ocfl structure] [flags] 12 | 13 | Aliases: 14 | validate, check 15 | 16 | Examples: 17 | gocfl validate ./archive.zip 18 | 19 | Flags: 20 | -h, --help help for validate 21 | --object-id string validate only the object with the specified id in storage root 22 | -o, --object-path string validate only the object at the specified path in storage root 23 | 24 | Global Flags: 25 | --config string config file (default is embedded) 26 | --log-file string log output file (default is console) 27 | --log-level string log level (CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG) 28 | --s3-access-key-id string Access Key ID for S3 Buckets 29 | --s3-endpoint string Endpoint for S3 Buckets 30 | --s3-region string Region for S3 Access 31 | --s3-secret-access-key string Secret Access Key for S3 Buckets 32 | ``` 33 | 34 | ## Fixtures (OCFL 1.1) 35 | Evalution of the [OCFL fixtures](https://github.com/OCFL/fixtures/tree/main/1.1) result in the following output: 36 | 37 | * [Bad Objects](bad-objects.txt) 38 | * [Warn Objects](warn-objects.txt) 39 | * [Good Objects](good-objects.txt) 40 | 41 | ## Examples 42 | 43 | ``` 44 | PS C:\daten\go\dev\gocfl> ../bin/gocfl.exe validate C:\temp\ocflroot --config ./config/gocfl.toml 45 | Using config file: ./config/gocfl.toml 46 | 2023-01-09T16:24:36.152 cmd::validate [validate.go:46] > INFO - validating 'C:/temp/ocflroot' 47 | 2023-01-09T16:24:36.153 ocfl::(*StorageRootBase).Check [storagerootbase.go:397] > INFO - StorageRoot with version '1.1' found 48 | object folder 'id=u003Ablah-blubb' 49 | 2023-01-09T16:24:36.154 ocfl::(*ObjectBase).Check [objectbase.go:1019] > INFO - object 'id:blah-blubb' with object version '1.1' found 50 | 51 | [storage root 'file://C:/temp/ocflroot'] 52 | #W013 - ‘In an OCFL Object, extension sub-directories SHOULD be named according to a registered extension name.’ [extension 'NNNN-direct-clean-path-layout' is not registered] 53 | #W013 - ‘In an OCFL Object, extension sub-directories SHOULD be named according to a registered extension name.’ [extension 'NNNN-direct-path-layout' is not registered] 54 | #W013 - ‘In an OCFL Object, extension sub-directories SHOULD be named according to a registered extension name.’ [extension 'NNNN-gocfl-extension-manager' is not registered] 55 | 56 | [object 'file://C:/temp/ocflroot/id=u003Ablah-blubb' - ''] 57 | #W013 - ‘In an OCFL Object, extension sub-directories SHOULD be named according to a registered extension name.’ [extension 'NNNN-direct-clean-path-layout' is not registered] 58 | #W013 - ‘In an OCFL Object, extension sub-directories SHOULD be named according to a registered extension name.’ [extension 'NNNN-gocfl-extension-manager' is not registered] 59 | 60 | no errors found 61 | 2023-01-09T16:24:40.858 cmd::validate.func1 [validate.go:44] > INFO - Duration: 4.7068474s 62 | ``` 63 | -------------------------------------------------------------------------------- /fixtures/new/1.1/warn-objects/W003_empty_content_folder/0=ocfl_object_1.1: -------------------------------------------------------------------------------- 1 | ocfl_object_1.1 2 | -------------------------------------------------------------------------------- /fixtures/new/1.1/warn-objects/W003_empty_content_folder/inventory.json: -------------------------------------------------------------------------------- 1 | { 2 | "digestAlgorithm": "sha512", 3 | "head": "v1", 4 | "id": "http://example.org/minimal_no_content", 5 | "manifest": { }, 6 | "type": "https://ocfl.io/1.1/spec/#inventory", 7 | "versions": { 8 | "v1": { 9 | "created": "2019-01-01T02:03:04Z", 10 | "message": "One version and no content", 11 | "state": { }, 12 | "user": { "address": "mailto:Person_A@example.org", "name": "Person A" } 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /fixtures/new/1.1/warn-objects/W003_empty_content_folder/inventory.json.sha512: -------------------------------------------------------------------------------- 1 | f0a161e1193f1770c8b9387f1a1373c3a3d2da71fda22986524629091120e8dae44c308fd12a3311c7f2c6110405b5b63dd3fd5323af9bedb49985813f2a243d inventory.json 2 | -------------------------------------------------------------------------------- /fixtures/new/1.1/warn-objects/W003_empty_content_folder/v1/inventory.json: -------------------------------------------------------------------------------- 1 | { 2 | "digestAlgorithm": "sha512", 3 | "head": "v1", 4 | "id": "http://example.org/minimal_no_content", 5 | "manifest": { }, 6 | "type": "https://ocfl.io/1.1/spec/#inventory", 7 | "versions": { 8 | "v1": { 9 | "created": "2019-01-01T02:03:04Z", 10 | "message": "One version and no content", 11 | "state": { }, 12 | "user": { "address": "mailto:Person_A@example.org", "name": "Person A" } 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /fixtures/new/1.1/warn-objects/W003_empty_content_folder/v1/inventory.json.sha512: -------------------------------------------------------------------------------- 1 | f0a161e1193f1770c8b9387f1a1373c3a3d2da71fda22986524629091120e8dae44c308fd12a3311c7f2c6110405b5b63dd3fd5323af9bedb49985813f2a243d inventory.json 2 | -------------------------------------------------------------------------------- /gocfl-info-1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "$ref": "#/$defs/Info", 4 | "$defs": { 5 | "Info": { 6 | "properties": { 7 | "signature": { 8 | "type": "string", 9 | "maxLength": 128, 10 | "minLength": 3, 11 | "pattern": "[a-zA-Z0-9/.-:-]+", 12 | "title": "archival signature", 13 | "description": "unique identifier within the archive system" 14 | }, 15 | "organisation_id": { 16 | "type": "string", 17 | "pattern": "[a-zA-Z0-9/.-:-]+", 18 | "title": "organisation identifier", 19 | "description": "id or abbreviation of organisation responsible for the object" 20 | }, 21 | "organisation": { 22 | "type": "string", 23 | "title": "organisation name", 24 | "description": "name of organisation responsible for the object" 25 | }, 26 | "collection_id": { 27 | "type": "string", 28 | "pattern": "[a-zA-Z0-9/._:-]+", 29 | "title": "collection identifier", 30 | "description": "id of collection the object belongs to" 31 | }, 32 | "collection": { 33 | "type": "string", 34 | "title": "collection name", 35 | "description": "name of collection the object belongs to" 36 | }, 37 | "sets": { 38 | "items": { 39 | "type": "string" 40 | }, 41 | "type": "array", 42 | "uniqueItems": true, 43 | "title": "sets", 44 | "description": "list of datasets object is belonging to" 45 | }, 46 | "identifiers": { 47 | "items": { 48 | "type": "string" 49 | }, 50 | "type": "array", 51 | "uniqueItems": true, 52 | "title": "identifiers", 53 | "description": "list of identifiers" 54 | }, 55 | "title": { 56 | "type": "string", 57 | "title": "title", 58 | "description": "title of object" 59 | }, 60 | "alternative_titles": { 61 | "items": { 62 | "type": "string" 63 | }, 64 | "type": "array", 65 | "uniqueItems": true, 66 | "title": "alternative titles", 67 | "description": "list of alternative titles of this object or parts of it" 68 | }, 69 | "description": { 70 | "type": "string" 71 | }, 72 | "keywords": { 73 | "items": { 74 | "type": "string" 75 | }, 76 | "type": "array", 77 | "uniqueItems": true 78 | }, 79 | "user": { 80 | "type": "string", 81 | "title": "user", 82 | "description": "name of person ingesting this object" 83 | }, 84 | "address": { 85 | "type": "string", 86 | "title": "address", 87 | "description": "adress of person ingesting this archive (email" 88 | }, 89 | "created": { 90 | "type": "string", 91 | "format": "date-time", 92 | "title": "creation date", 93 | "description": "date" 94 | }, 95 | "last_changed": { 96 | "type": "string", 97 | "format": "date-time", 98 | "title": "last changed", 99 | "description": "date" 100 | }, 101 | "deprecates": { 102 | "type": "string", 103 | "title": "deprecates", 104 | "description": "signature of object" 105 | }, 106 | "references": { 107 | "items": { 108 | "type": "string" 109 | }, 110 | "type": "array", 111 | "title": "references", 112 | "description": "list of signatures" 113 | }, 114 | "ingest_workflow": { 115 | "type": "string", 116 | "title": "ingest workflow", 117 | "description": "name of the workflow" 118 | }, 119 | "additional": { 120 | "type":["number","string","boolean","object","array", "null"], 121 | "title": "additional data", 122 | "description": "unstructured additional data" 123 | } 124 | }, 125 | "additionalProperties": false, 126 | "type": "object", 127 | "required": [ 128 | "signature", 129 | "organisation_id", 130 | "organisation", 131 | "title", 132 | "user", 133 | "address", 134 | "created", 135 | "last_changed" 136 | ] 137 | } 138 | } 139 | } -------------------------------------------------------------------------------- /gocfl/cmd/errors.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | const ( 4 | ERRORIDUnknownError = "IDUnknownError" 5 | ERRORTest = "Test" 6 | ERRORTest2 = "Test2" 7 | ) 8 | -------------------------------------------------------------------------------- /gocfl/cmd/validate.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "github.com/je4/filesystem/v3/pkg/writefs" 7 | "github.com/je4/utils/v2/pkg/zLogger" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/pkgerrors" 11 | "github.com/spf13/cobra" 12 | ublogger "gitlab.switch.ch/ub-unibas/go-ublogger/v2" 13 | "go.ub.unibas.ch/cloud/certloader/v2/pkg/loader" 14 | "io" 15 | "log" 16 | "os" 17 | ) 18 | 19 | var validateCmd = &cobra.Command{ 20 | Use: "validate [path to ocfl structure]", 21 | Aliases: []string{"check"}, 22 | Short: "validates an ocfl structure", 23 | //Long: "an utterly useless command for testing", 24 | Example: "gocfl validate ./archive.zip", 25 | Args: cobra.ExactArgs(1), 26 | Run: validate, 27 | } 28 | 29 | func initValidate() { 30 | validateCmd.Flags().StringP("object-path", "o", "", "validate only the object at the specified path in storage root") 31 | validateCmd.Flags().String("object-id", "", "validate only the object with the specified id in storage root") 32 | } 33 | 34 | func doValidateConf(cmd *cobra.Command) { 35 | if str := getFlagString(cmd, "object-path"); str != "" { 36 | conf.Validate.ObjectPath = str 37 | } 38 | if str := getFlagString(cmd, "object-id"); str != "" { 39 | conf.Validate.ObjectID = str 40 | } 41 | } 42 | 43 | func validate(cmd *cobra.Command, args []string) { 44 | ocflPath, err := ocfl.Fullpath(args[0]) 45 | if err != nil { 46 | cobra.CheckErr(err) 47 | return 48 | } 49 | 50 | // create logger instance 51 | hostname, err := os.Hostname() 52 | if err != nil { 53 | log.Fatalf("cannot get hostname: %v", err) 54 | } 55 | 56 | var loggerTLSConfig *tls.Config 57 | var loggerLoader io.Closer 58 | if conf.Log.Stash.TLS != nil { 59 | loggerTLSConfig, loggerLoader, err = loader.CreateClientLoader(conf.Log.Stash.TLS, nil) 60 | if err != nil { 61 | log.Fatalf("cannot create client loader: %v", err) 62 | } 63 | defer loggerLoader.Close() 64 | } 65 | 66 | zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack 67 | _logger, _logstash, _logfile, err := ublogger.CreateUbMultiLoggerTLS(conf.Log.Level, conf.Log.File, 68 | ublogger.SetDataset(conf.Log.Stash.Dataset), 69 | ublogger.SetLogStash(conf.Log.Stash.LogstashHost, conf.Log.Stash.LogstashPort, conf.Log.Stash.Namespace, conf.Log.Stash.LogstashTraceLevel), 70 | ublogger.SetTLS(conf.Log.Stash.TLS != nil), 71 | ublogger.SetTLSConfig(loggerTLSConfig), 72 | ) 73 | if err != nil { 74 | log.Fatalf("cannot create logger: %v", err) 75 | } 76 | if _logstash != nil { 77 | defer _logstash.Close() 78 | } 79 | 80 | if _logfile != nil { 81 | defer _logfile.Close() 82 | } 83 | 84 | l2 := _logger.With().Timestamp().Str("host", hostname).Logger() //.Output(output) 85 | var logger zLogger.ZLogger = &l2 86 | 87 | t := startTimer() 88 | defer func() { logger.Info().Msgf("Duration: %s", t.String()) }() 89 | 90 | doValidateConf(cmd) 91 | 92 | logger.Info().Msgf("validating '%s'", ocflPath) 93 | 94 | extensionParams := GetExtensionParamValues(cmd, conf) 95 | extensionFactory, err := InitExtensionFactory(extensionParams, "", false, nil, nil, nil, nil, (logger)) 96 | if err != nil { 97 | logger.Error().Stack().Err(err).Msg("cannot initialize extension factory") 98 | return 99 | } 100 | 101 | fsFactory, err := initializeFSFactory(nil, nil, nil, true, true, logger) 102 | if err != nil { 103 | logger.Error().Stack().Err(err).Msg("cannot create filesystem factory") 104 | return 105 | } 106 | 107 | destFS, err := fsFactory.Get(ocflPath, true) 108 | if err != nil { 109 | logger.Error().Stack().Err(err).Msgf("cannot get filesystem for '%s'", ocflPath) 110 | return 111 | } 112 | defer func() { 113 | if err := writefs.Close(destFS); err != nil { 114 | logger.Error().Stack().Err(err).Msgf("cannot close filesystem for '%s'", destFS) 115 | } 116 | }() 117 | 118 | ctx := ocfl.NewContextValidation(context.TODO()) 119 | storageRoot, err := ocfl.LoadStorageRoot(ctx, destFS, extensionFactory, logger) 120 | if err != nil { 121 | logger.Error().Stack().Err(err).Msg("cannot load storageroot") 122 | return 123 | } 124 | objectID := conf.Validate.ObjectID 125 | objectPath := conf.Validate.ObjectPath 126 | if objectID != "" && objectPath != "" { 127 | logger.Error().Msg("do not use object-path AND object-id at the same time") 128 | return 129 | } 130 | if objectID == "" && objectPath == "" { 131 | if err := storageRoot.Check(); err != nil { 132 | logger.Error().Stack().Err(err).Msg("ocfl not valid") 133 | return 134 | } 135 | } else { 136 | if objectID != "" { 137 | if err := storageRoot.CheckObjectByID(objectID); err != nil { 138 | logger.Error().Stack().Err(err).Msgf("ocfl object '%s' not valid", objectID) 139 | return 140 | } 141 | } else { 142 | if err := storageRoot.CheckObjectByFolder(objectPath); err != nil { 143 | logger.Error().Stack().Err(err).Msgf("ocfl object '%s' not valid", objectPath) 144 | return 145 | } 146 | } 147 | } 148 | _ = showStatus(ctx, logger) 149 | } 150 | -------------------------------------------------------------------------------- /gocfl/main.go: -------------------------------------------------------------------------------- 1 | //go:build !imagick && !vips 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/ocfl-archive/gocfl/v2/gocfl/cmd" 7 | ) 8 | 9 | /* 10 | func init() { 11 | os.Setenv("SIEGFRIED_HOME", "c:/temp") 12 | } 13 | */ 14 | 15 | func main() { 16 | cmd.Execute() 17 | } 18 | -------------------------------------------------------------------------------- /gocfl/main_imagick.go: -------------------------------------------------------------------------------- 1 | //go:build imagick 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/ocfl-archive/gocfl/v2/gocfl/cmd" 7 | "gopkg.in/gographics/imagick.v3/imagick" 8 | ) 9 | 10 | func main() { 11 | imagick.Initialize() 12 | defer imagick.Terminate() 13 | cmd.Execute() 14 | 15 | } 16 | -------------------------------------------------------------------------------- /gocfl/main_vips.go: -------------------------------------------------------------------------------- 1 | //go:build vips 2 | 3 | package main 4 | 5 | import "github.com/ocfl-archive/gocfl/v2/gocfl/cmd" 6 | import "github.com/davidbyttow/govips/v2/vips" 7 | 8 | func main() { 9 | vips.Startup(nil) 10 | defer vips.Shutdown() 11 | 12 | cmd.Execute() 13 | } 14 | -------------------------------------------------------------------------------- /internal/embed.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "embed" 5 | ) 6 | 7 | //go:embed siegfried/default.sig errors.toml 8 | var InternalFS embed.FS 9 | -------------------------------------------------------------------------------- /internal/errors.toml: -------------------------------------------------------------------------------- 1 | [[errors]] 2 | id = "Test2" 3 | type = "unknown" 4 | weight = 50 5 | message = "Testing two for error" 6 | 7 | [[errors]] 8 | id = "Test" 9 | type = "unknown" 10 | weight = 50 11 | message = "Testing for error" 12 | -------------------------------------------------------------------------------- /internal/siegfried/default.sig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ocfl-archive/gocfl/eb4a7c86f6aa461052adfb4e04e9390d19da5307/internal/siegfried/default.sig -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | # CLI helpers 2 | 3 | set shell := ["bash", "-uc"] 4 | 5 | # Get help 6 | help: 7 | @just -l 8 | 9 | # Fix imports 10 | imports: 11 | goimports -w ./... 12 | 13 | # Snapshot 14 | snapshot: 15 | goreleaser build --snapshot --single-target --clean -f .goreleaser.yml 16 | 17 | # Version 18 | version: 19 | dist/*/gocfl --version 20 | 21 | # Release 22 | release: 23 | goreleaser release --skip=publish --clean -f .goreleaser.yml 24 | 25 | # Single-target release 26 | target: 27 | goreleaser build --single-target --clean -f .goreleaser.yml 28 | -------------------------------------------------------------------------------- /pkg/dilcis/DILCISExtensionMETS.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /pkg/dilcis/DILCISExtensionSIPMETS.xsd: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /pkg/dilcis/mets/DILCISExtensionMETS.go: -------------------------------------------------------------------------------- 1 | // Code generated by xgen. DO NOT EDIT. 2 | 3 | package mets 4 | 5 | // CONTENTINFORMATIONTYPE ... 6 | type CONTENTINFORMATIONTYPE string 7 | 8 | // OTHERCONTENTINFORMATIONTYPE ... 9 | type OTHERCONTENTINFORMATIONTYPE string 10 | 11 | // OAISPACKAGETYPE ... 12 | type OAISPACKAGETYPE string 13 | 14 | // NOTETYPE ... 15 | type NOTETYPE string 16 | 17 | // OTHERTYPE ... 18 | type OTHERTYPE string 19 | -------------------------------------------------------------------------------- /pkg/dilcis/mets/DILCISExtensionSIPMETS.go: -------------------------------------------------------------------------------- 1 | // Code generated by xgen. DO NOT EDIT. 2 | 3 | package mets 4 | 5 | // FILEFORMATNAME ... 6 | type FILEFORMATNAME string 7 | 8 | // FILEFORMATVERSION ... 9 | type FILEFORMATVERSION string 10 | 11 | // FORMATREGISTRY ... 12 | type FORMATREGISTRY string 13 | 14 | // FORMATREGISTRYKEY ... 15 | type FORMATREGISTRYKEY string 16 | -------------------------------------------------------------------------------- /pkg/dilcis/mets/xlink.go: -------------------------------------------------------------------------------- 1 | // Code generated by xgen. DO NOT EDIT. 2 | 3 | package mets 4 | 5 | import ( 6 | "encoding/xml" 7 | ) 8 | 9 | // Href ... 10 | type Href string 11 | 12 | // Role ... 13 | type Role string 14 | 15 | // Arcrole ... 16 | type Arcrole string 17 | 18 | // Title ... 19 | type Title string 20 | 21 | // Show ... 22 | type Show string 23 | 24 | // Actuate ... 25 | type Actuate string 26 | 27 | // Label ... 28 | type Label string 29 | 30 | // From ... 31 | type From string 32 | 33 | // To ... 34 | type To string 35 | 36 | // SimpleLink ... 37 | type SimpleLink struct { 38 | // XMLName xml.Name `xml:"simpleLink"` 39 | TypeAttr string `xml:"xlink:type,attr,omitempty"` 40 | XlinkHrefAttr string `xml:"xlink:href,attr,omitempty"` 41 | XlinkRoleAttr string `xml:"xlink:role,attr,omitempty"` 42 | XlinkArcroleAttr string `xml:"xlink:arcrole,attr,omitempty"` 43 | XlinkTitleAttr string `xml:"xlink:title,attr,omitempty"` 44 | XlinkShowAttr string `xml:"xlink:show,attr,omitempty"` 45 | XlinkActuateAttr string `xml:"xlink:actuate,attr,omitempty"` 46 | } 47 | 48 | // ExtendedLink ... 49 | type ExtendedLink struct { 50 | XMLName xml.Name `xml:"extendedLink"` 51 | TypeAttr string `xml:"type,attr,omitempty"` 52 | XlinkRoleAttr string `xml:"xlink:role,attr,omitempty"` 53 | XlinkTitleAttr string `xml:"xlink:title,attr,omitempty"` 54 | } 55 | 56 | // LocatorLink ... 57 | type LocatorLink struct { 58 | XMLName xml.Name `xml:"locatorLink"` 59 | TypeAttr string `xml:"type,attr,omitempty"` 60 | XlinkHrefAttr string `xml:"xlink:href,attr"` 61 | XlinkRoleAttr string `xml:"xlink:role,attr,omitempty"` 62 | XlinkTitleAttr string `xml:"xlink:title,attr,omitempty"` 63 | XlinkLabelAttr string `xml:"xlink:label,attr,omitempty"` 64 | } 65 | 66 | // ArcLink ... 67 | type ArcLink struct { 68 | XMLName xml.Name `xml:"arcLink"` 69 | TypeAttr string `xml:"type,attr,omitempty"` 70 | XlinkArcroleAttr string `xml:"xlink:arcrole,attr,omitempty"` 71 | XlinkTitleAttr string `xml:"xlink:title,attr,omitempty"` 72 | XlinkShowAttr string `xml:"xlink:show,attr,omitempty"` 73 | XlinkActuateAttr string `xml:"xlink:actuate,attr,omitempty"` 74 | XlinkFromAttr string `xml:"xlink:from,attr,omitempty"` 75 | XlinkToAttr string `xml:"xlink:to,attr,omitempty"` 76 | } 77 | 78 | // ResourceLink ... 79 | type ResourceLink struct { 80 | XMLName xml.Name `xml:"resourceLink"` 81 | TypeAttr string `xml:"type,attr,omitempty"` 82 | XlinkRoleAttr string `xml:"xlink:role,attr,omitempty"` 83 | XlinkTitleAttr string `xml:"xlink:title,attr,omitempty"` 84 | XlinkLabelAttr string `xml:"xlink:label,attr,omitempty"` 85 | } 86 | 87 | // TitleLink ... 88 | type TitleLink struct { 89 | XMLName xml.Name `xml:"titleLink"` 90 | TypeAttr string `xml:"type,attr,omitempty"` 91 | } 92 | 93 | // EmptyLink ... 94 | type EmptyLink struct { 95 | XMLName xml.Name `xml:"emptyLink"` 96 | TypeAttr string `xml:"type,attr,omitempty"` 97 | } 98 | -------------------------------------------------------------------------------- /pkg/dilcis/premis/premisFunc.go: -------------------------------------------------------------------------------- 1 | package premis 2 | 3 | import ( 4 | "encoding/xml" 5 | ) 6 | 7 | func NewStringPlusAuthority(str string, authority string, authorityURI string, valueURI string) *StringPlusAuthority { 8 | return &StringPlusAuthority{ 9 | AuthorityAttr: authority, 10 | AuthorityURIAttr: authorityURI, 11 | ValueURIAttr: valueURI, 12 | Value: str, 13 | } 14 | } 15 | 16 | var locCryptoFunctions = map[string]string{ 17 | "crc32": "CRC32", 18 | "md5": "MD5", 19 | "sha1": "SHA-1", 20 | "sha256": "SHA-256", 21 | "sha384": "SHA-384", 22 | "sha512": "SHA-512", 23 | } 24 | 25 | var locCryptoFunctionsURI = map[string]string{ 26 | "crc32": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/crc32", 27 | "md5": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/md5", 28 | "sha1": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/sha1", 29 | "sha256": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/sha256", 30 | "sha384": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/sha384", 31 | "sha512": "https://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions/sha512", 32 | } 33 | 34 | func NewFixityComplexType(digestAlg, checksum, originator string) *FixityComplexType { 35 | fct := &FixityComplexType{ 36 | XMLName: xml.Name{}, 37 | MessageDigestAlgorithm: &StringPlusAuthority{ 38 | Value: digestAlg, 39 | }, 40 | MessageDigest: checksum, 41 | MessageDigestOriginator: &StringPlusAuthority{ 42 | //XMLName: xml.Name{}, 43 | Value: originator, 44 | }, 45 | } 46 | if a, ok := locCryptoFunctions[digestAlg]; ok { 47 | fct.MessageDigestAlgorithm.AuthorityAttr = "cryptographicHashFunctions" 48 | fct.MessageDigestAlgorithm.AuthorityURIAttr = "http://id.loc.gov/vocabulary/preservation/cryptographicHashFunctions" 49 | fct.MessageDigestAlgorithm.ValueURIAttr = locCryptoFunctionsURI[digestAlg] 50 | fct.MessageDigestAlgorithm.Value = a 51 | } 52 | return fct 53 | } 54 | 55 | func NewSignificantPropertiesComplexType(name, value string) *SignificantPropertiesComplexType { 56 | return &SignificantPropertiesComplexType{ 57 | XMLName: xml.Name{}, 58 | SignificantPropertiesType: NewStringPlusAuthority(name, "", "", ""), 59 | SignificantPropertiesValue: value, 60 | SignificantPropertiesExtension: nil, 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/dilcis/xlink.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /pkg/extension/0001-digest-algorithms.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/je4/filesystem/v3/pkg/writefs" 8 | "github.com/je4/utils/v2/pkg/checksum" 9 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 10 | "io" 11 | "io/fs" 12 | ) 13 | 14 | const DigestAlgorithmsName = "0001-digest-algorithms" 15 | const DigestAlgorithmsDescription = "controlled vocabulary of digest algorithm names that may be used to indicate the given algorithm in fixity blocks of OCFL Objects" 16 | 17 | var algorithms = []checksum.DigestAlgorithm{ 18 | checksum.DigestBlake2b160, 19 | checksum.DigestBlake2b256, 20 | checksum.DigestBlake2b384, 21 | checksum.DigestBlake2b512, 22 | checksum.DigestMD5, 23 | checksum.DigestSHA512, 24 | checksum.DigestSHA256, 25 | checksum.DigestSHA1, 26 | } 27 | 28 | func NewDigestAlgorithmsFS(fsys fs.FS) (*DigestAlgorithms, error) { 29 | fp, err := fsys.Open("config.json") 30 | if err != nil { 31 | return nil, errors.Wrap(err, "cannot open config.json") 32 | } 33 | defer fp.Close() 34 | data, err := io.ReadAll(fp) 35 | if err != nil { 36 | return nil, errors.Wrap(err, "cannot read config.json") 37 | } 38 | 39 | var config = &DigestAlgorithmsConfig{} 40 | if err := json.Unmarshal(data, config); err != nil { 41 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 42 | } 43 | return NewDigestAlgorithms(config) 44 | } 45 | 46 | func NewDigestAlgorithms(config *DigestAlgorithmsConfig) (*DigestAlgorithms, error) { 47 | sl := &DigestAlgorithms{DigestAlgorithmsConfig: config} 48 | if config.ExtensionName != sl.GetName() { 49 | return nil, errors.New(fmt.Sprintf("invalid extension name'%s'for extension %s", config.ExtensionName, sl.GetName())) 50 | } 51 | return sl, nil 52 | } 53 | 54 | type DigestAlgorithmsConfig struct { 55 | *ocfl.ExtensionConfig 56 | } 57 | type DigestAlgorithms struct { 58 | *DigestAlgorithmsConfig 59 | fsys fs.FS 60 | } 61 | 62 | func (sl *DigestAlgorithms) Terminate() error { 63 | return nil 64 | } 65 | 66 | func (sl *DigestAlgorithms) GetFS() fs.FS { 67 | return sl.fsys 68 | } 69 | 70 | func (sl *DigestAlgorithms) GetConfig() any { 71 | return sl.DigestAlgorithmsConfig 72 | } 73 | 74 | func (sl *DigestAlgorithms) IsRegistered() bool { 75 | return true 76 | } 77 | 78 | func (sl *DigestAlgorithms) GetFixityDigests() []checksum.DigestAlgorithm { 79 | return algorithms 80 | } 81 | 82 | func (sl *DigestAlgorithms) SetFS(fsys fs.FS, create bool) { 83 | sl.fsys = fsys 84 | } 85 | 86 | func (sl *DigestAlgorithms) SetParams(params map[string]string) error { 87 | return nil 88 | } 89 | 90 | func (sl *DigestAlgorithms) GetName() string { return DigestAlgorithmsName } 91 | func (sl *DigestAlgorithms) WriteConfig() error { 92 | if sl.fsys == nil { 93 | return errors.New("no filesystem set") 94 | } 95 | configWriter, err := writefs.Create(sl.fsys, "config.json") 96 | if err != nil { 97 | return errors.Wrap(err, "cannot open config.json") 98 | } 99 | defer configWriter.Close() 100 | jenc := json.NewEncoder(configWriter) 101 | jenc.SetIndent("", " ") 102 | if err := jenc.Encode(sl.ExtensionConfig); err != nil { 103 | return errors.Wrapf(err, "cannot encode config to file") 104 | } 105 | return nil 106 | } 107 | 108 | // check interface satisfaction 109 | var ( 110 | _ ocfl.Extension = &DigestAlgorithms{} 111 | _ ocfl.ExtensionFixityDigest = &DigestAlgorithms{} 112 | ) 113 | -------------------------------------------------------------------------------- /pkg/extension/0002-flat-direct-storage-layout.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/je4/filesystem/v3/pkg/writefs" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "io" 10 | "io/fs" 11 | ) 12 | 13 | const StorageLayoutFlatDirectName = "0002-flat-direct-storage-layout" 14 | const StorageLayoutFlatDirectDescription = "one to one mapping without changes" 15 | 16 | func NewStorageLayoutFlatDirectFS(fsys fs.FS) (*StorageLayoutFlatDirect, error) { 17 | fp, err := fsys.Open("config.json") 18 | if err != nil { 19 | return nil, errors.Wrap(err, "cannot open config.json") 20 | } 21 | defer fp.Close() 22 | data, err := io.ReadAll(fp) 23 | if err != nil { 24 | return nil, errors.Wrap(err, "cannot read config.json") 25 | } 26 | 27 | var config = &StorageLayoutFlatDirectConfig{} 28 | if err := json.Unmarshal(data, config); err != nil { 29 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 30 | } 31 | return NewStorageLayoutFlatDirect(config) 32 | } 33 | func NewStorageLayoutFlatDirect(config *StorageLayoutFlatDirectConfig) (*StorageLayoutFlatDirect, error) { 34 | sl := &StorageLayoutFlatDirect{StorageLayoutFlatDirectConfig: config} 35 | if config.ExtensionName != sl.GetName() { 36 | return nil, errors.New(fmt.Sprintf("invalid extension name'%s'for extension %s", config.ExtensionName, sl.GetName())) 37 | } 38 | return sl, nil 39 | } 40 | 41 | type StorageLayoutFlatDirectConfig struct { 42 | *ocfl.ExtensionConfig 43 | } 44 | type StorageLayoutFlatDirect struct { 45 | *StorageLayoutFlatDirectConfig 46 | fsys fs.FS 47 | } 48 | 49 | func (sl *StorageLayoutFlatDirect) Terminate() error { 50 | return nil 51 | } 52 | 53 | func (sl *StorageLayoutFlatDirect) GetFS() fs.FS { 54 | return sl.fsys 55 | } 56 | 57 | func (sl *StorageLayoutFlatDirect) GetConfig() any { 58 | return sl.StorageLayoutFlatDirectConfig 59 | } 60 | 61 | func (sl *StorageLayoutFlatDirect) IsRegistered() bool { 62 | return true 63 | } 64 | 65 | func (sl *StorageLayoutFlatDirect) Stat(w io.Writer, statInfo []ocfl.StatInfo) error { 66 | return nil 67 | } 68 | 69 | func (sl *StorageLayoutFlatDirect) SetFS(fsys fs.FS, create bool) { 70 | sl.fsys = fsys 71 | } 72 | 73 | func (sl *StorageLayoutFlatDirect) SetParams(params map[string]string) error { 74 | return nil 75 | } 76 | 77 | func (sl *StorageLayoutFlatDirect) GetName() string { return StorageLayoutFlatDirectName } 78 | func (sl *StorageLayoutFlatDirect) WriteConfig() error { 79 | if sl.fsys == nil { 80 | return errors.New("no filesystem set") 81 | } 82 | configWriter, err := writefs.Create(sl.fsys, "config.json") 83 | if err != nil { 84 | return errors.Wrap(err, "cannot open config.json") 85 | } 86 | defer configWriter.Close() 87 | jenc := json.NewEncoder(configWriter) 88 | jenc.SetIndent("", " ") 89 | if err := jenc.Encode(sl.ExtensionConfig); err != nil { 90 | return errors.Wrapf(err, "cannot encode config to file") 91 | } 92 | return nil 93 | } 94 | 95 | func (sl *StorageLayoutFlatDirect) WriteLayout(fsys fs.FS) error { 96 | configWriter, err := writefs.Create(fsys, "ocfl_layout.json") 97 | if err != nil { 98 | return errors.Wrap(err, "cannot open ocfl_layout.json") 99 | } 100 | defer configWriter.Close() 101 | jenc := json.NewEncoder(configWriter) 102 | jenc.SetIndent("", " ") 103 | if err := jenc.Encode(struct { 104 | Extension string `json:"extension"` 105 | Description string `json:"description"` 106 | }{ 107 | Extension: StorageLayoutFlatDirectName, 108 | Description: StorageLayoutFlatDirectDescription, 109 | }); err != nil { 110 | return errors.Wrapf(err, "cannot encode config to file") 111 | } 112 | return nil 113 | } 114 | 115 | func (sl *StorageLayoutFlatDirect) BuildStorageRootPath(storageRoot ocfl.StorageRoot, id string) (string, error) { 116 | return id, nil 117 | } 118 | 119 | // check interface satisfaction 120 | var ( 121 | _ ocfl.Extension = &StorageLayoutFlatDirect{} 122 | _ ocfl.ExtensionStorageRootPath = &StorageLayoutFlatDirect{} 123 | ) 124 | -------------------------------------------------------------------------------- /pkg/extension/0002-flat-direct-storage-layout_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestFlatDirectory(t *testing.T) { 9 | // https://ocfl.github.io/extensions/0002-flat-direct-storage-layout.html 10 | // Example 1 11 | l := StorageLayoutFlatDirect{} 12 | objectID := "object-01" 13 | testResult := "object-01" 14 | rootPath, err := l.BuildStorageRootPath(nil, objectID) 15 | if err != nil { 16 | t.Errorf("cannot convert %s", objectID) 17 | } 18 | if rootPath != testResult { 19 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 20 | } 21 | fmt.Printf("StorageLayoutFlatDirect(%s) -> %s\n", objectID, rootPath) 22 | 23 | objectID = "..hor_rib:lé-$id" 24 | testResult = "..hor_rib:lé-$id" 25 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 26 | if err != nil { 27 | t.Errorf("cannot convert %s - %v", objectID, err) 28 | } else { 29 | if rootPath != testResult { 30 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 31 | } else { 32 | fmt.Printf("StorageLayoutFlatDirect(%s) -> %s\n", objectID, rootPath) 33 | } 34 | } 35 | 36 | // https://ocfl.github.io/extensions/0002-flat-direct-storage-layout.html 37 | // Example 2 38 | objectID = "info:fedora/object-01" 39 | testResult = "info:fedora/object-01" 40 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 41 | if err != nil { 42 | t.Errorf("cannot convert %s - %v", objectID, err) 43 | } else { 44 | if rootPath != testResult { 45 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 46 | } else { 47 | fmt.Printf("StorageLayoutFlatDirect(%s) -> %s\n", objectID, rootPath) 48 | } 49 | } 50 | 51 | objectID = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij" 52 | testResult = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij" 53 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 54 | if err != nil { 55 | t.Errorf("cannot convert %s - %v", objectID, err) 56 | } else { 57 | if rootPath != testResult { 58 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 59 | } else { 60 | fmt.Printf("StorageLayoutFlatDirect(%s) -> %s\n", objectID, rootPath) 61 | } 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /pkg/extension/0003-hash-and-id-n-tuple-storage-layout_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "github.com/je4/utils/v2/pkg/checksum" 6 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 7 | "testing" 8 | ) 9 | 10 | func TestHashAndIdNTuple(t *testing.T) { 11 | // https://ocfl.github.io/extensions/0003-hash-and-id-n-tuple-storage-layout.html#encapsulation-directory 12 | // Example 1 13 | l, err := NewStorageLayoutHashAndIdNTuple(&StorageLayoutHashAndIdNTupleConfig{ 14 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0003-hash-and-id-n-tuple-storage-layout"}, 15 | DigestAlgorithm: string(checksum.DigestSHA256), 16 | TupleSize: 3, 17 | NumberOfTuples: 3, 18 | }) 19 | if err != nil { 20 | t.Errorf("error calling NewStorageLayoutHashAndIdNTuple(%s, %v, %v) - %v", checksum.DigestSHA256, 3, 3, err) 21 | return 22 | } 23 | fmt.Printf("\nNewStorageLayoutHashAndIdNTuple(%s, %v, %v)\n", checksum.DigestSHA256, 3, 3) 24 | objectID := "object-01" 25 | testResult := "3c0/ff4/240/object-01" 26 | rootPath, err := l.BuildStorageRootPath(nil, objectID) 27 | if err != nil { 28 | t.Errorf("cannot convert %s", objectID) 29 | } 30 | if rootPath != testResult { 31 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 32 | } 33 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 34 | 35 | objectID = "..hor/rib:le-$id" 36 | testResult = "487/326/d8c/%2e%2ehor%2frib%3ale-%24id" 37 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 38 | if err != nil { 39 | t.Errorf("cannot convert %s", objectID) 40 | } 41 | if rootPath != testResult { 42 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 43 | } 44 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 45 | 46 | // https://ocfl.github.io/extensions/0003-hash-and-id-n-tuple-storage-layout.html#encapsulation-directory 47 | // Example 2 48 | l, err = NewStorageLayoutHashAndIdNTuple(&StorageLayoutHashAndIdNTupleConfig{ 49 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0003-hash-and-id-n-tuple-storage-layout"}, 50 | DigestAlgorithm: string(checksum.DigestMD5), 51 | TupleSize: 2, 52 | NumberOfTuples: 15, 53 | }) 54 | if err != nil { 55 | t.Errorf("error calling NewStorageLayoutHashAndIdNTuple(%s, %v, %v) - %v", checksum.DigestMD5, 2, 15, err) 56 | return 57 | } 58 | fmt.Printf("\nNewStorageLayoutHashAndIdNTuple(%s, %v, %v)\n", checksum.DigestMD5, 2, 15) 59 | objectID = "object-01" 60 | testResult = "ff/75/53/44/92/48/5e/ab/b3/9f/86/35/67/28/88/object-01" 61 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 62 | if err != nil { 63 | t.Errorf("cannot convert %s", objectID) 64 | } 65 | if rootPath != testResult { 66 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 67 | } 68 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 69 | 70 | objectID = "..hor/rib:le-$id" 71 | testResult = "08/31/97/66/fb/6c/29/35/dd/17/5b/94/26/77/17/%2e%2ehor%2frib%3ale-%24id" 72 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 73 | if err != nil { 74 | t.Errorf("cannot convert %s", objectID) 75 | } 76 | if rootPath != testResult { 77 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 78 | } 79 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 80 | 81 | // https://ocfl.github.io/extensions/0003-hash-and-id-n-tuple-storage-layout.html#encapsulation-directory 82 | // Example 3 83 | l, err = NewStorageLayoutHashAndIdNTuple(&StorageLayoutHashAndIdNTupleConfig{ 84 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0003-hash-and-id-n-tuple-storage-layout"}, 85 | DigestAlgorithm: string(checksum.DigestSHA256), 86 | TupleSize: 0, 87 | NumberOfTuples: 0, 88 | }) 89 | if err != nil { 90 | t.Errorf("error calling NewStorageLayoutHashAndIdNTuple(%s, %v, %v) - %v", checksum.DigestSHA256, 0, 0, err) 91 | return 92 | } 93 | fmt.Printf("\nNewStorageLayoutHashAndIdNTuple(%s, %v, %v)\n", checksum.DigestSHA256, 0, 0) 94 | objectID = "object-01" 95 | testResult = "object-01" 96 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 97 | if err != nil { 98 | t.Errorf("cannot convert %s", objectID) 99 | } 100 | if rootPath != testResult { 101 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 102 | } 103 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 104 | 105 | objectID = "..hor/rib:le-$id" 106 | testResult = "%2e%2ehor%2frib%3ale-%24id" 107 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 108 | if err != nil { 109 | t.Errorf("cannot convert %s", objectID) 110 | } 111 | if rootPath != testResult { 112 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 113 | } 114 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 115 | 116 | } 117 | -------------------------------------------------------------------------------- /pkg/extension/0004-hashed-n-tuple-storage-layout_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "github.com/je4/utils/v2/pkg/checksum" 6 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 7 | "testing" 8 | ) 9 | 10 | func TestHashedNTuple(t *testing.T) { 11 | // https://ocfl.github.io/extensions/0004-hashed-n-tuple-storage-layout.html 12 | // Example 1 13 | l, err := NewStorageLayoutHashedNTuple(&StorageLayoutHashedNTupleConfig{ 14 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0004-hashed-n-tuple-storage-layout"}, 15 | DigestAlgorithm: string(checksum.DigestSHA256), 16 | TupleSize: 3, 17 | NumberOfTuples: 3, 18 | ShortObjectRoot: false, 19 | }, 20 | ) 21 | if err != nil { 22 | t.Errorf("error calling NewStorageLayoutHashedNTuple(%s, %v, %v, %v) - %v", checksum.DigestSHA256, 3, 3, false, err) 23 | return 24 | } 25 | fmt.Printf("\nNewStorageLayoutHashedNTuple(%s, %v, %v, %v) - %v\n", checksum.DigestSHA256, 3, 3, false, err) 26 | objectID := "object-01" 27 | testResult := "3c0/ff4/240/3c0ff4240c1e116dba14c7627f2319b58aa3d77606d0d90dfc6161608ac987d4" 28 | rootPath, err := l.BuildStorageRootPath(nil, objectID) 29 | if err != nil { 30 | t.Errorf("cannot convert %s", objectID) 31 | } 32 | if rootPath != testResult { 33 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 34 | } 35 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 36 | 37 | objectID = "..hor/rib:le-$id" 38 | testResult = "487/326/d8c/487326d8c2a3c0b885e23da1469b4d6671fd4e76978924b4443e9e3c316cda6d" 39 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 40 | if err != nil { 41 | t.Errorf("cannot convert %s", objectID) 42 | } 43 | if rootPath != testResult { 44 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 45 | } 46 | fmt.Printf("StorageLayoutHashAndIdNTuple(%s) -> %s\n", objectID, rootPath) 47 | 48 | // https://ocfl.github.io/extensions/0004-hashed-n-tuple-storage-layout.html 49 | // Example 2 50 | l, err = NewStorageLayoutHashedNTuple(&StorageLayoutHashedNTupleConfig{ 51 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0004-hashed-n-tuple-storage-layout"}, 52 | DigestAlgorithm: string(checksum.DigestMD5), 53 | TupleSize: 2, 54 | NumberOfTuples: 15, 55 | ShortObjectRoot: true, 56 | }) 57 | if err != nil { 58 | t.Errorf("error calling NewStorageLayoutHashedNTuple(%s, %v, %v, %v) - %v", checksum.DigestMD5, 2, 15, true, err) 59 | return 60 | } 61 | fmt.Printf("\nNewStorageLayoutHashedNTuple(%s, %v, %v, %v)\n", checksum.DigestMD5, 2, 15, true) 62 | objectID = "object-01" 63 | testResult = "ff/75/53/44/92/48/5e/ab/b3/9f/86/35/67/28/88/4e" 64 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 65 | if err != nil { 66 | t.Errorf("cannot convert %s", objectID) 67 | } 68 | if rootPath != testResult { 69 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 70 | } 71 | fmt.Printf("StorageLayoutHashedNTuple(%s) -> %s\n", objectID, rootPath) 72 | 73 | objectID = "..hor/rib:le-$id" 74 | testResult = "08/31/97/66/fb/6c/29/35/dd/17/5b/94/26/77/17/e0" 75 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 76 | if err != nil { 77 | t.Errorf("cannot convert %s", objectID) 78 | } 79 | if rootPath != testResult { 80 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 81 | } 82 | fmt.Printf("StorageLayoutHashedNTuple(%s) -> %s\n", objectID, rootPath) 83 | 84 | // https://ocfl.github.io/extensions/0004-hashed-n-tuple-storage-layout.html 85 | // Example 3 86 | l, err = NewStorageLayoutHashedNTuple(&StorageLayoutHashedNTupleConfig{ 87 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0004-hashed-n-tuple-storage-layout"}, 88 | DigestAlgorithm: string(checksum.DigestSHA256), 89 | TupleSize: 0, 90 | NumberOfTuples: 0, 91 | ShortObjectRoot: false, 92 | }, 93 | ) 94 | if err != nil { 95 | t.Errorf("error calling NewStorageLayoutHashedNTuple(%s, %v, %v, %v) - %v", checksum.DigestSHA256, 0, 0, false, err) 96 | return 97 | } 98 | fmt.Printf("\nNewStorageLayoutHashedNTuple(%s, %v, %v, %v)\n", checksum.DigestSHA256, 0, 0, false) 99 | objectID = "object-01" 100 | testResult = "3c0ff4240c1e116dba14c7627f2319b58aa3d77606d0d90dfc6161608ac987d4" 101 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 102 | if err != nil { 103 | t.Errorf("cannot convert %s", objectID) 104 | } 105 | if rootPath != testResult { 106 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 107 | } 108 | fmt.Printf("StorageLayoutHashedNTuple(%s) -> %s\n", objectID, rootPath) 109 | 110 | objectID = "..hor/rib:le-$id" 111 | testResult = "487326d8c2a3c0b885e23da1469b4d6671fd4e76978924b4443e9e3c316cda6d" 112 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 113 | if err != nil { 114 | t.Errorf("cannot convert %s", objectID) 115 | } 116 | if rootPath != testResult { 117 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 118 | } 119 | fmt.Printf("StorageLayoutHashedNTuple(%s) -> %s\n", objectID, rootPath) 120 | 121 | } 122 | -------------------------------------------------------------------------------- /pkg/extension/0006-flat-omit-prefix-storage-layout.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/je4/filesystem/v3/pkg/writefs" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "io" 10 | "io/fs" 11 | "strings" 12 | ) 13 | 14 | const FlatOmitPrefixStorageLayoutName = "0006-flat-omit-prefix-storage-layout" 15 | const FlatOmitPrefixStorageLayoutDescription = "removes prefix after last occurrence of delimiter" 16 | 17 | func NewFlatOmitPrefixStorageLayoutFS(fsys fs.FS) (*FlatOmitPrefixStorageLayout, error) { 18 | fp, err := fsys.Open("config.json") 19 | if err != nil { 20 | return nil, errors.Wrap(err, "cannot open config.json") 21 | } 22 | defer fp.Close() 23 | data, err := io.ReadAll(fp) 24 | if err != nil { 25 | return nil, errors.Wrap(err, "cannot read config.json") 26 | } 27 | 28 | var config = &FlatOmitPrefixStorageLayoutConfig{} 29 | if err := json.Unmarshal(data, config); err != nil { 30 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 31 | } 32 | return NewFlatOmitPrefixStorageLayout(config) 33 | } 34 | func NewFlatOmitPrefixStorageLayout(config *FlatOmitPrefixStorageLayoutConfig) (*FlatOmitPrefixStorageLayout, error) { 35 | sl := &FlatOmitPrefixStorageLayout{FlatOmitPrefixStorageLayoutConfig: config} 36 | if config.ExtensionName != sl.GetName() { 37 | return nil, errors.New(fmt.Sprintf("invalid extension name'%s'for extension %s", config.ExtensionName, sl.GetName())) 38 | } 39 | return sl, nil 40 | } 41 | 42 | type FlatOmitPrefixStorageLayoutConfig struct { 43 | *ocfl.ExtensionConfig 44 | Delimiter string `json:"delimiter"` 45 | } 46 | type FlatOmitPrefixStorageLayout struct { 47 | *FlatOmitPrefixStorageLayoutConfig 48 | fsys fs.FS 49 | } 50 | 51 | func (sl *FlatOmitPrefixStorageLayout) Terminate() error { 52 | return nil 53 | } 54 | 55 | func (sl *FlatOmitPrefixStorageLayout) GetFS() fs.FS { 56 | return sl.fsys 57 | } 58 | 59 | func (sl *FlatOmitPrefixStorageLayout) GetConfig() any { 60 | return sl.FlatOmitPrefixStorageLayoutConfig 61 | } 62 | 63 | func (sl *FlatOmitPrefixStorageLayout) IsRegistered() bool { 64 | return true 65 | } 66 | 67 | func (sl *FlatOmitPrefixStorageLayout) Stat(w io.Writer, statInfo []ocfl.StatInfo) error { 68 | return nil 69 | } 70 | 71 | func (sl *FlatOmitPrefixStorageLayout) SetFS(fsys fs.FS, create bool) { 72 | sl.fsys = fsys 73 | } 74 | 75 | func (sl *FlatOmitPrefixStorageLayout) SetParams(params map[string]string) error { 76 | return nil 77 | } 78 | 79 | func (sl *FlatOmitPrefixStorageLayout) GetName() string { return FlatOmitPrefixStorageLayoutName } 80 | func (sl *FlatOmitPrefixStorageLayout) WriteConfig() error { 81 | if sl.fsys == nil { 82 | return errors.New("no filesystem set") 83 | } 84 | configWriter, err := writefs.Create(sl.fsys, "config.json") 85 | if err != nil { 86 | return errors.Wrap(err, "cannot open config.json") 87 | } 88 | defer configWriter.Close() 89 | jenc := json.NewEncoder(configWriter) 90 | jenc.SetIndent("", " ") 91 | if err := jenc.Encode(sl.ExtensionConfig); err != nil { 92 | return errors.Wrapf(err, "cannot encode config to file") 93 | } 94 | return nil 95 | } 96 | 97 | func (sl *FlatOmitPrefixStorageLayout) WriteLayout(fsys fs.FS) error { 98 | configWriter, err := writefs.Create(fsys, "ocfl_layout.json") 99 | if err != nil { 100 | return errors.Wrap(err, "cannot open ocfl_layout.json") 101 | } 102 | defer configWriter.Close() 103 | jenc := json.NewEncoder(configWriter) 104 | jenc.SetIndent("", " ") 105 | if err := jenc.Encode(struct { 106 | Extension string `json:"extension"` 107 | Description string `json:"description"` 108 | }{ 109 | Extension: FlatOmitPrefixStorageLayoutName, 110 | Description: FlatOmitPrefixStorageLayoutDescription, 111 | }); err != nil { 112 | return errors.Wrapf(err, "cannot encode config to file") 113 | } 114 | return nil 115 | } 116 | 117 | func (sl *FlatOmitPrefixStorageLayout) BuildStorageRootPath(storageRoot ocfl.StorageRoot, id string) (string, error) { 118 | last := strings.LastIndex(id, sl.Delimiter) 119 | if last < 0 { 120 | return id, nil 121 | } 122 | return id[last+len(sl.Delimiter):], nil 123 | } 124 | 125 | // check interface satisfaction 126 | var ( 127 | _ ocfl.Extension = &FlatOmitPrefixStorageLayout{} 128 | _ ocfl.ExtensionStorageRootPath = &FlatOmitPrefixStorageLayout{} 129 | ) 130 | -------------------------------------------------------------------------------- /pkg/extension/0006-flat-omit-prefix-storage-layout_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 6 | "testing" 7 | ) 8 | 9 | func TestFlatOmitPrefixStorageLayout(t *testing.T) { 10 | // https://github.com/OCFL/extensions/blob/main/docs/0006-flat-omit-prefix-storage-layout.md 11 | // Example 1 12 | l := FlatOmitPrefixStorageLayout{ 13 | FlatOmitPrefixStorageLayoutConfig: &FlatOmitPrefixStorageLayoutConfig{ 14 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0006-flat-omit-prefix-storage-layout"}, 15 | Delimiter: ":", 16 | }, 17 | } 18 | objectID := "namespace:12887296" 19 | testResult := "12887296" 20 | rootPath, err := l.BuildStorageRootPath(nil, objectID) 21 | if err != nil { 22 | t.Errorf("cannot convert %s", objectID) 23 | } 24 | if rootPath != testResult { 25 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 26 | } 27 | fmt.Printf("FlatOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 28 | 29 | objectID = "urn:uuid:6e8bc430-9c3a-11d9-9669-0800200c9a66" 30 | testResult = "6e8bc430-9c3a-11d9-9669-0800200c9a66" 31 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 32 | if err != nil { 33 | t.Errorf("cannot convert %s - %v", objectID, err) 34 | } else { 35 | if rootPath != testResult { 36 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 37 | } else { 38 | fmt.Printf("FlatOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 39 | } 40 | } 41 | 42 | // Example 1 43 | l = FlatOmitPrefixStorageLayout{ 44 | FlatOmitPrefixStorageLayoutConfig: &FlatOmitPrefixStorageLayoutConfig{ 45 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0006-flat-omit-prefix-storage-layout"}, 46 | Delimiter: "edu/", 47 | }, 48 | } 49 | objectID = "https://institution.edu/3448793" 50 | testResult = "3448793" 51 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 52 | if err != nil { 53 | t.Errorf("cannot convert %s", objectID) 54 | } 55 | if rootPath != testResult { 56 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 57 | } 58 | fmt.Printf("FlatOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 59 | 60 | objectID = "https://institution.edu/abc/edu/f8.05v" 61 | testResult = "f8.05v" 62 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 63 | if err != nil { 64 | t.Errorf("cannot convert %s - %v", objectID, err) 65 | } else { 66 | if rootPath != testResult { 67 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 68 | } else { 69 | fmt.Printf("FlatOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /pkg/extension/0006-flat-omit-prefix-storage-layout_test.go_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 6 | "testing" 7 | ) 8 | 9 | func TestNTupleOmitPrefixStorageLayout(t *testing.T) { 10 | // https://github.com/OCFL/extensions/blob/main/docs/0006-flat-omit-prefix-storage-layout.md 11 | // Example 1 12 | l := NTupleOmitPrefixStorageLayout{ 13 | NTupleOmitPrefixStorageLayoutConfig: &NTupleOmitPrefixStorageLayoutConfig{ 14 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0006-flat-omit-prefix-storage-layout"}, 15 | Delimiter: ":", 16 | TupleSize: 4, 17 | NumberOfTuples: 2, 18 | ZeroPadding: "left", 19 | ReverseObjectRoot: true, 20 | }, 21 | } 22 | objectID := "namespace:12887296" 23 | testResult := "6927/8821/12887296" 24 | rootPath, err := l.BuildStorageRootPath(nil, objectID) 25 | if err != nil { 26 | t.Errorf("cannot convert %s", objectID) 27 | } 28 | if rootPath != testResult { 29 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 30 | } 31 | fmt.Printf("NTupleOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 32 | 33 | objectID = "urn:uuid:6e8bc430-9c3a-11d9-9669-0800200c9a66" 34 | testResult = "66a9/c002/6e8bc430-9c3a-11d9-9669-0800200c9a66" 35 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 36 | if err != nil { 37 | t.Errorf("cannot convert %s - %v", objectID, err) 38 | } else { 39 | if rootPath != testResult { 40 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 41 | } else { 42 | fmt.Printf("NTupleOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 43 | } 44 | } 45 | 46 | objectID = "abc123" 47 | testResult = "321c/ba00/abc123" 48 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 49 | if err != nil { 50 | t.Errorf("cannot convert %s - %v", objectID, err) 51 | } else { 52 | if rootPath != testResult { 53 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 54 | } else { 55 | fmt.Printf("NTupleOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 56 | } 57 | } 58 | 59 | // Example 1 60 | l = NTupleOmitPrefixStorageLayout{ 61 | NTupleOmitPrefixStorageLayoutConfig: &NTupleOmitPrefixStorageLayoutConfig{ 62 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "0006-flat-omit-prefix-storage-layout"}, 63 | Delimiter: "edu/", 64 | TupleSize: 3, 65 | NumberOfTuples: 3, 66 | ZeroPadding: "right", 67 | ReverseObjectRoot: false, 68 | }, 69 | } 70 | objectID = "https://institution.edu/3448793" 71 | testResult = "344/879/300/3448793" 72 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 73 | if err != nil { 74 | t.Errorf("cannot convert %s", objectID) 75 | } 76 | if rootPath != testResult { 77 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 78 | } 79 | fmt.Printf("NTupleOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 80 | 81 | objectID = "https://institution.edu/abc/edu/f8.05v" 82 | testResult = "f8./05v/000/f8.05v" 83 | rootPath, err = l.BuildStorageRootPath(nil, objectID) 84 | if err != nil { 85 | t.Errorf("cannot convert %s - %v", objectID, err) 86 | } else { 87 | if rootPath != testResult { 88 | t.Errorf("%s -> %s != %s", objectID, rootPath, testResult) 89 | } else { 90 | fmt.Printf("NTupleOmitPrefixStorageLayout(%s) -> %s\n", objectID, rootPath) 91 | } 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-direct-clean-path-layout.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "github.com/je4/utils/v2/pkg/checksum" 7 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 8 | "io" 9 | "io/fs" 10 | ) 11 | 12 | // fallback for object with unregigered naming 13 | 14 | const LegacyDirectCleanName = "NNNN-direct-clean-path-layout" 15 | const LegacyDirectCleanDescription = "Maps OCFL object identifiers to storage paths or as an object extension that maps logical paths to content paths. This is done by replacing or removing \"dangerous characters\" from names" 16 | 17 | func NewLegacyDirectCleanFS(fsys fs.FS) (ocfl.Extension, error) { 18 | fp, err := fsys.Open("config.json") 19 | if err != nil { 20 | return nil, errors.Wrap(err, "cannot open config.json") 21 | } 22 | defer fp.Close() 23 | data, err := io.ReadAll(fp) 24 | if err != nil { 25 | return nil, errors.Wrap(err, "cannot read config.json") 26 | } 27 | var config = &LegacyDirectCleanConfig{ 28 | DirectCleanConfig: &DirectCleanConfig{}, 29 | } 30 | if err := json.Unmarshal(data, config); err != nil { 31 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 32 | } 33 | // compatibility with old config 34 | if config.MaxFilenameLen > 0 && config.MaxPathnameLen == 0 { 35 | config.MaxPathnameLen = config.MaxFilenameLen 36 | config.MaxFilenameLen = 0 37 | } 38 | if config.FallbackSubFolders > 0 && config.NumberOfFallbackTuples == 0 { 39 | config.NumberOfFallbackTuples = config.FallbackSubFolders 40 | config.FallbackSubFolders = 0 41 | } 42 | return NewLegacyDirectClean(config) 43 | } 44 | 45 | func NewLegacyDirectClean(config *LegacyDirectCleanConfig) (ocfl.Extension, error) { 46 | if config.MaxPathnameLen == 0 { 47 | config.MaxPathnameLen = 32000 48 | } 49 | if config.MaxPathSegmentLen == 0 { 50 | config.MaxPathSegmentLen = 127 51 | } 52 | if config.FallbackDigestAlgorithm == "" { 53 | config.FallbackDigestAlgorithm = checksum.DigestSHA512 54 | } 55 | if config.FallbackFolder == "" { 56 | config.FallbackFolder = "fallback" 57 | } 58 | 59 | sl := &LegacyDirectClean{DirectClean: &DirectClean{DirectCleanConfig: config.DirectCleanConfig}} 60 | if config.ExtensionName != sl.GetName() { 61 | return nil, errors.Errorf("invalid extension name'%s'for extension %s", config.ExtensionName, sl.GetName()) 62 | } 63 | 64 | var err error 65 | if sl.hash, err = checksum.GetHash(config.FallbackDigestAlgorithm); err != nil { 66 | return nil, errors.Wrapf(err, "hash %s not supported", config.FallbackDigestAlgorithm) 67 | } 68 | 69 | return sl, nil 70 | } 71 | 72 | type LegacyDirectCleanConfig struct { 73 | *DirectCleanConfig 74 | } 75 | 76 | type LegacyDirectClean struct { 77 | *DirectClean 78 | } 79 | 80 | func (sl *LegacyDirectClean) IsRegistered() bool { 81 | return false 82 | } 83 | func (sl *LegacyDirectClean) GetName() string { return LegacyDirectCleanName } 84 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-filesystem_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | 3 | package extension 4 | 5 | import ( 6 | "emperror.dev/errors" 7 | "io/fs" 8 | "os" 9 | "runtime" 10 | "syscall" 11 | "time" 12 | ) 13 | 14 | func (fsm *FilesystemMeta) init(fullpath string, fileInfo fs.FileInfo) error { 15 | fsm.OS = runtime.GOOS 16 | sys := fileInfo.Sys() 17 | if sys == nil { 18 | return errors.New("fileInfo.Sys() is nil") 19 | } 20 | stat_t, ok := sys.(*syscall.Stat_t) 21 | if !ok { 22 | return errors.New("fileInfo.Sys() is not *syscall.Stat_t") 23 | } 24 | fsm.ATime = time.Unix(stat_t.Atimespec.Sec, stat_t.Atimespec.Nsec) 25 | fsm.CTime = time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec) 26 | fsm.MTime = time.Unix(stat_t.Mtimespec.Sec, stat_t.Mtimespec.Nsec) 27 | fsm.Size = uint64(stat_t.Size) 28 | fi, err := os.Lstat(fullpath) 29 | if err != nil { 30 | return errors.WithStack(err) 31 | } 32 | if fi.Mode()&os.ModeSymlink != 0 { 33 | fsm.Symlink, err = os.Readlink(fullpath) 34 | if err != nil { 35 | return errors.Wrapf(err, "cannot read Symlink %s", fullpath) 36 | } 37 | } 38 | unixPerms := fi.Mode() & os.ModePerm 39 | fsm.Attr = unixPerms.String() 40 | fsm.SystemStat = stat_t 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-filesystem_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows && !plan9 && !darwin 2 | 3 | package extension 4 | 5 | import ( 6 | "emperror.dev/errors" 7 | "io/fs" 8 | "os" 9 | "runtime" 10 | "syscall" 11 | "time" 12 | ) 13 | 14 | func (fsm *FilesystemMeta) init(fullpath string, fileInfo fs.FileInfo) error { 15 | fsm.OS = runtime.GOOS 16 | sys := fileInfo.Sys() 17 | if sys == nil { 18 | return errors.New("fileInfo.Sys() is nil") 19 | } 20 | stat_t, ok := sys.(*syscall.Stat_t) 21 | if !ok { 22 | return errors.New("fileInfo.Sys() is not *syscall.Stat_t") 23 | } 24 | fsm.ATime = time.Unix(stat_t.Atim.Sec, stat_t.Atim.Nsec) 25 | fsm.CTime = time.Unix(stat_t.Ctim.Sec, stat_t.Ctim.Nsec) 26 | fsm.MTime = time.Unix(stat_t.Mtim.Sec, stat_t.Mtim.Nsec) 27 | fsm.Size = uint64(stat_t.Size) 28 | fi, err := os.Lstat(fullpath) 29 | if err != nil { 30 | return errors.WithStack(err) 31 | } 32 | if fi.Mode()&os.ModeSymlink != 0 { 33 | fsm.Symlink, err = os.Readlink(fullpath) 34 | if err != nil { 35 | return errors.Wrapf(err, "cannot read Symlink %s", fullpath) 36 | } 37 | } 38 | unixPerms := fi.Mode() & os.ModePerm 39 | fsm.Attr = unixPerms.String() 40 | fsm.SystemStat = stat_t 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-filesystem_win.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package extension 4 | 5 | import ( 6 | "emperror.dev/errors" 7 | "github.com/gomiran/volmgmt/fileattr" 8 | "io/fs" 9 | "os" 10 | "runtime" 11 | "syscall" 12 | "time" 13 | ) 14 | 15 | func (fsm *FilesystemMeta) init(fullpath string, fileInfo fs.FileInfo) error { 16 | fsm.OS = runtime.GOOS 17 | sys := fileInfo.Sys() 18 | if sys == nil { 19 | return errors.New("fileInfo.Sys() is nil") 20 | } 21 | win32FileAttributeData, ok := sys.(*syscall.Win32FileAttributeData) 22 | if !ok { 23 | return errors.New("fileInfo.Sys() is not *syscall.Win32FileAttributeData") 24 | } 25 | fsm.CTime = time.Unix(0, win32FileAttributeData.CreationTime.Nanoseconds()) 26 | fsm.MTime = time.Unix(0, win32FileAttributeData.LastWriteTime.Nanoseconds()) 27 | fsm.ATime = time.Unix(0, win32FileAttributeData.LastAccessTime.Nanoseconds()) 28 | fsm.Size = uint64(win32FileAttributeData.FileSizeLow) 29 | 30 | attr := fileattr.Value(win32FileAttributeData.FileAttributes) 31 | fsm.Attr = attr.String() 32 | 33 | fi, err := os.Lstat(fullpath) 34 | if err != nil { 35 | return errors.WithStack(err) 36 | } 37 | if fi.Mode()&os.ModeSymlink != 0 { 38 | fsm.Symlink, err = os.Readlink(fullpath) 39 | if err != nil { 40 | return errors.Wrapf(err, "cannot read Symlink %s", fullpath) 41 | } 42 | } 43 | 44 | fsm.SystemStat = win32FileAttributeData 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-indexer-logging-object.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "fmt" 6 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 7 | "io" 8 | "io/fs" 9 | "net/url" 10 | ) 11 | 12 | const LoggingIndexerName = "NNNN-indexer-logging-object" 13 | 14 | type LoggingIndexerConfig struct { 15 | *Config 16 | } 17 | 18 | type LoggingIndexer struct { 19 | *LoggingIndexerConfig 20 | metadata map[string]any 21 | } 22 | 23 | func (sl *LoggingIndexer) Terminate() error { 24 | return nil 25 | } 26 | 27 | func (sl *LoggingIndexer) GetFS() fs.FS { 28 | //TODO implement me 29 | panic("implement me") 30 | } 31 | 32 | func (sl *LoggingIndexer) GetConfig() any { 33 | //TODO implement me 34 | panic("implement me") 35 | } 36 | 37 | func (sl *LoggingIndexer) IsRegistered() bool { 38 | return false 39 | } 40 | 41 | func (li *LoggingIndexer) SetFS(fsys fs.FS, create bool) { 42 | //TODO implement me 43 | panic("implement me") 44 | } 45 | 46 | func (li *LoggingIndexer) SetParams(params map[string]string) error { 47 | //TODO implement me 48 | panic("implement me") 49 | } 50 | 51 | func (li *LoggingIndexer) WriteConfig() error { 52 | //TODO implement me 53 | panic("implement me") 54 | } 55 | 56 | func NewLoggingIndexer(config *LoggingIndexerConfig) (*LoggingIndexer, error) { 57 | li := &LoggingIndexer{LoggingIndexerConfig: config, metadata: map[string]any{}} 58 | if config.ExtensionName != li.GetName() { 59 | return nil, errors.New(fmt.Sprintf("invalid extension name %s for extension %s", config.ExtensionName, li.GetName())) 60 | } 61 | return li, nil 62 | } 63 | 64 | func (li *LoggingIndexer) GetName() string { 65 | return LoggingIndexerName 66 | } 67 | func (li *LoggingIndexer) Start() error { 68 | li.metadata = map[string]any{} 69 | return nil 70 | } 71 | func (li *LoggingIndexer) AddFile(fullpath url.URL) error { 72 | return nil 73 | } 74 | 75 | func (li *LoggingIndexer) MoveFile(fullpath url.URL) error { 76 | return nil 77 | 78 | } 79 | 80 | func (li *LoggingIndexer) DeleteFile(fullpath url.URL) error { 81 | return nil 82 | 83 | } 84 | 85 | func (li *LoggingIndexer) WriteLog(logfile io.Writer) error { 86 | return nil 87 | 88 | } 89 | 90 | var ( 91 | _ ocfl.Extension = &LoggingIndexer{} 92 | ) 93 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-pairtree-storage-layout_test.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "github.com/je4/utils/v2/pkg/checksum" 6 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 7 | "testing" 8 | ) 9 | 10 | func TestPairtreeIDEncode(t *testing.T) { 11 | fmt.Printf("(NewPairTreeStorageLayout(%s, %s, %v, %s)\n", "", "", 2, checksum.DigestSHA256) 12 | 13 | ptsl, err := NewStorageLayoutPairTree(&StorageLayoutPairTreeConfig{ 14 | ExtensionConfig: &ocfl.ExtensionConfig{ExtensionName: "gocfl-pairtree"}, 15 | UriBase: "", 16 | StoreDir: "", 17 | ShortyLength: 2, 18 | DigestAlgorithm: string(checksum.DigestSHA256), 19 | }) 20 | if err != nil { 21 | t.Errorf("instantiate failed - %v", err) 22 | return 23 | } 24 | 25 | sourceID := "ark:/13030/xt12t3" 26 | testResult := "ar/k+/=1/30/30/=x/t1/2t/3" 27 | dest, _ := ptsl.BuildStorageRootPath(nil, sourceID) 28 | if dest != testResult { 29 | t.Errorf("IDEncode(%s) => %s != %s", sourceID, dest, testResult) 30 | } else { 31 | fmt.Printf("IDEncode(%s) => %s\n", sourceID, dest) 32 | } 33 | 34 | /* wrong example???? 35 | sourceID = "http://n2t.info/urn:nbn:se:kb:repos-1" 36 | testResult = "ht/tp/+=/=n/2t/,i/nf/o=/ur/n+/n/bn/+s/e+/kb/+/re/p/OS/-1" 37 | dest = ptsl.idToDirpath(sourceID) 38 | if dest != testResult { 39 | t.Errorf("IDEncode(%s) => %s != %s", sourceID, dest, testResult) 40 | } else { 41 | fmt.Printf("IDEncode(%s) => %s\n", sourceID, dest) 42 | } 43 | */ 44 | 45 | sourceID = "what-the-*@?#!^!?" 46 | testResult = "wh/at/-t/he/-^/2a/@^/3f/#!/^5/e!/^3/f" 47 | dest, _ = ptsl.BuildStorageRootPath(nil, sourceID) 48 | if dest != testResult { 49 | t.Errorf("IDEncode(%s) => %s != %s", sourceID, dest, testResult) 50 | } else { 51 | fmt.Printf("IDEncode(%s) => %s\n", sourceID, dest) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-thumbnail_imagick.go: -------------------------------------------------------------------------------- 1 | //go:build imagick && !vips && cgo 2 | 3 | package extension 4 | 5 | import ( 6 | "emperror.dev/errors" 7 | "fmt" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "gopkg.in/gographics/imagick.v3/imagick" 10 | "io" 11 | "slices" 12 | "strings" 13 | ) 14 | 15 | func (thumb *Thumbnail) StreamObject(object ocfl.Object, reader io.Reader, stateFiles []string, dest string) error { 16 | if len(stateFiles) == 0 { 17 | return errors.Errorf("no state files for object '%s'", object.GetID()) 18 | } 19 | if !slices.Contains([]string{"png", "jpeg"}, strings.ToLower(thumb.ThumbnailConfig.Ext)) { 20 | thumb.logger.Info().Msgf("unsupported target image format '%s'", thumb.ThumbnailConfig.Ext) 21 | return nil 22 | } 23 | inventory := object.GetInventory() 24 | head := inventory.GetHead() 25 | if _, ok := thumb.counter[head]; !ok { 26 | thumb.counter[head] = 0 27 | } 28 | if _, ok := thumb.streamInfo[head]; !ok { 29 | thumb.streamInfo[head] = map[string]*ThumbnailResult{} 30 | } 31 | infoName := fmt.Sprintf("%s/content/%s", head, stateFiles[0]) 32 | if _, ok := thumb.streamInfo[head][infoName]; ok { 33 | thumb.logger.Info().Msgf("thumbnail for '%s' already created", stateFiles[0]) 34 | return nil 35 | } 36 | //ext := filepath.Ext(stateFiles[0]) 37 | 38 | imgBytes, err := io.ReadAll(reader) 39 | if err != nil { 40 | return errors.Wrap(err, "cannot read image") 41 | } 42 | 43 | mw := imagick.NewMagickWand() 44 | defer mw.Destroy() 45 | 46 | if err := mw.ReadImageBlob(imgBytes); err != nil { 47 | thumb.logger.Info().Msgf("cannot decode image '%s': %v", stateFiles[0], err) 48 | return nil 49 | } 50 | imgBytes = nil // free memory 51 | width, height, err := mw.GetSize() 52 | if err != nil { 53 | thumb.logger.Info().Msgf("cannot get image size of '%s': %v", stateFiles[0], err) 54 | return nil 55 | } 56 | if width == 0 || height == 0 { 57 | thumb.logger.Info().Msgf("image '%s' has zero size", stateFiles[0]) 58 | return nil 59 | } 60 | thumb.logger.Info().Msgf("image '%s' format: %s, size: %d x %d", stateFiles[0], mw.GetFormat(), width, height) 61 | 62 | rectAspect := width / height 63 | thumbAspect := uint(thumb.Width) / uint(thumb.Height) 64 | newHeight := uint(thumb.Height) 65 | newWidth := uint(thumb.Width) 66 | if rectAspect > thumbAspect { 67 | newHeight = uint(height * uint(thumb.Width) / width) 68 | } else { 69 | newWidth = uint(width * uint(thumb.Height) / height) 70 | } 71 | 72 | if err := mw.ResizeImage(newWidth, newHeight, imagick.FILTER_LANCZOS); err != nil { 73 | thumb.logger.Info().Msgf("cannot resize image '%s': %v", stateFiles[0], err) 74 | return nil 75 | } 76 | 77 | if err := mw.SetImageFormat(thumb.ThumbnailConfig.Ext); err != nil { 78 | thumb.logger.Info().Msgf("cannot set image '%s' format '%s': %v", stateFiles[0], thumb.ThumbnailConfig.Ext, err) 79 | return nil 80 | } 81 | mw.ResetIterator() 82 | imgBytes = mw.GetImageBlob() 83 | 84 | fsys := object.GetFS() 85 | if fsys == nil { 86 | return errors.New("no filesystem set") 87 | } 88 | 89 | thumb.counter[head]++ 90 | targetFile, digest, err := thumb.storeThumbnail(object, head, io.NopCloser(strings.NewReader(string(imgBytes)))) 91 | if err != nil { 92 | return errors.Wrap(err, "cannot store thumbnail") 93 | } 94 | imgBytes = nil // free memory 95 | 96 | thumb.logger.Info().Msgf("thumbnail stored: %s", targetFile) 97 | ml := &ThumbnailResult{ 98 | //SourceDigest: cs, 99 | Filename: targetFile, 100 | Ext: thumb.ThumbnailConfig.Ext, 101 | Error: "", 102 | ID: "internal imagick", 103 | ThumbDigest: digest, 104 | } 105 | thumb.streamInfo[head][infoName] = ml 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/extension/NNNN-thumbnail_vips.go: -------------------------------------------------------------------------------- 1 | //go:build vips && !imagick && cgo 2 | 3 | package extension 4 | 5 | import ( 6 | "emperror.dev/errors" 7 | "fmt" 8 | "github.com/davidbyttow/govips/v2/vips" 9 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 10 | "io" 11 | "slices" 12 | "strings" 13 | ) 14 | 15 | func (thumb *Thumbnail) StreamObject(object ocfl.Object, reader io.Reader, stateFiles []string, dest string) error { 16 | if len(stateFiles) == 0 { 17 | return errors.Errorf("no state files for object '%s'", object.GetID()) 18 | } 19 | if !slices.Contains([]string{"png", "jpeg"}, strings.ToLower(thumb.ThumbnailConfig.Ext)) { 20 | thumb.logger.Info().Msgf("unsupported target image format '%s'", thumb.ThumbnailConfig.Ext) 21 | return nil 22 | } 23 | inventory := object.GetInventory() 24 | head := inventory.GetHead() 25 | if _, ok := thumb.counter[head]; !ok { 26 | thumb.counter[head] = 0 27 | } 28 | if _, ok := thumb.streamInfo[head]; !ok { 29 | thumb.streamInfo[head] = map[string]*ThumbnailResult{} 30 | } 31 | infoName := fmt.Sprintf("%s/content/%s", head, stateFiles[0]) 32 | if _, ok := thumb.streamInfo[head][infoName]; ok { 33 | thumb.logger.Info().Msgf("thumbnail for '%s' already created", stateFiles[0]) 34 | return nil 35 | } 36 | //ext := filepath.Ext(stateFiles[0]) 37 | 38 | img, err := vips.NewImageFromReader(reader) 39 | if err != nil { 40 | thumb.logger.Info().Msgf("cannot decode image '%s': %v", stateFiles[0], err) 41 | return nil 42 | } 43 | defer img.Close() 44 | 45 | width := img.Width() 46 | height := img.Height() 47 | if width == 0 || height == 0 { 48 | thumb.logger.Info().Msgf("image '%s' has zero size", stateFiles[0]) 49 | return nil 50 | } 51 | 52 | thumb.logger.Info().Msgf("image '%s' format: %s, size: %d x %d", stateFiles[0], img.Format().FileExt(), width, height) 53 | 54 | rectAspect := float64(width) / float64(height) 55 | thumbAspect := float64(thumb.Width) / float64(thumb.Height) 56 | newHeight := int(thumb.Height) 57 | newWidth := int(thumb.Width) 58 | _ = newWidth 59 | if rectAspect > thumbAspect { 60 | newHeight = (height * int(thumb.Width)) / width 61 | } else { 62 | newWidth = (width * int(thumb.Height)) / height 63 | } 64 | scale := float64(newHeight) / float64(height) 65 | if err := img.Resize(scale, vips.KernelLanczos3); err != nil { 66 | thumb.logger.Info().Msgf("cannot resize image '%s': %v", stateFiles[0], err) 67 | return nil 68 | } 69 | var imgBytes []byte 70 | var meta *vips.ImageMetadata 71 | switch thumb.ThumbnailConfig.Ext { 72 | case "png": 73 | imgBytes, meta, err = img.ExportPng(vips.NewPngExportParams()) 74 | case "jpeg": 75 | imgBytes, meta, err = img.ExportJpeg(vips.NewJpegExportParams()) 76 | default: 77 | thumb.logger.Info().Msgf("unsupported target image format '%s'", thumb.ThumbnailConfig.Ext) 78 | return nil 79 | } 80 | _ = meta 81 | 82 | fsys := object.GetFS() 83 | if fsys == nil { 84 | return errors.New("no filesystem set") 85 | } 86 | 87 | thumb.counter[head]++ 88 | targetFile, digest, err := thumb.storeThumbnail(object, head, io.NopCloser(strings.NewReader(string(imgBytes)))) 89 | if err != nil { 90 | return errors.Wrap(err, "cannot store thumbnail") 91 | } 92 | imgBytes = nil // free memory 93 | 94 | thumb.logger.Info().Msgf("thumbnail stored: %s", targetFile) 95 | ml := &ThumbnailResult{ 96 | //SourceDigest: cs, 97 | Filename: targetFile, 98 | Ext: thumb.ThumbnailConfig.Ext, 99 | Error: "", 100 | ID: "internal vips", 101 | ThumbDigest: digest, 102 | } 103 | thumb.streamInfo[head][infoName] = ml 104 | return nil 105 | } 106 | -------------------------------------------------------------------------------- /pkg/extension/config.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import "emperror.dev/errors" 4 | 5 | var ErrNotSupported = errors.New("extension not supported") 6 | 7 | type Config struct { 8 | ExtensionName string `json:"extensionName"` 9 | } 10 | -------------------------------------------------------------------------------- /pkg/extension/initial.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/je4/filesystem/v3/pkg/writefs" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "io" 10 | "io/fs" 11 | ) 12 | 13 | const InitialName = "initial" 14 | const InitialDescription = "initial extension defines the name of the extension manager" 15 | 16 | func GetInitialParams() []*ocfl.ExtensionExternalParam { 17 | return []*ocfl.ExtensionExternalParam{ 18 | { 19 | ExtensionName: InitialName, 20 | Functions: []string{"add"}, 21 | Param: "extension", 22 | Description: "name of the extension manager", 23 | }, 24 | } 25 | } 26 | 27 | func NewInitialFS(fsys fs.FS) (*Initial, error) { 28 | var config = &InitialConfig{ 29 | ExtensionConfig: &ocfl.ExtensionConfig{ 30 | ExtensionName: InitialName, 31 | }, 32 | Extension: ocfl.DefaultExtensionManagerName, 33 | } 34 | if fsys != nil { 35 | fp, err := fsys.Open("config.json") 36 | if err != nil { 37 | return nil, errors.Wrap(err, "cannot open config.json") 38 | } 39 | defer fp.Close() 40 | data, err := io.ReadAll(fp) 41 | if err != nil { 42 | return nil, errors.Wrap(err, "cannot read config.json") 43 | } 44 | 45 | if err := json.Unmarshal(data, config); err != nil { 46 | return nil, errors.Wrapf(err, "cannot unmarshal InitialConfig '%s'", string(data)) 47 | } 48 | } 49 | return NewInitial(config) 50 | } 51 | func NewInitial(config *InitialConfig) (*Initial, error) { 52 | sl := &Initial{ 53 | InitialConfig: config, 54 | } 55 | if config.ExtensionName != sl.GetName() { 56 | return nil, errors.New(fmt.Sprintf("invalid extension name'%s'for extension %s", config.ExtensionName, sl.GetName())) 57 | } 58 | return sl, nil 59 | } 60 | 61 | type InitialEntry struct { 62 | Path string `json:"path"` 63 | Description string `json:"description"` 64 | } 65 | 66 | type InitialConfig struct { 67 | *ocfl.ExtensionConfig 68 | Extension string `json:"extension"` 69 | } 70 | type Initial struct { 71 | *InitialConfig 72 | fsys fs.FS 73 | } 74 | 75 | func (sl *Initial) Terminate() error { 76 | return nil 77 | } 78 | 79 | func (sl *Initial) SetExtension(ext string) { 80 | sl.ExtensionName = ext 81 | } 82 | 83 | func (sl *Initial) GetExtension() string { 84 | return sl.InitialConfig.Extension 85 | } 86 | 87 | func (sl *Initial) GetFS() fs.FS { 88 | return sl.fsys 89 | } 90 | 91 | func (sl *Initial) GetConfig() any { 92 | return sl.InitialConfig 93 | } 94 | 95 | func (sl *Initial) IsRegistered() bool { 96 | return true 97 | } 98 | 99 | func (sl *Initial) SetFS(fsys fs.FS, create bool) { 100 | sl.fsys = fsys 101 | } 102 | 103 | func (sl *Initial) SetParams(params map[string]string) error { 104 | name := fmt.Sprintf("ext-%s-%s", InitialName, "extension") 105 | if p, ok := params[name]; ok { 106 | sl.InitialConfig.Extension = p 107 | } 108 | return nil 109 | } 110 | 111 | func (sl *Initial) GetName() string { return InitialName } 112 | 113 | func (sl *Initial) WriteConfig() error { 114 | if sl.fsys == nil { 115 | return errors.New("no filesystem set") 116 | } 117 | configWriter, err := writefs.Create(sl.fsys, "config.json") 118 | if err != nil { 119 | return errors.Wrap(err, "cannot open config.json") 120 | } 121 | defer configWriter.Close() 122 | jenc := json.NewEncoder(configWriter) 123 | jenc.SetIndent("", " ") 124 | if err := jenc.Encode(sl.InitialConfig); err != nil { 125 | return errors.Wrapf(err, "cannot encode config to file") 126 | } 127 | 128 | return nil 129 | } 130 | 131 | // check interface satisfaction 132 | var ( 133 | _ ocfl.Extension = &Initial{} 134 | _ ocfl.ExtensionInitial = &Initial{} 135 | ) 136 | -------------------------------------------------------------------------------- /pkg/extension/pathdirect.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/je4/filesystem/v3/pkg/writefs" 8 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 9 | "io" 10 | "io/fs" 11 | ) 12 | 13 | const PathDirectName = "NNNN-direct-path-layout" 14 | 15 | func NewPathDirectFS(fsys fs.FS) (ocfl.Extension, error) { 16 | fp, err := fsys.Open("config.json") 17 | if err != nil { 18 | return nil, errors.Wrap(err, "cannot open config.json") 19 | } 20 | defer fp.Close() 21 | data, err := io.ReadAll(fp) 22 | if err != nil { 23 | return nil, errors.Wrap(err, "cannot read config.json") 24 | } 25 | var config = &PathDirectConfig{} 26 | if err := json.Unmarshal(data, config); err != nil { 27 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 28 | } 29 | return NewPathDirect(config) 30 | } 31 | 32 | func NewPathDirect(config *PathDirectConfig) (*PathDirect, error) { 33 | sl := &PathDirect{PathDirectConfig: config} 34 | if config.ExtensionName != sl.GetName() { 35 | return nil, errors.New(fmt.Sprintf("invalid extension name %s for extension %s", config.ExtensionName, sl.GetName())) 36 | } 37 | return sl, nil 38 | } 39 | 40 | type PathDirectConfig struct { 41 | *Config 42 | fsys fs.FS 43 | } 44 | 45 | type PathDirect struct { 46 | *PathDirectConfig 47 | } 48 | 49 | func (sl *PathDirect) Terminate() error { 50 | return nil 51 | } 52 | 53 | func (sl *PathDirect) GetFS() fs.FS { 54 | return sl.fsys 55 | } 56 | 57 | func (sl *PathDirect) GetConfig() any { 58 | return sl.PathDirectConfig 59 | } 60 | 61 | func (sl *PathDirect) IsRegistered() bool { 62 | return false 63 | } 64 | 65 | func (sl *PathDirectConfig) SetFS(fsys fs.FS, create bool) { 66 | sl.fsys = fsys 67 | } 68 | 69 | func (sl *PathDirect) SetParams(params map[string]string) error { 70 | return nil 71 | } 72 | 73 | func (sl *PathDirect) GetName() string { return PathDirectName } 74 | 75 | func (sl *PathDirect) WriteLayout(fsys fs.FS) error { 76 | configWriter, err := writefs.Create(fsys, "ocfl_layout.json") 77 | if err != nil { 78 | return errors.Wrap(err, "cannot open ocfl_layout.json") 79 | } 80 | defer configWriter.Close() 81 | jenc := json.NewEncoder(configWriter) 82 | jenc.SetIndent("", " ") 83 | if err := jenc.Encode(struct { 84 | Extension string `json:"extension"` 85 | Description string `json:"description"` 86 | }{ 87 | Extension: StorageLayoutFlatDirectName, 88 | Description: StorageLayoutFlatDirectDescription, 89 | }); err != nil { 90 | return errors.Wrapf(err, "cannot encode config to file") 91 | } 92 | return nil 93 | } 94 | 95 | func (sl *PathDirect) WriteConfig() error { 96 | if sl.fsys == nil { 97 | return errors.New("no filesystem set") 98 | } 99 | configWriter, err := writefs.Create(sl.fsys, "config.json") 100 | if err != nil { 101 | return errors.Wrap(err, "cannot open config.json") 102 | } 103 | defer configWriter.Close() 104 | jenc := json.NewEncoder(configWriter) 105 | jenc.SetIndent("", " ") 106 | if err := jenc.Encode(sl.PathDirectConfig); err != nil { 107 | return errors.Wrapf(err, "cannot encode config to file") 108 | } 109 | return nil 110 | } 111 | 112 | func (sl *PathDirect) BuildStorageRootPath(storageRoot ocfl.StorageRoot, id string) (string, error) { 113 | return id, nil 114 | } 115 | func (sl *PathDirect) BuildObjectManifestPath(object ocfl.Object, originalPath string, area string) (string, error) { 116 | return originalPath, nil 117 | } 118 | 119 | // check interface satisfaction 120 | var ( 121 | _ ocfl.Extension = &PathDirect{} 122 | _ ocfl.ExtensionStorageRootPath = &PathDirect{} 123 | _ ocfl.ExtensionObjectContentPath = &PathDirect{} 124 | ) 125 | -------------------------------------------------------------------------------- /pkg/ocfl/extension.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "fmt" 5 | "github.com/je4/utils/v2/pkg/checksum" 6 | "io" 7 | "io/fs" 8 | ) 9 | 10 | type ExtensionConfig struct { 11 | ExtensionName string `json:"extensionName"` 12 | } 13 | 14 | type Extension interface { 15 | GetName() string 16 | SetFS(fsys fs.FS, create bool) 17 | GetFS() fs.FS 18 | SetParams(params map[string]string) error 19 | WriteConfig() error 20 | //GetConfigString() string 21 | GetConfig() any 22 | IsRegistered() bool 23 | // Stat(w io.Writer, statInfo []StatInfo) error 24 | Terminate() error 25 | } 26 | 27 | const ( 28 | ExtensionStorageRootPathName = "StorageRootPath" 29 | ExtensionObjectContentPathName = "ObjectContentPath" 30 | ExtensionObjectExtractPathName = "ObjectExtractPath" 31 | ExtensionObjectExternalPathName = "ObjectExternalPath" 32 | ExtensionContentChangeName = "ContentChange" 33 | ExtensionObjectChangeName = "ObjectChange" 34 | ExtensionFixityDigestName = "FixityDigest" 35 | ExtensionMetadataName = "Metadata" 36 | ExtensionAreaName = "Area" 37 | ExtensionStreamName = "Stream" 38 | ExtensionNewVersionName = "NewVersion" 39 | ExtensionInitialName = "Initial" 40 | ) 41 | 42 | type ExtensionInitial interface { 43 | Extension 44 | GetExtension() string 45 | SetExtension(ext string) 46 | } 47 | 48 | type ExtensionStorageRootPath interface { 49 | Extension 50 | WriteLayout(fsys fs.FS) error 51 | BuildStorageRootPath(storageRoot StorageRoot, id string) (string, error) 52 | } 53 | 54 | type ExtensionObjectContentPath interface { 55 | Extension 56 | BuildObjectManifestPath(object Object, originalPath string, area string) (string, error) 57 | } 58 | 59 | var ExtensionObjectExtractPathWrongAreaError = fmt.Errorf("invalid area") 60 | 61 | type ExtensionObjectExtractPath interface { 62 | Extension 63 | BuildObjectExtractPath(object Object, originalPath string, area string) (string, error) 64 | } 65 | 66 | type ExtensionObjectStatePath interface { 67 | Extension 68 | BuildObjectStatePath(object Object, originalPath string, area string) (string, error) 69 | } 70 | 71 | type ExtensionArea interface { 72 | Extension 73 | GetAreaPath(object Object, area string) (string, error) 74 | } 75 | 76 | type ExtensionStream interface { 77 | Extension 78 | StreamObject(object Object, reader io.Reader, stateFiles []string, dest string) error 79 | } 80 | 81 | type ExtensionContentChange interface { 82 | Extension 83 | AddFileBefore(object Object, sourceFS fs.FS, source string, dest string, area string, isDir bool) error 84 | UpdateFileBefore(object Object, sourceFS fs.FS, source, dest, area string, isDir bool) error 85 | DeleteFileBefore(object Object, dest string, area string) error 86 | AddFileAfter(object Object, sourceFS fs.FS, source []string, internalPath, digest, area string, isDir bool) error 87 | UpdateFileAfter(object Object, sourceFS fs.FS, source, dest, area string, isDir bool) error 88 | DeleteFileAfter(object Object, dest string, area string) error 89 | } 90 | 91 | type ExtensionObjectChange interface { 92 | Extension 93 | UpdateObjectBefore(object Object) error 94 | UpdateObjectAfter(object Object) error 95 | } 96 | 97 | type ExtensionFixityDigest interface { 98 | Extension 99 | GetFixityDigests() []checksum.DigestAlgorithm 100 | } 101 | 102 | type ExtensionMetadata interface { 103 | Extension 104 | GetMetadata(object Object) (map[string]any, error) 105 | } 106 | 107 | type ExtensionNewVersion interface { 108 | Extension 109 | NeedNewVersion(object Object) (bool, error) 110 | DoNewVersion(object Object) error 111 | } 112 | -------------------------------------------------------------------------------- /pkg/ocfl/extensionExternalParam.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ocfl-archive/gocfl/v2/config" 6 | "github.com/spf13/cobra" 7 | "golang.org/x/exp/slices" 8 | ) 9 | 10 | type ExtensionExternalParam struct { 11 | ExtensionName string 12 | Functions []string 13 | Param string 14 | // File string 15 | Description string 16 | Default string 17 | } 18 | 19 | func (eep *ExtensionExternalParam) SetParam(cmd *cobra.Command) { 20 | if !slices.Contains(eep.Functions, cmd.Name()) { 21 | return 22 | } 23 | name := eep.GetCobraName() 24 | cmd.Flags().String(name, eep.Default, eep.Description) 25 | } 26 | 27 | func (eep *ExtensionExternalParam) GetParam(cmd *cobra.Command, conf *config.GOCFLConfig) (name, value string) { 28 | if !slices.Contains(eep.Functions, cmd.Name()) { 29 | return 30 | } 31 | name = eep.GetCobraName() 32 | value, _ = cmd.Flags().GetString(name) 33 | confExt, ok := conf.Extension[eep.ExtensionName] 34 | if ok { 35 | if str, ok := confExt[eep.Param]; ok { 36 | if str != "" { 37 | value = str 38 | } 39 | } 40 | } 41 | return 42 | } 43 | 44 | func (eep *ExtensionExternalParam) GetCobraName() string { 45 | flagName := fmt.Sprintf("ext-%s-%s", eep.ExtensionName, eep.Param) 46 | return flagName 47 | } 48 | -------------------------------------------------------------------------------- /pkg/ocfl/extensionInitialDummy.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | ) 10 | 11 | func NewInitialDummyFS(fsys fs.FS) (Extension, error) { 12 | fp, err := fsys.Open("config.json") 13 | if err != nil { 14 | return nil, errors.Wrap(err, "cannot open config.json") 15 | } 16 | defer fp.Close() 17 | data, err := io.ReadAll(fp) 18 | if err != nil { 19 | return nil, errors.Wrap(err, "cannot read config.json") 20 | } 21 | var config = &ExtensionManagerConfig{} 22 | if err := json.Unmarshal(data, config); err != nil { 23 | return nil, errors.Wrapf(err, "cannot unmarshal DirectCleanConfig '%s'", string(data)) 24 | } 25 | return NewInitialDummy(config) 26 | } 27 | 28 | func NewInitialDummy(config *ExtensionManagerConfig) (*InitialDummy, error) { 29 | sl := &InitialDummy{ExtensionManagerConfig: config} 30 | if config.ExtensionName != sl.GetName() { 31 | return nil, errors.New(fmt.Sprintf("invalid extension name %s for extension %s", config.ExtensionName, sl.GetName())) 32 | } 33 | return sl, nil 34 | } 35 | 36 | type InitialDummy struct { 37 | *ExtensionManagerConfig 38 | } 39 | 40 | func (dummy *InitialDummy) Terminate() error { 41 | return nil 42 | } 43 | 44 | func (dummy *InitialDummy) GetFS() fs.FS { 45 | //TODO implement me 46 | panic("implement me") 47 | } 48 | 49 | func (dummy *InitialDummy) GetConfig() any { 50 | return dummy.ExtensionManagerConfig 51 | } 52 | 53 | func (dummy *InitialDummy) IsRegistered() bool { 54 | return false 55 | } 56 | 57 | func (dummy *InitialDummy) GetName() string { 58 | return "dummy" 59 | } 60 | 61 | func (dummy *InitialDummy) SetParams(params map[string]string) error { 62 | return nil 63 | } 64 | 65 | func (dummy *InitialDummy) GetConfigString() string { 66 | //TODO implement me 67 | panic("implement me") 68 | } 69 | 70 | func (dummy *InitialDummy) WriteConfig() error { 71 | panic("never call me") 72 | } 73 | 74 | func (dummy *InitialDummy) SetFS(fsys fs.FS, create bool) { 75 | panic("never call me") 76 | } 77 | 78 | var ( 79 | _ Extension = &InitialDummy{} 80 | ) 81 | -------------------------------------------------------------------------------- /pkg/ocfl/extensionManager.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import "io/fs" 4 | 5 | const DefaultExtensionManagerName = "NNNN-gocfl-extension-manager" 6 | const DefaultExtensionInitialName = "initial" 7 | 8 | type ExtensionManager interface { 9 | Extension 10 | ExtensionStorageRootPath 11 | ExtensionObjectContentPath 12 | ExtensionObjectStatePath 13 | ExtensionContentChange 14 | ExtensionObjectChange 15 | ExtensionFixityDigest 16 | ExtensionObjectExtractPath 17 | ExtensionMetadata 18 | ExtensionArea 19 | ExtensionStream 20 | ExtensionNewVersion 21 | GetConfig() any 22 | GetExtensions() []Extension 23 | Add(ext Extension) error 24 | Finalize() 25 | GetConfigName(extName string) (any, error) 26 | GetFSName(extName string) (fs.FS, error) 27 | StoreRootLayout(fsys fs.FS) error 28 | SetInitial(initial ExtensionInitial) 29 | } 30 | 31 | type ExtensionManagerConfig struct { 32 | *ExtensionConfig 33 | Sort map[string][]string `json:"sort"` 34 | Exclusion map[string][][]string `json:"exclusion"` 35 | } 36 | -------------------------------------------------------------------------------- /pkg/ocfl/fixity.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "github.com/je4/utils/v2/pkg/checksum" 5 | ) 6 | 7 | type Fixity map[checksum.DigestAlgorithm]map[string][]string 8 | 9 | func (f Fixity) Checksums(filename string) map[checksum.DigestAlgorithm]string { 10 | result := map[checksum.DigestAlgorithm]string{} 11 | for da, dfs := range f { 12 | for d, fs := range dfs { 13 | found := false 14 | for _, fname := range fs { 15 | if fname == filename { 16 | result[da] = d 17 | found = true 18 | break 19 | } 20 | if found { 21 | break 22 | } 23 | } 24 | if found { 25 | break 26 | } 27 | } 28 | } 29 | return result 30 | } 31 | -------------------------------------------------------------------------------- /pkg/ocfl/fs.go: -------------------------------------------------------------------------------- 1 | // Package ocfl for manipulating and checking Oxford Common Filesystem Layout 2 | // This Oxford Common File Layout (OCFL) specification describes an 3 | // application-independent approach to the storage of digital information in a 4 | // structured, transparent, and predictable manner. It is designed to promote 5 | // long-term object management best practices within digital repositories. 6 | // https://ocfl.io 7 | package ocfl 8 | 9 | /* 10 | type FileSeeker interface { 11 | io.Seeker 12 | fs.File 13 | //Stat() (fs.FileInfo, error) 14 | } 15 | 16 | type CloserAt interface { 17 | io.ReaderAt 18 | io.Closer 19 | } 20 | 21 | type OCFLFSRead interface { 22 | String() string 23 | OpenSeeker(name string) (FileSeeker, error) 24 | Open(name string) (fs.File, error) 25 | Stat(name string) (fs.FileInfo, error) 26 | ReadFile(name string) ([]byte, error) 27 | Close() error 28 | IsNotExist(err error) bool 29 | WalkDir(root string, fn fs.WalkDirFunc) error 30 | ReadDir(name string) ([]fs.DirEntry, error) 31 | HasContent() bool 32 | SubFS(subfolder string) (OCFLFSRead, error) 33 | } 34 | 35 | // Filesystem abstraction for OCFL access 36 | type OCFLFS interface { 37 | OCFLFSRead 38 | Create(name string) (io.WriteCloser, error) 39 | Delete(name string) error 40 | Discard() error 41 | SubFSRW(subfolder string) (OCFLFS, error) 42 | Rename(src, dest string) error 43 | } 44 | */ 45 | -------------------------------------------------------------------------------- /pkg/ocfl/inventory.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "encoding/json" 7 | "github.com/je4/utils/v2/pkg/checksum" 8 | "github.com/je4/utils/v2/pkg/zLogger" 9 | "golang.org/x/exp/slices" 10 | ) 11 | 12 | type StateFileCallback func(internal []string, external []string, digest string) error 13 | 14 | type Inventory interface { 15 | Finalize(inCreation bool) error 16 | IsEqual(i2 Inventory) bool 17 | Init(id string, digest checksum.DigestAlgorithm, fixity []checksum.DigestAlgorithm) error 18 | GetID() string 19 | GetContentDir() string 20 | GetRealContentDir() string 21 | GetHead() string 22 | GetSpec() InventorySpec 23 | CheckFiles(fileManifest map[checksum.DigestAlgorithm]map[string][]string) error 24 | 25 | DeleteFile(stateFilename string) error 26 | RenameFile(stateSource, stateDest string) error 27 | //Rename(oldVirtualFilename, newVirtualFilename string) error 28 | AddFile(stateFilenames []string, manifestFilename string, checksums map[checksum.DigestAlgorithm]string) error 29 | CopyFile(dest string, digest string) error 30 | 31 | IterateStateFiles(version string, fn StateFileCallback) error 32 | GetStateFiles(version string, cs string) ([]string, error) 33 | 34 | //GetContentDirectory() string 35 | GetVersionStrings() []string 36 | GetVersions() map[string]*Version 37 | GetFiles() map[string][]string 38 | GetManifest() map[string][]string 39 | GetFixity() Fixity 40 | GetFilesFlat() []string 41 | GetDigestAlgorithm() checksum.DigestAlgorithm 42 | GetFixityDigestAlgorithm() []checksum.DigestAlgorithm 43 | IsWriteable() bool 44 | IsModified() bool 45 | BuildManifestName(stateFilename string) string 46 | BuildManifestNameVersion(stateFilename string, version string) string 47 | NewVersion(msg, UserName, UserAddress string) error 48 | GetDuplicates(checksum string) []string 49 | AlreadyExists(stateFilename, checksum string) (bool, error) 50 | // IsUpdate(virtualFilename, checksum string) (bool, error) 51 | Clean() error 52 | 53 | VersionLessOrEqual(v1, v2 string) bool 54 | echoDelete(existing []string, pathprefix string) error 55 | } 56 | 57 | func newInventory(ctx context.Context, object Object, folder string, version OCFLVersion, logger zLogger.ZLogger) (Inventory, error) { 58 | switch version { 59 | case Version1_1: 60 | sr, err := newInventoryV1_1(ctx, object, folder, logger) 61 | if err != nil { 62 | return nil, errors.WithStack(err) 63 | } 64 | return sr, nil 65 | default: 66 | //case Version1_0: 67 | sr, err := newInventoryV1_0(ctx, object, folder, logger) 68 | if err != nil { 69 | return nil, errors.WithStack(err) 70 | } 71 | return sr, nil 72 | // return nil, errors.Finalize(fmt.Sprintf("Inventory Version %s not supported", version)) 73 | } 74 | } 75 | 76 | func InventoryIsEqual(i1, i2 Inventory) bool { 77 | data1, err := json.Marshal(i1) 78 | if err != nil { 79 | return false 80 | } 81 | 82 | data2, err := json.Marshal(i2) 83 | if err != nil { 84 | return false 85 | } 86 | return slices.Equal(data1, data2) 87 | } 88 | -------------------------------------------------------------------------------- /pkg/ocfl/inventory1_0.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "net/url" 8 | ) 9 | 10 | const ( 11 | ContentDirectory1_0 = "content" 12 | ) 13 | 14 | type InventoryV1_0 struct { 15 | *InventoryBase 16 | } 17 | 18 | func newInventoryV1_0(ctx context.Context, object Object, folder string, logger zLogger.ZLogger) (*InventoryV1_0, error) { 19 | ivUrl, _ := url.Parse(string(InventorySpec1_0)) 20 | ib, err := newInventoryBase(ctx, object, folder, ivUrl, "", logger) 21 | if err != nil { 22 | return nil, errors.Wrap(err, "cannot create InventoryBase") 23 | } 24 | 25 | i := &InventoryV1_0{InventoryBase: ib} 26 | return i, nil 27 | } 28 | 29 | func (i *InventoryV1_0) IsEqual(i2 Inventory) bool { 30 | i10_2, ok := i2.(*InventoryV1_0) 31 | if !ok { 32 | return false 33 | } 34 | 35 | return i.InventoryBase.isEqual(i10_2.InventoryBase) 36 | } 37 | 38 | var ( 39 | _ Inventory = &InventoryV1_0{} 40 | ) 41 | -------------------------------------------------------------------------------- /pkg/ocfl/inventory1_1.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "net/url" 8 | ) 9 | 10 | const ( 11 | ContentDirectory1_1 = "content" 12 | ) 13 | 14 | type InventoryV1_1 struct { 15 | *InventoryBase 16 | } 17 | 18 | func newInventoryV1_1(ctx context.Context, object Object, folder string, logger zLogger.ZLogger) (*InventoryV1_1, error) { 19 | ivUrl, _ := url.Parse(string(InventorySpec1_1)) 20 | ib, err := newInventoryBase(ctx, object, folder, ivUrl, "", logger) 21 | if err != nil { 22 | return nil, errors.Wrap(err, "cannot create InventoryBase") 23 | } 24 | 25 | i := &InventoryV1_1{InventoryBase: ib} 26 | return i, nil 27 | } 28 | 29 | func (i *InventoryV1_1) IsEqual(i2 Inventory) bool { 30 | i11_2, ok := i2.(*InventoryV1_1) 31 | if !ok { 32 | return false 33 | } 34 | return i.InventoryBase.isEqual(i11_2.InventoryBase) 35 | } 36 | 37 | var ( 38 | _ Inventory = &InventoryV1_1{} 39 | ) 40 | -------------------------------------------------------------------------------- /pkg/ocfl/metadata.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "github.com/je4/utils/v2/pkg/checksum" 5 | "time" 6 | ) 7 | 8 | type FileMetadata struct { 9 | Checksums map[checksum.DigestAlgorithm]string 10 | InternalName []string 11 | VersionName map[string][]string 12 | Extension map[string]any 13 | } 14 | 15 | type VersionMetadata struct { 16 | Created time.Time 17 | Message string 18 | Name string 19 | Address string 20 | } 21 | 22 | type ObjectMetadata struct { 23 | ID string 24 | DigestAlgorithm checksum.DigestAlgorithm 25 | Head string 26 | Versions map[string]*VersionMetadata 27 | Files map[string]*FileMetadata 28 | Extension any 29 | } 30 | 31 | type StorageRootMetadata struct { 32 | Objects map[string]*ObjectMetadata 33 | } 34 | -------------------------------------------------------------------------------- /pkg/ocfl/object.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "fmt" 7 | "github.com/je4/utils/v2/pkg/checksum" 8 | "github.com/je4/utils/v2/pkg/zLogger" 9 | "io" 10 | "io/fs" 11 | ) 12 | 13 | type NamesStruct struct { 14 | ExternalPaths []string 15 | InternalPath string 16 | ManifestPath string 17 | } 18 | 19 | type Object interface { 20 | LoadInventory(folder string) (Inventory, error) 21 | CreateInventory(id string, digest checksum.DigestAlgorithm, fixity []checksum.DigestAlgorithm) (Inventory, error) 22 | StoreInventory(version bool, objectRoot bool) error 23 | GetInventory() Inventory 24 | StoreExtensions() error 25 | Init(id string, digest checksum.DigestAlgorithm, fixity []checksum.DigestAlgorithm, manager ExtensionManager) error 26 | Load() error 27 | StartUpdate(sourceFS fs.FS, msg string, UserName string, UserAddress string, echo bool) (fs.FS, error) 28 | EndUpdate() error 29 | BeginArea(area string) 30 | EndArea() error 31 | AddFolder(fsys fs.FS, versionFS fs.FS, checkDuplicate bool, area string) error 32 | AddFile(fsys fs.FS, versionFS fs.FS, path string, checkDuplicate bool, area string, noExtensionHook bool, isDir bool) error 33 | AddData(data []byte, path string, checkDuplicate bool, area string, noExtensionHook bool, isDir bool) error 34 | AddReader(r io.ReadCloser, files []string, area string, noExtensionHook bool, isDir bool) (string, error) 35 | DeleteFile(virtualFilename string, digest string) error 36 | RenameFile(virtualFilenameSource, virtualFilenameDest string, digest string) error 37 | GetID() string 38 | GetVersion() OCFLVersion 39 | Check() error 40 | Close() error 41 | GetFS() fs.FS 42 | IsModified() bool 43 | Stat(w io.Writer, statInfo []StatInfo) error 44 | Extract(fsys fs.FS, version string, withManifest bool, area string) error 45 | GetMetadata() (*ObjectMetadata, error) 46 | GetAreaPath(area string) (string, error) 47 | GetExtensionManager() ExtensionManager 48 | BuildNames(files []string, area string) (*NamesStruct, error) 49 | } 50 | 51 | func GetObjectVersion(ctx context.Context, ofs fs.FS) (version OCFLVersion, err error) { 52 | files, err := fs.ReadDir(ofs, ".") 53 | if err != nil { 54 | return "", errors.Wrap(err, "cannot get files") 55 | } 56 | for _, file := range files { 57 | if file.IsDir() { 58 | continue 59 | } 60 | matches := objectVersionRegexp.FindStringSubmatch(file.Name()) 61 | if matches != nil { 62 | if version != "" { 63 | return "", errVersionMultiple 64 | } 65 | version = OCFLVersion(matches[1]) 66 | cnt, err := fs.ReadFile(ofs, file.Name()) 67 | if err != nil { 68 | return "", errors.Wrapf(err, "cannot read %s", file.Name()) 69 | } 70 | t := fmt.Sprintf("ocfl_object_%s", version) 71 | if string(cnt) != t+"\n" && string(cnt) != t+"\r\n" { 72 | // todo: which error version should be used??? 73 | addValidationErrors(ctx, GetValidationError(Version1_0, E007).AppendDescription("%s: %s != %s", file.Name(), cnt, t+"\\n").AppendContext("object folder '%s'", ofs)) 74 | } 75 | } 76 | } 77 | if version == "" { 78 | addValidationErrors(ctx, GetValidationError(Version1_0, E003).AppendDescription("no version file found in '%v'", ofs).AppendContext("object folder '%s'", ofs)) 79 | return "", nil 80 | } 81 | return version, nil 82 | } 83 | 84 | func newObject(ctx context.Context, fsys fs.FS, version OCFLVersion, storageRoot StorageRoot, extensionManager ExtensionManager, logger zLogger.ZLogger) (Object, error) { 85 | var err error 86 | if version == "" { 87 | version, err = GetObjectVersion(ctx, fsys) 88 | if err != nil { 89 | return nil, errors.Wrap(err, "cannot get version of object") 90 | } 91 | } 92 | switch version { 93 | case Version1_1: 94 | o, err := newObjectV1_1(ctx, fsys, storageRoot, extensionManager, logger) 95 | if err != nil { 96 | return nil, errors.WithStack(err) 97 | } 98 | return o, nil 99 | case Version2_0: 100 | o, err := newObjectV2_0(ctx, fsys, storageRoot, extensionManager, logger) 101 | if err != nil { 102 | return nil, errors.WithStack(err) 103 | } 104 | return o, nil 105 | default: 106 | o, err := newObjectV1_0(ctx, fsys, storageRoot, extensionManager, logger) 107 | if err != nil { 108 | return nil, errors.WithStack(err) 109 | } 110 | return o, nil 111 | // return nil, errors.Finalize(fmt.Sprintf("Object Version %s not supported", version)) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/ocfl/object1_0.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "io/fs" 8 | ) 9 | 10 | type ObjectV1_0 struct { 11 | *ObjectBase 12 | } 13 | 14 | func newObjectV1_0(ctx context.Context, fsys fs.FS, storageRoot StorageRoot, extensionManager ExtensionManager, logger zLogger.ZLogger) (*ObjectV1_0, error) { 15 | ob, err := newObjectBase(ctx, fsys, Version1_0, storageRoot, extensionManager, logger) 16 | if err != nil { 17 | return nil, errors.WithStack(err) 18 | } 19 | obv10 := &ObjectV1_0{ObjectBase: ob} 20 | return obv10, nil 21 | } 22 | 23 | var ( 24 | _ Object = &ObjectV1_0{} 25 | ) 26 | -------------------------------------------------------------------------------- /pkg/ocfl/object1_1.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "io/fs" 8 | ) 9 | 10 | const ObjectV11Version = "1.1" 11 | 12 | type ObjectV1_1 struct { 13 | *ObjectBase 14 | } 15 | 16 | func newObjectV1_1(ctx context.Context, fsys fs.FS, storageRoot StorageRoot, extensionManager ExtensionManager, logger zLogger.ZLogger) (*ObjectV1_1, error) { 17 | ob, err := newObjectBase(ctx, fsys, Version1_1, storageRoot, extensionManager, logger) 18 | if err != nil { 19 | return nil, errors.WithStack(err) 20 | } 21 | obv11 := &ObjectV1_1{ObjectBase: ob} 22 | return obv11, nil 23 | } 24 | 25 | var ( 26 | _ Object = &ObjectV1_1{} 27 | ) 28 | -------------------------------------------------------------------------------- /pkg/ocfl/object2_0.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "io/fs" 8 | ) 9 | 10 | const ObjectV20Version = "2.0" 11 | 12 | type ObjectV2_0 struct { 13 | *ObjectBase 14 | } 15 | 16 | func newObjectV2_0(ctx context.Context, fsys fs.FS, storageRoot StorageRoot, extensionManager ExtensionManager, logger zLogger.ZLogger) (*ObjectV2_0, error) { 17 | ob, err := newObjectBase(ctx, fsys, Version2_0, storageRoot, extensionManager, logger) 18 | if err != nil { 19 | return nil, errors.WithStack(err) 20 | } 21 | obv20 := &ObjectV2_0{ObjectBase: ob} 22 | return obv20, nil 23 | } 24 | 25 | var ( 26 | _ Object = &ObjectV2_0{} 27 | ) 28 | -------------------------------------------------------------------------------- /pkg/ocfl/stat.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | type StatInfo int64 4 | 5 | const ( 6 | StatObjectFolders StatInfo = iota 7 | StatExtension 8 | StatExtensionConfigs 9 | StatObjects 10 | StatObjectVersions 11 | StatObjectVersionState 12 | StatObjectManifest 13 | StatObjectExtension 14 | StatObjectExtensionConfigs 15 | ) 16 | 17 | var StatInfoString = map[string]StatInfo{ 18 | "ObjectFolders": StatObjectFolders, 19 | "Extension": StatExtension, 20 | "ExtensionConfigs": StatExtensionConfigs, 21 | "Objects": StatObjects, 22 | "ObjectVersions": StatObjectVersions, 23 | "ObjectVersionState": StatObjectVersionState, 24 | "ObjectManifest": StatObjectManifest, 25 | "ObjectExtension": StatObjectExtension, 26 | "ObjectExtensionConfigs": StatObjectExtensionConfigs, 27 | } 28 | -------------------------------------------------------------------------------- /pkg/ocfl/storageroot1_0.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "io/fs" 8 | ) 9 | 10 | const Version1_0 OCFLVersion = "1.0" 11 | 12 | type StorageRootV1_0 struct { 13 | *StorageRootBase 14 | } 15 | 16 | func NewStorageRootV1_0(ctx context.Context, fsys fs.FS, extensionFactory *ExtensionFactory, manager ExtensionManager, logger zLogger.ZLogger) (*StorageRootV1_0, error) { 17 | srb, err := NewStorageRootBase(ctx, fsys, Version1_0, extensionFactory, manager, logger) 18 | if err != nil { 19 | return nil, errors.Wrapf(err, "cannot create StorageRootBase Version %s", Version1_0) 20 | } 21 | 22 | sr := &StorageRootV1_0{StorageRootBase: srb} 23 | return sr, nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/ocfl/storageroot1_1.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/checksum" 7 | "github.com/je4/utils/v2/pkg/zLogger" 8 | "io/fs" 9 | ) 10 | 11 | const Version1_1 OCFLVersion = "1.1" 12 | 13 | type StorageRootV1_1 struct { 14 | *StorageRootBase 15 | } 16 | 17 | func NewStorageRootV1_1(ctx context.Context, fsys fs.FS, extensionFactory *ExtensionFactory, extensionManager ExtensionManager, logger zLogger.ZLogger) (*StorageRootV1_1, error) { 18 | srb, err := NewStorageRootBase(ctx, fsys, Version1_1, extensionFactory, extensionManager, logger) 19 | if err != nil { 20 | return nil, errors.Wrapf(err, "cannot create StorageRootBase Version %s", Version1_1) 21 | } 22 | 23 | sr := &StorageRootV1_1{StorageRootBase: srb} 24 | return sr, nil 25 | } 26 | 27 | func (osr *StorageRootV1_1) Init(version OCFLVersion, digest checksum.DigestAlgorithm, manager ExtensionManager) error { 28 | /* 29 | specFile := "ocfl_1.1.md" 30 | spec, err := writefs.Create(osr.fsys, specFile) 31 | if err != nil { 32 | return errors.Wrapf(err, "cannot create %s", specFile) 33 | } 34 | if _, err := spec.Write(specs.OCFL1_1); err != nil { 35 | _ = spec.Close() 36 | return errors.Wrapf(err, "cannot write into '%s'", specFile) 37 | } 38 | if err := spec.Close(); err != nil { 39 | return errors.Wrapf(err, "cannot close '%s'", specFile) 40 | } 41 | 42 | */ 43 | return osr.StorageRootBase.Init(version, digest, manager) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/ocfl/storageroot2_0.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/checksum" 7 | "github.com/je4/utils/v2/pkg/zLogger" 8 | "io/fs" 9 | ) 10 | 11 | const Version2_0 OCFLVersion = "2.0" 12 | 13 | type StorageRootV2_0 struct { 14 | *StorageRootBase 15 | } 16 | 17 | func NewStorageRootV2_0(ctx context.Context, fsys fs.FS, extensionFactory *ExtensionFactory, extensionManager ExtensionManager, logger zLogger.ZLogger) (*StorageRootV2_0, error) { 18 | srb, err := NewStorageRootBase(ctx, fsys, Version2_0, extensionFactory, extensionManager, logger) 19 | if err != nil { 20 | return nil, errors.Wrapf(err, "cannot create StorageRootBase Version %s", Version2_0) 21 | } 22 | 23 | sr := &StorageRootV2_0{StorageRootBase: srb} 24 | return sr, nil 25 | } 26 | 27 | func (osr *StorageRootV2_0) Init(version OCFLVersion, digest checksum.DigestAlgorithm, manager ExtensionManager) error { 28 | /* 29 | specFile := "ocfl_1.1.md" 30 | spec, err := writefs.Create(osr.fsys, specFile) 31 | if err != nil { 32 | return errors.Wrapf(err, "cannot create %s", specFile) 33 | } 34 | if _, err := spec.Write(specs.OCFL2_0); err != nil { 35 | _ = spec.Close() 36 | return errors.Wrapf(err, "cannot write into '%s'", specFile) 37 | } 38 | if err := spec.Close(); err != nil { 39 | return errors.Wrapf(err, "cannot close '%s'", specFile) 40 | } 41 | 42 | */ 43 | return osr.StorageRootBase.Init(version, digest, manager) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/ocfl/validation.go: -------------------------------------------------------------------------------- 1 | package ocfl 2 | 3 | type Validation interface { 4 | addValidationError(errno ValidationErrorCode, format string, a ...any) error 5 | addValidationWarning(errno ValidationErrorCode, format string, a ...any) error 6 | } 7 | -------------------------------------------------------------------------------- /pkg/subsystem/indexer/actions.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | ironmaiden "github.com/je4/indexer/v3/pkg/indexer" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "os" 8 | "time" 9 | ) 10 | 11 | func InitActions(relevance map[int]ironmaiden.MimeWeightString, siegfried *Siegfried, ffmpeg *FFMPEG, magick *ImageMagick, tika *Tika, logger zLogger.ZLogger) (*ironmaiden.ActionDispatcher, error) { 12 | ad := ironmaiden.NewActionDispatcher(relevance) 13 | if siegfried != nil && siegfried.Signature != "" { 14 | signatureData, err := os.ReadFile(siegfried.Signature) 15 | if err != nil { 16 | logger.Warningf("no siegfried signature file provided. using default signature file. please provide a recent signature file.") 17 | } 18 | logger.Info("indexer action siegfried added") 19 | _ = ironmaiden.NewActionSiegfried("siegfried", signatureData, siegfried.MimeMap, nil, ad) 20 | } 21 | if ffmpeg != nil && ffmpeg.Enabled { 22 | timeout, err := time.ParseDuration(ffmpeg.Timeout) 23 | if err != nil { 24 | return nil, errors.Wrapf(err, "cannot parse ffmpeg timeout '%s'", ffmpeg.Timeout) 25 | } 26 | _ = ironmaiden.NewActionFFProbe( 27 | "ffprobe", 28 | ffmpeg.FFProbe, 29 | ffmpeg.WSL, 30 | timeout, 31 | ffmpeg.Online, 32 | ffmpeg.Mime, 33 | nil, 34 | ad) 35 | logger.Info("indexer action ffprobe added") 36 | } 37 | if magick != nil && magick.Enabled { 38 | timeout, err := time.ParseDuration(magick.Timeout) 39 | if err != nil { 40 | return nil, errors.Wrapf(err, "cannot parse magick timeout '%s'", magick.Timeout) 41 | } 42 | _ = ironmaiden.NewActionIdentifyV2("identify", magick.Identify, magick.Convert, magick.WSL, timeout, magick.Online, nil, ad) 43 | logger.Info("indexer action identify added") 44 | } 45 | if tika != nil && tika.Enabled { 46 | timeout, err := time.ParseDuration(tika.Timeout) 47 | if err != nil { 48 | return nil, errors.Wrapf(err, "cannot parse magick timeout '%s'", magick.Timeout) 49 | } 50 | _ = ironmaiden.NewActionTika("tika", tika.AddressMeta, timeout, tika.RegexpMimeMeta, tika.RegexpMimeMetaNot, "", tika.Online, nil, ad) 51 | logger.Info("indexer action tika added") 52 | 53 | _ = ironmaiden.NewActionTika("fulltext", tika.AddressFulltext, timeout, tika.RegexpMimeFulltext, tika.RegexpMimeFulltextNot, "X-TIKA:content", tika.Online, nil, ad) 54 | logger.Info("indexer action fulltext added") 55 | 56 | } 57 | 58 | return ad, nil 59 | } 60 | -------------------------------------------------------------------------------- /pkg/subsystem/indexer/config.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | ironmaiden "github.com/je4/indexer/v3/pkg/indexer" 5 | ) 6 | 7 | type Siegfried struct { 8 | Signature string 9 | MimeMap map[string]string 10 | } 11 | 12 | type FFMPEG struct { 13 | FFProbe string 14 | WSL bool 15 | Timeout string 16 | Online bool 17 | Enabled bool 18 | Mime []ironmaiden.FFMPEGMime 19 | } 20 | 21 | type ImageMagick struct { 22 | Identify string 23 | Convert string 24 | WSL bool 25 | Timeout string 26 | Online bool 27 | Enabled bool 28 | } 29 | 30 | type Tika struct { 31 | AddressMeta string 32 | RegexpMimeFulltext string 33 | RegexpMimeFulltextNot string 34 | RegexpMimeMeta string 35 | RegexpMimeMetaNot string 36 | Timeout string 37 | Online bool 38 | Enabled bool 39 | AddressFulltext string 40 | } 41 | -------------------------------------------------------------------------------- /pkg/subsystem/migration/helper.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "github.com/google/shlex" 6 | "github.com/ocfl-archive/gocfl/v2/config" 7 | "github.com/ocfl-archive/gocfl/v2/pkg/ocfl" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | "regexp" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | func anyToStringMapString(dataAny any) (map[string]string, error) { 17 | result := map[string]string{} 18 | data, ok := dataAny.(map[string]interface{}) 19 | if !ok { 20 | return nil, errors.Errorf("cannot convert to map[string]interface{}") 21 | } 22 | for k, v := range data { 23 | str, ok := v.(string) 24 | if !ok { 25 | return nil, errors.Errorf("cannot convert '%s' to string", k) 26 | } 27 | result[strings.ToLower(k)] = str 28 | } 29 | return result, nil 30 | } 31 | 32 | func GetMigrations(conf *config.GOCFLConfig) (*Migration, error) { 33 | m := &Migration{ 34 | Functions: map[string]*Function{}, 35 | } 36 | 37 | for name, fn := range conf.Migration.Function { 38 | parts, err := shlex.Split(fn.Command) 39 | if err != nil { 40 | return nil, errors.Wrapf(err, "cannot parse Migration.Function.%s", name) 41 | } 42 | if len(parts) < 1 { 43 | return nil, errors.Errorf("Migration.Function.%s is empty", name) 44 | } 45 | re, err := regexp.Compile(fn.FilenameRegexp) 46 | if err != nil { 47 | return nil, errors.Wrapf(err, "cannot parse Migration.Function.%s", name) 48 | } 49 | timeout := fn.Timeout 50 | var pronoms []string 51 | for _, pro := range fn.Pronoms { 52 | pronoms = append(pronoms, strings.TrimSpace(pro)) 53 | } 54 | strategy, ok := Strategies[fn.Strategy] 55 | if !ok { 56 | return nil, errors.Errorf("unknown strategy '%s' in Migration.Function.%s", fn.Strategy, name) 57 | } 58 | m.Functions[name] = &Function{ 59 | title: fn.Title, 60 | id: fn.ID, 61 | command: parts[0], 62 | args: parts[1:], 63 | Strategy: strategy, 64 | regexp: re, 65 | replace: fn.FilenameReplacement, 66 | timeout: time.Duration(timeout), 67 | pronoms: pronoms, 68 | } 69 | } 70 | return m, nil 71 | } 72 | 73 | func DoMigrate(object ocfl.Object, mig *Function, ext string, targetNames []string, file io.ReadCloser) error { 74 | tmpFile, err := os.CreateTemp(os.TempDir(), "gocfl_*"+ext) 75 | if err != nil { 76 | return errors.Wrap(err, "cannot create temp file") 77 | } 78 | if _, err := io.Copy(tmpFile, file); err != nil { 79 | _ = tmpFile.Close() 80 | return errors.Wrap(err, "cannot copy file") 81 | } 82 | if err := file.Close(); err != nil { 83 | return errors.Wrap(err, "cannot close file") 84 | } 85 | tmpFilename := filepath.ToSlash(tmpFile.Name()) 86 | targetFilename := filepath.ToSlash(filepath.Join(filepath.Dir(tmpFilename), "target."+filepath.Base(tmpFilename)+filepath.Ext(targetNames[0]))) 87 | 88 | if err := tmpFile.Close(); err != nil { 89 | return errors.Wrap(err, "cannot close temp file") 90 | } 91 | defer func() { 92 | _ = os.Remove(tmpFilename) 93 | _ = os.Remove(targetFilename) 94 | }() 95 | if err := mig.Migrate(tmpFilename, targetFilename); err != nil { 96 | return errors.Wrapf(err, "cannot migrate file '%v' to object '%s'", targetNames, object.GetID()) 97 | } 98 | 99 | mFile, err := os.Open(targetFilename) 100 | if err != nil { 101 | return errors.Wrapf(err, "cannot open file '%s'", targetFilename) 102 | } 103 | if _, err := object.AddReader(mFile, targetNames, "content", false, false); err != nil { 104 | return errors.Wrapf(err, "cannot migrate file '%v' to object '%s'", targetNames, object.GetID()) 105 | } 106 | if err := mFile.Close(); err != nil { 107 | return errors.Wrapf(err, "cannot close file '%s'", targetFilename) 108 | } 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /pkg/subsystem/migration/migrate.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "fmt" 7 | "io/fs" 8 | "os/exec" 9 | "path/filepath" 10 | "regexp" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | type Strategy string 16 | 17 | const ( 18 | StrategyReplace Strategy = "replace" 19 | StrategyAdd Strategy = "add" 20 | StrategyFolder Strategy = "folder" 21 | ) 22 | 23 | var Strategies = map[string]Strategy{ 24 | "replace": StrategyReplace, 25 | "add": StrategyAdd, 26 | "folder": StrategyFolder, 27 | } 28 | 29 | type Function struct { 30 | command string 31 | args []string 32 | Strategy Strategy 33 | regexp *regexp.Regexp 34 | replace string 35 | timeout time.Duration 36 | title string 37 | id string 38 | pronoms []string 39 | } 40 | 41 | var migrationVersionRegexp = regexp.MustCompile(`^([^.]+)\.(.+)$`) 42 | 43 | func (f *Function) GetDestinationName(src string, head string, isMigrated bool) string { 44 | dest := f.regexp.ReplaceAllString(src, f.replace) 45 | if f.Strategy == StrategyFolder { 46 | if isMigrated { 47 | parts := migrationVersionRegexp.FindStringSubmatch(filepath.Base(src)) 48 | if parts == nil { 49 | return "" 50 | } 51 | dest = filepath.ToSlash(filepath.Join(filepath.Dir(src), fmt.Sprintf("%s.%s", head, parts[2]))) 52 | } else { 53 | dest = filepath.ToSlash(filepath.Join(src, fmt.Sprintf("%s.%s", head, dest))) 54 | } 55 | } 56 | return dest 57 | } 58 | 59 | func (f *Function) Migrate(source string, dest string) error { 60 | ctx, cancel := context.WithTimeout(context.Background(), f.timeout) 61 | defer cancel() 62 | args := []string{} 63 | for _, arg := range f.args { 64 | // arg = strings.ReplaceAll(arg, "{source}", filepath.Base(source)) 65 | // arg = strings.ReplaceAll(arg, "{destination}", filepath.Base(dest)) 66 | arg = strings.ReplaceAll(arg, "{source}", filepath.ToSlash(source)) 67 | arg = strings.ReplaceAll(arg, "{destination}", filepath.ToSlash(dest)) 68 | 69 | args = append(args, arg) 70 | } 71 | cmd := exec.CommandContext(ctx, f.command, args...) 72 | cmd.Dir = filepath.Dir(source) 73 | return errors.Wrapf(cmd.Run(), "cannot run command '%s %s'", f.command, strings.Join(args, " ")) 74 | } 75 | 76 | func (f *Function) GetID() string { 77 | return f.id 78 | } 79 | 80 | type Migration struct { 81 | Functions map[string]*Function 82 | //Sources map[string]string 83 | SourceFS fs.FS 84 | } 85 | 86 | func (m *Migration) GetFunctionByName(name string) (*Function, error) { 87 | if f, ok := m.Functions[strings.ToLower(name)]; ok { 88 | return f, nil 89 | } 90 | return nil, errors.Errorf("Migration.Function.%s does not exist", name) 91 | } 92 | 93 | func (m *Migration) GetFunctionByPronom(pronom string) (*Function, error) { 94 | for _, f := range m.Functions { 95 | for _, pro := range f.pronoms { 96 | if pro == pronom { 97 | return f, nil 98 | } 99 | } 100 | } 101 | return nil, errors.Errorf("Migration.Source.%s does not exist", pronom) 102 | } 103 | 104 | func (m *Migration) SetSourceFS(fs fs.FS) { 105 | m.SourceFS = fs 106 | } 107 | -------------------------------------------------------------------------------- /pkg/subsystem/thumbnail/helper.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "emperror.dev/errors" 5 | "github.com/google/shlex" 6 | "github.com/ocfl-archive/gocfl/v2/config" 7 | "regexp" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | func anyToStringMapString(dataAny any) (map[string]string, error) { 13 | result := map[string]string{} 14 | data, ok := dataAny.(map[string]interface{}) 15 | if !ok { 16 | return nil, errors.Errorf("cannot convert to map[string]interface{}") 17 | } 18 | for k, v := range data { 19 | str, ok := v.(string) 20 | if !ok { 21 | return nil, errors.Errorf("cannot convert '%s' to string", k) 22 | } 23 | result[strings.ToLower(k)] = str 24 | } 25 | return result, nil 26 | } 27 | 28 | func GetThumbnails(conf *config.GOCFLConfig) (*Thumbnail, error) { 29 | m := &Thumbnail{ 30 | Functions: map[string]*Function{}, 31 | Background: conf.Thumbnail.Background, 32 | } 33 | 34 | for name, fn := range conf.Thumbnail.Function { 35 | parts, err := shlex.Split(fn.Command) 36 | if err != nil { 37 | return nil, errors.Wrapf(err, "cannot parse Thumbnail.Function.%s", name) 38 | } 39 | if len(parts) < 1 { 40 | return nil, errors.Errorf("Thumbnail.Function.%s is empty", name) 41 | } 42 | timeout := fn.Timeout 43 | if err != nil { 44 | return nil, errors.Wrapf(err, "cannot parse timeout of Thumbnail.Function.%s", name) 45 | } 46 | var mimeRes = []*regexp.Regexp{} 47 | for _, mime := range fn.Mime { 48 | re, err := regexp.Compile(mime) 49 | if err != nil { 50 | return nil, errors.Wrapf(err, "cannot parse Migration.Function.%s", name) 51 | } 52 | mimeRes = append(mimeRes, re) 53 | } 54 | var pronoms []string 55 | for _, pro := range fn.Pronoms { 56 | pronoms = append(pronoms, strings.TrimSpace(pro)) 57 | } 58 | m.Functions[name] = &Function{ 59 | thumb: m, 60 | title: fn.Title, 61 | id: fn.ID, 62 | command: parts[0], 63 | args: parts[1:], 64 | timeout: time.Duration(timeout), 65 | pronoms: pronoms, 66 | mime: mimeRes, 67 | } 68 | } 69 | return m, nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/subsystem/thumbnail/thumbnail.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "context" 5 | "emperror.dev/errors" 6 | "github.com/je4/utils/v2/pkg/zLogger" 7 | "io/fs" 8 | "os/exec" 9 | "path/filepath" 10 | "regexp" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | type ThumbnailMeta struct { 17 | Ext string 18 | Width uint64 19 | Height uint64 20 | Mime string 21 | } 22 | 23 | type Function struct { 24 | thumb *Thumbnail 25 | command string 26 | args []string 27 | timeout time.Duration 28 | title string 29 | id string 30 | pronoms []string 31 | mime []*regexp.Regexp 32 | } 33 | 34 | func (f *Function) Thumbnail(source string, dest string, width uint64, height uint64, logger zLogger.ZLogger) error { 35 | ctx, cancel := context.WithTimeout(context.Background(), f.timeout) 36 | defer cancel() 37 | args := []string{} 38 | for _, arg := range f.args { 39 | arg = strings.ReplaceAll(arg, "{source}", filepath.ToSlash(source)) 40 | arg = strings.ReplaceAll(arg, "{destination}", filepath.ToSlash(dest)) 41 | arg = strings.ReplaceAll(arg, "{background}", f.thumb.Background) 42 | arg = strings.ReplaceAll(arg, "{width}", strconv.FormatUint(width, 10)) 43 | arg = strings.ReplaceAll(arg, "{height}", strconv.FormatUint(height, 10)) 44 | args = append(args, arg) 45 | } 46 | logger.Debug().Msgf("%s %v", f.command, args) 47 | cmd := exec.CommandContext(ctx, f.command, args...) 48 | cmd.Dir = filepath.Dir(source) 49 | return errors.Wrapf(cmd.Run(), "cannot run command '%s %s'", f.command, strings.Join(args, " ")) 50 | } 51 | 52 | func (f *Function) GetID() string { 53 | return f.id 54 | } 55 | 56 | type Thumbnail struct { 57 | Functions map[string]*Function 58 | SourceFS fs.FS 59 | Background string 60 | } 61 | 62 | func (m *Thumbnail) GetFunctionByName(name string) (*Function, error) { 63 | if f, ok := m.Functions[strings.ToLower(name)]; ok { 64 | return f, nil 65 | } 66 | return nil, errors.Errorf("Thumbnail.Function.%s does not exist", name) 67 | } 68 | 69 | func (m *Thumbnail) GetFunctionByPronom(pronom string) (*Function, error) { 70 | for _, f := range m.Functions { 71 | for _, pro := range f.pronoms { 72 | if pro == pronom { 73 | return f, nil 74 | } 75 | } 76 | } 77 | return nil, errors.Errorf("Thumbnail.Source.%s does not exist", pronom) 78 | } 79 | 80 | func (m *Thumbnail) GetFunctionByMimetype(mime string) (*Function, error) { 81 | for _, f := range m.Functions { 82 | for _, re := range f.mime { 83 | if re.MatchString(mime) { 84 | return f, nil 85 | } 86 | } 87 | } 88 | return nil, errors.Errorf("Thumbnail.Source.%s does not exist", mime) 89 | } 90 | 91 | func (m *Thumbnail) SetSourceFS(fs fs.FS) { 92 | m.SourceFS = fs 93 | } 94 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | var ( 4 | Version = "dev-0.0.0" 5 | Commit = "000000000000000000000000000000000badf00d" 6 | Date = "1970-01-01T00:00:01Z" 7 | BuiltBy = "dev" 8 | ) 9 | 10 | // ShortCommit returns a short commit hash. 11 | func ShortCommit() string { 12 | return Commit[:6] 13 | } 14 | --------------------------------------------------------------------------------