├── .gitignore ├── lib ├── logging_windows.go ├── logging.go ├── util_test.go ├── helpers_test.go ├── cli.go ├── model.go ├── search_test.go ├── util.go ├── image.go ├── search.go ├── cmdchain.go ├── rest.go ├── cmdchain_test.go ├── sqlitedb.go └── sqlitedb_test.go ├── web └── paperless-frontend │ ├── babel.config.js │ ├── public │ ├── favicon.ico │ └── index.html │ ├── src │ ├── assets │ │ └── logo.png │ ├── rest.js │ ├── main.js │ └── App.vue │ ├── .gitignore │ ├── vue.config.js │ ├── README.md │ └── package.json ├── docker ├── Dockerfile └── run.sh ├── clipper ├── qtclipper.pro └── qtclipper.cpp ├── go.mod ├── paperless.go ├── LICENSE ├── README.org ├── go.sum └── uploader └── uploader.go /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | paperless -------------------------------------------------------------------------------- /lib/logging_windows.go: -------------------------------------------------------------------------------- 1 | 2 | // +build windows 3 | 4 | package paperless 5 | 6 | func SetupLogging() { 7 | } 8 | -------------------------------------------------------------------------------- /web/paperless-frontend/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /web/paperless-frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kopoli/paperless/HEAD/web/paperless-frontend/public/favicon.ico -------------------------------------------------------------------------------- /web/paperless-frontend/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kopoli/paperless/HEAD/web/paperless-frontend/src/assets/logo.png -------------------------------------------------------------------------------- /web/paperless-frontend/src/rest.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | var base = '/api/v1/'; 4 | 5 | export const ImageApi = axios.create({ 6 | baseURL: base + 'image', 7 | timeout: 60000 8 | }); 9 | 10 | export const TagApi = axios.create({ 11 | baseURL: base + 'tag', 12 | timeout: 60000 13 | }); 14 | -------------------------------------------------------------------------------- /web/paperless-frontend/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | 4 | import vmodal from 'vue-js-modal' 5 | 6 | import Paginate from 'vuejs-paginate' 7 | Vue.component('paginate', Paginate) 8 | 9 | Vue.use(vmodal) 10 | 11 | new Vue({ 12 | el: '#app', 13 | render: h => h(App) 14 | }) 15 | -------------------------------------------------------------------------------- /web/paperless-frontend/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | pnpm-debug.log* 14 | 15 | # Editor directories and files 16 | .idea 17 | .vscode 18 | *.suo 19 | *.ntvs* 20 | *.njsproj 21 | *.sln 22 | *.sw? 23 | -------------------------------------------------------------------------------- /lib/logging.go: -------------------------------------------------------------------------------- 1 | 2 | // +build !windows 3 | 4 | package paperless 5 | 6 | import ( 7 | "log" 8 | "log/syslog" 9 | 10 | ) 11 | 12 | func SetupLogging() { 13 | syslg, err := syslog.New(syslog.LOG_ERR|syslog.LOG_DAEMON, "paperless") 14 | if err != nil { 15 | log.Fatal("Creating a syslog logger failed") 16 | } 17 | log.SetOutput(syslg) 18 | } 19 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | RUN \ 4 | /bin/sed -i -e 's,http://archive.ubuntu.com/ubuntu,mirror://mirrors.ubuntu.com/mirrors.txt,g' /etc/apt/sources.list && \ 5 | apt-get update && \ 6 | apt-get -y upgrade && \ 7 | apt-get install -y unpaper imagemagick tesseract-ocr tesseract-ocr-fin && \ 8 | rm -rf /var/lib/apt/lists/* 9 | 10 | EXPOSE 8078 11 | 12 | CMD [ "/paperless" ] 13 | -------------------------------------------------------------------------------- /web/paperless-frontend/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | publicPath: process.env.NODE_ENV === 'production' 3 | ? '/dist/' 4 | : '/', 5 | devServer: { 6 | proxy: { 7 | '/api/v1': { 8 | target: 'http://localhost:8078', 9 | secure: false 10 | }, 11 | '/static/': { 12 | target: 'http://localhost:8078', 13 | secure: false 14 | } 15 | } 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /web/paperless-frontend/README.md: -------------------------------------------------------------------------------- 1 | # jeejee 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Lints and fixes files 19 | ``` 20 | npm run lint 21 | ``` 22 | 23 | ### Customize configuration 24 | See [Configuration Reference](https://cli.vuejs.org/config/). 25 | -------------------------------------------------------------------------------- /docker/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | die() { 4 | echo "Error: $@" 1>&2 5 | exit 1 6 | } 7 | 8 | PAPERLESS=../paperless 9 | IMG=ubuntu-16.04-paperless:latest 10 | DATADIR=../docker-data 11 | 12 | set -x 13 | 14 | test -e "$PAPERLESS" || die "Build paperless before trying to run this" 15 | 16 | mkdir -p $DATADIR 17 | 18 | docker build . -t $IMG || die "Could not build the docker image" 19 | 20 | docker run -v $PWD/$PAPERLESS:/paperless -v $PWD/$DATADIR:/data \ 21 | --rm \ 22 | $IMG 23 | -------------------------------------------------------------------------------- /clipper/qtclipper.pro: -------------------------------------------------------------------------------- 1 | #------------------------------------------------- 2 | # 3 | # Project created by QtCreator 2012-10-07T16:31:27 4 | # 5 | #------------------------------------------------- 6 | 7 | QT += core gui 8 | 9 | greaterThan(QT_MAJOR_VERSION, 4): QT += widgets 10 | 11 | CONFIG += c++11 warn_on release 12 | 13 | release { 14 | DEFINES += QT_NO_DEBUG_OUTPUT 15 | } 16 | debug { 17 | QMAKE_CFLAGS_WARN_ON = -Wall -Werror -Wundef -Wextra 18 | QMAKE_CXXFLAGS_WARN_ON = $$QMAKE_CFLAGS_WARN_ON 19 | } 20 | 21 | TARGET = qtclipper 22 | TEMPLATE = app 23 | 24 | SOURCES += qtclipper.cpp 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kopoli/paperless 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 7 | github.com/gamegos/jsend v0.0.0-20151011171802-f47e169f3d76 8 | github.com/go-chi/chi v4.1.1+incompatible 9 | github.com/go-chi/docgen v1.0.5 10 | github.com/jawher/mow.cli v1.1.0 11 | github.com/jmoiron/sqlx v1.2.0 12 | github.com/kopoli/go-util v0.0.0-20180512085434-f355416491c0 13 | github.com/mattn/go-sqlite3 v2.0.3+incompatible 14 | github.com/pkg/errors v0.9.1 // indirect 15 | github.com/pmezard/go-difflib v1.0.0 16 | github.com/stretchr/testify v1.3.0 17 | google.golang.org/appengine v1.6.6 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /web/paperless-frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | <%= htmlWebpackPlugin.options.title %> 9 | 10 | 11 | 14 |
15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /lib/util_test.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "os" 5 | "github.com/stretchr/testify/assert" 6 | "io/ioutil" 7 | "testing" 8 | ) 9 | 10 | func TestChecksumSame(t *testing.T) { 11 | first := "First string" 12 | second := "Second string" 13 | sum1 := Checksum([]byte(first)) 14 | sum2 := Checksum([]byte(second)) 15 | 16 | assert.NotEmpty(t, sum1) 17 | assert.NotEqual(t, sum1, sum2) 18 | } 19 | 20 | func TestChecksumFileSame(t *testing.T) { 21 | first := "First string" 22 | sum1 := Checksum([]byte(first)) 23 | var sum2 string 24 | 25 | fp, err := ioutil.TempFile("", "testfile") 26 | assert.Nil(t, err) 27 | defer os.Remove(fp.Name()) 28 | defer fp.Close() 29 | fp.Write([]byte(first)) 30 | sum2, err = ChecksumFile(fp.Name()) 31 | assert.Nil(t, err) 32 | assert.Equal(t, sum1, sum2) 33 | } 34 | -------------------------------------------------------------------------------- /lib/helpers_test.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/davecgh/go-spew/spew" 7 | _ "github.com/mattn/go-sqlite3" 8 | "github.com/pmezard/go-difflib/difflib" 9 | ) 10 | 11 | // general testing functionality 12 | 13 | func structEquals(a, b interface{}) bool { 14 | return spew.Sdump(a) == spew.Sdump(b) 15 | } 16 | 17 | func diffStr(a, b interface{}) (ret string) { 18 | diff := difflib.UnifiedDiff{ 19 | A: difflib.SplitLines(spew.Sdump(a)), 20 | B: difflib.SplitLines(spew.Sdump(b)), 21 | FromFile: "Expected", 22 | ToFile: "Received", 23 | Context: 3, 24 | } 25 | 26 | ret, _ = difflib.GetUnifiedDiffString(diff) 27 | return 28 | } 29 | 30 | func compare(t *testing.T, msg string, a, b interface{}) { 31 | if !structEquals(a, b) { 32 | t.Error(msg, "\n", diffStr(a, b)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paperless.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | util "github.com/kopoli/go-util" 9 | "github.com/kopoli/paperless/lib" 10 | ) 11 | 12 | var ( 13 | version = "Undefined" 14 | timestamp = "Undefined" 15 | ) 16 | 17 | func printErr(err error, message string, arg ...string) { 18 | msg := "" 19 | if err != nil { 20 | msg = fmt.Sprintf(" (error: %s)", err) 21 | } 22 | fmt.Fprintf(os.Stderr, "Error: %s%s.%s\n", message, strings.Join(arg, " "), msg) 23 | } 24 | 25 | func fault(err error, message string, arg ...string) { 26 | printErr(err, message, arg...) 27 | os.Exit(1) 28 | } 29 | 30 | func main() { 31 | opts := util.NewOptions() 32 | opts.Set("program-name", os.Args[0]) 33 | opts.Set("program-version", version) 34 | 35 | err := paperless.Cli(opts, os.Args) 36 | if err != nil { 37 | fault(err, "Command line parsing failed") 38 | } 39 | 40 | err = paperless.StartWeb(opts) 41 | if err != nil { 42 | fault(err, "Starting paperless web server failed") 43 | } 44 | } 45 | 46 | // Local Variables: 47 | // outline-regexp: "^////*\\|^func\\|^import" 48 | // End: 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Kalle Kankare 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/cli.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/jawher/mow.cli" 8 | util "github.com/kopoli/go-util" 9 | ) 10 | 11 | func Cli(opts util.Options, args []string) error { 12 | progName := opts.Get("program-name", "paperless") 13 | progVersion := opts.Get("program-version", "undefined") 14 | app := cli.App(progName, "Paperless office") 15 | 16 | app.Version("version v", fmt.Sprintf("%s: %s\nBuilt with: %s/%s on %s/%s", 17 | progName, progVersion, runtime.Compiler, runtime.Version(), 18 | runtime.GOOS, runtime.GOARCH)) 19 | 20 | optDbFile := app.StringOpt("d dbfile", "data/paperless.sqlite3", 21 | "The database file.") 22 | optImageDir := app.StringOpt("i image-directory", "data/images", 23 | "Directory to save the images in") 24 | optListenAddr := app.StringOpt("a address", ":8078", "Listen address and port") 25 | 26 | optPrintRoutes := app.BoolOpt("print-routes", false, 27 | "A debug option to print the REST API") 28 | 29 | app.Action = func() { 30 | opts.Set("database-file", *optDbFile) 31 | opts.Set("image-directory", *optImageDir) 32 | opts.Set("listen-address", *optListenAddr) 33 | 34 | if *optPrintRoutes { 35 | opts.Set("print-routes", "t") 36 | } 37 | } 38 | 39 | return app.Run(args) 40 | } 41 | -------------------------------------------------------------------------------- /lib/model.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "time" 7 | ) 8 | 9 | type Image struct { 10 | // in img 11 | Id int 12 | Checksum string 13 | Fileid string 14 | ScanDate time.Time 15 | AddDate time.Time 16 | InterpretDate time.Time 17 | ProcessLog string 18 | Filename string 19 | 20 | // in imgtext 21 | Text string 22 | Comment string 23 | 24 | // in tags 25 | Tags []Tag 26 | } 27 | 28 | func (i *Image) imgFile(basedir, kind, extension string) string { 29 | ret, _ := filepath.Abs(filepath.Join(basedir, 30 | fmt.Sprintf("%05d-%s.%s", i.Id, kind, extension))) 31 | return ret 32 | } 33 | func (i *Image) OrigFile(basedir string) string { 34 | return i.imgFile(basedir, "original", i.Fileid) 35 | } 36 | 37 | func (i *Image) TxtFile(basedir string) string { 38 | return i.imgFile(basedir, "contents", "txt") 39 | } 40 | 41 | func (i *Image) CleanFile(basedir string) string { 42 | return i.imgFile(basedir, "clean", "jpg") 43 | } 44 | 45 | func (i *Image) ThumbFile(basedir string) string { 46 | return i.imgFile(basedir, "thumbnail", "jpg") 47 | } 48 | 49 | type Tag struct { 50 | Id int 51 | Name string 52 | Comment string 53 | } 54 | 55 | type Script struct { 56 | Id int 57 | Name string 58 | Script string 59 | } 60 | -------------------------------------------------------------------------------- /web/paperless-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paperless-frontend", 3 | "description": "Paperless Office", 4 | "version": "1.2.0", 5 | "author": "Kalle Kankare ", 6 | "private": true, 7 | "scripts": { 8 | "serve": "vue-cli-service serve", 9 | "build": "vue-cli-service build", 10 | "lint": "vue-cli-service lint" 11 | }, 12 | "dependencies": { 13 | "axios": "^0.19.2", 14 | "bootstrap": "^3.4.1", 15 | "domurl": "^2.3.4", 16 | "core-js": "^3.6.5", 17 | "vue": "^2.6.11", 18 | "vue-js-modal": "^1.3.35", 19 | "vuejs-paginate": "^2.1.0" 20 | }, 21 | "devDependencies": { 22 | "@vue/cli-plugin-babel": "~4.4.0", 23 | "@vue/cli-plugin-eslint": "~4.4.0", 24 | "@vue/cli-service": "~4.4.0", 25 | "babel-eslint": "^10.1.0", 26 | "eslint": "^6.7.2", 27 | "eslint-plugin-vue": "^6.2.2", 28 | "vue-template-compiler": "^2.6.11" 29 | }, 30 | "eslintConfig": { 31 | "root": true, 32 | "env": { 33 | "node": true 34 | }, 35 | "extends": [ 36 | "plugin:vue/essential", 37 | "eslint:recommended" 38 | ], 39 | "parserOptions": { 40 | "parser": "babel-eslint" 41 | }, 42 | "rules": {} 43 | }, 44 | "browserslist": [ 45 | "> 1%", 46 | "last 2 versions", 47 | "not dead" 48 | ] 49 | } 50 | -------------------------------------------------------------------------------- /lib/search_test.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/davecgh/go-spew/spew" 7 | ) 8 | 9 | type pstr map[int]string 10 | 11 | func Test_lexer_Init(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | input string 15 | output []TokenType 16 | posstr pstr 17 | }{ 18 | {"Empty search", "", 19 | []TokenType{TokEOF}, nil}, 20 | {"Text search", "some text", 21 | []TokenType{TokString, TokString, TokEOF}, pstr{0: "some", 1: "text"}}, 22 | {"Text within parentheses", "(text)", 23 | []TokenType{TokParOpen, TokString, TokParClose, TokEOF}, pstr{1: "text"}}, 24 | {"Text with unbalanced parens", "(text", 25 | []TokenType{TokParOpen, TokString, TokError, TokEOF}, 26 | pstr{1: "text"}}, 27 | {"Multiple parens", "(())", 28 | []TokenType{TokParOpen, TokParOpen, TokParClose, TokParClose, TokEOF}, 29 | pstr{0: "("}}, 30 | {"Multiple parens 2", "() ()", 31 | []TokenType{TokParOpen, TokParClose, TokParOpen, TokParClose, TokEOF}, nil}, 32 | {"Special operators", "a AND b OR c", 33 | []TokenType{TokString, TokAnd, TokString, TokOr, TokString, TokEOF}, 34 | pstr{0: "a", 4: "c"}}, 35 | {"Just AND", "AND", 36 | []TokenType{TokAnd, TokEOF}, nil}, 37 | } 38 | for _, tt := range tests { 39 | t.Run(tt.name, func(t *testing.T) { 40 | failed := false 41 | l := &lexer{} 42 | l.Init(tt.input) 43 | 44 | // var result []TokenType 45 | var tokens []Token 46 | result := make([]TokenType, 0, len(tt.output)) 47 | for { 48 | t := l.NextToken() 49 | tokens = append(tokens, t) 50 | result = append(result, t.Type) 51 | if t.Type == TokEOF { 52 | break 53 | } 54 | } 55 | 56 | if !structEquals(tt.output, result) { 57 | t.Error("List of tokens differs:\n", diffStr(tt.output, result)) 58 | failed = true 59 | } 60 | for i, val := range tt.posstr { 61 | if i >= len(tokens) { 62 | t.Error("List should contain token in position", i, "with value", val) 63 | failed = true 64 | continue 65 | } 66 | 67 | if tokens[i].Value != val { 68 | t.Error("Token", i, "Expected:", val, "Got:", tokens[i].Value) 69 | failed = true 70 | continue 71 | } 72 | } 73 | 74 | if failed { 75 | t.Log("Outputted list of tokens:\n", spew.Sdump(tokens)) 76 | } 77 | l.Deinit() 78 | }) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /lib/util.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "crypto/sha1" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | ) 11 | 12 | /// Generic functionality 13 | func Checksum(data []byte) string { 14 | return fmt.Sprintf("sha1:%x", sha1.Sum(data)) 15 | } 16 | 17 | func ChecksumFile(path string) (sum string, err error) { 18 | var data []byte 19 | data, err = ioutil.ReadFile(path) 20 | if err != nil { 21 | return "", err 22 | } 23 | return Checksum(data), err 24 | } 25 | 26 | // MkdirParents creates all parent directories of the given path or returns an 27 | // error if they couldn't be created 28 | func MkdirParents(filename string) error { 29 | dirs := filepath.Dir(filename) 30 | return os.MkdirAll(dirs, 0755) 31 | } 32 | 33 | // Similar to filepath.Abs except the root directory is given 34 | func PathAbs(rootdir, path string) string { 35 | if filepath.IsAbs(path) { 36 | return path 37 | } 38 | return filepath.Join(rootdir, path) 39 | } 40 | 41 | /// Parallel processing 42 | 43 | type Runner struct { 44 | RunnerCount int 45 | ChanBuffer int 46 | Job func(string) 47 | Finalize func(string) 48 | } 49 | 50 | func CreateRunner(runners int) Runner { 51 | return Runner{ 52 | RunnerCount: runners, 53 | ChanBuffer: 10, 54 | Finalize: func(string) {}, 55 | } 56 | } 57 | 58 | func (run Runner) Do(data []string) { 59 | var wg sync.WaitGroup 60 | input := make(chan string, run.ChanBuffer) 61 | 62 | wg.Add(run.RunnerCount) 63 | for i := 0; i < run.RunnerCount; i++ { 64 | go func() { 65 | defer wg.Done() 66 | for item := range input { 67 | func() { 68 | defer run.Finalize(item) 69 | run.Job(item) 70 | }() 71 | } 72 | }() 73 | } 74 | 75 | for _, item := range data { 76 | input <- item 77 | } 78 | close(input) 79 | wg.Wait() 80 | 81 | } 82 | 83 | type RunnerPool struct { 84 | runnerCount int 85 | jobChan chan Job 86 | jobChanSize int 87 | wait sync.WaitGroup 88 | } 89 | 90 | type Job struct { 91 | Job func() 92 | Finalize func() 93 | } 94 | 95 | func CreatePool(runners int) RunnerPool { 96 | ret := RunnerPool{ 97 | runnerCount: runners, 98 | jobChanSize: 10, 99 | jobChan: make(chan Job), 100 | } 101 | 102 | ret.wait.Add(runners) 103 | for i := 0; i < runners; i++ { 104 | go func() { 105 | defer ret.wait.Done() 106 | for item := range ret.jobChan { 107 | func() { 108 | defer item.Finalize() 109 | item.Job() 110 | }() 111 | } 112 | }() 113 | } 114 | return ret 115 | } 116 | 117 | func (pool *RunnerPool) Delete() { 118 | close(pool.jobChan) 119 | } 120 | 121 | func (pool *RunnerPool) Do(job Job) { 122 | pool.jobChan <- job 123 | } 124 | 125 | var Pool *RunnerPool 126 | 127 | func CreateDefaultPool(runners int) { 128 | pool := CreatePool(runners) 129 | Pool = &pool 130 | } 131 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | * Paperless office interface 2 | 3 | Reads the texts of images and makes them browsable and searchable via HTTP. 4 | 5 | ** Requirements 6 | 7 | On an Ubuntu system run the following to install prerequisites for the OCR 8 | and processing: 9 | 10 | #+begin_src shell 11 | sudo apt-get install sqlite3 tesseract-ocr-osd tesseract-ocr-fin tesseract-ocr imagemagick unpaper 12 | #+end_src 13 | 14 | Also Golang is required. Tested with go 1.8. 15 | 16 | ** Building the application 17 | 18 | Clone this repository and run the following command: 19 | 20 | #+begin_src shell 21 | go build 22 | #+end_src 23 | 24 | ** Usage 25 | 26 | Start the web server with the following: 27 | 28 | #+begin_src shell 29 | ./paperless 30 | #+end_src 31 | 32 | This starts a web-server to the port 8078. See the '--help' argument for 33 | command line options 34 | 35 | File uploading happens with a browser. There is the '+' button which opens 36 | a panel where one can drag-and-drop images to OCR. 37 | 38 | ** Running this inside Docker 39 | 40 | If you have Docker properly set up, you can run this inside docker with the 41 | following: 42 | 43 | #+begin_src shell 44 | cd docker 45 | ./run.sh 46 | #+end_src 47 | 48 | This should start an Ubuntu 16.04 Docker instance where the program is 49 | running. 50 | 51 | ** Uploader application 52 | 53 | In the uploader directory there is a go-application that can be used to 54 | send batches of tagged images to the server. 55 | 56 | Build it with the following: 57 | 58 | #+begin_src shell 59 | cd uploader 60 | go build 61 | #+end_src 62 | 63 | Example run: 64 | 65 | #+begin_src shell 66 | ./uploader -t important,dontremove http://localhost:8078 important-01.jpg important-02.jpg 67 | #+end_src 68 | 69 | Usage is printed with the --help argument. 70 | 71 | ** Developing the frontend 72 | 73 | The frontend development requires Nodejs and NPM. Therefore the 74 | environment can be set up with: 75 | 76 | #+begin_src shell 77 | cd web/paperless-frontend 78 | npm install 79 | #+end_src 80 | 81 | To set up a running environment do the following: 82 | 83 | 1. Start the paperless -application to get the backend running in port 8078. 84 | 85 | 2. Start the webpack-dev-server with: 86 | 87 | #+begin_src shell 88 | cd web/paperless-frontend 89 | npm run serve 90 | #+end_src 91 | 92 | 3. This will start the frontend to port 8080 that connects to the backend 93 | in port 8078. 94 | 95 | 96 | The frontend is embedded in the final binary. To update the changes from 97 | the frontend development files to the binary, do the following: 98 | 99 | 1. Install the 'esc' file embedder, so the esc can be found in the $PATH. 100 | 101 | #+begin_src shell 102 | go get -i github.com/mjibson/esc 103 | #+end_src 104 | 105 | 2. Build the dist-package of the frontend: 106 | 107 | #+begin_src shell 108 | cd web/paperless-frontend 109 | npm run build 110 | #+end_src 111 | 112 | 3. Regenerate the lib/web-generated.go with: 113 | 114 | #+begin_src shell 115 | cd lib 116 | go generate 117 | #+end_src 118 | 119 | 4. Build normally, test and commit the generated files. 120 | 121 | ** License 122 | 123 | MIT license 124 | -------------------------------------------------------------------------------- /lib/image.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | util "github.com/kopoli/go-util" 14 | ) 15 | 16 | // SaveImage saves the image to db and starts to process it 17 | func SaveImage(filename string, data []byte, db *db, destdir string, tags string) (ret Image, err error) { 18 | 19 | supportedTypes := map[string]string{ 20 | "image/gif": "gif", 21 | "image/png": "png", 22 | "image/jpeg": "jpg", 23 | "image/bmp": "bmp", 24 | } 25 | ft := http.DetectContentType(data) 26 | var ok bool 27 | if ret.Fileid, ok = supportedTypes[ft]; !ok { 28 | err = util.E.New("Unsupported image type: %s", ft) 29 | return 30 | } 31 | 32 | ret.Checksum = Checksum(data) 33 | ret.AddDate = time.Now() 34 | ret.ScanDate = time.Now() // TODO 35 | ret.Filename = filename 36 | 37 | taglist := strings.Split(tags, ",") 38 | if len(taglist) > 0 { 39 | ret.Tags = make([]Tag, len(taglist)) 40 | 41 | for i := range taglist { 42 | ret.Tags[i].Name = strings.Trim(taglist[i], " \t\n\r") 43 | 44 | // Add a tag, ignore errors 45 | _, _ = db.addTag(ret.Tags[i]) 46 | } 47 | } 48 | 49 | ret, err = db.addImage(ret) 50 | if err != nil { 51 | return 52 | } 53 | 54 | file := bytes.NewReader(data) 55 | fp, err := os.OpenFile(ret.OrigFile(destdir), os.O_WRONLY|os.O_CREATE, 0666) 56 | if err != nil { 57 | _ = db.deleteImage(ret) 58 | return 59 | } 60 | defer fp.Close() 61 | 62 | _, err = io.Copy(fp, file) 63 | return 64 | } 65 | 66 | func ProcessImage(img *Image, scriptname string, db *db, destdir string) (err error) { 67 | script := ` 68 | unpaper --version 69 | convert -version 70 | tesseract --version 71 | 72 | convert -depth 8 $input pnm:$tmpUnpaper.pnm 73 | 74 | unpaper -vv -s a4 -l single -dv 3.0 -dr 80.0 --overwrite $tmpUnpaper.pnm $tmpConvert 75 | 76 | convert -normalize -colorspace Gray pnm:$tmpConvert pnm:$tmpTesseract 77 | 78 | tesseract -l fin -psm 1 $tmpTesseract stdout > $contents 79 | 80 | convert -trim -quality 80% +repage -type optimize pnm:$tmpConvert $cleanout 81 | 82 | convert -trim -quality 80% +repage -type optimize -thumbnail 200x200> pnm:$tmpConvert $thumbout 83 | 84 | ` 85 | 86 | ch, err := NewCmdChainScript(script) 87 | if err != nil { 88 | return 89 | } 90 | 91 | buf := &bytes.Buffer{} 92 | s := Status{ 93 | Environment: ch.Environment, 94 | Log: buf, 95 | } 96 | s.Constants = map[string]string{ 97 | "input": img.OrigFile(destdir), 98 | "contents": img.TxtFile(destdir), 99 | "cleanout": img.CleanFile(destdir), 100 | "thumbout": img.ThumbFile(destdir), 101 | } 102 | s.AllowedCommands = map[string]bool{ 103 | "convert": true, 104 | "unpaper": true, 105 | "tesseract": true, 106 | "file": true, 107 | "cat": true, 108 | } 109 | 110 | fmt.Fprintln(s.Log, "# Running the script named:", scriptname) 111 | 112 | err = RunCmdChain(ch, &s) 113 | if err != nil { 114 | return 115 | } 116 | 117 | data, err := ioutil.ReadFile(img.TxtFile(destdir)) 118 | // Ignore the error if the text-file was not generated 119 | if err != nil { 120 | data = []byte{} 121 | } 122 | 123 | img.InterpretDate = time.Now() 124 | img.ProcessLog = buf.String() 125 | img.Text = string(data) 126 | 127 | err = db.updateImage(*img) 128 | return 129 | } 130 | 131 | // DeleteImage deletes the image's files and data from the database 132 | func DeleteImage(img *Image, db *db, destdir string) error { 133 | var err error 134 | ret := util.NewErrorList("Deleting image data failed") 135 | 136 | err = db.deleteImage(*img) 137 | if err != nil { 138 | ret.Append(err) 139 | } 140 | 141 | files := []string{ 142 | img.OrigFile(destdir), 143 | img.TxtFile(destdir), 144 | img.CleanFile(destdir), 145 | img.ThumbFile(destdir), 146 | } 147 | 148 | for i := range files { 149 | err = os.Remove(files[i]) 150 | if err != nil { 151 | ret.Append(util.E.Annotate(err, "Removing file ", files[i], "failed")) 152 | } 153 | } 154 | 155 | if ret.IsEmpty() { 156 | return nil 157 | } 158 | 159 | return ret 160 | } 161 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/gamegos/jsend v0.0.0-20151011171802-f47e169f3d76 h1:I+EQEdxMrj5Wg+lAN99Ev8sCAmzHhr39Ez5hmSE9AYo= 5 | github.com/gamegos/jsend v0.0.0-20151011171802-f47e169f3d76/go.mod h1:HqmpnMATlmwXZIzrCMuMRlmYo8l3SoxJHIzew1sl1dU= 6 | github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 7 | github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4= 8 | github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 9 | github.com/go-chi/docgen v1.0.5 h1:TiGvJAuVPZJ9zFSwoF52eORe0SztOYqf9C79LVw/xbY= 10 | github.com/go-chi/docgen v1.0.5/go.mod h1:Nm4H4RaynSlvTexxWYWwXBzrwZKRE00MrkIIcJelhWM= 11 | github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= 12 | github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= 13 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 14 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/jawher/mow.cli v1.1.0 h1:NdtHXRc0CwZQ507wMvQ/IS+Q3W3x2fycn973/b8Zuk8= 16 | github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= 17 | github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= 18 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 19 | github.com/kopoli/go-util v0.0.0-20180512085434-f355416491c0 h1:EmaG3ddioKokz81XVk5X5c4uD5u7pXGAS6xw8ZwPzD8= 20 | github.com/kopoli/go-util v0.0.0-20180512085434-f355416491c0/go.mod h1:8C22fL3vKl0U3JGHuCIf3LG7TH2NNbBqIjDpbZYW990= 21 | github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= 22 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 23 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 24 | github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= 25 | github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 26 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 27 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 31 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 32 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 33 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 34 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 35 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM= 36 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 37 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= 38 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 39 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 40 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 42 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 43 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 44 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 45 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= 46 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 49 | -------------------------------------------------------------------------------- /lib/search.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "unicode" 7 | "unicode/utf8" 8 | ) 9 | 10 | type TokenType int 11 | 12 | const ( 13 | eof rune = -1 14 | TokEOF TokenType = -1 15 | TokError TokenType = iota 16 | TokCanceled 17 | TokAnd 18 | TokOr 19 | TokNot 20 | TokQuote 21 | TokString 22 | TokParOpen 23 | TokParClose 24 | ) 25 | 26 | type Query struct { 27 | } 28 | 29 | type Token struct { 30 | Type TokenType 31 | Value string 32 | Pos int 33 | } 34 | 35 | type stateFunc func(*lexer) stateFunc 36 | 37 | type lexer struct { 38 | input string 39 | start int 40 | pos int 41 | width int 42 | 43 | states []stateFunc 44 | initialized bool 45 | 46 | tokens chan Token 47 | } 48 | 49 | // Lexer public interface 50 | 51 | func (l *lexer) Init(input string) { 52 | l.Deinit() 53 | l.input = input 54 | l.start = 0 55 | l.pos = 0 56 | l.width = 0 57 | l.states = nil 58 | l.tokens = make(chan Token) 59 | l.initialized = true 60 | 61 | go l.run() 62 | } 63 | 64 | func (l *lexer) NextToken() (ret Token) { 65 | ret, ok := <-l.tokens 66 | if !ok { 67 | ret = Token{ 68 | Type: TokEOF, 69 | } 70 | } 71 | return 72 | } 73 | 74 | func (l *lexer) Deinit() { 75 | if l.initialized { 76 | for t := l.NextToken(); t.Type != TokEOF; { 77 | } 78 | l.initialized = false 79 | } 80 | } 81 | 82 | func (l *lexer) emit(t TokenType) { 83 | if l.hasContents() { 84 | l.tokens <- Token{t, l.input[l.start:l.pos], l.start} 85 | l.start = l.pos 86 | } 87 | } 88 | 89 | func (l *lexer) next() rune { 90 | if l.pos >= len(l.input) { 91 | l.width = 0 92 | return eof 93 | } 94 | 95 | var ret rune 96 | ret, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) 97 | l.pos += l.width 98 | return ret 99 | } 100 | 101 | func (l *lexer) rewind() { 102 | l.pos -= l.width 103 | } 104 | 105 | func (l *lexer) ignore() { 106 | l.start = l.pos 107 | } 108 | 109 | func (l *lexer) hasPrefix(prefix string) bool { 110 | if strings.HasPrefix(l.input[l.pos:], prefix) { 111 | return true 112 | } 113 | return false 114 | } 115 | 116 | func (l *lexer) hasContents() bool { 117 | return l.pos > l.start 118 | } 119 | 120 | func (l *lexer) isEqual(s string) bool { 121 | return s == l.input[l.start:l.pos] 122 | } 123 | 124 | func (l *lexer) errorf(format string, args ...interface{}) stateFunc { 125 | l.tokens <- Token{TokError, fmt.Sprintf(format, args...), l.pos} 126 | l.ignore() 127 | return nil 128 | } 129 | 130 | func (l *lexer) push(s stateFunc) { 131 | l.states = append(l.states, s) 132 | } 133 | 134 | func (l *lexer) pop() stateFunc { 135 | if l.states == nil || len(l.states) == 0 { 136 | return l.errorf("Internal error: Popping an empty stack") 137 | } 138 | 139 | ret := l.states[len(l.states)-1] 140 | l.states = l.states[:len(l.states)-1] 141 | return ret 142 | } 143 | 144 | func (l *lexer) lexHandleContent(r rune, this stateFunc) stateFunc { 145 | switch { 146 | case r == eof: 147 | return l.errorf("Unexpected end of string") 148 | case r == '"': 149 | l.push(lexTop) 150 | return lexQuoted 151 | case r == '(': 152 | l.emit(TokParOpen) 153 | l.push(this) 154 | return lexParentheses 155 | case unicode.IsSpace(r): 156 | l.ignore() 157 | } 158 | l.push(this) 159 | return lexWord 160 | } 161 | 162 | func lexTop(l *lexer) stateFunc { 163 | for { 164 | r := l.next() 165 | if r == eof { 166 | if l.hasContents() { 167 | l.emit(TokString) 168 | } 169 | break 170 | } 171 | return l.lexHandleContent(r, lexTop) 172 | } 173 | 174 | l.emit(TokEOF) 175 | return nil 176 | } 177 | 178 | func lexWord(l *lexer) stateFunc { 179 | r := l.next() 180 | switch { 181 | case r == eof || unicode.IsSpace(r) || r == '(' || r == ')' || r == '"': 182 | l.rewind() 183 | reserved := map[string]TokenType{ 184 | "AND": TokAnd, 185 | "OR": TokOr, 186 | } 187 | for k, v := range reserved { 188 | if l.isEqual(k) { 189 | l.emit(v) 190 | return l.pop() 191 | } 192 | } 193 | 194 | l.emit(TokString) 195 | return l.pop() 196 | } 197 | return lexWord 198 | } 199 | 200 | func lexParentheses(l *lexer) stateFunc { 201 | r := l.next() 202 | switch r { 203 | case eof: 204 | return l.errorf("Unmatched parenthesis") 205 | case ')': 206 | l.emit(TokParClose) 207 | return l.pop() 208 | } 209 | return l.lexHandleContent(r, lexParentheses) 210 | } 211 | 212 | func lexQuoted(l *lexer) stateFunc { 213 | r := l.next() 214 | switch r { 215 | case '"': 216 | l.emit(TokString) 217 | return l.pop() 218 | case eof: 219 | return l.errorf("Unbalanced quotes") 220 | } 221 | return lexQuoted 222 | } 223 | 224 | func (l *lexer) run() { 225 | for s := lexTop; s != nil; { 226 | s = s(l) 227 | } 228 | close(l.tokens) 229 | } 230 | -------------------------------------------------------------------------------- /uploader/uploader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "mime/multipart" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "path/filepath" 13 | "runtime" 14 | "sync" 15 | "time" 16 | 17 | cli "github.com/jawher/mow.cli" 18 | util "github.com/kopoli/go-util" 19 | ) 20 | 21 | const ( 22 | APIPath string = "/api/v1/image" 23 | ) 24 | 25 | type Config struct { 26 | opts util.Options 27 | files []string 28 | jobCount int 29 | timeout int 30 | } 31 | 32 | type JsendMsg struct { 33 | Message string `json:message` 34 | Status string `json:status` 35 | } 36 | 37 | func runCli(c *Config, args []string) (err error) { 38 | progName := c.opts.Get("program-name", "paperless-uploader") 39 | progVersion := c.opts.Get("program-version", "undefined") 40 | app := cli.App(progName, "Upload tool to Paperless Office server.") 41 | 42 | app.Version("version", fmt.Sprintf("%s: %s\nBuilt with: %s/%s on %s/%s", 43 | progName, progVersion, runtime.Compiler, runtime.Version(), 44 | runtime.GOOS, runtime.GOARCH)) 45 | 46 | app.Spec = "[OPTIONS] URL FILES..." 47 | 48 | optJobs := app.IntOpt("j jobs", runtime.NumCPU(), "Number of concurrent uploads") 49 | optVerbose := app.BoolOpt("v verbose", false, "Print upload statuses") 50 | optTags := app.StringOpt("t tag", "", "Comma separated list of tags.") 51 | optTimeout := app.IntOpt("timeout", 60, "HTTP timeout in seconds") 52 | argURL := app.StringArg("URL", "", "The upload HTTP URL.") 53 | argFiles := app.StringsArg("FILES", []string{}, "Image files to upload.") 54 | 55 | app.Action = func() { 56 | c.opts.Set("tags", *optTags) 57 | if *optVerbose { 58 | c.opts.Set("verbose", "t") 59 | } 60 | c.opts.Set("url", *argURL) 61 | c.jobCount = *optJobs 62 | c.timeout = *optTimeout 63 | 64 | c.files = *argFiles 65 | } 66 | 67 | err = app.Run(args) 68 | if err != nil { 69 | return 70 | } 71 | 72 | return 73 | } 74 | 75 | func checkArguments(c *Config) (err error) { 76 | var u *url.URL 77 | 78 | urlstr := c.opts.Get("url", "") 79 | u, err = url.Parse(urlstr) 80 | if err != nil { 81 | return 82 | } 83 | 84 | apipath, err := url.Parse(APIPath) 85 | if err != nil { 86 | return 87 | } 88 | 89 | // Set the proper URL here 90 | urlstr = u.ResolveReference(apipath).String() 91 | c.opts.Set("url", urlstr) 92 | 93 | if !u.IsAbs() { 94 | err = util.E.New("Supplied URL must be absolute: %s", urlstr) 95 | return 96 | } 97 | 98 | for i := range c.files { 99 | var st os.FileInfo 100 | st, err = os.Stat(c.files[i]) 101 | if err != nil || !st.Mode().IsRegular() { 102 | err = util.E.New("Invalid file: %s", c.files[i]) 103 | return 104 | } 105 | } 106 | 107 | if c.opts.IsSet("verbose") { 108 | fmt.Println("Uploading to URL:", urlstr) 109 | } 110 | 111 | return 112 | } 113 | 114 | func uploadFile(c *Config, file string) (err error) { 115 | fp, err := os.Open(file) 116 | if err != nil { 117 | return 118 | } 119 | defer fp.Close() 120 | 121 | body := &bytes.Buffer{} 122 | writer := multipart.NewWriter(body) 123 | part, err := writer.CreateFormFile("image", filepath.Base(file)) 124 | if err != nil { 125 | return 126 | } 127 | 128 | _, err = io.Copy(part, fp) 129 | if err != nil { 130 | return 131 | } 132 | 133 | err = writer.WriteField("tags", c.opts.Get("tags", "")) 134 | if err != nil { 135 | return 136 | } 137 | err = writer.Close() 138 | if err != nil { 139 | return 140 | } 141 | 142 | req, err := http.NewRequest("POST", c.opts.Get("url", ""), body) 143 | if err != nil { 144 | return 145 | } 146 | 147 | req.Header.Set("Content-Type", writer.FormDataContentType()) 148 | 149 | client := &http.Client{ 150 | Timeout: time.Duration(c.timeout) * time.Second, 151 | } 152 | resp, err := client.Do(req) 153 | if err != nil { 154 | return 155 | } 156 | 157 | body.Reset() 158 | _, err = body.ReadFrom(resp.Body) 159 | if err != nil { 160 | return 161 | } 162 | resp.Body.Close() 163 | 164 | var jmsg JsendMsg 165 | 166 | err = json.Unmarshal(body.Bytes(), &jmsg) 167 | if err != nil { 168 | // Ignore improper jsend messages 169 | jmsg.Message = "" 170 | } 171 | 172 | if c.opts.IsSet("verbose") { 173 | fmt.Println("Uploaded:", file, "Response:", body.String()) 174 | } 175 | 176 | if resp.StatusCode != http.StatusCreated { 177 | msg := "" 178 | if jmsg.Message != "" { 179 | msg = fmt.Sprintf(" (%s)", jmsg.Message) 180 | } 181 | err = util.E.New("Server responded with: %d%s", resp.StatusCode, msg) 182 | return 183 | } 184 | 185 | return 186 | } 187 | 188 | func upload(c *Config) (err error) { 189 | jobs := make(chan string, 10) 190 | wg := sync.WaitGroup{} 191 | worker := func(jobs <-chan string) { 192 | var err error 193 | for file := range jobs { 194 | err = uploadFile(c, file) 195 | if err != nil { 196 | fmt.Printf("%s: failed: %s\n", file, err) 197 | } else { 198 | fmt.Printf("%s: Uploaded ok\n", file) 199 | } 200 | } 201 | wg.Done() 202 | } 203 | 204 | for i := 0; i < c.jobCount; i++ { 205 | wg.Add(1) 206 | go worker(jobs) 207 | } 208 | 209 | for i := range c.files { 210 | jobs <- c.files[i] 211 | } 212 | 213 | close(jobs) 214 | wg.Wait() 215 | 216 | return nil 217 | } 218 | 219 | func main() { 220 | config := &Config{ 221 | opts: util.NewOptions(), 222 | } 223 | 224 | config.opts.Set("program-name", os.Args[0]) 225 | 226 | err := runCli(config, os.Args) 227 | if err != nil { 228 | err = util.E.Annotate(err, "Command line parsing failed") 229 | goto error 230 | } 231 | 232 | err = checkArguments(config) 233 | if err != nil { 234 | err = util.E.Annotate(err, "Invalid arguments") 235 | goto error 236 | } 237 | 238 | err = upload(config) 239 | if err != nil { 240 | goto error 241 | } 242 | 243 | os.Exit(0) 244 | 245 | error: 246 | fmt.Fprintln(os.Stderr, "Error:", err) 247 | os.Exit(1) 248 | } 249 | -------------------------------------------------------------------------------- /lib/cmdchain.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "regexp" 10 | "strings" 11 | "unicode" 12 | 13 | "github.com/kopoli/go-util" 14 | ) 15 | 16 | // Environment is the run environment for each command. It is supplied as part 17 | // of Status for a Link when it is Run or Validated. 18 | type Environment struct { 19 | 20 | // Constants are variables that are defined before a command chain is run 21 | Constants map[string]string 22 | 23 | // Tempfiles are constants that house a name of a temporary file. The 24 | // files are created before the chain is run and they are removed at 25 | // the end. 26 | TempFiles []string 27 | 28 | // The directory where the commands are run. This is a created 29 | // temporary directory 30 | RootDir string 31 | 32 | // AllowedCommands contain the commands that are allowed. If this is 33 | // nil, all commands are allowed. 34 | AllowedCommands map[string]bool 35 | 36 | initialized bool 37 | } 38 | 39 | func (e *Environment) initEnv() (err error) { 40 | if e.initialized { 41 | return 42 | } 43 | 44 | if e.Constants == nil { 45 | return util.E.New("field Constants not initialized") 46 | } 47 | 48 | e.RootDir, err = ioutil.TempDir("", "chain") 49 | if err != nil { 50 | return util.E.Annotate(err, "rootdir creation failed") 51 | } 52 | 53 | var fp *os.File 54 | for _, name := range e.TempFiles { 55 | fp, err = ioutil.TempFile(e.RootDir, "tmp") 56 | if err != nil { 57 | err = util.E.Annotate(err, "tempfile creation failed") 58 | e.initialized = true 59 | e2 := e.deinitEnv() 60 | if e2 != nil { 61 | err = util.E.Annotate(err, "temproot removal failed:", e2) 62 | e.initialized = false 63 | } 64 | return 65 | } 66 | e.Constants[name] = fp.Name() 67 | fp.Close() 68 | } 69 | 70 | e.initialized = true 71 | return 72 | } 73 | 74 | func (e *Environment) deinitEnv() (err error) { 75 | if !e.initialized { 76 | return 77 | } 78 | 79 | if !strings.HasPrefix(e.RootDir, os.TempDir()) || e.RootDir == os.TempDir() { 80 | err = util.E.New("Temporary directory path is corrupted: %s", e.RootDir) 81 | return 82 | } 83 | 84 | err = os.RemoveAll(e.RootDir) 85 | if err != nil { 86 | err = util.E.Annotate(err, "tempdir removal failed:") 87 | } 88 | 89 | // Clear the temporary file and directory names 90 | e.RootDir = "" 91 | for _, n := range e.TempFiles { 92 | e.Constants[n] = "" 93 | } 94 | 95 | e.initialized = false 96 | return 97 | } 98 | 99 | func (e *Environment) validate() (err error) { 100 | if len(e.RootDir) == 0 { 101 | return util.E.New("the RootDir must be defined") 102 | } 103 | info, err := os.Stat(e.RootDir) 104 | if err != nil || info.Mode()&os.ModeDir == 0 { 105 | return util.E.Annotate(err, "file ", e.RootDir, " is not a proper directory") 106 | } 107 | 108 | return 109 | } 110 | 111 | // Status is the runtime status of the command chain 112 | type Status struct { 113 | // The log output will be written to this 114 | Log io.Writer 115 | 116 | Environment 117 | } 118 | 119 | type Link interface { 120 | Validate(*Environment) error 121 | Run(*Status) error 122 | } 123 | 124 | type CmdChain struct { 125 | 126 | //TODO remove this (this should come from outside) 127 | Environment 128 | 129 | Links []Link 130 | } 131 | 132 | func (c *CmdChain) Validate(e *Environment) (err error) { 133 | for _, l := range c.Links { 134 | err = l.Validate(e) 135 | if err != nil { 136 | return 137 | } 138 | } 139 | return 140 | } 141 | 142 | func (c *CmdChain) Run(s *Status) (err error) { 143 | err = c.Validate(&s.Environment) 144 | if err != nil { 145 | return 146 | } 147 | 148 | for i := range c.Links { 149 | err = c.Links[i].Run(s) 150 | if err != nil { 151 | return 152 | } 153 | } 154 | 155 | return 156 | } 157 | 158 | func RunCmdChain(c *CmdChain, s *Status) (err error) { 159 | err = s.Environment.initEnv() 160 | if err != nil { 161 | return 162 | } 163 | 164 | err = c.Run(s) 165 | e2 := s.Environment.deinitEnv() 166 | if e2 != nil { 167 | err = util.E.Annotate(err, "cmdchain deinit failed: ", e2) 168 | } 169 | 170 | return 171 | } 172 | 173 | //////////////////////////////////////////////////////////// 174 | 175 | type Cmd struct { 176 | Cmd []string 177 | } 178 | 179 | // NewCmd creates a new Cmd from given command string 180 | func NewCmd(cmdstr string) (c *Cmd, err error) { 181 | command := splitWsQuote(cmdstr) 182 | 183 | if len(command) == 0 { 184 | return nil, util.E.New("A command could not be parsed from: %s", cmdstr) 185 | } 186 | 187 | c = &Cmd{command} 188 | 189 | _, err = exec.LookPath(c.Cmd[0]) 190 | if err != nil { 191 | return nil, util.E.Annotate(err, "Command", c.Cmd[0], "could not be found") 192 | 193 | } 194 | 195 | return 196 | } 197 | 198 | // Validate makes sure the command is proper and can be run 199 | func (c *Cmd) Validate(e *Environment) (err error) { 200 | if len(c.Cmd) == 0 { 201 | return util.E.New("command string must be non-empty") 202 | } 203 | 204 | if e.AllowedCommands != nil { 205 | if _, ok := e.AllowedCommands[c.Cmd[0]]; ok != true { 206 | return util.E.New("command is not allowed") 207 | } 208 | } 209 | 210 | _, err = exec.LookPath(c.Cmd[0]) 211 | if err != nil { 212 | return 213 | } 214 | 215 | err = e.validate() 216 | 217 | for idx, a := range c.Cmd { 218 | consts := parseConsts(a) 219 | if len(consts) > 0 { 220 | for _, co := range consts { 221 | if _, ok := e.Constants[co]; !ok { 222 | return util.E.New("constant \"%s\" not defined", co) 223 | } 224 | } 225 | } 226 | 227 | // Output redirection to a file 228 | if a == ">" && (idx == len(c.Cmd)-1 || c.Cmd[idx+1] == "") { 229 | return util.E.New("The output redirection requires a string") 230 | } 231 | } 232 | 233 | return 234 | } 235 | 236 | func (c *Cmd) Run(s *Status) (err error) { 237 | err = c.Validate(&s.Environment) 238 | if err != nil { 239 | return 240 | } 241 | 242 | var args []string 243 | for i := range c.Cmd { 244 | args = append(args, expandConsts(c.Cmd[i], s.Constants)) 245 | } 246 | 247 | if s.Log != nil { 248 | fmt.Fprintln(s.Log, "# Running command:", strings.Join(args, " ")) 249 | } 250 | 251 | var output io.Writer = s.Log 252 | 253 | redirout, pos := getRedirectFile(">", args) 254 | if redirout != "" { 255 | var fp *os.File 256 | redirout = PathAbs(s.RootDir, redirout) 257 | fp, err = os.OpenFile(redirout, os.O_WRONLY | os.O_CREATE, 0666) 258 | if err != nil { 259 | err = util.E.Annotate(err, "Could not open file",redirout,"for redirection") 260 | return 261 | } 262 | defer fp.Close() 263 | output = fp 264 | 265 | // Remove the redirection and the file argument from the command 266 | args = append(args[:pos], args[pos+2:]...) 267 | } 268 | 269 | cmd := exec.Command(args[0], args[1:]...) 270 | cmd.Dir = s.RootDir 271 | cmd.Stdout = output 272 | cmd.Stderr = s.Log 273 | 274 | return cmd.Run() 275 | } 276 | 277 | //////////////////////////////////////////////////////////// 278 | 279 | var ( 280 | constRe = regexp.MustCompile(`\$(\w+)`) 281 | tmpfileConstRe = regexp.MustCompile(`\$(tmp\w+)`) 282 | commentRe = regexp.MustCompile(`#.*$`) 283 | preWhitespaceRe = regexp.MustCompile(`^\s+`) 284 | ) 285 | 286 | // parseConsts parses the constants from a string. Returns a list of constant names 287 | func parseConsts(s string) (ret []string) { 288 | ret = []string{} 289 | 290 | matches := constRe.FindAllStringSubmatch(s, -1) 291 | if matches == nil { 292 | return 293 | } 294 | for _, m := range matches { 295 | ret = append(ret, m[1]) 296 | } 297 | 298 | return 299 | } 300 | 301 | func expandConsts(s string, constants map[string]string) string { 302 | return constRe.ReplaceAllStringFunc(s, func(match string) string { 303 | cs := parseConsts(match) 304 | if len(cs) != 1 { 305 | panic("Invalid Regexp parsing") 306 | } 307 | 308 | ret, ok := constants[cs[0]] 309 | if !ok { 310 | ret = "" 311 | } 312 | 313 | return ret 314 | }) 315 | } 316 | 317 | // Gets the string after the given redir string. If not found, returns empty 318 | // string. 319 | func getRedirectFile(redir string, args []string) (file string, pos int) { 320 | for i := range args { 321 | if args[i] == redir { 322 | if i+1 < len(args) { 323 | file = args[i+1] 324 | pos = i 325 | } 326 | return 327 | } 328 | } 329 | return 330 | } 331 | 332 | // NewCmdChainScript creates a CmdChain from a script where each command is on a separate line. The following syntax elements are supported: 333 | // 334 | // - Empty lines are filtered out. 335 | // 336 | // - Comments start with # and end with EOL. 337 | // 338 | // - Constants are strings that begin with $ and they can be set before running the cmdchain. 339 | // 340 | // - Temporary files are strings that start with $tmp and they are automatically created before running the cmdchain and removed afterwards. 341 | func NewCmdChainScript(script string) (c *CmdChain, err error) { 342 | c = &CmdChain{} 343 | c.Constants = make(map[string]string) 344 | 345 | for _, line := range strings.Split(script, "\n") { 346 | line = commentRe.ReplaceAllString(line, "") 347 | line = preWhitespaceRe.ReplaceAllString(line, "") 348 | 349 | if len(line) == 0 { 350 | continue 351 | } 352 | 353 | constants := parseConsts(line) 354 | for _, co := range constants { 355 | c.Constants[co] = "" 356 | if tmpfileConstRe.MatchString("$" + co) { 357 | c.TempFiles = append(c.TempFiles, co) 358 | } 359 | } 360 | 361 | var cmd *Cmd 362 | cmd, err = NewCmd(line) 363 | if err != nil { 364 | return nil, util.E.Annotate(err, "improper command") 365 | } 366 | 367 | c.Links = append(c.Links, cmd) 368 | } 369 | 370 | e := c.Environment 371 | e.RootDir = "/" 372 | 373 | err = c.Validate(&e) 374 | if err != nil { 375 | return nil, util.E.Annotate(err, "invalid command chain") 376 | } 377 | 378 | return 379 | } 380 | 381 | // splitWsQuote splits a string by whitespace, but takes doublequotes into 382 | // account 383 | func splitWsQuote(s string) []string { 384 | 385 | quote := rune(0) 386 | 387 | return strings.FieldsFunc(s, func(r rune) bool { 388 | switch { 389 | case r == quote: 390 | quote = rune(0) 391 | return true 392 | case quote != rune(0): 393 | return false 394 | case unicode.In(r, unicode.Quotation_Mark): 395 | quote = r 396 | return true 397 | default: 398 | return unicode.IsSpace(r) 399 | } 400 | }) 401 | } 402 | -------------------------------------------------------------------------------- /lib/rest.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | //go:generate esc -o web-generated.go -pkg paperless -private -prefix ../web/paperless-frontend ../web/paperless-frontend/dist/ 4 | 5 | import ( 6 | "bytes" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "os" 13 | "path/filepath" 14 | "strconv" 15 | "strings" 16 | "time" 17 | 18 | "github.com/gamegos/jsend" 19 | "github.com/go-chi/chi" 20 | "github.com/go-chi/chi/middleware" 21 | "github.com/go-chi/docgen" 22 | 23 | "github.com/kopoli/go-util" 24 | ) 25 | 26 | type backend struct { 27 | options util.Options 28 | db *db 29 | imgdir string 30 | 31 | staticURL string 32 | } 33 | 34 | /// JSON responding 35 | 36 | func requestJson(r *http.Request, data interface{}) (err error) { 37 | text, err := ioutil.ReadAll(r.Body) 38 | if err != nil { 39 | goto requestError 40 | } 41 | err = json.Unmarshal(text, data) 42 | if err != nil { 43 | goto requestError 44 | } 45 | 46 | return 47 | requestError: 48 | 49 | err = util.E.Annotate(err, "Converting HTTP request to JSON failed") 50 | return 51 | } 52 | 53 | func todoHandler(w http.ResponseWriter, r *http.Request) { 54 | w.Write([]byte("{ item: \"todo\" }")) 55 | } 56 | 57 | func (b *backend) respondErr(w http.ResponseWriter, code int, err error) { 58 | jsend.Wrap(w).Status(code).Message(err.Error()).Send() 59 | } 60 | 61 | func getPaging(r *http.Request) (ret *Page) { 62 | since, err := strconv.Atoi(r.URL.Query().Get("since")) 63 | if err != nil { 64 | since = 0 65 | } 66 | count, err := strconv.Atoi(r.URL.Query().Get("count")) 67 | if err != nil { 68 | count = 20 69 | } 70 | 71 | if count > 0 { 72 | ret = &Page{SinceId: since, Count: count} 73 | } 74 | 75 | return 76 | } 77 | 78 | /// Tag handling 79 | func (b *backend) tagHandler(w http.ResponseWriter, r *http.Request) { 80 | var err error 81 | annotate := func(arg ...interface{}) { 82 | err = util.E.Annotate(err, arg...) 83 | } 84 | 85 | switch r.Method { 86 | case "POST": 87 | var t Tag 88 | err = requestJson(r, &t) 89 | if err != nil { 90 | annotate("JSON parsing failed") 91 | goto requestError 92 | } 93 | t, err = b.db.addTag(t) 94 | if err != nil { 95 | annotate("Adding tag to db failed") 96 | goto requestError 97 | } 98 | 99 | jsend.Wrap(w).Status(http.StatusCreated).Data(t).Send() 100 | case "GET": 101 | p := getPaging(r) 102 | 103 | tags, err := b.db.getTags(p) 104 | if err != nil { 105 | util.E.Annotate(err) 106 | annotate("Getting tags from db failed") 107 | goto requestError 108 | } 109 | 110 | jsend.Wrap(w).Status(http.StatusOK).Data(tags).Send() 111 | } 112 | 113 | return 114 | 115 | requestError: 116 | b.respondErr(w, http.StatusBadRequest, err) 117 | return 118 | } 119 | 120 | func (b *backend) singleTagHandler(w http.ResponseWriter, r *http.Request) { 121 | 122 | var err error 123 | annotate := func(arg ...interface{}) { 124 | err = util.E.Annotate(err, arg...) 125 | } 126 | 127 | var t Tag 128 | 129 | tagid, err := strconv.Atoi(chi.URLParam(r, "tagID")) 130 | if err == nil { 131 | t, err = b.db.getTag(tagid) 132 | } 133 | if err != nil { 134 | annotate("Invalid tag ID from URL") 135 | goto requestError 136 | } 137 | 138 | switch r.Method { 139 | case "GET": 140 | jsend.Wrap(w).Status(http.StatusOK).Data(t).Send() 141 | case "PUT": 142 | var t2 Tag 143 | err = requestJson(r, &t) 144 | if err != nil { 145 | annotate("JSON parsing failed") 146 | goto requestError 147 | } 148 | t.Comment = t2.Comment 149 | err = b.db.updateTag(t) 150 | if err != nil { 151 | annotate("Updating tag in db failed") 152 | goto requestError 153 | } 154 | jsend.Wrap(w).Status(http.StatusOK).Data(t).Send() 155 | case "DELETE": 156 | err = b.db.deleteTag(t) 157 | if err != nil { 158 | annotate("Deleting tag from db failed") 159 | goto requestError 160 | } 161 | jsend.Wrap(w).Status(http.StatusOK).Message("Deleted").Send() 162 | } 163 | 164 | return 165 | 166 | requestError: 167 | b.respondErr(w, http.StatusBadRequest, err) 168 | return 169 | } 170 | 171 | // Image handling 172 | 173 | type resultimg struct { 174 | PageResult 175 | 176 | Images []restimg 177 | } 178 | 179 | type restimg struct { 180 | Image 181 | 182 | OrigImg string 183 | CleanImg string 184 | ThumbImg string 185 | } 186 | 187 | func (b *backend) wrapImage(img *Image) (ret restimg) { 188 | strip := func(s string) string { 189 | return b.staticURL + "/" + filepath.Base(s) 190 | } 191 | ret.Image = *img 192 | ret.OrigImg = strip(img.OrigFile("")) 193 | ret.CleanImg = strip(img.CleanFile("")) 194 | ret.ThumbImg = strip(img.ThumbFile("")) 195 | return 196 | } 197 | 198 | func (b *backend) wrapImages(imgs ImageResult) (ret resultimg) { 199 | ret.PageResult = imgs.PageResult 200 | 201 | ret.Images = make([]restimg, len(imgs.Images)) 202 | for i := range imgs.Images { 203 | ret.Images[i] = b.wrapImage(&imgs.Images[i]) 204 | } 205 | 206 | return 207 | } 208 | 209 | func (b *backend) imageHandler(w http.ResponseWriter, r *http.Request) { 210 | var err error 211 | annotate := func(arg ...interface{}) { 212 | err = util.E.Annotate(err, arg...) 213 | } 214 | 215 | switch r.Method { 216 | case "POST": 217 | err = r.ParseMultipartForm(20 * 1024 * 1024) 218 | if err != nil { 219 | annotate("Parsing multipartform failed") 220 | goto requestError 221 | } 222 | file, header, e2 := r.FormFile("image") 223 | if e2 != nil { 224 | err = e2 225 | annotate("Could not find image from POST data") 226 | goto requestError 227 | } 228 | tags := r.FormValue("tags") 229 | if tags == "" { 230 | err = util.E.New("Tags are required when uploading.") 231 | goto requestError 232 | } 233 | buf := &bytes.Buffer{} 234 | _, err = io.Copy(buf, file) 235 | if err != nil { 236 | annotate("Could not copy image data to buffer") 237 | goto requestError 238 | } 239 | 240 | img, e2 := SaveImage(header.Filename, buf.Bytes(), b.db, b.imgdir, tags) 241 | if e2 != nil { 242 | err = e2 243 | annotate("Could not save image") 244 | goto requestError 245 | } 246 | 247 | err = ProcessImage(&img, "default", b.db, b.imgdir) 248 | if err != nil { 249 | annotate("Could not process image") 250 | // Ignore errors with this as the data could be 251 | // incomplete before deletion 252 | _ = DeleteImage(&img, b.db, b.imgdir) 253 | goto requestError 254 | } 255 | 256 | jsend.Wrap(w).Status(http.StatusCreated).Data(img).Send() 257 | case "GET": 258 | p := getPaging(r) 259 | query := r.URL.Query().Get("q") 260 | tag := r.URL.Query().Get("t") 261 | 262 | s := &Search{ 263 | Match: query, 264 | Tag: tag, 265 | } 266 | 267 | images, e2 := b.db.getImages(p, s) 268 | if e2 != nil { 269 | err = e2 270 | annotate("Getting images from db failed") 271 | goto requestError 272 | } 273 | 274 | jsend.Wrap(w).Status(http.StatusOK).Data(b.wrapImages(images)).Send() 275 | } 276 | 277 | return 278 | 279 | requestError: 280 | b.respondErr(w, http.StatusBadRequest, err) 281 | return 282 | } 283 | 284 | func (b *backend) singleImageHandler(w http.ResponseWriter, r *http.Request) { 285 | var err error 286 | annotate := func(arg ...interface{}) { 287 | err = util.E.Annotate(err, arg...) 288 | } 289 | 290 | var img Image 291 | 292 | id, err := strconv.Atoi(chi.URLParam(r, "imageID")) 293 | if err == nil { 294 | img, err = b.db.getImage(id) 295 | } 296 | if err != nil { 297 | annotate("Invalid image ID from URL") 298 | goto requestError 299 | } 300 | 301 | switch r.Method { 302 | case "GET": 303 | jsend.Wrap(w).Status(http.StatusOK).Data(b.wrapImage(&img)).Send() 304 | case "PUT": 305 | var img2 Image 306 | err = requestJson(r, &img2) 307 | if err != nil { 308 | annotate("JSON parsing failed") 309 | goto requestError 310 | } 311 | img.Text = img2.Text 312 | img.Comment = img2.Comment 313 | err = b.db.updateImage(img) 314 | if err != nil { 315 | annotate("Updating image in db failed") 316 | goto requestError 317 | } 318 | jsend.Wrap(w).Status(http.StatusOK).Data(b.wrapImage(&img)).Send() 319 | case "DELETE": 320 | err = DeleteImage(&img, b.db, b.imgdir) 321 | if err != nil { 322 | annotate("Deleting image failed") 323 | goto requestError 324 | } 325 | jsend.Wrap(w).Status(http.StatusOK).Message("Deleted").Send() 326 | } 327 | 328 | return 329 | 330 | requestError: 331 | b.respondErr(w, http.StatusBadRequest, err) 332 | return 333 | } 334 | 335 | /// Script handling 336 | 337 | func (b *backend) loadScriptCtx(next http.Handler) http.Handler { 338 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 339 | }) 340 | } 341 | 342 | func (b *backend) versionHandler(w http.ResponseWriter, r *http.Request) { 343 | w.Write([]byte("{ \"version\": \"" + b.options.Get("version", "unversioned") + "\" }")) 344 | } 345 | 346 | func corsHandler(next http.Handler) http.Handler { 347 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 348 | w.Header().Set("Access-Control-Allow-Origin", "*") 349 | 350 | next.ServeHTTP(w, r) 351 | }) 352 | } 353 | 354 | // FileServer conveniently sets up a http.FileServer handler to serve static 355 | // files from a http.FileSystem. As chi updated to 3.x, the equivalent 356 | // function was removed. This one is copied from the example in: 357 | // https://github.com/go-chi/chi/blob/master/_examples/fileserver/main.go 358 | func FileServer(r chi.Router, path string, root http.FileSystem) { 359 | if strings.ContainsAny(path, "{}*") { 360 | panic("FileServer does not permit URL parameters.") 361 | } 362 | 363 | fs := http.StripPrefix(path, http.FileServer(root)) 364 | 365 | if path != "/" && path[len(path)-1] != '/' { 366 | r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP) 367 | path += "/" 368 | } 369 | path += "*" 370 | 371 | r.Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 372 | fs.ServeHTTP(w, r) 373 | })) 374 | } 375 | 376 | func StartWeb(o util.Options) (err error) { 377 | 378 | db, err := openDbFile(o.Get("database-file", "paperless.sqlite3")) 379 | if err != nil { 380 | return 381 | } 382 | defer db.Close() 383 | 384 | imgdir := o.Get("image-directory", "images") 385 | err = os.MkdirAll(imgdir, 0755) 386 | if err != nil { 387 | return 388 | } 389 | 390 | back := &backend{o, db, imgdir, "/static"} 391 | 392 | r := chi.NewRouter() 393 | 394 | r.Use(middleware.RequestID) 395 | r.Use(middleware.RealIP) 396 | r.Use(middleware.Logger) 397 | r.Use(middleware.Recoverer) 398 | r.Use(middleware.Timeout(600 * time.Second)) 399 | r.Use(corsHandler) 400 | 401 | // REST API 402 | r.Route("/api/v1", func(r chi.Router) { 403 | r.Get("/version", back.versionHandler) 404 | r.Route("/image", func(r chi.Router) { 405 | r.Get("/", back.imageHandler) 406 | r.Post("/", back.imageHandler) 407 | r.Route("/{imageID}", func(r chi.Router) { 408 | r.Get("/", back.singleImageHandler) 409 | r.Put("/", back.singleImageHandler) 410 | r.Delete("/", back.singleImageHandler) 411 | }) 412 | }) 413 | 414 | r.Route("/tag", func(r chi.Router) { 415 | r.Get("/", back.tagHandler) 416 | r.Post("/", back.tagHandler) 417 | r.Route("/{tagID}", func(r chi.Router) { 418 | r.Get("/", back.singleTagHandler) 419 | r.Put("/", back.singleTagHandler) 420 | r.Delete("/", back.singleTagHandler) 421 | }) 422 | }) 423 | r.Route("/script", func(r chi.Router) { 424 | r.Get("/", todoHandler) 425 | r.Post("/", todoHandler) 426 | r.Route("/{scriptID}", func(r chi.Router) { 427 | r.Use(back.loadScriptCtx) 428 | r.Get("/", todoHandler) 429 | r.Put("/", todoHandler) 430 | r.Delete("/", todoHandler) 431 | }) 432 | }) 433 | }) 434 | 435 | FileServer(r, back.staticURL, http.Dir(imgdir)) 436 | FileServer(r, "/dist", _escDir(false, "/dist/")) 437 | 438 | r.Get("/*", func(w http.ResponseWriter, r *http.Request) { 439 | fs := _escFS(false) 440 | httpfile, _ := fs.Open("/dist/index.html") 441 | st, _ := httpfile.Stat() 442 | http.ServeContent(w, r, "index.html", st.ModTime(), httpfile) 443 | }) 444 | 445 | if o.IsSet("print-routes") { 446 | fmt.Println(docgen.JSONRoutesDoc(r)) 447 | return 448 | } 449 | 450 | http.ListenAndServe(o.Get("listen-address", ":8078"), r) 451 | 452 | return 453 | } 454 | -------------------------------------------------------------------------------- /lib/cmdchain_test.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | func Test_parseConsts(t *testing.T) { 11 | type args struct { 12 | s string 13 | } 14 | tests := []struct { 15 | name string 16 | args args 17 | want []string 18 | }{ 19 | {"No variables", args{"something else"}, []string{}}, 20 | {"Single variable", args{"$a"}, []string{"a"}}, 21 | {"Many variables", args{"$a $b $c"}, []string{"a", "b", "c"}}, 22 | {"Combined", args{"$first$second"}, []string{"first", "second"}}, 23 | } 24 | for _, tt := range tests { 25 | t.Run(tt.name, func(t *testing.T) { 26 | if got := parseConsts(tt.args.s); !reflect.DeepEqual(got, tt.want) { 27 | t.Errorf("parseConsts() = %v, want %v", got, tt.want) 28 | } 29 | }) 30 | } 31 | } 32 | 33 | func Test_expandConsts(t *testing.T) { 34 | type args struct { 35 | s string 36 | constants map[string]string 37 | } 38 | tests := []struct { 39 | name string 40 | args args 41 | want string 42 | }{ 43 | {"Empty", args{"", map[string]string{}}, ""}, 44 | {"Single constant", args{"$abc", map[string]string{ 45 | "abc": "something", 46 | }}, "something"}, 47 | {"Multiple constants", args{"$a$b", map[string]string{ 48 | "a": "some", 49 | "b": "thing", 50 | }}, "something"}, 51 | {"Other stuff", args{"$a other $b", map[string]string{ 52 | "a": "some", 53 | "b": "thing", 54 | }}, "some other thing"}, 55 | {"Undefined", args{"$a$undefined$b", map[string]string{ 56 | "a": "some", 57 | "b": "thing", 58 | }}, "something"}, 59 | } 60 | for _, tt := range tests { 61 | t.Run(tt.name, func(t *testing.T) { 62 | if got := expandConsts(tt.args.s, tt.args.constants); got != tt.want { 63 | t.Errorf("expandConsts() = %v, want %v", got, tt.want) 64 | } 65 | }) 66 | } 67 | } 68 | 69 | func TestNewCmdChainScript(t *testing.T) { 70 | type args struct { 71 | script string 72 | } 73 | tests := []struct { 74 | name string 75 | args args 76 | wantC *CmdChain 77 | wantErr bool 78 | }{ 79 | {"Empty", args{""}, &CmdChain{ 80 | Environment: Environment{Constants: map[string]string{}}, 81 | }, false}, 82 | 83 | {"Comment and empty line", args{` 84 | # comment`}, &CmdChain{ 85 | Environment: Environment{Constants: map[string]string{}}, 86 | }, false}, 87 | 88 | {"Single command", args{"true"}, &CmdChain{ 89 | Links: []Link{&Cmd{[]string{"true"}}}, 90 | Environment: Environment{Constants: map[string]string{}}, 91 | }, false}, 92 | 93 | {"Two commands", args{"true\nfalse"}, &CmdChain{ 94 | Links: []Link{ 95 | &Cmd{[]string{"true"}}, 96 | &Cmd{[]string{"false"}}, 97 | }, 98 | Environment: Environment{Constants: map[string]string{}}, 99 | }, false}, 100 | 101 | {"Arguments", args{"true first second"}, &CmdChain{ 102 | Links: []Link{ 103 | &Cmd{[]string{"true", "first", "second"}}, 104 | }, 105 | Environment: Environment{Constants: map[string]string{}}, 106 | }, false}, 107 | 108 | {"Quoted arguments", args{"true 'first second'"}, &CmdChain{ 109 | Links: []Link{ 110 | &Cmd{[]string{"true", "first second"}}, 111 | }, 112 | Environment: Environment{Constants: map[string]string{}}, 113 | }, false}, 114 | 115 | {"Included a constant", args{"true $variable"}, &CmdChain{ 116 | Environment: Environment{ 117 | Constants: map[string]string{ 118 | "variable": "", 119 | }, 120 | }, 121 | Links: []Link{ 122 | &Cmd{[]string{"true", "$variable"}}, 123 | }, 124 | }, false}, 125 | 126 | {"Command not found", args{"this-command-is-not-found"}, nil, true}, 127 | 128 | {"Included a temporary file", args{"true $tmpSomething"}, &CmdChain{ 129 | Environment: Environment{ 130 | Constants: map[string]string{ 131 | "tmpSomething": "", 132 | }, 133 | TempFiles: []string{"tmpSomething"}, 134 | }, 135 | Links: []Link{ 136 | &Cmd{[]string{"true", "$tmpSomething"}}, 137 | }, 138 | }, false}, 139 | } 140 | for _, tt := range tests { 141 | t.Run(tt.name, func(t *testing.T) { 142 | gotC, err := NewCmdChainScript(tt.args.script) 143 | if (err != nil) != tt.wantErr { 144 | t.Errorf("NewCmdChainScript() error = %v, wantErr %v", err, tt.wantErr) 145 | return 146 | } 147 | if !reflect.DeepEqual(gotC, tt.wantC) { 148 | t.Errorf("NewCmdChainScript() = %v, want %v", gotC, tt.wantC) 149 | } 150 | }) 151 | } 152 | } 153 | 154 | func Test_splitWsQuote(t *testing.T) { 155 | type args struct { 156 | s string 157 | } 158 | tests := []struct { 159 | name string 160 | args args 161 | want []string 162 | }{ 163 | {"Empty", args{""}, []string{}}, 164 | {"One item", args{"jep"}, []string{"jep"}}, 165 | {"Two items", args{"jep something"}, []string{"jep", "something"}}, 166 | {"Quoted", args{"'sth abc'"}, []string{"sth abc"}}, 167 | {"Quoted two", args{"'sth' abc"}, []string{"sth", "abc"}}, 168 | {"Mixed quotes", args{"'a b c' \"c e\""}, []string{"a b c", "c e"}}, 169 | } 170 | for _, tt := range tests { 171 | t.Run(tt.name, func(t *testing.T) { 172 | if got := splitWsQuote(tt.args.s); !reflect.DeepEqual(got, tt.want) { 173 | t.Errorf("splitWsQuote() = %v, want %v", got, tt.want) 174 | } 175 | }) 176 | } 177 | } 178 | 179 | func TestCmd_Validate(t *testing.T) { 180 | type fields struct { 181 | Cmd []string 182 | } 183 | type args struct { 184 | e Environment 185 | } 186 | tests := []struct { 187 | name string 188 | fields fields 189 | args args 190 | wantErr bool 191 | }{ 192 | {"Proper command", fields{[]string{"true"}}, args{}, false}, 193 | {"Empty command", fields{[]string{""}}, args{}, true}, 194 | // {"LastErr already set", fields{[]string{"true"}}, args{Status{LastErr: errors.New("abc")}}, true}, 195 | {"Command not found", fields{[]string{"command-is-not-found"}}, args{}, true}, 196 | {"Proper output redirection", fields{[]string{"echo", ">", "outfile"}}, args{}, false}, 197 | {"Redirecting syntax error", fields{[]string{"echo", ">"}}, args{}, true}, 198 | {"Redirecting to empty", fields{[]string{"echo", ">", ""}}, args{}, true}, 199 | {"Command is allowed", fields{[]string{"true"}}, 200 | args{Environment{ 201 | AllowedCommands: map[string]bool{ 202 | "true": true, 203 | }, 204 | }}, false}, 205 | {"Command not allowed", fields{[]string{"true"}}, 206 | args{Environment{ 207 | AllowedCommands: map[string]bool{ 208 | "b": true, 209 | }, 210 | }}, true}, 211 | {"Constant is defined", fields{[]string{"true", "$something"}}, 212 | args{Environment{ 213 | Constants: map[string]string{ 214 | "something": "value", 215 | }, 216 | }}, false}, 217 | 218 | {"Constant is not defined", fields{[]string{"true", "$else"}}, args{}, true}, 219 | 220 | {"Commands cannot be read from a constant", fields{[]string{"$cmd"}}, 221 | args{Environment{ 222 | Constants: map[string]string{ 223 | "cmd": "true", 224 | }, 225 | }}, true}, 226 | } 227 | for _, tt := range tests { 228 | t.Run(tt.name, func(t *testing.T) { 229 | c := &Cmd{ 230 | Cmd: tt.fields.Cmd, 231 | } 232 | tt.args.e.RootDir = "/" 233 | 234 | if err := c.Validate(&tt.args.e); (err != nil) != tt.wantErr { 235 | t.Errorf("Cmd.Validate() error = %v, wantErr %v", err, tt.wantErr) 236 | } 237 | }) 238 | } 239 | } 240 | 241 | func envIsProper(e *Environment) bool { 242 | return e.initialized && e.validate() == nil 243 | } 244 | 245 | func envTempfilesExist(e *Environment) (ret bool) { 246 | if e.validate() != nil { 247 | return false 248 | } 249 | 250 | if len(e.TempFiles) == 0 { 251 | return false 252 | } 253 | 254 | ret = true 255 | for _, n := range e.TempFiles { 256 | _, ok := e.Constants[n] 257 | if !ok { 258 | ret = false 259 | continue 260 | } 261 | _, err := os.Stat(e.Constants[n]) 262 | if err != nil { 263 | ret = false 264 | } 265 | } 266 | return 267 | } 268 | 269 | func TestEnvironment_initEnv(t *testing.T) { 270 | tests := []struct { 271 | name string 272 | fields Environment 273 | wantErr bool 274 | validate func(*Environment) bool 275 | }{ 276 | {"Empty Environment", Environment{}, true, nil}, 277 | {"Already initialized", Environment{ 278 | initialized: true, 279 | }, false, nil}, 280 | {"Proper but empty", Environment{ 281 | Constants: map[string]string{}, 282 | }, false, envIsProper}, 283 | {"Proper with tempfiles", Environment{ 284 | Constants: map[string]string{ 285 | "a": "", 286 | }, 287 | TempFiles: []string{ 288 | "a", 289 | }, 290 | }, false, envTempfilesExist}, 291 | } 292 | for _, tt := range tests { 293 | t.Run(tt.name, func(t *testing.T) { 294 | var err error 295 | e := &tt.fields 296 | if err = e.initEnv(); (err != nil) != tt.wantErr { 297 | t.Errorf("Environment.initEnv() error = %v, wantErr %v", err, tt.wantErr) 298 | } 299 | 300 | if tt.validate != nil && !tt.validate(e) { 301 | t.Errorf("Environment should be proper") 302 | } 303 | 304 | if err == nil { 305 | e.deinitEnv() 306 | } 307 | }) 308 | } 309 | } 310 | 311 | func tempfilesShouldntExist(orig, deinit *Environment) (ret bool) { 312 | if len(orig.TempFiles) == 0 { 313 | return false 314 | } 315 | for _, n := range orig.TempFiles { 316 | fname := orig.Constants[n] 317 | _, err := os.Stat(fname) 318 | if err == nil { 319 | return false 320 | } 321 | } 322 | 323 | return true 324 | } 325 | 326 | func TestEnvironment_deinitEnv(t *testing.T) { 327 | tests := []struct { 328 | name string 329 | fields Environment 330 | shouldInit bool 331 | wantErr bool 332 | validate func(orig, deinit *Environment) bool 333 | }{ 334 | {"Empty Environment", Environment{}, false, false, nil}, 335 | {"Proper deinit", Environment{ 336 | Constants: map[string]string{ 337 | "a": "", 338 | }, 339 | TempFiles: []string{ 340 | "a", 341 | }, 342 | }, true, false, tempfilesShouldntExist}, 343 | } 344 | for _, tt := range tests { 345 | t.Run(tt.name, func(t *testing.T) { 346 | e := &tt.fields 347 | if tt.shouldInit { 348 | err := e.initEnv() 349 | if err != nil { 350 | t.Errorf("initEnv should succeed") 351 | return 352 | } 353 | } 354 | orig := *e 355 | 356 | if err := e.deinitEnv(); (err != nil) != tt.wantErr { 357 | t.Errorf("Environment.deinitEnv() error = %v, wantErr %v", err, tt.wantErr) 358 | } 359 | 360 | if tt.validate != nil && !tt.validate(&orig, e) { 361 | t.Errorf("Environment should be deinitialized properly") 362 | } 363 | 364 | }) 365 | } 366 | } 367 | 368 | func TestRunCmdChain(t *testing.T) { 369 | tests := []struct { 370 | name string 371 | script string 372 | consts map[string]string 373 | valid bool 374 | wantOutput bool 375 | output string 376 | wantErr bool 377 | }{ 378 | {"Empty script", "", nil, true, true, "", false}, 379 | {"Echo command", "echo piip", nil, true, true, 380 | "# Running command: echo piip\npiip\n", false}, 381 | {"Output redirection", "echo piip > a\ncat a", nil, true, true, 382 | "# Running command: echo piip > a\n# Running command: cat a\npiip\n", false}, 383 | {"Echo with a constant", "echo $msg", map[string]string{ 384 | "msg": "piip", 385 | }, true, true, "# Running command: echo piip\npiip\n", false}, 386 | {"Existing temporary file", "echo $tmpmsg\ncat $tmpmsg", nil, 387 | true, false, "", false}, 388 | {"Failing commands", "true\nfalse\ntrue", nil, true, false, "", true}, 389 | } 390 | for _, tt := range tests { 391 | t.Run(tt.name, func(t *testing.T) { 392 | 393 | ch, err := NewCmdChainScript(tt.script) 394 | if (err != nil) == tt.valid { 395 | t.Errorf("NewCmdChainScript() error = %v, valid %v", err, tt.valid) 396 | } 397 | 398 | s := Status{Environment: ch.Environment} 399 | if tt.wantOutput { 400 | s.Log = &bytes.Buffer{} 401 | } 402 | if tt.consts != nil { 403 | s.Constants = tt.consts 404 | } 405 | 406 | err = RunCmdChain(ch, &s) 407 | if (err != nil) != tt.wantErr { 408 | t.Errorf("RunCmdChain() error = %v, wantErr %v", err, tt.wantErr) 409 | } 410 | 411 | if tt.wantOutput && s.Log.(*bytes.Buffer).String() != tt.output { 412 | t.Errorf("RunCmdChain() = [%v], want [%v]", 413 | s.Log.(*bytes.Buffer).String(), 414 | tt.output) 415 | } 416 | }) 417 | } 418 | } 419 | -------------------------------------------------------------------------------- /lib/sqlitedb.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/jmoiron/sqlx" 9 | _ "github.com/mattn/go-sqlite3" 10 | 11 | "github.com/kopoli/go-util" 12 | ) 13 | 14 | type db struct { 15 | file string 16 | *sqlx.DB 17 | } 18 | 19 | type dbTx struct { 20 | *sqlx.Tx 21 | } 22 | 23 | // Pagination support 24 | type Page struct { 25 | // Id that was the last of the previous page 26 | SinceId int 27 | 28 | // Count is the number of items in the page 29 | Count int 30 | } 31 | 32 | // PageResult adds navigation information to the paginated data 33 | type PageResult struct { 34 | // Number of items returned by the whole search 35 | ResultCount int 36 | 37 | // IDs of the items that are the first of their page 38 | SinceIDs []int 39 | 40 | // Count is the number of items in the page 41 | Count int 42 | } 43 | 44 | type ImageResult struct { 45 | PageResult 46 | Images []Image 47 | } 48 | 49 | type Search struct { 50 | ID int 51 | OrderBy string 52 | Match string 53 | Tag string 54 | } 55 | 56 | func openDbFile(dbfile string) (ret *db, err error) { 57 | create := false 58 | 59 | dbfile = filepath.Clean(dbfile) 60 | 61 | i, err := os.Stat(dbfile) 62 | if err == nil && i.IsDir() { 63 | err = util.E.New("Given path is a directory") 64 | return 65 | } 66 | 67 | err = MkdirParents(dbfile) 68 | if err != nil { 69 | err = util.E.Annotate(err, 70 | "Could not create dbfile's parent directories") 71 | return 72 | } 73 | 74 | if _, err = os.Stat(dbfile); os.IsNotExist(err) { 75 | create = true 76 | err = nil 77 | } 78 | 79 | d, err := sqlx.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", dbfile)) 80 | if err != nil { 81 | err = util.E.Annotate(err, "Opening sqlite dbfile failed") 82 | return 83 | } 84 | 85 | if create { 86 | _, err = d.Exec(` 87 | CREATE TABLE IF NOT EXISTS tag ( 88 | id INTEGER PRIMARY KEY ASC AUTOINCREMENT, 89 | name TEXT DEFAULT "" NOT NULL UNIQUE ON CONFLICT ABORT, 90 | comment TEXT DEFAULT "" 91 | ); 92 | 93 | -- The image data 94 | CREATE TABLE IF NOT EXISTS image ( 95 | id INTEGER PRIMARY KEY ASC AUTOINCREMENT, 96 | checksum TEXT UNIQUE NOT NULL ON CONFLICT ABORT,-- checksum of the file 97 | fileid TEXT DEFAULT "", -- used to construct the processed image, 98 | -- thumbnail and text files 99 | scandate DATETIME, -- timestamp when it was scanned 100 | adddate DATETIME DEFAULT CURRENT_TIMESTAMP, -- timestamp when it was created in db 101 | interpretdate DATETIME, -- timestamp when it was interpret 102 | 103 | processlog TEXT DEFAULT "", -- Log of processing 104 | filename TEXT DEFAULT "" -- The original filename 105 | ); 106 | 107 | CREATE VIRTUAL TABLE IF NOT EXISTS imgtext USING fts4 ( 108 | text DEFAULT "", -- the OCR'd text 109 | comment DEFAULT "" -- freeform comment 110 | ); 111 | 112 | -- Tags for an image 113 | CREATE TABLE IF NOT EXISTS imgtag ( 114 | tagid INTEGER REFERENCES tag(id) NOT NULL, 115 | imgid INTEGER REFERENCES img(id) NOT NULL, 116 | UNIQUE (tagid, imgid) 117 | ); 118 | 119 | -- Script for processing the images 120 | CREATE TABLE IF NOT EXISTS script ( 121 | id INTEGER PRIMARY KEY ASC AUTOINCREMENT, 122 | name TEXT UNIQUE ON CONFLICT ABORT, 123 | script TEXT DEFAULT "" 124 | ); 125 | 126 | `) 127 | if err != nil { 128 | goto initfail 129 | } 130 | 131 | } 132 | _, err = d.Exec("PRAGMA busy_timeout=10000") 133 | if err != nil { 134 | goto initfail 135 | } 136 | 137 | // Work around the multiple access problems 138 | d.SetMaxOpenConns(1) 139 | 140 | ret = &db{dbfile, d} 141 | return 142 | 143 | initfail: 144 | d.Close() 145 | err = util.E.Annotate(err, "Initializing the database failed") 146 | ret = nil 147 | return 148 | } 149 | 150 | func (db *db) getTag(id int) (ret Tag, err error) { 151 | err = db.Get(&ret, "SELECT * from tag WHERE id = $1", id) 152 | return 153 | } 154 | 155 | func (db *db) getScript(id int) (ret Script, err error) { 156 | err = db.Get(&ret, "SELECT * from script WHERE id = $1", id) 157 | return 158 | } 159 | 160 | func (db *db) getImage(id int) (ret Image, err error) { 161 | if id < 0 { 162 | err = util.E.New("Negative ID for image is invalid") 163 | return 164 | } 165 | 166 | imgs, err := db.getImages(nil, &Search{ID: id}) 167 | if err != nil { 168 | return 169 | } 170 | 171 | if len(imgs.Images) == 0 { 172 | err = util.E.New("No image found with id %d", id) 173 | return 174 | } 175 | if len(imgs.Images) > 1 { 176 | err = util.E.New("Internal error: Multiple images with the same id") 177 | return 178 | } 179 | ret = imgs.Images[0] 180 | return 181 | } 182 | 183 | func (db *db) getTags(p *Page) (ret []Tag, err error) { 184 | query := "SELECT * from tag" 185 | order := " ORDER BY name ASC" 186 | sel := func() error { 187 | return db.Select(&ret, query+order) 188 | } 189 | 190 | if p != nil { 191 | query += " WHERE (id > ?) " + order + " LIMIT ?" 192 | sel = func() error { 193 | return db.Select(&ret, query, p.SinceId, p.Count) 194 | } 195 | } 196 | 197 | err = sel() 198 | return 199 | } 200 | 201 | func (db *db) addTag(t Tag) (ret Tag, err error) { 202 | _, err = db.Exec("INSERT INTO tag(name, comment) VALUES($1, $2)", t.Name, t.Comment) 203 | if err != nil { 204 | return 205 | } 206 | err = db.Get(&ret, "SELECT * FROM tag WHERE name = $1", t.Name) 207 | 208 | return 209 | } 210 | 211 | func (db *db) updateTag(t Tag) (err error) { 212 | _, err = db.Exec("UPDATE tag SET comment = $1 WHERE name = $2", t.Comment, t.Name) 213 | return 214 | } 215 | 216 | func (db *db) deleteTag(t Tag) (err error) { 217 | _, err = db.Exec("DELETE FROM tag WHERE name = $1", t.Name) 218 | return 219 | } 220 | 221 | func (db *db) getScripts(p *Page) (ret []Script, err error) { 222 | query := "SELECT * from script" 223 | order := " ORDER BY name ASC" 224 | sel := func() error { 225 | return db.Select(&ret, query+order) 226 | } 227 | 228 | if p != nil { 229 | query += " WHERE (id > ?) " + order + " LIMIT ?" 230 | sel = func() error { 231 | return db.Select(&ret, query, p.SinceId, p.Count) 232 | } 233 | } 234 | 235 | err = sel() 236 | return 237 | } 238 | 239 | func (db *db) addScript(s Script) (ret Script, err error) { 240 | _, err = db.Exec("INSERT INTO script(name, script) VALUES($1, $2)", s.Name, s.Script) 241 | if err != nil { 242 | return 243 | } 244 | 245 | err = db.Get(&ret, "SELECT * FROM script WHERE name = $1", s.Name) 246 | return 247 | } 248 | 249 | func (db *db) updateScript(s Script) (err error) { 250 | _, err = db.Exec("UPDATE script SET script = $1 WHERE name = $2", s.Script, s.Name) 251 | return 252 | } 253 | 254 | func (db *db) deleteScript(s Script) (err error) { 255 | _, err = db.Exec("DELETE FROM script WHERE name = $1", s.Name) 256 | return 257 | } 258 | 259 | func withTx(db *db, f func(*sqlx.Tx) error) (err error) { 260 | tx, err := db.Beginx() 261 | if err != nil { 262 | return 263 | } 264 | 265 | err = f(tx) 266 | if err != nil { 267 | tx.Rollback() 268 | return 269 | } 270 | 271 | err = tx.Commit() 272 | return 273 | } 274 | 275 | func (db *db) getImages(p *Page, s *Search) (ret ImageResult, err error) { 276 | query := "SELECT image.id FROM image, imgtext" 277 | order := " ORDER BY image.id ASC" 278 | 279 | where := " WHERE imgtext.rowid = image.id" 280 | 281 | args := map[string]interface{}{} 282 | 283 | if s != nil { 284 | if s.ID != 0 { 285 | where = where + " AND image.id = :id" 286 | args["id"] = fmt.Sprintf("%d", s.ID) 287 | } 288 | if s.Match != "" { 289 | where = where + " AND imgtext.text MATCH :match" 290 | args["match"] = s.Match 291 | } 292 | if s.Tag != "" { 293 | query = query + ", tag, imgtag" 294 | where = where + " AND tag.name = :tag AND imgtag.tagid = tag.id AND imgtag.imgid = image.id" 295 | args["tag"] = s.Tag 296 | } 297 | if s.OrderBy != "" { 298 | order = " ORDER BY :order ASC" 299 | args["order"] = s.OrderBy 300 | } 301 | } 302 | query = query + where + order 303 | 304 | nstmt, err := db.PrepareNamed(query) 305 | if err != nil { 306 | return 307 | } 308 | 309 | var ids []int 310 | 311 | err = nstmt.Select(&ids, args) 312 | if err != nil { 313 | nstmt.Close() 314 | return 315 | } 316 | nstmt.Close() 317 | 318 | ret.ResultCount = len(ids) 319 | 320 | // No images found 321 | if ret.ResultCount == 0 { 322 | ret.SinceIDs = make([]int, 0) 323 | return 324 | } 325 | 326 | if p == nil { 327 | ret.Count = 0 328 | ret.SinceIDs = make([]int, 1) 329 | ret.SinceIDs[0] = ids[0] 330 | } else { 331 | ret.Count = p.Count 332 | pages := ret.ResultCount / ret.Count 333 | if (ret.ResultCount % ret.Count) > 0 { 334 | pages += 1 335 | } 336 | 337 | ret.SinceIDs = make([]int, pages) 338 | for i := range ret.SinceIDs { 339 | ret.SinceIDs[i] = ids[ret.Count*i] 340 | } 341 | 342 | var start int = -1 343 | var realcount int = 0 344 | for i := range ids { 345 | if ids[i] == p.SinceId { 346 | start = i + 1 347 | break 348 | } 349 | } 350 | if start == -1 || start >= len(ids) { 351 | start = 0 352 | } 353 | 354 | realcount = ret.ResultCount - start 355 | if realcount > ret.Count { 356 | realcount = ret.Count 357 | } 358 | 359 | ids = ids[start : start+realcount] 360 | } 361 | 362 | q, qargs, err := sqlx.In(`SELECT * from image, imgtext WHERE imgtext.rowid = image.id AND image.id IN (?)`, ids) 363 | if err != nil { 364 | return 365 | } 366 | err = db.Select(&ret.Images, q, qargs...) 367 | if err != nil { 368 | return 369 | } 370 | 371 | err = withTx(db, func(tx *sqlx.Tx) (err error) { 372 | for i := range ret.Images { 373 | err = tx.Select(&ret.Images[i].Tags, `SELECT tag.id, tag.name, tag.comment FROM tag, imgtag 374 | WHERE imgtag.tagid = tag.id AND imgtag.imgid = $1 `, ret.Images[i].Id) 375 | if err != nil { 376 | return 377 | } 378 | } 379 | return 380 | }) 381 | return 382 | } 383 | 384 | func syncTagsToImage(tx *sqlx.Tx, i Image) (err error) { 385 | _, err = tx.NamedExec(`DELETE FROM imgtag WHERE imgid = :id`, i) 386 | if err != nil { 387 | return 388 | } 389 | 390 | for _, t := range i.Tags { 391 | _, err = tx.Exec(`INSERT INTO imgtag(imgid, tagid) SELECT $1, tag.id FROM tag WHERE tag.name = $2`, i.Id, t.Name) 392 | if err != nil { 393 | return 394 | } 395 | } 396 | return 397 | } 398 | 399 | func (db *db) addImage(i Image) (ret Image, err error) { 400 | err = withTx(db, func(tx *sqlx.Tx) (err error) { 401 | _, err = tx.NamedExec(`INSERT INTO 402 | image( checksum, fileid, scandate, adddate, interpretdate, processlog, filename) 403 | VALUES(:checksum, :fileid, :scandate, :adddate, :interpretdate, :processlog, :filename)`, i) 404 | if err != nil { 405 | return 406 | } 407 | 408 | var id int 409 | err = tx.Get(&id, "SELECT id FROM image WHERE checksum=$1", i.Checksum) 410 | if err != nil { 411 | return 412 | } 413 | i.Id = id 414 | 415 | _, err = tx.NamedExec(`INSERT INTO imgtext(rowid, text, comment) VALUES (:id, :text, :comment)`, i) 416 | if err != nil { 417 | return 418 | } 419 | 420 | err = syncTagsToImage(tx, i) 421 | ret = i 422 | return 423 | }) 424 | return 425 | } 426 | 427 | func (db *db) updateImage(i Image) (err error) { 428 | err = withTx(db, func(tx *sqlx.Tx) (err error) { 429 | _, err = tx.NamedExec(`UPDATE image SET 430 | interpretdate = :interpretdate, 431 | processlog = :processlog 432 | WHERE image.id = :id`, i) 433 | if err != nil { 434 | return 435 | } 436 | 437 | _, err = tx.NamedExec(`UPDATE imgtext SET 438 | text = :text, 439 | comment = :comment 440 | WHERE rowid = :id`, i) 441 | if err != nil { 442 | return 443 | } 444 | 445 | err = syncTagsToImage(tx, i) 446 | return 447 | }) 448 | return 449 | } 450 | 451 | func (db *db) deleteImage(s Image) (err error) { 452 | err = withTx(db, func(tx *sqlx.Tx) (err error) { 453 | _, err = tx.Exec(`DELETE FROM imgtag WHERE imgid IN 454 | (SELECT id FROM image WHERE image.checksum = $1)`, s.Checksum) 455 | if err != nil { 456 | return 457 | } 458 | _, err = tx.Exec(`DELETE FROM imgtext WHERE rowid IN 459 | (SELECT id FROM image WHERE image.checksum = $1)`, s.Checksum) 460 | if err != nil { 461 | return 462 | } 463 | _, err = tx.Exec(`DELETE FROM image WHERE image.checksum = $1`, s.Checksum) 464 | return 465 | }) 466 | return 467 | } 468 | -------------------------------------------------------------------------------- /lib/sqlitedb_test.go: -------------------------------------------------------------------------------- 1 | package paperless 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "testing" 7 | "time" 8 | 9 | _ "github.com/mattn/go-sqlite3" 10 | ) 11 | 12 | var dbfile = ":memory:" 13 | 14 | func setupDb() (*db, error) { 15 | return openDbFile(dbfile) 16 | } 17 | 18 | func clearDbFile(dbfile string) error { 19 | return nil 20 | } 21 | 22 | func teardownDb() (err error) { 23 | return clearDbFile(dbfile) 24 | } 25 | 26 | func Test_db_openDbFile(t *testing.T) { 27 | tests := []struct { 28 | name string 29 | dbfile string 30 | wantErr bool 31 | }{ 32 | {"Empty filename", "", true}, 33 | {"Improper filename", "././", true}, 34 | {"Proper filename", "test.sqlite", false}, 35 | } 36 | for _, tt := range tests { 37 | t.Run(tt.name, func(t *testing.T) { 38 | gotRet, err := openDbFile(tt.dbfile) 39 | if (err != nil) != tt.wantErr { 40 | t.Errorf("openDbFile() error = %v, wantErr %v", err, tt.wantErr) 41 | return 42 | } 43 | if err != nil { 44 | return 45 | } 46 | if gotRet == nil { 47 | t.Errorf("openDbFile() returns nil and no error") 48 | return 49 | } 50 | err = gotRet.Close() 51 | if err != nil { 52 | t.Errorf("db.Close() error = %v", err) 53 | } 54 | _, err = os.Stat(tt.dbfile) 55 | if err != nil { 56 | t.Errorf("Statting %s errors = %v", tt.dbfile, err) 57 | } 58 | err = clearDbFile(tt.dbfile) 59 | if err != nil { 60 | t.Errorf("clearDbFile() error = %v", err) 61 | } 62 | }) 63 | } 64 | } 65 | 66 | type testOp interface { 67 | run(*db) error 68 | } 69 | 70 | type testFunc func(*db) error 71 | 72 | func (t testFunc) run(d *db) error { 73 | return t(d) 74 | } 75 | 76 | func Test_db_Tag(t *testing.T) { 77 | at := func(name, comment string) testFunc { 78 | return func(d *db) error { 79 | _, err := d.addTag(Tag{Name: name, Comment: comment}) 80 | return err 81 | } 82 | } 83 | 84 | dt := func(name string) testFunc { 85 | return func(d *db) error { 86 | return d.deleteTag(Tag{Name: name}) 87 | } 88 | } 89 | 90 | ut := func(name, comment string) testFunc { 91 | return func(d *db) error { 92 | return d.updateTag(Tag{Name: name, Comment: comment}) 93 | } 94 | } 95 | 96 | tests := []struct { 97 | name string 98 | ops []testOp 99 | wantErr bool 100 | paging *Page 101 | wantTags []Tag 102 | }{ 103 | {"Add empty tag", []testOp{at("", "")}, false, nil, []Tag{Tag{Id: 1}}}, 104 | {"Add tag with contents", []testOp{at("name", "")}, false, nil, []Tag{Tag{Id: 1, Name: "name"}}}, 105 | {"Add tag and remove it", []testOp{ 106 | at("name", ""), at("abc", ""), dt("name"), 107 | }, false, nil, []Tag{Tag{Id: 2, Name: "abc"}}}, 108 | {"Add tag and update it", []testOp{ 109 | at("name", ""), ut("name", "comment"), 110 | }, false, nil, []Tag{Tag{Id: 1, Name: "name", Comment: "comment"}}}, 111 | {"Update a tag", []testOp{ 112 | at("name", ""), ut("name", "comment"), 113 | }, false, nil, []Tag{Tag{Id: 1, Name: "name", Comment: "comment"}}}, 114 | {"Add duplicate", []testOp{ 115 | at("name", ""), at("name", "other"), 116 | }, true, nil, []Tag{Tag{Id: 1, Name: "name"}}}, 117 | {"Pagination", []testOp{ 118 | at("f1", ""), at("f2", ""), at("f3", ""), at("f4", ""), 119 | }, false, &Page{SinceId: 2, Count: 5}, []Tag{Tag{Id: 3, Name: "f3"}, Tag{Id: 4, Name: "f4"}}}, 120 | } 121 | for _, tt := range tests { 122 | db, err := setupDb() 123 | if err != nil { 124 | t.Errorf("Setting up db failed with error = %v", err) 125 | return 126 | } 127 | t.Run(tt.name, func(t *testing.T) { 128 | 129 | var failed bool = false 130 | fail := struct { 131 | failed bool 132 | err error 133 | i int 134 | }{} 135 | 136 | for i, op := range tt.ops { 137 | err := op.run(db) 138 | failed = failed || (err != nil) 139 | if failed && !fail.failed { 140 | fail.failed = true 141 | fail.err = err 142 | fail.i = i 143 | } 144 | } 145 | if failed != tt.wantErr { 146 | t.Errorf("op no.%d error = %v, wantErr %v", fail.i, fail.err, tt.wantErr) 147 | return 148 | } 149 | 150 | tags, err := db.getTags(tt.paging) 151 | if err != nil { 152 | t.Errorf("db.getTags() error = %v", err) 153 | } 154 | 155 | compare(t, "db.getTags() not expected", tt.wantTags, tags) 156 | }) 157 | db.Close() 158 | err = teardownDb() 159 | if err != nil { 160 | t.Errorf("Could not remove database file: %v", err) 161 | } 162 | } 163 | } 164 | 165 | func Test_db_Script(t *testing.T) { 166 | at := func(name, script string) testFunc { 167 | return func(d *db) error { 168 | _, err := d.addScript(Script{Name: name, Script: script}) 169 | return err 170 | } 171 | } 172 | 173 | dt := func(name string) testFunc { 174 | return func(d *db) error { 175 | return d.deleteScript(Script{Name: name}) 176 | } 177 | } 178 | 179 | ut := func(name, script string) testFunc { 180 | return func(d *db) error { 181 | return d.updateScript(Script{Name: name, Script: script}) 182 | } 183 | } 184 | 185 | tests := []struct { 186 | name string 187 | ops []testOp 188 | wantErr bool 189 | paging *Page 190 | wantScripts []Script 191 | }{ 192 | {"Add empty script", []testOp{at("", "")}, false, nil, []Script{Script{Id: 1}}}, 193 | {"Add script with contents", []testOp{at("name", "")}, false, nil, []Script{Script{Id: 1, Name: "name"}}}, 194 | {"Add script and remove it", []testOp{ 195 | at("name", ""), at("abc", ""), dt("name"), 196 | }, false, nil, []Script{Script{Id: 2, Name: "abc"}}}, 197 | {"Add script and update it", []testOp{ 198 | at("name", ""), ut("name", "script"), 199 | }, false, nil, []Script{Script{Id: 1, Name: "name", Script: "script"}}}, 200 | {"Update a script", []testOp{ 201 | at("name", "script"), ut("name", "toinen"), 202 | }, false, nil, []Script{Script{Id: 1, Name: "name", Script: "toinen"}}}, 203 | {"Add duplicate", []testOp{ 204 | at("name", ""), at("name", "other"), 205 | }, true, nil, []Script{Script{Id: 1, Name: "name"}}}, 206 | {"Pagination", []testOp{ 207 | at("f1", ""), at("f2", ""), at("f3", ""), at("f4", ""), 208 | }, false, &Page{SinceId: 2, Count: 5}, []Script{Script{Id: 3, Name: "f3"}, Script{Id: 4, Name: "f4"}}}, 209 | } 210 | for _, tt := range tests { 211 | db, err := setupDb() 212 | if err != nil { 213 | t.Errorf("Setting up db failed with error = %v", err) 214 | return 215 | } 216 | t.Run(tt.name, func(t *testing.T) { 217 | 218 | var failed bool = false 219 | fail := struct { 220 | failed bool 221 | err error 222 | i int 223 | }{} 224 | 225 | for i, op := range tt.ops { 226 | err := op.run(db) 227 | failed = failed || (err != nil) 228 | if failed && !fail.failed { 229 | fail.failed = true 230 | fail.err = err 231 | fail.i = i 232 | } 233 | } 234 | if failed != tt.wantErr { 235 | t.Errorf("op no.%d error = %v, wantErr %v", fail.i, fail.err, tt.wantErr) 236 | return 237 | } 238 | 239 | scripts, err := db.getScripts(tt.paging) 240 | if err != nil { 241 | t.Errorf("db.getScripts() error = %v", err) 242 | } 243 | 244 | compare(t, "db.getScripts() not expected", tt.wantScripts, scripts) 245 | }) 246 | db.Close() 247 | err = teardownDb() 248 | if err != nil { 249 | t.Errorf("Could not remove database file: %v", err) 250 | } 251 | } 252 | } 253 | 254 | func Test_db_Image(t *testing.T) { 255 | at := func(name, comment string) testFunc { 256 | return func(d *db) error { 257 | _, err := d.addTag(Tag{Name: name, Comment: comment}) 258 | return err 259 | } 260 | } 261 | 262 | ai := func(i Image) testFunc { 263 | return func(d *db) error { 264 | _, err := d.addImage(i) 265 | return err 266 | } 267 | } 268 | 269 | di := func(checksum string) testFunc { 270 | return func(d *db) error { 271 | return d.deleteImage(Image{Checksum: checksum}) 272 | } 273 | } 274 | 275 | ui := func(i Image) testFunc { 276 | return func(d *db) error { 277 | return d.updateImage(i) 278 | } 279 | } 280 | 281 | cmp := func(t *testing.T, i1, i2 []Image) { 282 | for n := range i1 { 283 | i1[n].AddDate = time.Time{} 284 | } 285 | for n := range i2 { 286 | i2[n].AddDate = time.Time{} 287 | } 288 | 289 | compare(t, "db.getImages() not expected", i1, i2) 290 | } 291 | 292 | tests := []struct { 293 | name string 294 | ops []testOp 295 | wantErr bool 296 | paging *Page 297 | search *Search 298 | wantImages []Image 299 | }{ 300 | {"Add an image", []testOp{ 301 | ai(Image{Checksum: "a", Fileid: "fid"}), 302 | }, false, nil, nil, []Image{Image{Id: 1, Checksum: "a", Fileid: "fid"}}}, 303 | {"Add an images with text", []testOp{ 304 | ai(Image{Checksum: "a", Fileid: "fid"}), ai(Image{Checksum: "b", Text: "b"}), 305 | }, false, nil, nil, []Image{Image{Id: 1, Checksum: "a", Fileid: "fid"}, Image{Id: 2, Checksum: "b", Text: "b"}}}, 306 | {"Add image and remove it", []testOp{ 307 | ai(Image{Checksum: "a", Text: "fid"}), ai(Image{Checksum: "b", ProcessLog: "pl"}), di("a"), 308 | }, false, nil, nil, []Image{Image{Id: 2, Checksum: "b", ProcessLog: "pl"}}}, 309 | {"Add image and update it", []testOp{ 310 | ai(Image{Checksum: "a", Text: "fid"}), ui(Image{Id: 1, Checksum: "a", Text: "other"}), 311 | }, false, nil, nil, []Image{Image{Id: 1, Checksum: "a", Text: "other"}}}, 312 | {"Add a duplicate", []testOp{ 313 | ai(Image{Checksum: "a", Text: "jeje"}), ai(Image{Checksum: "a", Text: "b"}), 314 | }, true, nil, nil, []Image{Image{Id: 1, Checksum: "a", Text: "jeje"}}}, 315 | {"Pagination", []testOp{ 316 | ai(Image{Checksum: "f1"}), ai(Image{Checksum: "f2"}), 317 | ai(Image{Checksum: "f3"}), ai(Image{Checksum: "f4"}), 318 | }, false, &Page{SinceId: 2, Count: 5}, nil, []Image{Image{Id: 3, Checksum: "f3"}, Image{Id: 4, Checksum: "f4"}}}, 319 | {"Add image with tags", []testOp{ 320 | at("eka", ""), at("toka", ""), 321 | ai(Image{Checksum: "a", Text: "jeje", Tags: []Tag{Tag{Name: "toka"}}}), 322 | }, false, nil, nil, []Image{Image{Id: 1, Checksum: "a", Text: "jeje", Tags: []Tag{Tag{Id: 2, Name: "toka"}}}}}, 323 | {"Update image with tags", []testOp{ 324 | at("eka", ""), 325 | ai(Image{Checksum: "a"}), 326 | ui(Image{Id: 1, Checksum: "a", Tags: []Tag{Tag{Id: 1, Name: "eka"}}}), 327 | }, false, nil, nil, []Image{Image{Id: 1, Checksum: "a", Tags: []Tag{Tag{Id: 1, Name: "eka"}}}}}, 328 | {"Add images and search them", []testOp{ 329 | ai(Image{Checksum: "a", Text: "first"}), 330 | ai(Image{Checksum: "b", Text: "second"}), 331 | }, false, nil, &Search{Match: "seco*"}, 332 | []Image{Image{Id: 2, Checksum: "b", Text: "second"}}}, 333 | } 334 | for _, tt := range tests { 335 | db, err := setupDb() 336 | if err != nil { 337 | t.Errorf("Setting up db failed with error = %v", err) 338 | return 339 | } 340 | t.Run(tt.name, func(t *testing.T) { 341 | 342 | var failed bool = false 343 | fail := struct { 344 | failed bool 345 | err error 346 | i int 347 | }{} 348 | 349 | for i, op := range tt.ops { 350 | err := op.run(db) 351 | failed = failed || (err != nil) 352 | if failed && !fail.failed { 353 | fail.failed = true 354 | fail.err = err 355 | fail.i = i 356 | } 357 | } 358 | if failed != tt.wantErr { 359 | t.Errorf("op no.%d error = %v, wantErr %v", fail.i, fail.err, tt.wantErr) 360 | return 361 | } 362 | 363 | images, err := db.getImages(tt.paging, tt.search) 364 | if err != nil { 365 | t.Errorf("db.getImages() error = %v", err) 366 | } 367 | 368 | cmp(t, tt.wantImages, images.Images) 369 | }) 370 | db.Close() 371 | err = teardownDb() 372 | if err != nil { 373 | t.Errorf("Could not remove database file: %v", err) 374 | } 375 | } 376 | } 377 | 378 | func withDb(f func(*db) error) (err error) { 379 | _ = teardownDb() 380 | db, err := setupDb() 381 | if err != nil { 382 | return 383 | } 384 | 385 | e2 := f(db) 386 | 387 | db.Close() 388 | err = teardownDb() 389 | if err != nil { 390 | return 391 | } 392 | 393 | return e2 394 | } 395 | 396 | func addImages(db *db, count int) (err error) { 397 | for i := 0; i < count; i++ { 398 | idStr := "id:" + strconv.Itoa(i+1) 399 | _, err = db.addImage(Image{ 400 | Checksum: idStr, 401 | Text: "jep " + idStr, 402 | ProcessLog: "jeje", 403 | }) 404 | if err != nil { 405 | return 406 | } 407 | } 408 | return 409 | } 410 | 411 | func Benchmark_getImages(b *testing.B) { 412 | 413 | var countImages int = 100 414 | 415 | err := withDb(func(db *db) (err error) { 416 | err = addImages(db, countImages) 417 | if err != nil { 418 | b.Errorf("Adding images failed: %v", err) 419 | } 420 | 421 | b.ResetTimer() 422 | 423 | tests := []struct { 424 | name string 425 | page *Page 426 | search *Search 427 | count int 428 | }{ 429 | {"Get all images", nil, nil, countImages}, 430 | {"Get page of 10 images", &Page{SinceId: 3, Count: 10}, nil, 10}, 431 | {"Get all images with string 7", nil, &Search{Match: "*7*"}, 11}, 432 | } 433 | 434 | for _, tt := range tests { 435 | b.Run(tt.name, func(b *testing.B) { 436 | var is ImageResult 437 | for i := 0; i < b.N; i++ { 438 | is, err = db.getImages(tt.page, tt.search) 439 | if err != nil { 440 | b.Errorf("Getting images failed: %v", err) 441 | return 442 | } 443 | if len(is.Images) != tt.count { 444 | b.Errorf("Expected %d images got %d images", tt.count, len(is.Images)) 445 | } 446 | } 447 | }) 448 | } 449 | 450 | return 451 | }) 452 | 453 | if err != nil { 454 | b.Errorf("Database handling failed with: %v", err) 455 | } 456 | } 457 | -------------------------------------------------------------------------------- /web/paperless-frontend/src/App.vue: -------------------------------------------------------------------------------- 1 | 185 | 186 | 187 | 454 | 455 | 584 | -------------------------------------------------------------------------------- /clipper/qtclipper.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 Kalle Kankare 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #define CLIPPER_VERSION "1.0" 38 | 39 | struct Options { 40 | QStringList infiles; 41 | QString outpattern; 42 | bool discardNoChange; 43 | }; 44 | 45 | class ClipperScene; 46 | 47 | class SceneChanger 48 | { 49 | public: 50 | virtual QString saveImage(const QImage &, int, bool) = 0; 51 | virtual bool deleteImage(const QString&) = 0; 52 | virtual bool popScene() = 0; 53 | 54 | enum Direction { 55 | NEXT, 56 | PREVIOUS, 57 | SUBIMAGE 58 | }; 59 | 60 | virtual bool pushScene(const QImage&, Direction = NEXT, int = 0) = 0; 61 | 62 | virtual ClipperScene* getScene() = 0; 63 | virtual void pushCmd(QUndoCommand *) = 0; 64 | virtual void redo() = 0; 65 | virtual void undo() = 0; 66 | 67 | virtual void quit() = 0; 68 | }; 69 | 70 | enum CustomItemTypes { 71 | ClipperRectType = QGraphicsItem::UserType + 1, 72 | }; 73 | 74 | class ClipperRect : public QGraphicsRectItem 75 | { 76 | public: 77 | enum Style 78 | { 79 | LASSO_SELECTION, 80 | SELECTED, 81 | SAVED, 82 | }; 83 | 84 | struct rect_t { 85 | QRectF rect; 86 | Style style; 87 | int position; 88 | QTransform transform; 89 | QTransform createTransform; 90 | }; 91 | 92 | ClipperRect(QRectF rect = QRectF(), QGraphicsItem *parent = 0) : 93 | QGraphicsRectItem(rect, parent), m_style(LASSO_SELECTION), m_pos(0) 94 | {} 95 | 96 | void setStyle(Style style) 97 | { 98 | m_style = style; 99 | QPen pen; 100 | QBrush brush; 101 | 102 | switch(m_style) 103 | { 104 | case LASSO_SELECTION: 105 | pen.setColor(Qt::black); 106 | pen.setWidth(3); 107 | break; 108 | case SELECTED: 109 | pen.setColor(Qt::green); 110 | pen.setWidth(2); 111 | brush.setColor(Qt::darkGreen); 112 | brush.setStyle(Qt::BDiagPattern); 113 | setBrush(brush); 114 | break; 115 | case SAVED: 116 | pen.setColor(Qt::gray); 117 | pen.setWidth(2); 118 | brush.setColor(Qt::gray); 119 | brush.setStyle(Qt::BDiagPattern); 120 | setBrush(brush); 121 | default: 122 | break; 123 | } 124 | setPen(pen); 125 | } 126 | Style style() const { return m_style; } 127 | void setPosition(const int pos) {m_pos = pos;} 128 | int position() const { return m_pos; } 129 | virtual int type() const { return ClipperRectType; } 130 | QTransform createTransform() const { return m_createTransform; } 131 | void setCreateTransform(QTransform transform) { m_createTransform = transform; } 132 | 133 | QRectF createRect() 134 | { 135 | QTransform bak = transform(); 136 | QRectF ret; 137 | setTransform(m_createTransform); 138 | ret = boundingRect(); 139 | setTransform(bak); 140 | return ret; 141 | } 142 | 143 | rect_t save() 144 | { 145 | rect_t ret; 146 | ret.rect = rect(); 147 | ret.style = m_style; 148 | ret.position = m_pos; 149 | ret.transform = transform(); 150 | ret.createTransform = m_createTransform; 151 | return ret; 152 | } 153 | 154 | void load(const rect_t& r) 155 | { 156 | setRect(r.rect); 157 | m_style = r.style; 158 | m_pos = r.position; 159 | setTransform(r.transform); 160 | m_createTransform = r.createTransform; 161 | } 162 | private: 163 | Style m_style; 164 | int m_pos; 165 | QTransform m_createTransform; 166 | }; 167 | 168 | template 169 | class ClipperCommand : public QUndoCommand 170 | { 171 | public: 172 | typedef std::tuple locals_t; 173 | 174 | template 175 | ClipperCommand(C&& cmd): QUndoCommand(0) 176 | { 177 | m_cmd = std::bind(cmd, std::placeholders::_1, this); 178 | } 179 | 180 | template 181 | void set(Locals2&&... locals) 182 | { 183 | m_local = std::forward_as_tuple(locals...); 184 | } 185 | 186 | template 187 | typename std::tuple_element::type get() 188 | { 189 | return std::get(m_local); 190 | } 191 | 192 | virtual void undo() { m_cmd(false); } 193 | virtual void redo() { m_cmd(true); } 194 | private: 195 | 196 | locals_t m_local; 197 | std::function m_cmd; 198 | }; 199 | 200 | class ClipperScene : public QGraphicsScene 201 | { 202 | public: 203 | ClipperScene(const QImage &img, SceneChanger* changer, QObject *parent = 0) : 204 | QGraphicsScene(parent), changer(changer), m_pic(0), m_rectpos(0), 205 | m_changed(false), m_size(img.size()) 206 | { 207 | m_pic = addPixmap(QPixmap::fromImage(img)); 208 | fitPicInView(); 209 | } 210 | 211 | protected: 212 | 213 | virtual void keyPressEvent(QKeyEvent *event) 214 | { 215 | switch(event->key()) 216 | { 217 | case Qt::Key_Q: 218 | changer->quit(); 219 | break; 220 | 221 | // toggle fullscreen 222 | case Qt::Key_F: 223 | { 224 | QGraphicsView *view = getView(); 225 | if(view->isFullScreen()) 226 | view->showNormal(); 227 | else 228 | view->showFullScreen(); 229 | break; 230 | } 231 | 232 | case Qt::Key_U: 233 | changer->undo(); 234 | break; 235 | case Qt::Key_R: 236 | changer->redo(); 237 | break; 238 | default: 239 | break; 240 | } 241 | } 242 | 243 | virtual void mouseReleaseEvent(QGraphicsSceneMouseEvent *event) 244 | { 245 | debugMouseEvent("release",event); 246 | 247 | QPointF p1 = event->buttonDownScenePos(event->button()), 248 | p2 = event->scenePos(); 249 | 250 | SceneChanger *ch = changer; 251 | 252 | switch(event->button()) 253 | { 254 | case Qt::LeftButton: { 255 | QRectF rect(p1,p2); 256 | rect = rect.normalized(); 257 | 258 | // Add a new rectangle 259 | if(!rect.size().isNull()) 260 | { 261 | qDebug() << "Added rect of size " << rect; 262 | typedef ClipperCommand cmd_rect_t; 263 | changer->pushCmd(new cmd_rect_t( 264 | [rect, ch] (bool redo, cmd_rect_t* ptr) { 265 | if(redo) { 266 | int pos = -1; 267 | if (ptr->get<0>()) 268 | pos = ptr->get<0>(); 269 | ClipperRect *cr = ch->getScene()->addRect(rect, pos); 270 | ptr->set(cr->position()); 271 | } else { 272 | ch->getScene()->removeRect(ptr->get<0>()); 273 | } 274 | })); 275 | } 276 | // A normal click 277 | else 278 | { 279 | ClipperRect *cr = getClipperRect(event->scenePos()); 280 | 281 | // Save the whole current image 282 | if(!cr) 283 | { 284 | rect = sceneRect(); 285 | bool changed = m_changed; 286 | QTransform tr = m_pic->transform(); 287 | typedef ClipperCommand cmd_saveimg_t; 288 | changer->pushCmd(new cmd_saveimg_t( 289 | [rect, ch, changed, tr] (bool redo, cmd_saveimg_t* ptr) { 290 | if(redo){ 291 | QImage img = ch->getScene()->createImage(rect, tr); 292 | RectData saved = ch->getScene()->saveRects(); 293 | ptr->set(ch->saveImage(img, 0, changed), img, saved); 294 | ch->popScene(); 295 | } else { 296 | ch->deleteImage(ptr->get<0>()); 297 | ch->pushScene(ptr->get<1>(), SceneChanger::PREVIOUS); 298 | ch->getScene()->loadRects(ptr->get<2>()); 299 | } 300 | })); 301 | } 302 | // save a subimage limited by the ClipperRect 303 | else 304 | { 305 | rect = cr->boundingRect(); 306 | typedef ClipperCommand cmd_savesubimg_t; 307 | changer->pushCmd(new cmd_savesubimg_t( 308 | [rect, cr, ch] (bool redo, 309 | cmd_savesubimg_t* ptr){ 310 | if(redo){ 311 | QImage img = ch->getScene()->createImage(rect, 312 | cr->createTransform()); 313 | ptr->set(ch->saveImage(img, cr->position(), true)); 314 | cr->setStyle(ClipperRect::SAVED); 315 | } else { 316 | ch->deleteImage(ptr->get<0>()); 317 | cr->setStyle(ClipperRect::SELECTED); 318 | } 319 | })); 320 | } 321 | } 322 | break; 323 | } 324 | case Qt::RightButton: 325 | { 326 | ClipperRect *cr = getClipperRect(event->scenePos()); 327 | QRectF rect; 328 | // Discard the whole image and switch to the next 329 | if (!cr) 330 | { 331 | rect = sceneRect(); 332 | QTransform tr = m_pic->transform(); 333 | typedef ClipperCommand cmd_discardimg_t; 334 | changer->pushCmd(new cmd_discardimg_t ( 335 | [rect, ch, tr](bool redo, cmd_discardimg_t *ptr) { 336 | if (redo) { 337 | ptr->set(ch->getScene()->createImage(rect, tr), ch->getScene()->saveRects()); 338 | ch->popScene(); 339 | } else { 340 | ch->pushScene(ptr->get<0>(), SceneChanger::PREVIOUS); 341 | ch->getScene()->loadRects(ptr->get<1>()); 342 | } 343 | })); 344 | } 345 | // Discard a rectangle unless it has already been saved 346 | else if(cr->style() != ClipperRect::SAVED) 347 | { 348 | rect = cr->boundingRect(); 349 | int rectpos = cr->position(); 350 | typedef ClipperCommand cmd_removerect_t; 351 | changer->pushCmd(new cmd_removerect_t( 352 | [rect, rectpos, ch] (bool redo, cmd_removerect_t* ptr) { 353 | if (redo) { 354 | int pos = rectpos; 355 | if (ptr->get<1>() != 0) 356 | pos = ptr->get<1>(); 357 | ch->getScene()->removeRect(pos); 358 | ptr->set(rect, pos); 359 | } else { 360 | ClipperRect *re = ch->getScene()->addRect( 361 | ptr->get<0>(), ptr->get<1>()); 362 | ptr->set(ptr->get<0>(), re->position()); 363 | } 364 | })); 365 | } 366 | break; 367 | } 368 | case Qt::MiddleButton: 369 | { 370 | ClipperRect *cr = getClipperRect(event->scenePos()); 371 | // zoom to a given rectangle 372 | if (cr) 373 | { 374 | QRectF rect = cr->createRect(); 375 | int imagepos = cr->position(); 376 | typedef ClipperCommand<> cmd_zoomimg_t; 377 | QTransform tr = cr->createTransform(); 378 | 379 | qDebug() << "Current transform" << m_pic->transform() << "createtransform" << tr; 380 | changer->pushCmd(new cmd_zoomimg_t( 381 | [rect, ch, imagepos, tr](bool redo, cmd_zoomimg_t*){ 382 | if (redo) { 383 | QImage img = ch->getScene()->createImage(rect, tr); 384 | ch->pushScene(img, SceneChanger::SUBIMAGE, imagepos); 385 | } else { 386 | ch->popScene(); 387 | }})); 388 | } 389 | break; 390 | } 391 | default: 392 | break; 393 | } 394 | } 395 | 396 | virtual void wheelEvent(QGraphicsSceneWheelEvent *event) 397 | { 398 | qDebug() << "Wheel: " << " modifiers: " << event->modifiers() << " delta: " << 399 | event->delta() << " orientation: " << event->orientation(); 400 | 401 | QTransform tr; 402 | 403 | // rotation 404 | if(event->modifiers() & Qt::ShiftModifier) 405 | { 406 | qreal amount = 2; 407 | if(event->modifiers() & Qt::ControlModifier) 408 | amount = 90; 409 | qreal deg = amount * sgn(event->delta()); 410 | tr = QTransform() 411 | .translate(m_size.width() / 2, m_size.height() / 2) 412 | .rotate(deg) 413 | .translate(- m_size.width() / 2, - m_size.height() / 2); 414 | m_changed = true; 415 | } 416 | #if 0 // disable scaling for now 417 | // scaling 418 | else if (event->modifiers() & Qt::ControlModifier) 419 | { 420 | qreal factor = 1.15; 421 | if(event->delta() < 0 ) 422 | factor = 1.0 / factor; 423 | tr = QTransform().scale(factor,factor); 424 | m_changed = true; 425 | } 426 | #endif 427 | m_pic->setTransform(tr, true); 428 | for (int i = 0; i < rects.size(); i++) 429 | rects.at(i)->setTransform(tr, true); 430 | } 431 | 432 | private: 433 | 434 | template int sgn(T val) 435 | { 436 | return (T(0) < val) - (val < T(0)); 437 | } 438 | 439 | ClipperRect *addRect(const QRectF &coords, int pos = -1) 440 | { 441 | ClipperRect *rect = new ClipperRect(); 442 | 443 | if (pos < 0) 444 | pos = ++m_rectpos; 445 | 446 | rect->setRect(coords); 447 | rect->setStyle(ClipperRect::SELECTED); 448 | rect->setPosition(pos); 449 | rect->setCreateTransform(m_pic->transform()); 450 | rects.push_back(rect); 451 | addItem(rect); 452 | rect->show(); 453 | return rect; 454 | } 455 | 456 | ClipperRect *addRect(const ClipperRect::rect_t &r) 457 | { 458 | ClipperRect *ret = addRect(r.rect, r.position); 459 | ret->load(r); 460 | return ret; 461 | } 462 | 463 | void removeRect(int pos) 464 | { 465 | QList::iterator it = rects.begin(); 466 | for(;it != rects.end(); ++it) 467 | if((*it)->position() == pos) 468 | { 469 | removeItem(*it); 470 | rects.erase(it); 471 | break; 472 | } 473 | } 474 | 475 | void displayRects(bool visible) 476 | { 477 | for (int i = 0; i < rects.size(); i++) 478 | if(!visible) 479 | rects.at(i)->hide(); 480 | else 481 | rects.at(i)->show(); 482 | } 483 | 484 | QGraphicsItem* getItem(QPointF at) 485 | { 486 | return itemAt(at,QTransform()); 487 | } 488 | 489 | bool pointsAtPic(QPointF at) 490 | { 491 | return getItem(at) == m_pic; 492 | } 493 | 494 | QRectF getSubItemRect(QPointF at) 495 | { 496 | QGraphicsItem *it = getItem(at); 497 | QRectF ret; 498 | if (!it || it->type() != ClipperRectType) 499 | return ret; 500 | 501 | return it->boundingRect(); 502 | } 503 | ClipperRect *getClipperRect(QPointF at) 504 | { 505 | QGraphicsItem *it = getItem(at); 506 | if(!it || it->type() != ClipperRectType) 507 | return 0; 508 | return static_cast(it); 509 | } 510 | 511 | QGraphicsView *getView() 512 | { 513 | return (views().count()) ? views().first() : 0; 514 | } 515 | 516 | void fitPicInView() 517 | { 518 | if(!m_pic || !views().count()) 519 | return; 520 | 521 | QGraphicsView *view = getView(); 522 | view->centerOn(m_pic); 523 | view->fitInView(m_pic,Qt::KeepAspectRatio); 524 | } 525 | 526 | QImage createImage(const QRectF &rect, QTransform tr = QTransform()) 527 | { 528 | QImage img(rect.size().toSize(),QImage::Format_ARGB32); 529 | img.fill(Qt::white); 530 | 531 | QPainter pt(&img); 532 | displayRects(false); 533 | QTransform bak = m_pic->transform(); 534 | m_pic->setTransform(tr); 535 | render(&pt, QRectF(), rect); 536 | m_pic->setTransform(bak); 537 | displayRects(true); 538 | return img; 539 | } 540 | 541 | void debugMouseEvent(char const* name, QGraphicsSceneMouseEvent *event) 542 | { 543 | qDebug() << name << ":" << event->button() << "pos: " << 544 | event->buttonDownScenePos(event->button()) << " last: " << 545 | event->scenePos(); 546 | } 547 | 548 | typedef QList RectData; 549 | 550 | RectData saveRects() 551 | { 552 | RectData ret; 553 | for (int i = 0; i < rects.size(); i++) 554 | ret.push_back(rects.at(i)->save()); 555 | return ret; 556 | } 557 | void loadRects(const RectData &data) 558 | { 559 | for (int i = 0; i < data.size(); i++) 560 | addRect(data.at(i)); 561 | } 562 | 563 | SceneChanger *changer; 564 | QGraphicsPixmapItem *m_pic; 565 | ClipperRect selectionRect; 566 | QList rects; 567 | int m_rectpos; 568 | bool m_changed; 569 | QSize m_size; 570 | }; 571 | 572 | 573 | class ClipperView : public QGraphicsView 574 | { 575 | public: 576 | explicit ClipperView(QWidget *parent = 0) : QGraphicsView(parent) 577 | { 578 | setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); 579 | setDragMode(QGraphicsView::RubberBandDrag); 580 | } 581 | 582 | void setClipperScene(ClipperScene *scene) 583 | { 584 | setScene(scene); 585 | fitInView(scene->sceneRect(),Qt::KeepAspectRatio); 586 | } 587 | void resizeEvent(QResizeEvent *) 588 | { 589 | fitInView(scene()->sceneRect(),Qt::KeepAspectRatio); 590 | } 591 | }; 592 | 593 | class ClipperJsonFile 594 | { 595 | QJsonArray images; 596 | public: 597 | 598 | void add(const QString& name, const QString& parent = QString()) 599 | { 600 | QJsonObject item; 601 | item["name"] = name; 602 | if(!parent.isEmpty()) 603 | item["parent"] = parent; 604 | 605 | images.push_back(item); 606 | } 607 | void remove(const QString& name) 608 | { 609 | QJsonArray::iterator it = images.begin(); 610 | 611 | for (; it != images.end(); it++) 612 | { 613 | QJsonObject ob = (*it).toObject(); 614 | if (ob["name"].toString() == name) 615 | { 616 | images.erase(it); 617 | break; 618 | } 619 | } 620 | } 621 | bool save(const QString& filename) 622 | { 623 | QJsonDocument doc(images); 624 | 625 | QString tmp = ""; 626 | tmp.append(doc.toJson()); 627 | qDebug() << "Generating the following json into " << filename << ":\n" << tmp; 628 | 629 | QFile fp(filename); 630 | if(!fp.open(QIODevice::WriteOnly)) 631 | { 632 | qWarning() << "Could not open file " << filename << " for writing."; 633 | return false; 634 | } 635 | return fp.write(doc.toJson()) >= 0; 636 | } 637 | }; 638 | 639 | class SceneHandler : public SceneChanger 640 | { 641 | public: 642 | SceneHandler(Options &opts) : sceneStack(), view(), opts(opts) 643 | { 644 | scenePosStack.push(0); 645 | popScene(); 646 | cmdStack.setUndoLimit(10); 647 | } 648 | 649 | virtual QString saveImage(const QImage &img, int subImageNumber, bool changed) 650 | { 651 | if(img.isNull() || (!changed && opts.discardNoChange)) 652 | return QString(); 653 | 654 | QString name = opts.outpattern; 655 | QString parent; 656 | for (int i = 0; i < scenePosStack.size(); i++) 657 | name += QString("-") + QString::number(scenePosStack.at(i)); 658 | 659 | if (subImageNumber > 0) 660 | { 661 | parent = name + ".jpg"; 662 | name = name + "+" + QString::number(subImageNumber); 663 | } 664 | name += ".jpg"; 665 | qDebug() << "Saving image to " << name; 666 | json.add(name, parent); 667 | img.save(name); 668 | 669 | return name; 670 | } 671 | 672 | virtual bool deleteImage(const QString& name) 673 | { 674 | qDebug() << "removing file " << name; 675 | if(name.isNull()) 676 | return false; 677 | 678 | json.remove(name); 679 | return QFile::remove(name); 680 | } 681 | 682 | void quit() 683 | { 684 | json.save("clipper-images.json"); 685 | QCoreApplication::exit(); 686 | } 687 | 688 | // replaces current scene with a new one or quits if not available. 689 | bool popScene() 690 | { 691 | QGraphicsScene *top = 0; 692 | if(!sceneStack.empty()) 693 | top = sceneStack.pop(); 694 | 695 | if(sceneStack.empty()) 696 | { 697 | QString fname; 698 | do { 699 | if(opts.infiles.empty()) 700 | { 701 | this->quit(); 702 | return false; 703 | } 704 | fname = opts.infiles.first(); 705 | opts.infiles.removeFirst(); 706 | ++scenePosStack.top(); 707 | qDebug()<< "Filename is here" << fname << "and position " << 708 | scenePosStack.top(); 709 | } while(newScene(fname).isNull()); 710 | } 711 | else if(scenePosStack.size() > 1) 712 | scenePosStack.pop(); 713 | view.setClipperScene(sceneStack.top()); 714 | view.show(); 715 | 716 | if(top) 717 | top->deleteLater(); 718 | 719 | return true; 720 | } 721 | 722 | 723 | bool pushScene(const QImage& img, Direction dir = NEXT, int pos = 0) 724 | { 725 | if (dir == PREVIOUS) 726 | scenePosStack.top()--; 727 | else if(dir == SUBIMAGE) 728 | scenePosStack.push(pos); 729 | 730 | newScene(img); 731 | view.setClipperScene(sceneStack.top()); 732 | 733 | return true; 734 | } 735 | 736 | ClipperScene *getScene() 737 | { 738 | return !sceneStack.empty() ? sceneStack.top() : 0; 739 | } 740 | 741 | void pushCmd(QUndoCommand *cmd) { cmdStack.push(cmd); } 742 | void undo() { cmdStack.undo(); } 743 | void redo() { cmdStack.redo(); } 744 | 745 | private: 746 | 747 | QImage newScene(const QImage &img) 748 | { 749 | sceneStack.push(new ClipperScene(img, this)); 750 | return img; 751 | } 752 | 753 | QImage newScene(const QString &name) 754 | { 755 | QImage img; 756 | if (!img.load(name)) 757 | return QImage(); 758 | return newScene(img); 759 | } 760 | 761 | QStack sceneStack; 762 | QUndoStack cmdStack; 763 | QStack scenePosStack; 764 | ClipperView view; 765 | Options opts; 766 | ClipperJsonFile json; 767 | }; 768 | 769 | int main(int argc, char **argv) 770 | { 771 | QApplication app(argc, argv); 772 | QCommandLineParser parser; 773 | Options opts; 774 | 775 | // Command line handling 776 | QCoreApplication::setApplicationName(argv[0]); 777 | QCoreApplication::setApplicationVersion(CLIPPER_VERSION); 778 | parser.addVersionOption(); 779 | parser.addHelpOption(); 780 | parser.setApplicationDescription("Image clipping application."); 781 | QCommandLineOption OutOpt(QStringList() << "o" << "out", 782 | "Pattern of output files.", "outpattern"); 783 | parser.addOption(OutOpt); 784 | QCommandLineOption DiscardNoChange(QStringList() << "d" << 785 | "discard-if-not-changed", "Discard the image when saving if not changed."); 786 | parser.addOption(DiscardNoChange); 787 | parser.process(app); 788 | opts.infiles = parser.positionalArguments(); 789 | if (!parser.isSet(OutOpt)) 790 | { 791 | qCritical() << "Error: --out is a required parameter.\n"; 792 | parser.showHelp(1); 793 | Q_UNREACHABLE(); 794 | } 795 | opts.outpattern = parser.value(OutOpt); 796 | 797 | QFileInfo info(opts.outpattern); 798 | if (!info.dir().exists()) 799 | { 800 | qCritical() << "Error: --out directory" << info.dir().path() << 801 | " does not exist.\n"; 802 | parser.showHelp(1); 803 | Q_UNREACHABLE(); 804 | } 805 | 806 | opts.discardNoChange = false; 807 | if (parser.isSet(DiscardNoChange)) 808 | opts.discardNoChange = true; 809 | 810 | if (opts.infiles.empty()) 811 | return 0; 812 | 813 | //DEBUG 814 | qDebug() << "infiles:"; 815 | for (int i = 0; i < opts.infiles.size(); ++i) 816 | qDebug() << opts.infiles.at(i).toLocal8Bit().constData(); 817 | qDebug() << "outpattern: " << opts.outpattern.toLocal8Bit().constData(); 818 | 819 | // The processing 820 | SceneHandler sh(opts); 821 | return app.exec(); 822 | } 823 | --------------------------------------------------------------------------------