├── .cfignore ├── .dockerignore ├── .drone.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── cmd ├── account.go ├── add.go ├── delete.go ├── export.go ├── import.go ├── open.go ├── pocket.go ├── print.go ├── search.go ├── serve │ ├── root.go │ ├── web-handler-api.go │ ├── web-handler-ui.go │ └── web-handler.go ├── update.go └── utils.go ├── database ├── database.go ├── migration │ └── m1.go ├── migrations.go ├── structs.go └── xorm.go ├── docs ├── README.md ├── databases.md ├── installation.md ├── override.css └── usage.md ├── go.mod ├── go.sum ├── main.go ├── model └── model.go ├── package-lock.json ├── package.json ├── renovate.json ├── src ├── cache.html ├── cache.js ├── component │ ├── variable.less │ ├── yla-dialog.js │ ├── yla-dialog.less │ ├── yla-tooltip.js │ └── yla-tooltip.less ├── index.html ├── index.js ├── less │ └── stylesheet.less ├── login.html ├── login.js ├── page │ └── base.js ├── res │ ├── apple-touch-icon-144x144.png │ ├── apple-touch-icon-152x152.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ └── favicon.ico ├── submit.html └── submit.js └── utils ├── colours.go ├── errors.go ├── parse.go ├── strings.go └── urls.go /.cfignore: -------------------------------------------------------------------------------- 1 | cmd/* 2 | database/* 3 | dist/* 4 | docs/* 5 | packrd/* 6 | model/* 7 | node_modules/* 8 | src/* 9 | .cache/* 10 | _docpress/* 11 | go.mod 12 | go.sum 13 | main.go 14 | package.json 15 | package-lock.json -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | README.md 3 | Dockerfile 4 | LICENSE 5 | README.md 6 | screenshot/* 7 | shiori 8 | shiori.db 9 | vendor/* 10 | dist/* 11 | node_modules/* 12 | _docpress/* -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | name: default 4 | 5 | platform: 6 | os: linux 7 | arch: amd64 8 | 9 | steps: 10 | - name: check 11 | image: golang:1.15 12 | commands: 13 | - make fmt-check 14 | 15 | - name: docker-check 16 | image: plugins/docker 17 | settings: 18 | repo: techknowlogick/shiori 19 | dry_run: true 20 | when: 21 | event: 22 | - pull_request 23 | 24 | - name: build-node 25 | image: node:10 26 | commands: 27 | - make dep-node 28 | 29 | - name: build-go 30 | image: golang:1.15 31 | environment: 32 | GO111MODULE: on 33 | commands: 34 | - GO111MODULE=off go get -u github.com/markbates/pkger/cmd/pkger 35 | - go mod download 36 | - pkger 37 | - make build 38 | - ./shiori 39 | - ./shiori add https://example.com 40 | - ./shiori print 41 | - ./shiori delete 1 42 | - ./shiori add https://src.techknowlogick.com 43 | - ./shiori add https://www.thestar.com/news/gta/2019/02/13/woman-accused-of-throwing-a-chair-off-downtown-toronto-balcony-turns-herself-in.html 44 | - ./shiori print 45 | 46 | - name: docker-publish-nightly 47 | image: plugins/docker 48 | settings: 49 | username: 50 | from_secret: docker_username 51 | password: 52 | from_secret: docker_password 53 | repo: techknowlogick/shiori 54 | tags: latest, nightly 55 | when: 56 | event: push 57 | branch: master 58 | 59 | - name: docker-publish-tag 60 | image: plugins/docker 61 | settings: 62 | username: 63 | from_secret: docker_username 64 | password: 65 | from_secret: docker_password 66 | repo: techknowlogick/shiori 67 | auto_tag: true 68 | default_tags: true 69 | when: 70 | event: tag 71 | 72 | - name: cross 73 | image: techknowlogick/xgo 74 | environment: 75 | # This path does not exist. However, when we set the gopath to /go, the build fails. Not sure why. 76 | # Leaving this here until we know how to resolve this properly. 77 | GOPATH: /srv/app 78 | commands: 79 | - rm -rf /source 80 | - ln -s $CI_WORKSPACE /source 81 | - go get 82 | - make cross 83 | when: 84 | event: 85 | exclude: 86 | - pull_request 87 | 88 | - name: compress-and-checksum 89 | image: golang:1.15 90 | commands: 91 | - make release 92 | when: 93 | event: 94 | exclude: 95 | - pull_request 96 | 97 | - name: publish 98 | image: plugins/github-release 99 | settings: 100 | api_key: 101 | from_secret: github_token 102 | files: dist/release/* 103 | when: 104 | event: tag 105 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Exclude sample 2 | sample.txt 3 | 4 | # Exclude config file 5 | .vscode/ 6 | 7 | # Exclude generated file 8 | shiori* 9 | *.db 10 | thumb/ 11 | dist/* 12 | .cache/* 13 | cmd/serve/serve-packr.go 14 | packrd/* 15 | release/* 16 | 17 | # Exclude nodejs packages 18 | node_modules/* 19 | 20 | # Exclude generated documentation 21 | _docpress/* 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts-alpine as nodebuilder 2 | RUN apk --no-cache add python2 make bash git 3 | 4 | WORKDIR /app 5 | 6 | COPY . . 7 | 8 | RUN make dep-node 9 | 10 | FROM golang:1.15-alpine as gobuilder 11 | 12 | RUN apk update \ 13 | && apk --no-cache add git build-base make bash 14 | 15 | WORKDIR /go/src/src.techknowlogick.com/shiori 16 | COPY . . 17 | COPY --from=nodebuilder /app/dist /go/src/src.techknowlogick.com/shiori/dist/ 18 | RUN go get -u github.com/markbates/pkger/cmd/pkger 19 | RUN pkger && make build 20 | 21 | FROM alpine:3.12 22 | 23 | ENV ENV_SHIORI_DIR /srv/shiori/ 24 | 25 | RUN apk --no-cache add dumb-init ca-certificates 26 | COPY --from=gobuilder /go/src/src.techknowlogick.com/shiori/shiori /usr/local/bin/shiori 27 | COPY --from=gobuilder /go/src/src.techknowlogick.com/shiori/dist /dist 28 | 29 | WORKDIR /srv/ 30 | RUN mkdir shiori 31 | 32 | EXPOSE 8080 33 | 34 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 35 | CMD ["/usr/local/bin/shiori", "serve"] 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Radhi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DIST := dist 2 | IMPORT := src.techknowlogick.com/shiori 3 | GO ?= go 4 | SED_INPLACE := sed -i 5 | GOFILES := $(shell find . -name "*.go" -type f ! -path "./vendor/*" ! -path "*/*-packr.go") 6 | GOFMT ?= gofmt -s 7 | SHASUM := shasum -a 256 8 | 9 | GO111MODULE ?= on 10 | 11 | TAGS ?= 12 | 13 | ifneq ($(DRONE_TAG),) 14 | VERSION ?= $(subst v,,$(DRONE_TAG)) 15 | SHIORI_VERSION := $(VERSION) 16 | else 17 | ifneq ($(DRONE_BRANCH),) 18 | VERSION ?= $(subst release/v,,$(DRONE_BRANCH)) 19 | else 20 | VERSION ?= master 21 | endif 22 | SHIORI_VERSION := $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') 23 | endif 24 | 25 | LDFLAGS := -X "main.Version=$(SHIORI_VERSION)" -X "main.Tags=$(TAGS)" 26 | 27 | ifeq ($(OS), Windows_NT) 28 | EXECUTABLE := shiori.exe 29 | else 30 | EXECUTABLE := shiori 31 | endif 32 | 33 | # $(call strip-suffix,filename) 34 | strip-suffix = $(firstword $(subst ., ,$(1))) 35 | 36 | .PHONY: fmt 37 | fmt: 38 | $(GOFMT) -w $(GOFILES) 39 | 40 | .PHONY: fmt-check 41 | fmt-check: 42 | # get all go files and run go fmt on them 43 | @diff=$$($(GOFMT) -d $(GOFILES)); \ 44 | if [ -n "$$diff" ]; then \ 45 | echo "Please run 'make fmt' and commit the result:"; \ 46 | echo "$${diff}"; \ 47 | exit 1; \ 48 | fi; 49 | 50 | .PHONY: build-clean 51 | build-clean: 52 | rm -rf ./_docpress 53 | 54 | .PHONY: build-docs 55 | build-docs: build-clean 56 | npx docpress@0.7.6 build . 57 | npx replace-x 'https?://[^"]+' '#not-remote-font' ./_docpress/ --include="*.css" -q -r 58 | 59 | .PHONY: build 60 | build: $(EXECUTABLE) 61 | 62 | $(EXECUTABLE): $(SOURCES) 63 | $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ 64 | 65 | # dist step is kept for backwords compatibility 66 | .PHONY: dist 67 | dist: dep-node dep-go 68 | 69 | .PHONY: dep 70 | dep: dep-node dep-go 71 | 72 | .PHONY: dep-node 73 | dep-node: 74 | @hash npx > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 75 | echo "Please install npm version 5.2+"; \ 76 | exit 1; \ 77 | fi; 78 | npm install 79 | npx parcel build src/*.html --public-url /dist/ 80 | #npx replace-x '(\.\./){1,3}(shiori|app)' '/dist' ./dist/ --include="*.css" -q -r 81 | 82 | .PHONY: dep-go 83 | dep-go: 84 | @hash packr2 > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 85 | $(GO) get -u github.com/markbates/pkger/cmd/pkger; \ 86 | fi 87 | pkger 88 | 89 | .PHONY: cross 90 | cross: release-dirs release-windows release-darwin release-linux release-copy 91 | 92 | .PHONY: release 93 | release: release-compress release-check 94 | 95 | .PHONY: release-dirs 96 | release-dirs: 97 | mkdir -p $(DIST)/binaries $(DIST)/release 98 | 99 | .PHONY: release-windows 100 | release-windows: 101 | @hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 102 | $(GO) get -u src.techknowlogick.com/xgo; \ 103 | fi 104 | xgo -dest $(DIST) -tags 'netgo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'windows/*' -out shiori . 105 | ifeq ($(CI),drone) 106 | cp /build/* $(DIST)/binaries 107 | endif 108 | 109 | .PHONY: release-darwin 110 | release-darwin: 111 | @hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 112 | $(GO) get -u src.techknowlogick.com/xgo; \ 113 | fi 114 | xgo -dest $(DIST) -tags 'netgo $(TAGS)' -ldflags '$(LDFLAGS)' -targets 'darwin/*' -out shiori . 115 | ifeq ($(CI),drone) 116 | cp /build/* $(DIST)/binaries 117 | endif 118 | 119 | .PHONY: release-linux 120 | release-linux: 121 | @hash xgo > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 122 | $(GO) get -u src.techknowlogick.com/xgo; \ 123 | fi 124 | xgo -dest $(DIST) -tags 'netgo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'linux/*' -out shiori . 125 | ifeq ($(CI),drone) 126 | cp /build/* $(DIST)/binaries 127 | endif 128 | 129 | .PHONY: release-copy 130 | release-copy: 131 | cd $(DIST); for file in `find /build -type f -name "*"`; do cp $${file} ./release/; done; 132 | 133 | .PHONY: release-check 134 | release-check: 135 | cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done; 136 | 137 | .PHONY: release-compress 138 | release-compress: 139 | @hash gxz > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ 140 | $(GO) get -u github.com/ulikunitz/xz/cmd/gxz; \ 141 | fi 142 | cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && gxz -k -9 $${file}; done; 143 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: ./shiori serve 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shiori 2 | 3 | [](https://cloud.drone.io/techknowlogick/shiori) 4 | [](https://goreportcard.com/report/src.techknowlogick.com/shiori) 5 | [](https://godoc.org/src.techknowlogick.com/shiori) 6 | [](https://github.com/techknowlogick/shiori/releases/latest) 7 | 8 | Shiori is a simple bookmarks manager written in Go language. Intended as a simple clone of [Pocket](https://getpocket.com/). You can use it as command line application or as web application. This application is distributed as a single binary, which means it can be installed and used easily. 9 | 10 | ## Features 11 | 12 | - Simple and clean command line interface. 13 | - Basic bookmarks management i.e. add, edit and delete. 14 | - Search bookmarks by their title, tags, url and page content. 15 | - Import and export bookmarks from and to Netscape Bookmark file. 16 | - Portable, thanks to its single binary format and sqlite3 database 17 | - Simple web interface for those who don't want to use a command line app. 18 | - Where possible, by default `shiori` will download a static copy of the webpage in simple text and HTML format, which later can be used as an offline archive for that page. 19 | 20 | ## Demo 21 | 22 | A demo can be found at: https://shiori-demo.techknowlogick.com/ where the user/password are both `demo`. The database will be wiped every so often, so please don't use this for anything other than quick evaluation. 23 | 24 | ## FAQs 25 | 26 | ### Why did you make this fork? 27 | 28 | My goals for this fork were to change things for my personal preferences. 29 | 30 | ### What are the changes made to this repo? 31 | 32 | Please see the [list of changes](https://github.com/techknowlogick/shiori/issues/82) made to this project compared to the original. 33 | 34 | ### Is the original Shiori not being updated anymore? 35 | 36 | Sadly the original maintainer of Shiori is no longer able to maintain the project anymore. This project was forked during a prior hiatus and changes with the original project means this project couldn't be merged back. Please see the [answer to this question provided by Radhi](https://github.com/go-shiori/shiori/issues/256) for more details about why the original project is no longer maintained. 37 | 38 | ## License 39 | 40 | Shiori is distributed using [MIT license](https://choosealicense.com/licenses/mit/), which means you can use and modify it however you want. However, if you make an enhancement or a bug fix, if possible, please send a pull request. 41 | -------------------------------------------------------------------------------- /cmd/account.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "syscall" 7 | 8 | "src.techknowlogick.com/shiori/utils" 9 | 10 | "github.com/urfave/cli" 11 | "golang.org/x/crypto/ssh/terminal" 12 | ) 13 | 14 | var ( 15 | CmdAccount = cli.Command{ 16 | Name: "account", 17 | Usage: "Manage account for accessing web interface", 18 | Subcommands: []cli.Command{ 19 | subcmdAddAccount, 20 | subcmdPrintAccounts, 21 | subcmdDeleteAccounts, 22 | }, 23 | } 24 | 25 | subcmdAddAccount = cli.Command{ 26 | Name: "add", 27 | Usage: "Create new account", 28 | Action: runAddAccount, 29 | } 30 | 31 | subcmdPrintAccounts = cli.Command{ 32 | Name: "print", 33 | Usage: "List all accounts", 34 | Aliases: []string{"list", "ls"}, 35 | Flags: []cli.Flag{ 36 | cli.StringFlag{ 37 | Name: "search, s", 38 | Usage: "Search accounts by username", 39 | }, 40 | }, 41 | Action: runPrintAccount, 42 | } 43 | 44 | subcmdDeleteAccounts = cli.Command{ 45 | Name: "delete", 46 | Aliases: []string{"rm"}, 47 | Description: "Delete accounts. " + 48 | "Accepts space-separated list of usernames. " + 49 | "If no arguments, all records will be deleted.", 50 | Flags: []cli.Flag{ 51 | cli.StringFlag{ 52 | Name: "yes, y", 53 | Usage: "Skip confirmation prompt and delete ALL accounts", 54 | }, 55 | }, 56 | Action: runDeleteAccount, 57 | } 58 | ) 59 | 60 | func runAddAccount(c *cli.Context) error { 61 | // TODO: check for duplicate account already 62 | args := c.Args() 63 | db, err := getDbConnection(c) 64 | 65 | if err != nil { 66 | return errors.New(utils.CErrorSprint(err)) 67 | } 68 | 69 | if len(args) < 1 { 70 | return errors.New(utils.CErrorSprint("Username must not be empty")) 71 | } 72 | 73 | username := args[0] 74 | if username == "" { 75 | return errors.New(utils.CErrorSprint("Username must not be empty")) 76 | } 77 | 78 | fmt.Println("Username: " + username) 79 | 80 | // Read and validate password 81 | fmt.Print("Password: ") 82 | bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) 83 | if err != nil { 84 | return errors.New(utils.CErrorSprint(err)) 85 | } 86 | 87 | fmt.Println() 88 | strPassword := string(bytePassword) 89 | if len(strPassword) < 8 { 90 | return errors.New(utils.CErrorSprint("Password must be at least 8 characters")) 91 | } 92 | 93 | // Save account to database 94 | err = db.CreateAccount(username, strPassword) 95 | if err != nil { 96 | return errors.New(utils.CErrorSprint(err)) 97 | } 98 | return nil 99 | } 100 | 101 | func runPrintAccount(c *cli.Context) error { 102 | // Parse flags 103 | db, err := getDbConnection(c) 104 | 105 | if err != nil { 106 | return errors.New(utils.CErrorSprint(err)) 107 | } 108 | keyword := c.String("search") 109 | 110 | // Fetch list accounts in database 111 | accounts, err := db.GetAccounts(keyword) 112 | if err != nil { 113 | return errors.New(utils.CErrorSprint(err)) 114 | } 115 | 116 | // Show list accounts 117 | for _, account := range accounts { 118 | utils.CIndex.Print("- ") 119 | fmt.Println(account.Username) 120 | } 121 | return nil 122 | } 123 | 124 | func runDeleteAccount(c *cli.Context) error { 125 | args := c.Args() 126 | skipConfirm := c.Bool("yes") 127 | db, err := getDbConnection(c) 128 | 129 | if err != nil { 130 | return errors.New(utils.CErrorSprint(err)) 131 | } 132 | 133 | // If no arguments (i.e all accounts going to be deleted), 134 | // confirm to user 135 | if len(args) == 0 && !skipConfirm { 136 | confirmDelete := "" 137 | fmt.Print("Remove ALL accounts? (y/n): ") 138 | fmt.Scanln(&confirmDelete) 139 | 140 | if confirmDelete != "y" { 141 | fmt.Println("No accounts deleted") 142 | return nil 143 | } 144 | } 145 | 146 | // Delete accounts in database 147 | err = db.DeleteAccounts(args...) 148 | if err != nil { 149 | return errors.New(utils.CErrorSprint(err)) 150 | } 151 | 152 | fmt.Println("Account(s) have been deleted") 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /cmd/add.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math" 7 | nurl "net/url" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | 12 | "src.techknowlogick.com/shiori/utils" 13 | 14 | valid "github.com/asaskevich/govalidator" 15 | "github.com/go-shiori/go-readability" 16 | "github.com/gofrs/uuid" 17 | "github.com/urfave/cli" 18 | "src.techknowlogick.com/shiori/model" 19 | ) 20 | 21 | var ( 22 | CmdAdd = cli.Command{ 23 | Name: "add", 24 | Usage: "Bookmark the specified URL", 25 | Flags: []cli.Flag{ 26 | cli.StringFlag{ 27 | Name: "title, i", 28 | Usage: "Custom title for this bookmark", 29 | }, 30 | cli.StringFlag{ 31 | Name: "excerpt, e", 32 | Usage: "Custom excerpt for this bookmark", 33 | }, 34 | cli.StringSliceFlag{ 35 | Name: "tags, t", 36 | Usage: "Comma-separated tags for this bookmark", 37 | }, 38 | cli.BoolFlag{ 39 | Name: "offline, o", 40 | Usage: "Save bookmark without fetching data from internet", 41 | }, 42 | }, 43 | Action: runAddBookmark, 44 | } 45 | ) 46 | 47 | func runAddBookmark(c *cli.Context) error { 48 | // Read flag and arguments 49 | args := c.Args() 50 | dataDir := c.GlobalString("data-dir") 51 | title := c.String("title") 52 | excerpt := c.String("excerpt") 53 | tags := c.StringSlice("tags") 54 | 55 | url := args[0] 56 | 57 | db, err := getDbConnection(c) 58 | 59 | if err != nil { 60 | return errors.New(utils.CErrorSprint(err)) 61 | } 62 | 63 | // Make sure URL valid 64 | parsedURL, err := nurl.Parse(url) 65 | if err != nil || !valid.IsRequestURL(url) { 66 | return errors.New(utils.CErrorSprint("URL is not valid")) 67 | } 68 | 69 | // Clear fragment and UTM parameters from URL 70 | parsedURL.Fragment = "" 71 | utils.ClearUTMParams(parsedURL) 72 | 73 | // Create bookmark item 74 | book := model.Bookmark{ 75 | URL: parsedURL.String(), 76 | Title: normalizeSpace(title), 77 | Excerpt: normalizeSpace(excerpt), 78 | } 79 | 80 | // Set bookmark tags 81 | book.Tags = make([]model.Tag, len(tags)) 82 | for i, tag := range tags { 83 | book.Tags[i].Name = strings.TrimSpace(tag) 84 | } 85 | 86 | // fetch data from internet 87 | article, _ := readability.FromURL(parsedURL.String(), 20*time.Second) 88 | 89 | book.Author = article.Byline 90 | book.MinReadTime = int(math.Floor(float64(article.Length)/(987+188) + 0.5)) 91 | book.MaxReadTime = int(math.Floor(float64(article.Length)/(987-188) + 0.5)) 92 | book.Content = article.TextContent 93 | book.HTML = article.Content 94 | 95 | // If title and excerpt doesnt have submitted value, use from article 96 | if book.Title == "" { 97 | book.Title = article.Title 98 | } 99 | 100 | if book.Excerpt == "" { 101 | book.Excerpt = strings.Map(utils.FixUtf, article.Excerpt) 102 | } 103 | 104 | // Make sure title is not empty 105 | if book.Title == "" { 106 | book.Title = book.URL 107 | } 108 | 109 | // Save bookmark image to local disk 110 | u2, err := uuid.NewV4() 111 | if err != nil { 112 | return errors.New(utils.CErrorSprint(err)) 113 | } 114 | imgPath := filepath.Join(dataDir, "thumb", u2.String()) 115 | err = downloadFile(article.Image, imgPath, 20*time.Second) 116 | if err == nil { 117 | book.ImageURL = fmt.Sprintf("/thumb/%s", u2) 118 | } 119 | 120 | // Save bookmark to database 121 | err = db.InsertBookmark(&book) 122 | if err != nil { 123 | return errors.New(utils.CErrorSprint(err)) 124 | } 125 | 126 | printBookmarks(book) 127 | 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /cmd/delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "src.techknowlogick.com/shiori/utils" 10 | 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | var ( 15 | CmdDelete = cli.Command{ 16 | Name: "delete", 17 | Usage: "Delete the saved bookmarks", 18 | Description: "Delete bookmarks. " + 19 | "When a record is deleted, the last record is moved to the removed index. " + 20 | "Accepts space-separated list of indices (e.g. 5 6 23 4 110 45), hyphenated range (e.g. 100-200) or both (e.g. 1-3 7 9). " + 21 | "If no arguments, all records will be deleted.", 22 | Flags: []cli.Flag{ 23 | cli.BoolFlag{ 24 | Name: "yes, y", 25 | Usage: "Skip confirmation prompt and delete ALL bookmarks", 26 | }, 27 | }, 28 | Action: runDeleteBookmark, 29 | } 30 | ) 31 | 32 | func runDeleteBookmark(c *cli.Context) error { 33 | // Read flag and arguments 34 | args := c.Args() 35 | dataDir := c.GlobalString("data-dir") 36 | skipConfirm := c.Bool("yes") 37 | 38 | db, err := getDbConnection(c) 39 | 40 | if err != nil { 41 | return errors.New(utils.CErrorSprint(err)) 42 | } 43 | 44 | // If no arguments (i.e all bookmarks going to be deleted), 45 | // confirm to user 46 | if len(args) == 0 && !skipConfirm { 47 | confirmDelete := "" 48 | fmt.Print("Remove ALL bookmarks? (y/n): ") 49 | fmt.Scanln(&confirmDelete) 50 | if confirmDelete != "y" { 51 | return errors.New(utils.CErrorSprint("No bookmarks deleted")) 52 | } 53 | } 54 | 55 | // Convert args to ids 56 | ids, err := utils.ParseIndexList(args) 57 | if err != nil { 58 | return errors.New(utils.CErrorSprint(err)) 59 | } 60 | 61 | // Delete bookmarks from database 62 | err = db.DeleteBookmarks(ids...) 63 | if err != nil { 64 | utils.CError.Println(err) 65 | } 66 | 67 | // Delete thumbnail image from local disk 68 | for _, id := range ids { 69 | // TODO: this logic is broken due to bookmark images using UUIDs 70 | imgPath := filepath.Join(dataDir, "thumb", fmt.Sprintf("%d", id)) 71 | os.Remove(imgPath) 72 | } 73 | 74 | fmt.Println("Bookmark(s) have been deleted") 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /cmd/export.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "html/template" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | 12 | "src.techknowlogick.com/shiori/database" 13 | "src.techknowlogick.com/shiori/model" 14 | "src.techknowlogick.com/shiori/utils" 15 | 16 | "github.com/urfave/cli" 17 | ) 18 | 19 | var ( 20 | CmdExport = cli.Command{ 21 | Name: "export", 22 | Usage: "Export bookmarks into HTML file in Netscape Bookmark format", 23 | Action: runExportBookmarks, 24 | } 25 | ) 26 | 27 | func runExportBookmarks(c *cli.Context) error { 28 | args := c.Args() 29 | 30 | if len(args) != 1 { 31 | return errors.New(utils.CErrorSprint("Please set path to target-file")) 32 | } 33 | 34 | db, err := getDbConnection(c) 35 | 36 | if err != nil { 37 | return errors.New(utils.CErrorSprint(err)) 38 | } 39 | 40 | // Fetch bookmarks from database 41 | bookmarks, err := db.GetBookmarks(database.BookmarkOptions{}) 42 | if err != nil { 43 | return errors.New(utils.CErrorSprint(err)) 44 | } 45 | 46 | if len(bookmarks) == 0 { 47 | return errors.New(utils.CErrorSprint("No saved bookmarks yet")) 48 | } 49 | 50 | // Make sure destination directory exist 51 | dstDir := filepath.Dir(args[0]) 52 | os.MkdirAll(dstDir, os.ModePerm) 53 | 54 | // Open destination file 55 | dstFile, err := os.Create(args[0]) 56 | if err != nil { 57 | return errors.New(utils.CErrorSprint(err)) 58 | } 59 | defer dstFile.Close() 60 | 61 | // Create template 62 | funcMap := template.FuncMap{ 63 | "unix": func(t time.Time) int64 { 64 | return t.Unix() 65 | }, 66 | "combine": func(tags []model.Tag) string { 67 | strTags := make([]string, len(tags)) 68 | for i, tag := range tags { 69 | strTags[i] = tag.Name 70 | } 71 | 72 | return strings.Join(strTags, ",") 73 | }, 74 | } 75 | 76 | tplContent := `` + 77 | `` + 78 | `
` + 81 | `{{range $book := .}}` + 82 | `
`
85 |
86 | tpl, err := template.New("export").Funcs(funcMap).Parse(tplContent)
87 | if err != nil {
88 | return errors.New(utils.CErrorSprint(err))
89 | }
90 |
91 | // Execute template
92 | err = tpl.Execute(dstFile, &bookmarks)
93 | if err != nil {
94 | return errors.New(utils.CErrorSprint(err))
95 | }
96 |
97 | fmt.Println("Export finished")
98 | return nil
99 | }
100 |
--------------------------------------------------------------------------------
/cmd/import.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | nurl "net/url"
7 | "os"
8 | "strconv"
9 | "strings"
10 | "time"
11 |
12 | "src.techknowlogick.com/shiori/model"
13 | "src.techknowlogick.com/shiori/utils"
14 |
15 | "github.com/PuerkitoBio/goquery"
16 | valid "github.com/asaskevich/govalidator"
17 | "github.com/urfave/cli"
18 | )
19 |
20 | var (
21 | CmdImport = cli.Command{
22 | Name: "import",
23 | Usage: "Import bookmarks from HTML file in Netscape Bookmark format",
24 | Flags: []cli.Flag{
25 | cli.BoolFlag{
26 | Name: "generate-tag, t",
27 | Usage: "Auto generate tag from bookmark's category",
28 | },
29 | },
30 | Action: runImportBookmarks,
31 | }
32 | )
33 |
34 | func runImportBookmarks(c *cli.Context) error {
35 | // Parse flags
36 | generateTag := c.Bool("generate-tag")
37 | args := c.Args()
38 |
39 | db, err := getDbConnection(c)
40 |
41 | if err != nil {
42 | return errors.New(utils.CErrorSprint(err))
43 | }
44 |
45 | // If user doesn't specify, ask if tag need to be generated
46 | if !generateTag {
47 | var submit string
48 | fmt.Print("Add parents folder as tag? (y/n): ")
49 | fmt.Scanln(&submit)
50 |
51 | generateTag = submit == "y"
52 | }
53 |
54 | // Open bookmark's file
55 | srcFile, err := os.Open(args[0])
56 | if err != nil {
57 | return errors.New(utils.CErrorSprint(err))
58 | }
59 | defer srcFile.Close()
60 |
61 | // Parse bookmark's file
62 | doc, err := goquery.NewDocumentFromReader(srcFile)
63 | if err != nil {
64 | return errors.New(utils.CErrorSprint(err))
65 | }
66 |
67 | bookmarks := []model.Bookmark{}
68 | doc.Find("dt>a").Each(func(_ int, a *goquery.Selection) {
69 | // Get related elements
70 | dt := a.Parent()
71 | dl := dt.Parent()
72 |
73 | // Get metadata
74 | title := a.Text()
75 | url, _ := a.Attr("href")
76 | strTags, _ := a.Attr("tags")
77 | strModified, _ := a.Attr("last_modified")
78 | intModified, _ := strconv.ParseInt(strModified, 10, 64)
79 | modified := time.Unix(intModified, 0)
80 |
81 | // Make sure URL valid
82 | parsedURL, err := nurl.Parse(url)
83 | if err != nil || !valid.IsRequestURL(url) {
84 | utils.CError.Printf("%s will be skipped: URL is not valid\n\n", url)
85 | return
86 | }
87 |
88 | // Clear fragment and UTM parameters from URL
89 | parsedURL.Fragment = ""
90 | utils.ClearUTMParams(parsedURL)
91 |
92 | // Get bookmark tags
93 | tags := []model.Tag{}
94 | for _, strTag := range strings.Split(strTags, ",") {
95 | if strTag != "" {
96 | tags = append(tags, model.Tag{Name: strTag})
97 | }
98 | }
99 |
100 | // Get bookmark excerpt
101 | excerpt := ""
102 | if dd := dt.Next(); dd.Is("dd") {
103 | excerpt = dd.Text()
104 | }
105 |
106 | // Get category name for this bookmark
107 | // and add it as tags (if necessary)
108 | category := ""
109 | if dtCategory := dl.Prev(); dtCategory.Is("h3") {
110 | category = dtCategory.Text()
111 | category = normalizeSpace(category)
112 | category = strings.ToLower(category)
113 | category = strings.Replace(category, " ", "-", -1)
114 | }
115 |
116 | if category != "" && generateTag {
117 | tags = append(tags, model.Tag{Name: category})
118 | }
119 |
120 | // Add item to list
121 | bookmark := model.Bookmark{
122 | URL: parsedURL.String(),
123 | Title: normalizeSpace(title),
124 | Excerpt: normalizeSpace(excerpt),
125 | Modified: modified,
126 | Tags: tags,
127 | }
128 |
129 | bookmarks = append(bookmarks, bookmark)
130 | })
131 |
132 | // Save bookmarks to database
133 | for _, book := range bookmarks {
134 | // Save book to database
135 | err = db.InsertBookmark(&book)
136 | if err != nil {
137 | return errors.New(utils.CErrorSprint(fmt.Sprintf("%s is skipped: %v\n\n", book.URL, err)))
138 | }
139 |
140 | printBookmarks(book)
141 | }
142 |
143 | return nil
144 | }
145 |
--------------------------------------------------------------------------------
/cmd/open.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "strings"
7 |
8 | "src.techknowlogick.com/shiori/database"
9 | "src.techknowlogick.com/shiori/utils"
10 |
11 | "github.com/urfave/cli"
12 | )
13 |
14 | var (
15 | CmdOpen = cli.Command{
16 | Name: "open",
17 | Usage: "Open the saved bookmarks",
18 | Description: "Open bookmarks in browser. " +
19 | "Accepts space-separated list of indices (e.g. 5 6 23 4 110 45), hyphenated range (e.g. 100-200) or both (e.g. 1-3 7 9). " +
20 | "If no arguments, ALL bookmarks will be opened.",
21 | Flags: []cli.Flag{
22 | cli.BoolFlag{
23 | Name: "yes, y",
24 | Usage: "Skip confirmation prompt and open ALL bookmarks",
25 | },
26 | cli.BoolFlag{
27 | Name: "cache, c",
28 | Usage: "Open the bookmark's cache in text-only mode",
29 | },
30 | cli.BoolFlag{
31 | Name: "trim-space",
32 | Usage: "Trim all spaces and newlines from the bookmark's cache",
33 | },
34 | },
35 | Action: runOpenBookmark,
36 | }
37 | )
38 |
39 | func runOpenBookmark(c *cli.Context) error {
40 | cacheMode := c.Bool("cache")
41 | trimSpace := c.Bool("trim-space")
42 | skipConfirm := c.Bool("yes")
43 | args := c.Args()
44 |
45 | db, err := getDbConnection(c)
46 |
47 | if err != nil {
48 | return errors.New(utils.CErrorSprint(err))
49 | }
50 |
51 | // If no arguments (i.e all bookmarks will be opened),
52 | // confirm to user
53 | if len(args) == 0 && !skipConfirm {
54 | confirmOpen := ""
55 | fmt.Print("Open ALL bookmarks? (y/n): ")
56 | fmt.Scanln(&confirmOpen)
57 |
58 | if confirmOpen != "y" {
59 | return nil
60 | }
61 | }
62 |
63 | // Convert args to ids
64 | ids, err := utils.ParseIndexList(args)
65 | if err != nil {
66 | return errors.New(utils.CErrorSprint(err))
67 | }
68 | bookmarks, err := db.GetBookmarks(database.BookmarkOptions{}, ids...)
69 | if err != nil {
70 | return errors.New(utils.CErrorSprint(err))
71 | }
72 |
73 | if len(bookmarks) == 0 {
74 | if len(args) > 0 {
75 | return errors.New(utils.CErrorSprint("No matching index found"))
76 | } else {
77 | return errors.New(utils.CErrorSprint("No saved bookmarks yet"))
78 | }
79 | }
80 |
81 | // If not cache mode, open bookmarks in browser
82 | if !cacheMode {
83 | for _, book := range bookmarks {
84 | err = openBrowser(book.URL)
85 | if err != nil {
86 | return errors.New(utils.CErrorSprint("Failed to open %s: %v\n", book.URL, err))
87 | }
88 | }
89 | return nil
90 | }
91 |
92 | termWidth := getTerminalWidth()
93 | if termWidth < 50 {
94 | termWidth = 50
95 | }
96 |
97 | for _, book := range bookmarks {
98 | if trimSpace {
99 | words := strings.Fields(book.Content)
100 | book.Content = strings.Join(words, " ")
101 | }
102 |
103 | utils.CIndex.Printf("%d. ", book.ID)
104 | utils.CTitle.Println(book.Title)
105 | fmt.Println()
106 |
107 | if book.Content == "" {
108 | utils.CError.Println("This bookmark doesn't have any cached content")
109 | } else {
110 | fmt.Println(book.Content)
111 | }
112 |
113 | fmt.Println()
114 | utils.CSymbol.Println(strings.Repeat("-", termWidth))
115 | fmt.Println()
116 | }
117 |
118 | return nil
119 | }
120 |
--------------------------------------------------------------------------------
/cmd/pocket.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "errors"
5 | nurl "net/url"
6 | "os"
7 | "strconv"
8 | "strings"
9 | "time"
10 |
11 | "src.techknowlogick.com/shiori/model"
12 | "src.techknowlogick.com/shiori/utils"
13 |
14 | "github.com/PuerkitoBio/goquery"
15 | valid "github.com/asaskevich/govalidator"
16 | "github.com/urfave/cli"
17 | )
18 |
19 | var (
20 | CmdPocket = cli.Command{
21 | Name: "pocket",
22 | Usage: "Import bookmarks from Pocket's exported HTML file",
23 | Action: runImportPocket,
24 | }
25 | )
26 |
27 | func runImportPocket(c *cli.Context) error {
28 | args := c.Args()
29 |
30 | db, err := getDbConnection(c)
31 |
32 | if err != nil {
33 | return errors.New(utils.CErrorSprint(err))
34 | }
35 |
36 | if len(args) != 1 {
37 | return errors.New(utils.CErrorSprint("Please set path to source-file"))
38 | }
39 |
40 | // Open bookmark's file
41 | srcFile, err := os.Open(args[0])
42 | if err != nil {
43 | return errors.New(utils.CErrorSprint(err))
44 | }
45 | defer srcFile.Close()
46 |
47 | // Parse bookmark's file
48 | doc, err := goquery.NewDocumentFromReader(srcFile)
49 | if err != nil {
50 | return errors.New(utils.CErrorSprint(err))
51 | }
52 |
53 | bookmarks := []model.Bookmark{}
54 | doc.Find("a").Each(func(_ int, a *goquery.Selection) {
55 | // Get metadata
56 | title := a.Text()
57 | url, _ := a.Attr("href")
58 | strTags, _ := a.Attr("tags")
59 | strModified, _ := a.Attr("time_added")
60 | intModified, _ := strconv.ParseInt(strModified, 10, 64)
61 | modified := time.Unix(intModified, 0)
62 |
63 | // Make sure URL valid
64 | parsedURL, err := nurl.Parse(url)
65 | if err != nil || !valid.IsRequestURL(url) {
66 | utils.CError.Printf("%s will be skipped: URL is not valid\n\n", url)
67 | return
68 | }
69 |
70 | // Clear fragment and UTM parameters from URL
71 | parsedURL.Fragment = ""
72 | utils.ClearUTMParams(parsedURL)
73 |
74 | // Get bookmark tags
75 | tags := []model.Tag{}
76 | for _, strTag := range strings.Split(strTags, ",") {
77 | if strTag != "" {
78 | tags = append(tags, model.Tag{Name: strTag})
79 | }
80 | }
81 |
82 | // Add item to list
83 | bookmark := model.Bookmark{
84 | URL: parsedURL.String(),
85 | Title: normalizeSpace(title),
86 | Modified: modified,
87 | Tags: tags,
88 | }
89 |
90 | bookmarks = append(bookmarks, bookmark)
91 | })
92 |
93 | // Save bookmarks to database
94 | for _, book := range bookmarks {
95 | // Save book to database
96 | err = db.InsertBookmark(&book)
97 | if err != nil {
98 | return errors.New(utils.CErrorSprint("%s is skipped: %v\n\n", book.URL, err))
99 | }
100 |
101 | printBookmarks(book)
102 | }
103 | return nil
104 | }
105 |
--------------------------------------------------------------------------------
/cmd/print.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 |
8 | "src.techknowlogick.com/shiori/database"
9 | "src.techknowlogick.com/shiori/utils"
10 |
11 | "github.com/urfave/cli"
12 | )
13 |
14 | var (
15 | CmdPrint = cli.Command{
16 | Name: "print",
17 | Usage: "Print the saved bookmarks to command line",
18 | Aliases: []string{"list", "ls"},
19 | Description: "Show the saved bookmarks by its DB index. " +
20 | "Accepts space-separated list of indices (e.g. 5 6 23 4 110 45), hyphenated range (e.g. 100-200) or both (e.g. 1-3 7 9). " +
21 | "If no arguments, all records with actual index from DB are shown.",
22 | // hdl.printBookmarks
23 | Flags: []cli.Flag{
24 | cli.BoolFlag{
25 | Name: "index-only, i",
26 | Usage: "Only print the index of bookmarks",
27 | },
28 | cli.BoolFlag{
29 | Name: "json, j",
30 | Usage: "Output data in JSON format",
31 | },
32 | },
33 | Action: runPrintBookmarks,
34 | }
35 | )
36 |
37 | func runPrintBookmarks(c *cli.Context) error {
38 | // Read flags
39 | useJSON := c.Bool("json")
40 | indexOnly := c.Bool("index-only")
41 | args := c.Args()
42 |
43 | db, err := getDbConnection(c)
44 |
45 | if err != nil {
46 | return errors.New(utils.CErrorSprint(err))
47 | }
48 |
49 | // Convert args to ids
50 | ids, err := utils.ParseIndexList(args)
51 | if err != nil {
52 | return errors.New(utils.CErrorSprint(err))
53 | }
54 |
55 | // Read bookmarks from database
56 | bookmarks, err := db.GetBookmarks(database.BookmarkOptions{}, ids...)
57 | if err != nil {
58 | return errors.New(utils.CErrorSprint(err))
59 | }
60 |
61 | if len(bookmarks) == 0 {
62 | if len(args) > 0 {
63 | return errors.New(utils.CErrorSprint("No matching index found"))
64 | } else {
65 | return errors.New(utils.CErrorSprint("No bookmarks saved yet"))
66 | }
67 | }
68 |
69 | // Print data
70 | if useJSON {
71 | bt, err := json.MarshalIndent(&bookmarks, "", " ")
72 | if err != nil {
73 | return errors.New(utils.CErrorSprint(err))
74 | }
75 |
76 | fmt.Println(string(bt))
77 | return nil
78 | }
79 |
80 | if indexOnly {
81 | for _, bookmark := range bookmarks {
82 | fmt.Printf("%d ", bookmark.ID)
83 | }
84 | fmt.Println()
85 | return nil
86 | }
87 |
88 | printBookmarks(bookmarks...)
89 | return nil
90 | }
91 |
--------------------------------------------------------------------------------
/cmd/search.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 |
8 | "src.techknowlogick.com/shiori/database"
9 | "src.techknowlogick.com/shiori/utils"
10 |
11 | "github.com/urfave/cli"
12 | )
13 |
14 | var (
15 | CmdSearch = cli.Command{
16 | Name: "search",
17 | Usage: "Search bookmarks by submitted keyword",
18 | Description: "Search bookmarks by looking for matching keyword in bookmark's title and content. " +
19 | "If no keyword submitted, print all saved bookmarks. ",
20 | Flags: []cli.Flag{
21 | cli.BoolFlag{
22 | Name: "index-only, i",
23 | Usage: "Only print the index of bookmarks",
24 | },
25 | cli.BoolFlag{
26 | Name: "json, j",
27 | Usage: "Output data in JSON format",
28 | },
29 | cli.StringSliceFlag{
30 | Name: "tags, t",
31 | Usage: "Search bookmarks with specified tag(s)",
32 | },
33 | },
34 | Action: runSearchBookmarks,
35 | }
36 | )
37 |
38 | func runSearchBookmarks(c *cli.Context) error {
39 | // Read flags
40 | tags := c.StringSlice("tags")
41 | useJSON := c.Bool("json")
42 | indexOnly := c.Bool("index-only")
43 | args := c.Args()
44 |
45 | // Fetch keyword
46 | keyword := ""
47 | if len(args) > 0 {
48 | keyword = args[0]
49 | }
50 |
51 | db, err := getDbConnection(c)
52 |
53 | if err != nil {
54 | return errors.New(utils.CErrorSprint(err))
55 | }
56 |
57 | // Read bookmarks from database
58 | bookmarks, err := db.SearchBookmarks(database.BookmarkOptions{Keyword: keyword}, tags...)
59 | if err != nil {
60 | return errors.New(utils.CErrorSprint(err))
61 | }
62 |
63 | if len(bookmarks) == 0 {
64 | return errors.New(utils.CErrorSprint("No matching bookmarks found"))
65 | }
66 |
67 | // Print data
68 | if useJSON {
69 | bt, err := json.MarshalIndent(&bookmarks, "", " ")
70 | if err != nil {
71 | return errors.New(utils.CErrorSprint(err))
72 | }
73 |
74 | fmt.Println(string(bt))
75 | return nil
76 | }
77 |
78 | if indexOnly {
79 | for _, bookmark := range bookmarks {
80 | fmt.Printf("%d ", bookmark.ID)
81 | }
82 | fmt.Println()
83 | return nil
84 | }
85 |
86 | printBookmarks(bookmarks...)
87 | return nil
88 | }
89 |
--------------------------------------------------------------------------------
/cmd/serve/root.go:
--------------------------------------------------------------------------------
1 | package serve
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 | "net/http"
8 | "path/filepath"
9 | "time"
10 |
11 | "src.techknowlogick.com/shiori/database"
12 | "src.techknowlogick.com/shiori/utils"
13 |
14 | "github.com/gin-gonic/contrib/commonlog"
15 | "github.com/gin-gonic/gin"
16 | "github.com/gofrs/uuid"
17 | "github.com/sirupsen/logrus"
18 | "github.com/urfave/cli"
19 | )
20 |
21 | var (
22 | CmdServe = cli.Command{
23 | Name: "serve",
24 | Usage: "Serve web app for managing bookmarks",
25 | Description: "Run a simple annd performant web server which serves the site for managing bookmarks.",
26 | Flags: []cli.Flag{
27 | cli.StringFlag{
28 | Name: "listen, l",
29 | Usage: "Address the server listens to",
30 | EnvVar: "SHIORI_LISTEN_ADDRESS",
31 | },
32 | cli.StringFlag{
33 | Name: "jwt-secret",
34 | Usage: "JWT Secret fof session protection (Default: Randon each start)",
35 | EnvVar: "SHIORI_JWT_SECRET",
36 | Hidden: true,
37 | },
38 | cli.StringFlag{
39 | Name: "server-log-type",
40 | Usage: "Type of logs that will be output to stdout (json, plain, gin-default, disabled)",
41 | EnvVar: "SHIORI_SERVER_LOG_TYPE",
42 | Value: "disabled",
43 | Hidden: true,
44 | },
45 | cli.IntFlag{
46 | Name: "port, p",
47 | Value: 8080,
48 | Usage: "Port that used by server",
49 | EnvVar: "SHIORI_PORT,PORT",
50 | },
51 | cli.BoolFlag{
52 | Name: "insecure-default-user",
53 | Usage: "For demo service this creates a temporary default user. Very insecure, do not use this flag.",
54 | Hidden: true,
55 | EnvVar: "SHIORI_INSECURE_DEMO_USER",
56 | },
57 | cli.BoolFlag{
58 | Name: "server-debug",
59 | Usage: "Enable Gin (webserver) debug mode",
60 | Hidden: true,
61 | EnvVar: "SHIORI_SERVER_DEBUG",
62 | },
63 | },
64 | Action: func(c *cli.Context) error {
65 | db, err := getDbConnection(c)
66 |
67 | if err != nil {
68 | return errors.New(utils.CErrorSprint(err))
69 | }
70 |
71 | demoUser, _ := db.GetAccount("demo")
72 | if demoUser.ID == 0 && c.Bool("insecure-default-user") {
73 | db.CreateAccount("demo", "demo")
74 | }
75 |
76 | dataDir := c.GlobalString("data-dir")
77 | hdl, err := newWebHandler(&handlerOptions{db: db, dataDir: dataDir, jwtSecret: c.String("jwt-secret")})
78 | // Parse flags
79 | listenAddress := c.String("listen")
80 | port := c.Int("port")
81 |
82 | // Create router
83 | if !c.Bool("server-debug") {
84 | gin.SetMode(gin.ReleaseMode)
85 | }
86 |
87 | router := gin.New()
88 |
89 | // Add request ID to logs (currently only shows in json)
90 | router.Use(func(c *gin.Context) {
91 | u, _ := uuid.NewV4()
92 | requestID := u.String()
93 | c.Set("request_id", requestID)
94 | c.Header("X-Request-Id", requestID)
95 | c.Next()
96 | })
97 |
98 | switch c.String("server-log-type") {
99 | case "json":
100 | router.Use(gin.LoggerWithConfig(gin.LoggerConfig{Formatter: func(param gin.LogFormatterParams) string {
101 | logFormat := map[string]interface{}{
102 | "type": "server-request-log",
103 | "timestamp": param.TimeStamp.Format("2006/01/02 - 15:04:05"),
104 | "status_code": param.StatusCode,
105 | "latency": param.Latency,
106 | "client_ip": param.ClientIP,
107 | "method": param.Method,
108 | "path": param.Path,
109 | "error_message": param.ErrorMessage,
110 | "keys": param.Keys,
111 | }
112 |
113 | bytes, err := json.Marshal(logFormat)
114 | if err != nil {
115 | utils.CheckError(err)
116 | }
117 | return fmt.Sprintf("%s\n", string(bytes))
118 | }}))
119 | case "gin-default":
120 | router.Use(gin.Logger())
121 | case "plain":
122 | router.Use(commonlog.New())
123 | default:
124 | // disabled
125 | }
126 |
127 | router.Use(gin.Recovery())
128 |
129 | router.GET("/dist/*filepath", hdl.serveFiles)
130 |
131 | router.GET("/", hdl.serveIndexPage)
132 | router.GET("/login", hdl.serveLoginPage)
133 | router.GET("/bookmark/:id", hdl.serveBookmarkCache)
134 | router.GET("/thumb/:id", hdl.serveThumbnailImage)
135 | router.GET("/submit", hdl.serveSubmitPage)
136 |
137 | router.POST("/api/login", hdl.apiLogin)
138 | router.GET("/api/bookmarks", hdl.apiGetBookmarks)
139 | router.GET("/api/tags", hdl.apiGetTags)
140 | router.POST("/api/bookmarks", hdl.apiInsertBookmark)
141 | router.PUT("/api/cache", hdl.apiUpdateCache)
142 | router.PUT("/api/bookmarks", hdl.apiUpdateBookmark)
143 | router.PUT("/api/bookmarks/tags", hdl.apiUpdateBookmarkTags)
144 | router.DELETE("/api/bookmarks", hdl.apiDeleteBookmark)
145 |
146 | // Create server
147 | url := fmt.Sprintf("%s:%d", listenAddress, port)
148 | svr := &http.Server{
149 | Addr: url,
150 | Handler: router,
151 | ReadTimeout: 10 * time.Second,
152 | WriteTimeout: 20 * time.Second,
153 | }
154 |
155 | // Serve app
156 | logrus.Infoln("Serve shiori in", url)
157 | return svr.ListenAndServe()
158 | },
159 | }
160 | )
161 |
162 | func getDbConnection(c *cli.Context) (database.Database, error) {
163 | dbType := c.GlobalString("db-type")
164 | dbDsn := c.GlobalString("db-dsn")
165 | dataDir := c.GlobalString("data-dir")
166 |
167 | if dbType == "sqlite3" && dbDsn == "shiori.db" {
168 | dbDsn = filepath.Join(dataDir, dbDsn)
169 | }
170 |
171 | db, err := database.OpenXormDatabase(&database.XormOptions{DbDsn: dbDsn, DbType: dbType, ShowSQL: c.GlobalBool("show-sql-log")})
172 | return db, err
173 |
174 | }
175 |
--------------------------------------------------------------------------------
/cmd/serve/web-handler-api.go:
--------------------------------------------------------------------------------
1 | package serve
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io"
7 | "math"
8 | "net/http"
9 | nurl "net/url"
10 | "os"
11 | fp "path/filepath"
12 | "strings"
13 | "sync"
14 | "time"
15 |
16 | "src.techknowlogick.com/shiori/database"
17 | "src.techknowlogick.com/shiori/model"
18 | "src.techknowlogick.com/shiori/utils"
19 |
20 | valid "github.com/asaskevich/govalidator"
21 | jwt "github.com/dgrijalva/jwt-go"
22 | "github.com/gin-gonic/gin"
23 | "github.com/go-shiori/go-readability"
24 | "github.com/gofrs/uuid"
25 | "golang.org/x/crypto/bcrypt"
26 | )
27 |
28 | // login is handler for POST /api/login
29 | func (h *webHandler) apiLogin(c *gin.Context) {
30 | // Decode request
31 | var request model.LoginRequest
32 | err := json.NewDecoder(c.Request.Body).Decode(&request)
33 | utils.CheckError(err)
34 |
35 | // Get account data from database
36 | account, err := h.db.GetAccount(request.Username)
37 | if err != nil {
38 | c.String(http.StatusBadRequest, "Username or Password incorrect")
39 | return
40 | }
41 |
42 | // Compare password with database
43 | err = bcrypt.CompareHashAndPassword([]byte(account.Password), []byte(request.Password))
44 | if err != nil {
45 | c.String(http.StatusBadRequest, "Username or Password incorrect")
46 | return
47 | }
48 |
49 | // Calculate expiration time
50 | nbf := time.Now()
51 | exp := time.Now().Add(12 * time.Hour)
52 | if request.Remember {
53 | exp = time.Now().Add(7 * 24 * time.Hour)
54 | }
55 |
56 | // Create token
57 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
58 | "nbf": nbf.Unix(),
59 | "exp": exp.Unix(),
60 | "sub": account.ID,
61 | })
62 |
63 | tokenString, err := token.SignedString(h.jwtKey)
64 | utils.CheckError(err)
65 |
66 | // Return tokenc.Request
67 | fmt.Fprint(c.Writer, tokenString)
68 | }
69 |
70 | // apiGetBookmarks is handler for GET /api/bookmarks
71 | func (h *webHandler) apiGetBookmarks(c *gin.Context) {
72 | // Check token
73 | err := h.checkAPIToken(c.Request)
74 | utils.CheckError(err)
75 |
76 | // Get URL queries
77 | keyword := c.Request.URL.Query().Get("keyword")
78 | strTags := c.Request.URL.Query().Get("tags")
79 | tags := strings.Split(strTags, ",")
80 | if len(tags) == 1 && tags[0] == "" {
81 | tags = []string{}
82 | }
83 |
84 | // Fetch all matching bookmarks
85 | bookmarks, err := h.db.SearchBookmarks(database.BookmarkOptions{Keyword: keyword}, tags...)
86 | utils.CheckError(err)
87 |
88 | err = json.NewEncoder(c.Writer).Encode(&bookmarks)
89 | utils.CheckError(err)
90 | }
91 |
92 | // apiGetTags is handler for GET /api/tags
93 | func (h *webHandler) apiGetTags(c *gin.Context) {
94 | // Check token
95 | err := h.checkAPIToken(c.Request)
96 | utils.CheckError(err)
97 |
98 | // Fetch all tags
99 | tags, err := h.db.GetTags()
100 | utils.CheckError(err)
101 |
102 | err = json.NewEncoder(c.Writer).Encode(&tags)
103 | utils.CheckError(err)
104 | }
105 |
106 | // apiInsertBookmark is handler for POST /api/bookmark
107 | func (h *webHandler) apiInsertBookmark(c *gin.Context) {
108 | // Enable CORS for this endpoint
109 | c.Header("Access-Control-Allow-Origin", "*")
110 | c.Header("Access-Control-Allow-Methods", "POST")
111 | c.Header("Access-Control-Allow-Headers", "Content-Type")
112 |
113 | // Check token
114 | err := h.checkAPIToken(c.Request)
115 | utils.CheckError(err)
116 |
117 | // Decode request
118 | book := model.Bookmark{}
119 | err = json.NewDecoder(c.Request.Body).Decode(&book)
120 | utils.CheckError(err)
121 |
122 | // Make sure URL valid
123 | parsedURL, err := nurl.Parse(book.URL)
124 | if err != nil || !valid.IsRequestURL(book.URL) {
125 | panic(fmt.Errorf("URL is not valid"))
126 | }
127 |
128 | // Clear fragment and UTM parameters from URL
129 | parsedURL.Fragment = ""
130 | utils.ClearUTMParams(parsedURL)
131 | book.URL = parsedURL.String()
132 |
133 | // Fetch data from internet
134 | article, _ := readability.FromURL(parsedURL.String(), 20*time.Second)
135 |
136 | book.Author = article.Byline
137 | book.MinReadTime = int(math.Floor(float64(article.Length)/(987+188) + 0.5))
138 | book.MaxReadTime = int(math.Floor(float64(article.Length)/(987-188) + 0.5))
139 | book.Content = article.TextContent
140 | book.HTML = article.Content
141 |
142 | // If title and excerpt doesnt have submitted value, use from article
143 | if book.Title == "" {
144 | book.Title = article.Title
145 | }
146 |
147 | if book.Excerpt == "" {
148 | book.Excerpt = strings.Map(utils.FixUtf, article.Excerpt)
149 | }
150 |
151 | // Make sure title is not empty
152 | if book.Title == "" {
153 | book.Title = book.URL
154 | }
155 |
156 | // Check if book has content
157 | if book.Content != "" {
158 | book.HasContent = true
159 | }
160 |
161 | // Save bookmark image to local disk
162 | u2, err := uuid.NewV4()
163 | if err != nil {
164 | utils.CheckError(err)
165 | }
166 | imgPath := fp.Join(h.dataDir, "thumb", u2.String())
167 | err = downloadFile(article.Image, imgPath, 20*time.Second)
168 | if err == nil {
169 | book.ImageURL = fmt.Sprintf("/thumb/%s", u2)
170 | }
171 |
172 | // Save bookmark to database
173 | err = h.db.InsertBookmark(&book)
174 | if err != nil {
175 | utils.CheckError(err)
176 | }
177 |
178 | // Return new saved result
179 | err = json.NewEncoder(c.Writer).Encode(&book)
180 | utils.CheckError(err)
181 | }
182 |
183 | // apiDeleteBookmarks is handler for DELETE /api/bookmark
184 | func (h *webHandler) apiDeleteBookmark(c *gin.Context) {
185 | // Check token
186 | err := h.checkAPIToken(c.Request)
187 | utils.CheckError(err)
188 |
189 | // Decode request
190 | ids := []int{}
191 | err = json.NewDecoder(c.Request.Body).Decode(&ids)
192 | utils.CheckError(err)
193 |
194 | // Delete bookmarks
195 | err = h.db.DeleteBookmarks(ids...)
196 | utils.CheckError(err)
197 |
198 | // Delete thumbnail image from local disk
199 | for _, id := range ids {
200 | imgPath := fp.Join(h.dataDir, "thumb", fmt.Sprintf("%d", id))
201 | os.Remove(imgPath)
202 | }
203 |
204 | fmt.Fprint(c.Writer, 1)
205 | }
206 |
207 | // apiUpdateBookmark is handler for PUT /api/bookmarks
208 | func (h *webHandler) apiUpdateBookmark(c *gin.Context) {
209 | // Check token
210 | err := h.checkAPIToken(c.Request)
211 | utils.CheckError(err)
212 |
213 | // Decode request
214 | request := model.Bookmark{}
215 | err = json.NewDecoder(c.Request.Body).Decode(&request)
216 | utils.CheckError(err)
217 |
218 | // Validate input
219 | if request.Title == "" {
220 | panic(fmt.Errorf("Title must not empty"))
221 | }
222 |
223 | // Get existing bookmark from database
224 | reqID := request.ID
225 | bookmarks, err := h.db.GetBookmarks(database.BookmarkOptions{}, reqID)
226 | utils.CheckError(err)
227 | if len(bookmarks) == 0 {
228 | panic(fmt.Errorf("No bookmark with matching index"))
229 | }
230 |
231 | // Set new bookmark data
232 | book := bookmarks[0]
233 | book.Title = request.Title
234 | book.Excerpt = request.Excerpt
235 |
236 | // Set new tags
237 | for i := range book.Tags {
238 | book.Tags[i].Deleted = true
239 | }
240 |
241 | for _, newTag := range request.Tags {
242 | for i, oldTag := range book.Tags {
243 | if newTag.Name == oldTag.Name {
244 | newTag.ID = oldTag.ID
245 | book.Tags[i].Deleted = false
246 | break
247 | }
248 | }
249 |
250 | if newTag.ID == 0 {
251 | book.Tags = append(book.Tags, newTag)
252 | }
253 | }
254 |
255 | // Update database
256 | res, err := h.db.UpdateBookmarks(book)
257 | utils.CheckError(err)
258 |
259 | // Return new saved result
260 | err = json.NewEncoder(c.Writer).Encode(&res[0])
261 | utils.CheckError(err)
262 | }
263 |
264 | // apiUpdateBookmarkTags is handler for PUT /api/bookmarks/tags
265 | func (h *webHandler) apiUpdateBookmarkTags(c *gin.Context) {
266 | // Check token
267 | err := h.checkAPIToken(c.Request)
268 | utils.CheckError(err)
269 |
270 | // Decode request
271 | request := struct {
272 | IDs []int `json:"ids"`
273 | Tags []model.Tag `json:"tags"`
274 | }{}
275 |
276 | err = json.NewDecoder(c.Request.Body).Decode(&request)
277 | utils.CheckError(err)
278 |
279 | // Validate input
280 | if len(request.IDs) == 0 || len(request.Tags) == 0 {
281 | panic(fmt.Errorf("IDs and tags must not empty"))
282 | }
283 |
284 | // Get existing bookmark from database
285 | bookmarks, err := h.db.GetBookmarks(database.BookmarkOptions{}, request.IDs...)
286 | utils.CheckError(err)
287 | if len(bookmarks) == 0 {
288 | panic(fmt.Errorf("No bookmark with matching index"))
289 | }
290 |
291 | // Set new tags
292 | for i, book := range bookmarks {
293 | for _, newTag := range request.Tags {
294 | for _, oldTag := range book.Tags {
295 | if newTag.Name == oldTag.Name {
296 | newTag.ID = oldTag.ID
297 | break
298 | }
299 | }
300 |
301 | if newTag.ID == 0 {
302 | book.Tags = append(book.Tags, newTag)
303 | }
304 | }
305 |
306 | bookmarks[i] = book
307 | }
308 |
309 | // Update database
310 | res, err := h.db.UpdateBookmarks(bookmarks...)
311 | utils.CheckError(err)
312 |
313 | // Return new saved result
314 | err = json.NewEncoder(c.Writer).Encode(&res)
315 | utils.CheckError(err)
316 | }
317 |
318 | // apiUpdateCache is handler for PUT /api/cache
319 | func (h *webHandler) apiUpdateCache(c *gin.Context) {
320 | // Check token
321 | err := h.checkAPIToken(c.Request)
322 | utils.CheckError(err)
323 |
324 | // Decode request
325 | ids := []int{}
326 | err = json.NewDecoder(c.Request.Body).Decode(&ids)
327 | utils.CheckError(err)
328 |
329 | // Prepare wait group and mutex
330 | wg := sync.WaitGroup{}
331 |
332 | // Fetch bookmarks from database
333 | books, err := h.db.GetBookmarks(database.BookmarkOptions{}, ids...)
334 | utils.CheckError(err)
335 |
336 | // Download new cache data
337 | for _, book := range books {
338 | wg.Add(1)
339 |
340 | go func(book *model.Bookmark) {
341 | // Make sure to stop wait group
342 | defer wg.Done()
343 |
344 | // Parse URL
345 | parsedURL, err := nurl.Parse(book.URL)
346 | if err != nil || !valid.IsRequestURL(book.URL) {
347 | return
348 | }
349 |
350 | // Fetch data from internet
351 | article, err := readability.FromURL(parsedURL.String(), 20*time.Second)
352 | if err != nil {
353 | return
354 | }
355 |
356 | book.Excerpt = article.Excerpt
357 | book.Author = article.Byline
358 | book.MinReadTime = int(math.Floor(float64(article.Length)/(987+188) + 0.5))
359 | book.MaxReadTime = int(math.Floor(float64(article.Length)/(987-188) + 0.5))
360 | book.Content = article.TextContent
361 | book.HTML = article.Content
362 |
363 | // Make sure title is not empty
364 | if article.Title != "" {
365 | book.Title = article.Title
366 | }
367 |
368 | // Check if book has content
369 | if book.Content != "" {
370 | book.HasContent = true
371 | }
372 |
373 | // Update bookmark image in local disk
374 | u2, err := uuid.NewV4()
375 | if err != nil {
376 | utils.CheckError(err)
377 | }
378 | imgPath := fp.Join(h.dataDir, "thumb", u2.String())
379 | err = downloadFile(article.Image, imgPath, 20*time.Second)
380 | if err == nil {
381 | book.ImageURL = fmt.Sprintf("/thumb/%s", u2)
382 | }
383 | }(&book)
384 | }
385 |
386 | // Wait until all finished
387 | wg.Wait()
388 |
389 | // Update database
390 | res, err := h.db.UpdateBookmarks(books...)
391 | utils.CheckError(err)
392 |
393 | // Return new saved result
394 | err = json.NewEncoder(c.Writer).Encode(&res)
395 | utils.CheckError(err)
396 | }
397 |
398 | func downloadFile(url, dstPath string, timeout time.Duration) error {
399 | // Fetch data from URL
400 | client := &http.Client{Timeout: timeout}
401 | resp, err := client.Get(url)
402 | if err != nil {
403 | return err
404 | }
405 | defer resp.Body.Close()
406 |
407 | // Make sure destination directory exist
408 | err = os.MkdirAll(fp.Dir(dstPath), os.ModePerm)
409 | if err != nil {
410 | return err
411 | }
412 |
413 | // Create destination file
414 | dst, err := os.Create(dstPath)
415 | if err != nil {
416 | return err
417 | }
418 | defer dst.Close()
419 |
420 | // Write response body to the file
421 | _, err = io.Copy(dst, resp.Body)
422 | if err != nil {
423 | return err
424 | }
425 |
426 | return nil
427 | }
428 |
--------------------------------------------------------------------------------
/cmd/serve/web-handler-ui.go:
--------------------------------------------------------------------------------
1 | package serve
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "html/template"
7 | "io"
8 | "mime"
9 | "net/http"
10 | nurl "net/url"
11 | "os"
12 | fp "path/filepath"
13 | "strconv"
14 |
15 | "src.techknowlogick.com/shiori/database"
16 | "src.techknowlogick.com/shiori/utils"
17 |
18 | "github.com/gin-gonic/gin"
19 | "github.com/gobuffalo/packr/v2"
20 | )
21 |
22 | // serveFiles serve files
23 | func (h *webHandler) serveFiles(c *gin.Context) {
24 | err := serveFile(c, c.Param("filepath"))
25 | utils.CheckError(err)
26 | }
27 |
28 | // serveIndexPage is handler for GET /
29 | func (h *webHandler) serveIndexPage(c *gin.Context) {
30 | // Check token
31 | err := h.checkToken(c.Request)
32 | if err != nil {
33 | redirectPage(c, "/login")
34 | return
35 | }
36 |
37 | bookmarks, err := h.db.GetBookmarks(database.BookmarkOptions{})
38 | utils.CheckError(err)
39 |
40 | // Create template
41 | funcMap := template.FuncMap{
42 | "html": func(s string) template.HTML {
43 | return template.HTML(s)
44 | },
45 | "hostname": func(s string) string {
46 | parsed, err := nurl.ParseRequestURI(s)
47 | if err != nil || len(parsed.Scheme) == 0 {
48 | return s
49 | }
50 |
51 | return parsed.Hostname()
52 | },
53 | }
54 |
55 | tplCache, err := createTemplate("index.html", funcMap)
56 | utils.CheckError(err)
57 |
58 | bt, err := json.Marshal(&bookmarks)
59 | utils.CheckError(err)
60 |
61 | // Execute template
62 | strBt := string(bt)
63 | err = tplCache.Execute(c.Writer, &strBt)
64 | utils.CheckError(err)
65 |
66 | }
67 |
68 | // serveSubmitPage is handler for GET /submit
69 | func (h *webHandler) serveSubmitPage(c *gin.Context) {
70 | err := serveFile(c, "submit.html")
71 | utils.CheckError(err)
72 | }
73 |
74 | // serveLoginPage is handler for GET /login
75 | func (h *webHandler) serveLoginPage(c *gin.Context) {
76 | // Check token
77 | err := h.checkToken(c.Request)
78 | if err == nil {
79 | redirectPage(c, "/")
80 | return
81 | }
82 |
83 | err = serveFile(c, "login.html")
84 | utils.CheckError(err)
85 | }
86 |
87 | // serveBookmarkCache is handler for GET /bookmark/:id
88 | func (h *webHandler) serveBookmarkCache(c *gin.Context) {
89 | // Get bookmark ID from URL
90 | strID := c.Param("id")
91 | id, err := strconv.Atoi(strID)
92 | utils.CheckError(err)
93 |
94 | // Get bookmarks in database
95 | bookmarks, err := h.db.GetBookmarks(database.BookmarkOptions{}, id)
96 | utils.CheckError(err)
97 |
98 | if len(bookmarks) == 0 {
99 | panic(fmt.Errorf("No bookmark with matching index"))
100 | }
101 |
102 | // Create template
103 | funcMap := template.FuncMap{
104 | "html": func(s string) template.HTML {
105 | return template.HTML(s)
106 | },
107 | "hostname": func(s string) string {
108 | parsed, err := nurl.ParseRequestURI(s)
109 | if err != nil || len(parsed.Scheme) == 0 {
110 | return s
111 | }
112 |
113 | return parsed.Hostname()
114 | },
115 | }
116 |
117 | tplCache, err := createTemplate("cache.html", funcMap)
118 | utils.CheckError(err)
119 |
120 | bt, err := json.Marshal(bookmarks[0])
121 | utils.CheckError(err)
122 |
123 | err = tplCache.Execute(c.Writer, string(bt))
124 | utils.CheckError(err)
125 | }
126 |
127 | // serveThumbnailImage is handler for GET /thumb/:id
128 | func (h *webHandler) serveThumbnailImage(c *gin.Context) {
129 | // Get bookmark ID from URL
130 | id := c.Param("id")
131 |
132 | // Open image
133 | imgPath := fp.Join(h.dataDir, "thumb", id)
134 | img, err := os.Open(imgPath)
135 | utils.CheckError(err)
136 | defer img.Close()
137 |
138 | // Get image type from its 512 first bytes
139 | buffer := make([]byte, 512)
140 | _, err = img.Read(buffer)
141 | utils.CheckError(err)
142 |
143 | mimeType := http.DetectContentType(buffer)
144 | c.Header("Content-Type", mimeType)
145 |
146 | // Serve image
147 | img.Seek(0, 0)
148 | _, err = io.Copy(c.Writer, img)
149 | utils.CheckError(err)
150 | }
151 |
152 | func serveFile(c *gin.Context, path string) error {
153 | // Open file
154 | box := packr.New("views", "../../dist")
155 | _, fname := fp.Split(path)
156 | src, err := box.Find(fname)
157 | if err != nil {
158 | return err
159 | }
160 |
161 | // Get content type
162 | ext := fp.Ext(fname)
163 | mimeType := mime.TypeByExtension(ext)
164 | if mimeType != "" {
165 | c.Header("Content-Type", mimeType)
166 | }
167 |
168 | // Serve file
169 | c.Writer.Write(src)
170 | return nil
171 | }
172 |
--------------------------------------------------------------------------------
/cmd/serve/web-handler.go:
--------------------------------------------------------------------------------
1 | package serve
2 |
3 | import (
4 | "crypto/rand"
5 | "fmt"
6 | "html/template"
7 | "net/http"
8 |
9 | "src.techknowlogick.com/shiori/database"
10 |
11 | jwt "github.com/dgrijalva/jwt-go"
12 | "github.com/dgrijalva/jwt-go/request"
13 | "github.com/gin-gonic/gin"
14 | "github.com/gobuffalo/packr/v2"
15 | )
16 |
17 | // webHandler is handler for every API and routes to web page
18 | type webHandler struct {
19 | db database.Database
20 | dataDir string
21 | jwtKey []byte
22 | tplCache *template.Template
23 | }
24 |
25 | type handlerOptions struct {
26 | db database.Database
27 | dataDir string
28 | jwtSecret string
29 | }
30 |
31 | // newWebHandler returns new webHandler
32 | func newWebHandler(options *handlerOptions) (*webHandler, error) {
33 | // Create JWT key
34 | jwtKey := make([]byte, 32)
35 | _, err := rand.Read(jwtKey)
36 | if err != nil {
37 | return nil, err
38 | }
39 | if len(options.jwtSecret) != 0 {
40 | jwtKey = []byte(options.jwtSecret)
41 | }
42 |
43 | // Create handler
44 | handler := &webHandler{
45 | db: options.db,
46 | dataDir: options.dataDir,
47 | jwtKey: jwtKey,
48 | }
49 |
50 | return handler, nil
51 | }
52 |
53 | func (h *webHandler) checkToken(r *http.Request) error {
54 | tokenCookie, err := r.Cookie("token")
55 | if err != nil {
56 | return fmt.Errorf("Token error: Token does not exist")
57 | }
58 |
59 | token, err := jwt.Parse(tokenCookie.Value, h.jwtKeyFunc)
60 | if err != nil {
61 | return fmt.Errorf("Token error: %v", err)
62 | }
63 |
64 | claims := token.Claims.(jwt.MapClaims)
65 | err = claims.Valid()
66 | if err != nil {
67 | return fmt.Errorf("Token error: %v", err)
68 | }
69 |
70 | return nil
71 | }
72 |
73 | func (h *webHandler) checkAPIToken(r *http.Request) error {
74 | token, err := request.ParseFromRequest(r,
75 | request.AuthorizationHeaderExtractor,
76 | h.jwtKeyFunc)
77 | if err != nil {
78 | // Try to check in cookie
79 | return h.checkToken(r)
80 | }
81 |
82 | claims := token.Claims.(jwt.MapClaims)
83 | err = claims.Valid()
84 | if err != nil {
85 | return fmt.Errorf("Token error: %v", err)
86 | }
87 |
88 | return nil
89 | }
90 |
91 | func (h *webHandler) jwtKeyFunc(token *jwt.Token) (interface{}, error) {
92 | if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
93 | return nil, fmt.Errorf("Unexpected signing method")
94 | }
95 |
96 | return h.jwtKey, nil
97 | }
98 |
99 | func createTemplate(filename string, funcMap template.FuncMap) (*template.Template, error) {
100 | // Open file
101 | box := packr.New("views", "../../dist")
102 | src, err := box.Find(filename)
103 | if err != nil {
104 | return nil, err
105 | }
106 |
107 | // Create template
108 | return template.New(filename).Delims("$|", "|$").Funcs(funcMap).Parse(string(src))
109 | }
110 |
111 | func redirectPage(c *gin.Context, url string) {
112 | c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
113 | c.Header("Pragma", "no-cache")
114 | c.Header("Expires", "0")
115 | c.Redirect(http.StatusMovedPermanently, url)
116 | }
117 |
--------------------------------------------------------------------------------
/cmd/update.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "math"
7 | nurl "net/url"
8 | fp "path/filepath"
9 | "strings"
10 | "sync"
11 | "time"
12 |
13 | "src.techknowlogick.com/shiori/database"
14 | "src.techknowlogick.com/shiori/utils"
15 |
16 | valid "github.com/asaskevich/govalidator"
17 | "github.com/go-shiori/go-readability"
18 | "github.com/gofrs/uuid"
19 | "github.com/gosuri/uiprogress"
20 | "github.com/urfave/cli"
21 | "src.techknowlogick.com/shiori/model"
22 | )
23 |
24 | var (
25 | CmdUpdate = cli.Command{
26 | Name: "update",
27 | Usage: "Update the saved bookmarks",
28 | Description: "Update fields of an existing bookmark. " +
29 | "Accepts space-separated list of indices (e.g. 5 6 23 4 110 45), hyphenated range (e.g. 100-200) or both (e.g. 1-3 7 9). " +
30 | "If no arguments, ALL bookmarks will be updated. Update works differently depending on the flags:\n" +
31 | "- If indices are passed without any flags (--url, --title, --tag and --excerpt), read the URLs from DB and update titles from web.\n" +
32 | "- If --url is passed (and --title is omitted), update the title from web using the URL. While using this flag, update only accept EXACTLY one index.\n" +
33 | "While updating bookmark's tags, you can use - to remove tag (e.g. -nature to remove nature tag from this bookmark).",
34 | Flags: []cli.Flag{
35 | cli.StringFlag{
36 | Name: "url, u",
37 | Usage: "New URL for this bookmark",
38 | },
39 | cli.StringFlag{
40 | Name: "title, i",
41 | Usage: "New title for this bookmark",
42 | },
43 | cli.StringFlag{
44 | Name: "excerpt, e",
45 | Usage: "New excerpt for this bookmark",
46 | },
47 | cli.StringSliceFlag{
48 | Name: "tags, t",
49 | Usage: "Comma-separated tags for this bookmark",
50 | },
51 | cli.BoolFlag{
52 | Name: "offline, o",
53 | Usage: "Update bookmark without fetching data from internet",
54 | },
55 | cli.BoolFlag{
56 | Name: "yes, y",
57 | Usage: "Skip confirmation prompt and update ALL bookmarks",
58 | },
59 | cli.BoolFlag{
60 | Name: "dont-overwrite",
61 | Usage: "Don't overwrite existing metadata. Useful when only want to update bookmark's content",
62 | },
63 | },
64 | Action: runUpdateBookmarks,
65 | }
66 | )
67 |
68 | func runUpdateBookmarks(c *cli.Context) error {
69 | // Parse flags
70 | args := c.Args()
71 | dataDir := c.GlobalString("data-dir")
72 | url := c.String("url")
73 | title := c.String("title")
74 | excerpt := c.String("excerpt")
75 | tags := c.StringSlice("tags")
76 | offline := c.Bool("offline")
77 | skipConfirm := c.Bool("yes")
78 | dontOverwrite := c.Bool("dont-overwrite")
79 |
80 | title = normalizeSpace(title)
81 | excerpt = normalizeSpace(excerpt)
82 |
83 | db, err := getDbConnection(c)
84 |
85 | if err != nil {
86 | return errors.New(utils.CErrorSprint(err))
87 | }
88 |
89 | // Convert args to ids
90 | ids, err := utils.ParseIndexList(args)
91 | if err != nil {
92 | return errors.New(utils.CErrorSprint(err))
93 | }
94 |
95 | // Check if --url flag is used
96 | if c.IsSet("url") {
97 | // Make sure URL is valid
98 | parsedURL, err := nurl.Parse(url)
99 | if err != nil || !valid.IsRequestURL(url) {
100 | return errors.New(utils.CErrorSprint("URL is not valid"))
101 | }
102 |
103 | // Clear fragment and UTM parameters from URL
104 | parsedURL.Fragment = ""
105 | utils.ClearUTMParams(parsedURL)
106 | url = parsedURL.String()
107 |
108 | // Make sure there is only one arguments
109 | if len(ids) != 1 {
110 | return errors.New(utils.CErrorSprint("Update only accepts one index while using --url flag"))
111 | }
112 | }
113 |
114 | // If no arguments (i.e all bookmarks will be updated),
115 | // confirm to user
116 | if len(args) == 0 && !skipConfirm {
117 | confirmUpdate := ""
118 | fmt.Print("Update ALL bookmarks? (y/n): ")
119 | fmt.Scanln(&confirmUpdate)
120 |
121 | if confirmUpdate != "y" {
122 | return errors.New(utils.CErrorSprint("No bookmarks updated"))
123 | }
124 | }
125 |
126 | // Prepare wait group and mutex
127 | mx := sync.Mutex{}
128 | wg := sync.WaitGroup{}
129 |
130 | // Fetch bookmarks from database
131 | bookmarks, err := db.GetBookmarks(database.BookmarkOptions{}, ids...)
132 | if err != nil {
133 | return errors.New(utils.CErrorSprint(err))
134 | }
135 |
136 | if len(bookmarks) == 0 {
137 | return errors.New(utils.CErrorSprint("No matching index found"))
138 | }
139 |
140 | // If not offline, fetch articles from internet
141 | listErrorMsg := []string{}
142 | if !offline {
143 | fmt.Println("Fetching new bookmarks data")
144 |
145 | // Start progress bar
146 | uiprogress.Start()
147 | bar := uiprogress.AddBar(len(bookmarks)).AppendCompleted().PrependElapsed()
148 |
149 | for i, book := range bookmarks {
150 | wg.Add(1)
151 |
152 | go func(pos int, book model.Bookmark) {
153 | // Make sure to increase bar
154 | defer func() {
155 | bar.Incr()
156 | wg.Done()
157 | }()
158 |
159 | // If used, use submitted URL
160 | if url != "" {
161 | book.URL = url
162 | }
163 |
164 | // Parse URL
165 | parsedURL, err := nurl.Parse(book.URL)
166 | if err != nil || !valid.IsRequestURL(book.URL) {
167 | mx.Lock()
168 | errorMsg := fmt.Sprintf("Failed to fetch %s: URL is not valid", book.URL)
169 | listErrorMsg = append(listErrorMsg, errorMsg)
170 | mx.Unlock()
171 | return
172 | }
173 |
174 | // Fetch data from internet
175 | article, err := readability.FromURL(parsedURL.String(), 20*time.Second)
176 | if err != nil {
177 | mx.Lock()
178 | errorMsg := fmt.Sprintf("Failed to fetch %s: %v", book.URL, err)
179 | listErrorMsg = append(listErrorMsg, errorMsg)
180 | mx.Unlock()
181 | return
182 | }
183 |
184 | book.Author = article.Byline
185 | book.MinReadTime = int(math.Floor(float64(article.Length)/(987+188) + 0.5))
186 | book.MaxReadTime = int(math.Floor(float64(article.Length)/(987-188) + 0.5))
187 | book.Content = article.TextContent
188 | book.HTML = article.Content
189 |
190 | if !dontOverwrite {
191 | book.Title = article.Title
192 | book.Excerpt = article.Excerpt
193 | }
194 |
195 | // Save bookmark image to local disk
196 | u2, err := uuid.NewV4()
197 | if err != nil {
198 | mx.Lock()
199 | errorMsg := fmt.Sprintf("Failed generate uuid")
200 | listErrorMsg = append(listErrorMsg, errorMsg)
201 | mx.Unlock()
202 | return
203 | }
204 | imgPath := fp.Join(dataDir, "thumb", u2.String())
205 | err = downloadFile(article.Image, imgPath, 20*time.Second)
206 | if err == nil {
207 | book.ImageURL = fmt.Sprintf("/thumb/%s", u2)
208 | }
209 |
210 | // Update list of bookmarks
211 | mx.Lock()
212 | bookmarks[pos] = book
213 | mx.Unlock()
214 | }(i, book)
215 | }
216 |
217 | wg.Wait()
218 | uiprogress.Stop()
219 |
220 | // Print error message
221 | fmt.Println()
222 | for _, errorMsg := range listErrorMsg {
223 | utils.CError.Println(errorMsg + "\n")
224 | }
225 | }
226 |
227 | // Map the tags to be added or deleted from flag --tags
228 | addedTags := make(map[string]struct{})
229 | deletedTags := make(map[string]struct{})
230 | for _, tag := range tags {
231 | tagName := strings.ToLower(tag)
232 | tagName = strings.TrimSpace(tagName)
233 |
234 | if strings.HasPrefix(tagName, "-") {
235 | tagName = strings.TrimPrefix(tagName, "-")
236 | deletedTags[tagName] = struct{}{}
237 | } else {
238 | addedTags[tagName] = struct{}{}
239 | }
240 | }
241 |
242 | // Set title, excerpt and tags from user submitted value
243 | for i, bookmark := range bookmarks {
244 | // Check if user submit his own title or excerpt
245 | if title != "" {
246 | bookmark.Title = title
247 | }
248 |
249 | if excerpt != "" {
250 | bookmark.Excerpt = excerpt
251 | }
252 |
253 | // Make sure title is not empty
254 | if bookmark.Title == "" {
255 | bookmark.Title = bookmark.URL
256 | }
257 |
258 | // Generate new tags
259 | tempAddedTags := make(map[string]struct{})
260 | for key, value := range addedTags {
261 | tempAddedTags[key] = value
262 | }
263 |
264 | newTags := []model.Tag{}
265 | for _, tag := range bookmark.Tags {
266 | if _, isDeleted := deletedTags[tag.Name]; isDeleted {
267 | tag.Deleted = true
268 | }
269 |
270 | if _, alreadyExist := addedTags[tag.Name]; alreadyExist {
271 | delete(tempAddedTags, tag.Name)
272 | }
273 |
274 | newTags = append(newTags, tag)
275 | }
276 |
277 | for tag := range tempAddedTags {
278 | newTags = append(newTags, model.Tag{Name: tag})
279 | }
280 |
281 | bookmark.Tags = newTags
282 |
283 | // Set bookmark new data
284 | bookmarks[i] = bookmark
285 | }
286 |
287 | // Update database
288 | result, err := db.UpdateBookmarks(bookmarks...)
289 | if err != nil {
290 | return errors.New(utils.CErrorSprint(err))
291 | }
292 |
293 | // Print update result
294 | printBookmarks(result...)
295 | return nil
296 | }
297 |
--------------------------------------------------------------------------------
/cmd/utils.go:
--------------------------------------------------------------------------------
1 | package cmd
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "net/http"
7 | "os"
8 | "os/exec"
9 | "path/filepath"
10 | fp "path/filepath"
11 | "runtime"
12 | "strings"
13 | "time"
14 |
15 | "src.techknowlogick.com/shiori/database"
16 | "src.techknowlogick.com/shiori/model"
17 | "src.techknowlogick.com/shiori/utils"
18 |
19 | "github.com/urfave/cli"
20 | "golang.org/x/crypto/ssh/terminal"
21 | )
22 |
23 | func normalizeSpace(str string) string {
24 | return strings.Join(strings.Fields(str), " ")
25 | }
26 |
27 | func downloadFile(url, dstPath string, timeout time.Duration) error {
28 | // Fetch data from URL
29 | client := &http.Client{Timeout: timeout}
30 | resp, err := client.Get(url)
31 | if err != nil {
32 | return err
33 | }
34 | defer resp.Body.Close()
35 |
36 | // Make sure destination directory exist
37 | err = os.MkdirAll(fp.Dir(dstPath), os.ModePerm)
38 | if err != nil {
39 | return err
40 | }
41 |
42 | // Create destination file
43 | dst, err := os.Create(dstPath)
44 | if err != nil {
45 | return err
46 | }
47 | defer dst.Close()
48 |
49 | // Write response body to the file
50 | _, err = io.Copy(dst, resp.Body)
51 | if err != nil {
52 | return err
53 | }
54 |
55 | return nil
56 | }
57 |
58 | // openBrowser tries to open the URL in a browser,
59 | // and returns whether it succeed in doing so.
60 | func openBrowser(url string) error {
61 | var args []string
62 | switch runtime.GOOS {
63 | case "darwin":
64 | args = []string{"open"}
65 | case "windows":
66 | args = []string{"cmd", "/c", "start"}
67 | default:
68 | args = []string{"xdg-open"}
69 | }
70 |
71 | cmd := exec.Command(args[0], append(args[1:], url)...)
72 | return cmd.Run()
73 | }
74 |
75 | func getTerminalWidth() int {
76 | width, _, _ := terminal.GetSize(int(os.Stdin.Fd()))
77 | return width
78 | }
79 |
80 | func getDbConnection(c *cli.Context) (database.Database, error) {
81 | dbType := c.GlobalString("db-type")
82 | dbDsn := c.GlobalString("db-dsn")
83 | dataDir := c.GlobalString("data-dir")
84 |
85 | if dbType == "sqlite3" && dbDsn == "shiori.db" {
86 | dbDsn = filepath.Join(dataDir, dbDsn)
87 | }
88 |
89 | db, err := database.OpenXormDatabase(&database.XormOptions{DbDsn: dbDsn, DbType: dbType, ShowSQL: c.GlobalBool("show-sql-log")})
90 | return db, err
91 |
92 | }
93 |
94 | func printBookmarks(bookmarks ...model.Bookmark) {
95 | for _, bookmark := range bookmarks {
96 | // Create bookmark index
97 | strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID)
98 | strSpace := strings.Repeat(" ", len(strBookmarkIndex))
99 |
100 | // Print bookmark title
101 | utils.CIndex.Print(strBookmarkIndex)
102 | utils.CTitle.Print(bookmark.Title)
103 |
104 | // Print read time
105 | if bookmark.MinReadTime > 0 {
106 | readTime := fmt.Sprintf(" (%d-%d minutes)", bookmark.MinReadTime, bookmark.MaxReadTime)
107 | if bookmark.MinReadTime == bookmark.MaxReadTime {
108 | readTime = fmt.Sprintf(" (%d minutes)", bookmark.MinReadTime)
109 | }
110 | utils.CReadTime.Println(readTime)
111 | } else {
112 | fmt.Println()
113 | }
114 |
115 | // Print bookmark URL
116 | utils.CSymbol.Print(strSpace + "> ")
117 | utils.CURL.Println(bookmark.URL)
118 |
119 | // Print bookmark excerpt
120 | if bookmark.Excerpt != "" {
121 | utils.CSymbol.Print(strSpace + "+ ")
122 | utils.CExcerpt.Println(bookmark.Excerpt)
123 | }
124 |
125 | // Print bookmark tags
126 | if len(bookmark.Tags) > 0 {
127 | utils.CSymbol.Print(strSpace + "# ")
128 | for i, tag := range bookmark.Tags {
129 | if i == len(bookmark.Tags)-1 {
130 | utils.CTag.Println(tag.Name)
131 | } else {
132 | utils.CTag.Print(tag.Name + ", ")
133 | }
134 | }
135 | }
136 |
137 | // Append new line
138 | fmt.Println()
139 | }
140 | }
141 |
--------------------------------------------------------------------------------
/database/database.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "src.techknowlogick.com/shiori/model"
5 | )
6 |
7 | // Database is interface for manipulating data in database.
8 | type Database interface {
9 | // InsertBookmark inserts new bookmark to database.
10 | InsertBookmark(bookmark *model.Bookmark) error
11 |
12 | // GetBookmarks fetch list of bookmarks based on submitted ids.
13 | GetBookmarks(options BookmarkOptions, ids ...int) ([]model.Bookmark, error)
14 |
15 | // GetTags fetch list of tags and their frequency
16 | GetTags() ([]model.Tag, error)
17 |
18 | // DeleteBookmarks removes all record with matching ids from database.
19 | DeleteBookmarks(ids ...int) error
20 |
21 | // SearchBookmarks search bookmarks by the keyword or tags.
22 | SearchBookmarks(options BookmarkOptions, tags ...string) ([]model.Bookmark, error)
23 |
24 | // UpdateBookmarks updates the saved bookmark in database.
25 | UpdateBookmarks(bookmarks ...model.Bookmark) ([]model.Bookmark, error)
26 |
27 | // CreateAccount creates new account in database
28 | CreateAccount(username, password string) error
29 |
30 | // GetAccount fetch account with matching username
31 | GetAccount(username string) (model.Account, error)
32 |
33 | // GetAccounts fetch list of accounts with matching keyword
34 | GetAccounts(keyword string) ([]model.Account, error)
35 |
36 | // DeleteAccounts removes all record with matching usernames
37 | DeleteAccounts(usernames ...string) error
38 |
39 | // GetBookmarkID fetchs bookmark ID based by its url
40 | GetBookmarkID(url string) int
41 | }
42 |
--------------------------------------------------------------------------------
/database/migration/m1.go:
--------------------------------------------------------------------------------
1 | package migration
2 |
3 | import (
4 | "time"
5 |
6 | "src.techknowlogick.com/xormigrate"
7 |
8 | "xorm.io/xorm"
9 | )
10 |
11 | type M1Tag struct {
12 | ID int `xorm:"'id' pk autoincr"`
13 | Name string
14 | Deleted bool
15 | NBookmark int `xorm:"n_bookmarks"`
16 | Created time.Time `xorm:"created"`
17 | Updated time.Time `xorm:"updated"`
18 | }
19 |
20 | func (m M1Tag) TableName() string {
21 | return "tag"
22 | }
23 |
24 | type M1Bookmark struct {
25 | ID int `xorm:"'id' pk autoincr"`
26 | URL string `xorm:"url"`
27 | Title string `xorm:"'title' NOT NULL"`
28 | ImageURL string `xorm:"'image_url' NOT NULL"`
29 | Excerpt string `xorm:"'excerpt' NOT NULL"`
30 | Author string `xorm:"'author' NOT NULL"`
31 | MinReadTime int `xorm:"'min_read_time' DEFAULT 0"`
32 | MaxReadTime int `xorm:"'max_read_time' DEFAULT 0"`
33 | Modified time.Time `xorm:"modified"`
34 | Content string `xorm:"TEXT 'content'"`
35 | HTML string `xorm:"TEXT 'html'"`
36 | HasContent bool `xorm:"has_content"`
37 | Created time.Time `xorm:"created"`
38 | Updated time.Time `xorm:"updated"`
39 | }
40 |
41 | func (m M1Bookmark) TableName() string {
42 | return "bookmark"
43 | }
44 |
45 | type M1BookmarkTag struct {
46 | BookmarkID int `xorm:"bookmark_id"`
47 | TagID int `xorm:"tag_id"`
48 | }
49 |
50 | func (m M1BookmarkTag) TableName() string {
51 | return "bookmark_tag"
52 | }
53 |
54 | type M1Account struct {
55 | ID int `xorm:"'id' pk autoincr"`
56 | Username string
57 | Password string
58 | Created time.Time `xorm:"created"`
59 | Updated time.Time `xorm:"updated"`
60 | }
61 |
62 | func (m M1Account) TableName() string {
63 | return "account"
64 | }
65 |
66 | var (
67 | M1 = &xormigrate.Migration{
68 | ID: "initial-migration",
69 | Description: "[M1] Create base set of tables",
70 | Migrate: func(tx *xorm.Engine) error {
71 | // Sync2 instead of CreateTables because tables may already exist
72 | return tx.Sync2(new(M1Tag), new(M1Bookmark), new(M1BookmarkTag), new(M1Account))
73 | },
74 | Rollback: func(tx *xorm.Engine) error {
75 | return tx.DropTables(new(M1Tag), new(M1Bookmark), new(M1BookmarkTag), new(M1Account))
76 | },
77 | }
78 | )
79 |
--------------------------------------------------------------------------------
/database/migrations.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "src.techknowlogick.com/shiori/database/migration"
5 | "src.techknowlogick.com/xormigrate"
6 | )
7 |
8 | var (
9 | migrations = []*xormigrate.Migration{
10 | migration.M1,
11 | }
12 | )
13 |
--------------------------------------------------------------------------------
/database/structs.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | // BookmarkOptions
4 | type BookmarkOptions struct {
5 | WithContent bool
6 | MaxID int
7 | PerPage int
8 | Keyword string
9 | }
10 |
--------------------------------------------------------------------------------
/database/xorm.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "fmt"
5 | "math"
6 | "strings"
7 | "time"
8 |
9 | "src.techknowlogick.com/shiori/model"
10 | "src.techknowlogick.com/xormigrate"
11 |
12 | _ "github.com/denisenkom/go-mssqldb"
13 | _ "github.com/go-sql-driver/mysql"
14 | _ "github.com/lib/pq"
15 | _ "github.com/mattn/go-sqlite3"
16 | "golang.org/x/crypto/bcrypt"
17 | "xorm.io/builder"
18 | "xorm.io/xorm"
19 | )
20 |
21 | // XormDatabase is implementation of Database interface for connecting to database.
22 | type XormDatabase struct {
23 | *xorm.Engine
24 | dbType string
25 | }
26 |
27 | type XormOptions struct {
28 | DbDsn string
29 | DbType string
30 | ShowSQL bool
31 | }
32 |
33 | // OpenXormDatabase creates and open connection to new database.
34 | func OpenXormDatabase(options *XormOptions) (*XormDatabase, error) {
35 | // Open database and start transaction
36 | db, err := xorm.NewEngine(options.DbType, options.DbDsn)
37 | if err != nil {
38 | return &XormDatabase{}, err
39 | }
40 | db.ShowSQL(options.ShowSQL)
41 | m := xormigrate.New(db, migrations)
42 | if err = m.Migrate(); err != nil {
43 | return &XormDatabase{}, fmt.Errorf("Could not migrate: %v", err)
44 | }
45 |
46 | return &XormDatabase{db, options.DbType}, nil
47 | }
48 |
49 | // InsertBookmark inserts new bookmark to database. Returns new ID and error if any happened.
50 | func (db *XormDatabase) InsertBookmark(bookmark *model.Bookmark) error {
51 | // Check URL and title
52 | if bookmark.URL == "" {
53 | return fmt.Errorf("URL must not be empty")
54 | }
55 |
56 | if bookmark.Title == "" {
57 | return fmt.Errorf("Title must not be empty")
58 | }
59 |
60 | bookmark.Modified = time.Now()
61 |
62 | session := db.NewSession()
63 | defer session.Close()
64 |
65 | // add Begin() before any action
66 | if err := session.Begin(); err != nil {
67 | // if returned then will rollback automatically
68 | return err
69 | }
70 |
71 | // create bookmark & get ID
72 | session.Insert(bookmark)
73 | for i := 0; i < len(bookmark.Tags); i++ {
74 | var tag model.Tag
75 | tag.Name = bookmark.Tags[i].Name
76 | has, err := session.Exist(&tag)
77 | if err != nil {
78 | return err
79 | }
80 | if !has {
81 | // create tag
82 | session.Insert(&tag)
83 | } else {
84 | session.Where("name = ?", tag.Name).Get(&tag)
85 | }
86 | bookmark.Tags[i] = tag
87 | // add bookmark_tag relation
88 | session.Insert(&model.BookmarkTag{BookmarkID: bookmark.ID, TagID: tag.ID})
89 | }
90 | session.Commit()
91 | return nil
92 | }
93 |
94 | // GetBookmarks fetch list of bookmarks based on submitted ids.
95 | func (db *XormDatabase) GetBookmarks(options BookmarkOptions, ids ...int) ([]model.Bookmark, error) {
96 | bookmarks := make([]model.Bookmark, 0)
97 | var err error
98 | if len(ids) > 0 {
99 | err = db.In("id", ids).Find(&bookmarks)
100 | } else {
101 | searchCond := builder.NewCond()
102 | if options.MaxID != 0 {
103 | searchCond.And(builder.Lt{"id": options.MaxID})
104 | }
105 | limit := 30
106 | if options.PerPage > 0 {
107 | limit = options.PerPage
108 | }
109 | err = db.Where(searchCond).Limit(limit).Find(&bookmarks)
110 | }
111 | for i := 0; i < len(bookmarks); i++ {
112 | bookmarks[i].Tags = make([]model.Tag, 0)
113 | tags := make([]model.Tag, 0)
114 | bookmark := bookmarks[i]
115 | db.Join("left", "bookmark_tag", "bookmark_tag.tag_id = tag.id").Where(builder.Eq{"bookmark_tag.bookmark_id": bookmark.ID}).Find(&tags)
116 | bookmarks[i].Tags = tags
117 | }
118 | return bookmarks, err
119 | }
120 |
121 | // DeleteBookmarks removes all record with matching ids from database.
122 | func (db *XormDatabase) DeleteBookmarks(ids ...int) error {
123 | if len(ids) == 0 {
124 | return db.deleteBookmarks()
125 | }
126 |
127 | page := 0
128 | for len(ids) > page*100 {
129 | upperIndex := int(math.Min(float64(page*100+100), float64(len(ids))))
130 | err := db.deleteBookmarks(ids[page*100 : upperIndex]...)
131 | if err != nil {
132 | fmt.Println(err)
133 | }
134 | page = page + 1
135 | }
136 | return nil
137 | }
138 |
139 | // deleteBookmarks removes all record with matching ids from database
140 | func (db *XormDatabase) deleteBookmarks(ids ...int) error {
141 | var bookmark model.Bookmark
142 | var err error
143 | if len(ids) > 0 {
144 | _, err = db.In("id", ids).Delete(&bookmark)
145 | } else {
146 | _, err = db.Delete(&bookmark)
147 | }
148 | return err
149 | }
150 |
151 | // SearchBookmarks search bookmarks by the keyword or tags.
152 | func (db *XormDatabase) SearchBookmarks(options BookmarkOptions, tags ...string) ([]model.Bookmark, error) {
153 | //var bookmarks []model.Bookmark
154 | bookmarks := make([]model.Bookmark, 0)
155 | searchCond := builder.NewCond()
156 |
157 | if len(options.Keyword) > 0 {
158 | keyword := strings.TrimSpace(options.Keyword)
159 | lowerKeyword := strings.ToLower(keyword)
160 | exprCond := builder.Or(
161 | builder.Like{"title", lowerKeyword},
162 | builder.Like{"content", lowerKeyword},
163 | )
164 | keywordCond := builder.Or(
165 | builder.Like{"url", lowerKeyword},
166 | exprCond,
167 | )
168 | searchCond = searchCond.And(keywordCond)
169 | }
170 |
171 | if len(tags) > 0 {
172 | tagsCond := builder.In("id", builder.Select("bookmark_id").From("bookmark_tag").LeftJoin("tag", builder.Expr("tag.id = bookmark_tag.tag_id")).Where(builder.In("tag.name", tags)))
173 | searchCond = searchCond.And(tagsCond)
174 | }
175 |
176 | err := db.Where(searchCond).Desc("created").Find(&bookmarks)
177 |
178 | for i := 0; i < len(bookmarks); i++ {
179 | bookmarks[i].Tags = make([]model.Tag, 0)
180 | tags := make([]model.Tag, 0)
181 | bookmark := bookmarks[i]
182 | db.Join("left", "bookmark_tag", "bookmark_tag.tag_id = tag.id").Where(builder.Eq{"bookmark_tag.bookmark_id": bookmark.ID}).Find(&tags)
183 | bookmarks[i].Tags = tags
184 | }
185 |
186 | return bookmarks, err
187 | }
188 |
189 | // UpdateBookmarks updates the saved bookmark in database.
190 | func (db *XormDatabase) UpdateBookmarks(bookmarks ...model.Bookmark) (result []model.Bookmark, err error) {
191 | result = []model.Bookmark{}
192 | session := db.NewSession()
193 | defer session.Close()
194 |
195 | // add Begin() before any action
196 | if err := session.Begin(); err != nil {
197 | // if returned then will rollback automatically
198 | return []model.Bookmark{}, err
199 | }
200 | for _, bookmark := range bookmarks {
201 | // create bookmark & get ID
202 | session.Where("id = ?", bookmark.ID).Update(&bookmark)
203 | // clear existing tag assignments
204 | session.Where("bookmark_id = ?", bookmark.ID).Delete(&model.BookmarkTag{})
205 | // insert & assign tag assignments
206 | for i := 0; i < len(bookmark.Tags); i++ {
207 | var tag model.Tag
208 | tag.Name = bookmark.Tags[i].Name
209 | has, err := session.Exist(&tag)
210 | if err != nil {
211 | return []model.Bookmark{}, err
212 | }
213 | if !has {
214 | // create tag
215 | session.Insert(&tag)
216 | } else {
217 | session.Where("name = ?", tag.Name).Get(&tag)
218 | }
219 | bookmark.Tags[i] = tag
220 | // add bookmark_tag relation
221 | session.Insert(&model.BookmarkTag{BookmarkID: bookmark.ID, TagID: tag.ID})
222 | }
223 | result = append(result, bookmark)
224 | }
225 | session.Commit()
226 | return result, nil
227 | }
228 |
229 | // CreateAccount saves new account to database. Returns new ID and error if any happened.
230 | func (db *XormDatabase) CreateAccount(username, password string) error {
231 | // Hash password with bcrypt
232 | hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 10)
233 | if err != nil {
234 | return err
235 | }
236 | _, err = db.Insert(&model.Account{Username: username, Password: string(hashedPassword)})
237 | return err
238 | }
239 |
240 | // GetAccount fetch account with matching username
241 | func (db *XormDatabase) GetAccount(username string) (model.Account, error) {
242 | var account model.Account
243 | has, err := db.Where("username = ?", username).Get(&account)
244 | if !has && err == nil {
245 | err = fmt.Errorf("user doesn't exist")
246 | }
247 | return account, err
248 | }
249 |
250 | // GetAccounts fetch list of accounts with matching keyword
251 | func (db *XormDatabase) GetAccounts(keyword string) ([]model.Account, error) {
252 | var accounts []model.Account
253 | var err error
254 | if keyword != "" {
255 | err = db.Where(builder.Like{"username", keyword}).Find(&accounts)
256 | } else {
257 | err = db.Find(&accounts)
258 | }
259 | return accounts, err
260 | }
261 |
262 | // DeleteAccounts removes all record with matching usernames
263 | func (db *XormDatabase) DeleteAccounts(usernames ...string) error {
264 | var account model.Account
265 | var err error
266 | if len(usernames) > 0 {
267 | _, err = db.In("username", usernames).Delete(&account)
268 | } else {
269 | _, err = db.Delete(&account)
270 | }
271 | return err
272 | }
273 |
274 | // GetTags fetch list of tags and their frequency
275 | func (db *XormDatabase) GetTags() ([]model.Tag, error) {
276 | tags := make([]model.Tag, 0)
277 | err := db.Table("tag").Select("bookmark_tag.tag_id as id, tag.name, COUNT(bookmark_tag.tag_id) as n_bookmarks").
278 | Join("left", "bookmark_tag", "bookmark_tag.tag_id = tag.id").
279 | GroupBy("bookmark_tag.tag_id, tag.name").Find(&tags)
280 |
281 | return tags, err
282 | }
283 |
284 | // GetBookmarkID fetchs bookmark ID based by its url
285 | func (db *XormDatabase) GetBookmarkID(url string) int {
286 | var bookmark model.Bookmark
287 | db.Where("url = ?", url).Get(&bookmark)
288 | return bookmark.ID
289 | }
290 |
--------------------------------------------------------------------------------
/docs/README.md:
--------------------------------------------------------------------------------
1 | * [shiori](../README.md)
2 | * [Installation](installation.md)
3 | * [Databases](databases.md)
4 | * [Usage](usage.md)
5 |
--------------------------------------------------------------------------------
/docs/databases.md:
--------------------------------------------------------------------------------
1 | ## Using other Databases
2 |
3 | To use another Database there are two options. You can use parameters while calling the shiori command itself or you can set environment variables.
4 |
5 | ### Using command parameters
6 |
7 | | Parameter | Default | Values | Description |
8 | |----------|--------|-------|---|
9 | | --db-type | sqlite3 | sqlite3, mssql, mysql, postgres | since this Shiori fork uses xorm as database layer, you can use several Databases. Check out the xorm docs for more informations |
10 | | --db-dsn | shiori.db | user:password@(localhost)/dbname?charset=utf8&parseTime=True&loc=Local | Your Database Connection string |
11 | | --show-sql-log | false | - | Bool, if set, Shiori will output all SQL Queries to the CLI |
12 |
13 | ### Using environment Variables
14 |
15 | | Variable | Default | Values | Description |
16 | |----------|--------|-------|---|
17 | | SHIORI_DBTYPE | sqlite3 | sqlite3, mssql, mysql, postgres | since this Shiori fork uses xorm as database layer, you can use several Databases. Check out the xorm docs for more informations |
18 | | SHIORI_DSN | shiori.db | user:password@(localhost)/dbname?charset=utf8&parseTime=True&loc=Local | Your Database Connection string |
19 | | SHIORI_SHOW_SQL | false | - | Bool, if set, Shiori will output all SQL Queries to the CLI |
20 |
21 |
--------------------------------------------------------------------------------
/docs/installation.md:
--------------------------------------------------------------------------------
1 | There are several installation methods available :
2 |
3 | - [Using precompiled binary](#using-precompiled-binary)
4 | - [Building from source](#building-from-source)
5 | - [Using Docker image](#using-docker-image)
6 |
7 | ## Using Precompiled Binary
8 |
9 | Download the latest version of `shiori` from [the release page](https://github.com/techknowlogick/shiori/releases/latest), then put it in your `PATH`.
10 |
11 | On Linux or MacOS, you can do it by adding this line to your profile file (either `$HOME/.bash_profile` or `$HOME/.profile`):
12 |
13 | ```
14 | export PATH=$PATH:/path/to/shiori
15 | ```
16 |
17 | Note that this will not automatically update your path for the remainder of the session. To do this, you should run:
18 |
19 | ```
20 | source $HOME/.bash_profile
21 | or
22 | source $HOME/.profile
23 | ```
24 |
25 | On Windows, you can simply set the `PATH` by using the advanced system settings.
26 |
27 | ## Building From Source
28 |
29 | Make sure you have `go >= 1.12` installed, then run :
30 |
31 | ```
32 | go get -u -d src.techknowlogick.com/shiori
33 | cd $GOPATH/src/src.techknowlogick.com/shiori
34 | GO111MODULE=on make dep build
35 | ```
36 |
37 | ## Using Docker Image
38 |
39 | To use Docker image, you can pull the latest automated build from Docker Hub :
40 |
41 | ```
42 | docker pull techknowlogick/shiori
43 | ```
44 |
45 | If you want to build the Docker image on your own, Shiori already has its [Dockerfile](https://github.com/techknowlogick/shiori/blob/master/Dockerfile), so you can build the Docker image by running :
46 |
47 | ```
48 | docker build -t shiori .
49 | ```
50 |
--------------------------------------------------------------------------------
/docs/override.css:
--------------------------------------------------------------------------------
1 | .menu-toggle::before {
2 | content: ">"!important;
3 | }
4 | .footer-nav .left a::before {
5 | content: ""!important;
6 | margin-right: 0!important;
7 | }
8 | .footer-nav .right a::after {
9 | content: ""!important;
10 | margin-left: 0!important;
11 | }
--------------------------------------------------------------------------------
/docs/usage.md:
--------------------------------------------------------------------------------
1 | Before using `shiori`, make sure it has been installed on your system. By default, `shiori` will store its data in directory `$HOME/.local/share/shiori`. If you want to set the data directory to another location, you can set the environment variable `SHIORI_DIR` to your desired path.
2 |
3 | - [Running Docker Container](#running-docker-container)
4 | - [Using Command Line Interface](#using-command-line-interface)
5 | - [Using Web Application](#using-web-application)
6 | - [CLI Examples](#cli-examples)
7 |
8 | ## Running Docker Container
9 |
10 | > If you are not using `shiori` from Docker image, you can skip this section.
11 |
12 | After building the image you will be able to start a container from it. To
13 | preserve the data, you need to bind the directory for storing database and thumbnails. In this example we're binding the data directory to our current working directory :
14 |
15 | ```
16 | docker run -d --rm --name shiori -p 8080:8080 -v $(pwd):/srv/shiori techknowlogick/shiori
17 | ```
18 |
19 | The above command will :
20 |
21 | - Creates a new container from image `techknowlogick/shiori`.
22 | - Set the container name to `shiori` (option `--name`).
23 | - Bind the host current working directory to `/srv/shiori` inside container (option `-v`).
24 | - Expose port `8080` in container to port `8080` in host machine (option `-p`).
25 | - Run the container in background (option `-d`).
26 | - Automatically remove the container when it stopped (option `--rm`).
27 |
28 | After you've run the container in background, you can access console of the container :
29 |
30 | ```
31 | docker exec -it shiori sh
32 | ```
33 |
34 | Now you can use `shiori` like normal. If you've finished, you can stop and remove the container by running :
35 |
36 | ```
37 | docker stop shiori
38 | ```
39 |
40 | ## Using Command Line Interface
41 |
42 | ```
43 | Simple command-line bookmark manager built with Go
44 |
45 | Usage:
46 | shiori [command]
47 |
48 | Available Commands:
49 | account Manage account for accessing web interface
50 | add Bookmark the specified URL
51 | delete Delete the saved bookmarks
52 | export Export bookmarks into HTML file in Netscape Bookmark format
53 | help Help about any command
54 | import Import bookmarks from HTML file in Netscape Bookmark format
55 | open Open the saved bookmarks
56 | pocket Import bookmarks from Pocket's exported HTML file
57 | print Print the saved bookmarks
58 | search Search bookmarks by submitted keyword
59 | serve Serve web app for managing bookmarks
60 | update Update the saved bookmarks
61 |
62 | Flags:
63 | -h, --help help for shiori
64 |
65 | Use "shiori [command] --help" for more information about a command.
66 | ```
67 |
68 | ## Using Web Application
69 |
70 | To access web application, you need to have at least one account. To create new account, run this command :
71 |
72 | ```
73 | shiori account add {{selected.length}} Items selected Page {{maxPage+1}} {{book.title}} {{book.excerpt}} {{book.id}} Page {{maxPage+1}}
155 | Shiori is a simple bookmarks manager written in Go language, developed by
156 | Radhi Fadlillah and other
157 | contributors. The source code is available on
158 | GitHub and released under MIT license.
159 | For ease of use, you can install the Shiori Bookmarklet by dragging this link (
161 | +Shiori) to your bookmark bar.
162 |
21 | 栞shiori
22 | simple bookmark manager Username: Password:
107 |