├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cmd └── repocket.go ├── go.mod ├── go.sum └── pkg ├── pocket └── pocket.go ├── repocket ├── config.go └── repocket.go └── util └── util.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: srvaroa 4 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | - package-ecosystem: "github-actions" 13 | directory: "/" 14 | schedule: 15 | interval: "daily" 16 | open-pull-requests-limit: 5 17 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.13 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | 29 | - name: Build 30 | run: go build -v cmd/repocket.go 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 https://github.com/srvaroa/repocket 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do 10 | so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN_DIR=./bin 2 | BINARY=repocket 3 | LINUX_ARCHS=386 amd64 arm arm64 4 | DARWIN_ARCHS=386 amd64 5 | 6 | VERSION=0.1 7 | 8 | build_all: 9 | mkdir -p ${BIN_DIR} 10 | $(foreach GOARCH, $(LINUX_ARCHS), \ 11 | $(shell export GOOS=linux; export GOARCH=$(GOARCH); export GO111MODULE=auto; go build -v -o $(BIN_DIR)/$(BINARY)-linux-$(GOARCH)-$(VERSION) cmd/repocket.go) \ 12 | ) 13 | $(foreach GOARCH, $(DARWIN_ARCHS), \ 14 | $(shell export GOOS=darwin; export GOARCH=$(GOARCH); export GO111MODULE=auto; go build -v -o $(BIN_DIR)/$(BINARY)-darwin-$(GOARCH)-$(VERSION) cmd/repocket.go) \ 15 | ) 16 | 17 | clean: 18 | rm -r ${BIN_DIR} || true 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Repocket 2 | ======== 3 | 4 | **Repocket** is a (very, very raw) tool to manage a local copy of articles 5 | from [GetPocket](https://getpocket.com). Because, you know, The 6 | Cloud is not a reliable storage. Also, grep. Local copies contain a 7 | header in plain YAML with the [metadata provided by 8 | Pocket](https://getpocket.com/developer/docs/v3/retrieve) 9 | 10 | **Repocket** maintains a simple folder structure to represent states. 11 | Articles are rendered as text files into two folders for unread and 12 | favourite articles. You can move articles to a deleted and archived 13 | folders to sync those statuses back to Pocket. 14 | 15 | **Caveat emptor** 16 | 17 | * Local copies are plain text, rendered with 18 | [w3m](http://w3m.sourceforge.net/), so expect images to be lost. I'm 19 | generally OK with this, and the original URLs are still there. 20 | * This workflow is not meant to be particularly user friendly, it just 21 | fits me well. I use the "favourite" state to mark articles that I 22 | want to keep a copy for. 23 | * It works in Linux and OSX, and will very likely not in Windows 24 | (although it should be trivial to fix.) 25 | 26 | Building 27 | -------- 28 | 29 | Go >= 1.11. 30 | 31 | You *MUST* have [w3m](http://w3m.sourceforge.net/) installed. Repocket 32 | uses [w3m](http://w3m.sourceforge.net/) to render the articles into 33 | plain text. 34 | 35 | Configuration 36 | ------------- 37 | 38 | You must create the config file before running `Repocket`. It's a very 39 | simple yaml stored at `~/.config/repocket/config` with these contents: 40 | 41 | consumer_key: 85480-9793dd8ed508561cb941d987 42 | favs_dir: 43 | unread_dir: 44 | deleted_dir: 45 | archived_dir: 46 | 47 | * The `consumer_key` indicates the GetPocket Application. You can leave 48 | the sample value above (my own app), but if you prefer to use your own 49 | just [create a new 50 | application](https://getpocket.com/developer/apps/new). 51 | 52 | The rest are directories to store articles. All expect an absolute 53 | path. Two of these are synced up and downstream: 54 | 55 | * `unread_dir` is the directory where unread, non archived articles will 56 | be downloaded. If a downloaded article is marked as archived in 57 | Pocket, then the file will be deleted. 58 | * `favs_dir` contains favourited articles. You move articles here from 59 | the `unread_dir`, in the next sync they will be marked as favourite 60 | and archived. Articles in your local fav directory are *never* 61 | deleted (even if you unfav them upstream.) 62 | 63 | These directories *only sync upstream*: 64 | 65 | * `deleted_dir` contains articles that should be deleted in the next run 66 | of the `delete` command. You move articles here when you want to 67 | delete them from Pocket. 68 | * `archived_dir` contains articles that should be archived in the next run 69 | of the `archive` command. You move articles here when you want to 70 | archive them in Pocket, but don't want to keep a local copy. 71 | 72 | Files in both `deleted_dir` and `archived_dir` are removed after a sync. 73 | 74 | Set up 75 | ------ 76 | 77 | When you first run `Repocket`, it will authenticate against the Pocket 78 | API. It will ask you to browse to a URL where you can grant permissions 79 | to read your list n articles. The message looks something like this: 80 | 81 | 2019/09/09 20:40:12 Browse to this URL, you may ignore errors: 82 | https://getpocket.com/auth/authorize?request_token=62074b8c-ed8a-b5e5-71f3-586bcf&redirect_uri=localhost 83 | 84 | Click on the link and accept the authorisation. Once you do this the 85 | first time, you simply need to click the link and ignore the browser. 86 | 87 | This step will write a new `access_token` property to your config file 88 | so you don't need to auth again. 89 | 90 | Synchronizing 91 | ------------- 92 | 93 | Run 94 | 95 | GO111MODULE=on go run ./cmd/repocket [favs|delete|archive|unread|sync] 96 | 97 | And **Repocket** will sync the folder associated to the given action. 98 | 99 | * `delete`: will delete articles added to `favs_dir` as deleted in 100 | Pocket. 101 | * `archive`: will delete articles added to `favs_dir` as deleted in 102 | Pocket. 103 | * `favs`: will mark articles added to `favs_dir` as favourited in 104 | Pocket. 105 | * `unread`: will download all unread articles to `unread_dir`. 106 | * `sync`: will execute the previous three actions, in the same order as 107 | shown in this list. 108 | 109 | TODO 110 | ---- 111 | 112 | Some things I'd like to implement: 113 | 114 | * Prepend the source URL to the file. 115 | -------------------------------------------------------------------------------- /cmd/repocket.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/srvaroa/repocket/pkg/repocket" 8 | "github.com/srvaroa/repocket/pkg/util" 9 | ) 10 | 11 | func usageAndExit() { 12 | log.Fatal("Usage: %s [favs|delete|archive|unread|sync]\n", os.Args[0]) 13 | } 14 | 15 | func main() { 16 | 17 | log.SetFlags(0) 18 | log.SetOutput(new(util.LogWriter)) 19 | 20 | if len(os.Args) != 2 { 21 | usageAndExit() 22 | } 23 | 24 | cmd := os.Args[1] 25 | 26 | r := repocket.Repocket{} 27 | err := r.Load() 28 | if err != nil { 29 | log.Fatalf("Unable to load configuration!", err) 30 | } 31 | r.Authenticate() 32 | 33 | switch cmd { 34 | case "delete": 35 | r.SyncDeletions() 36 | break 37 | case "archive": 38 | r.SyncArchived() 39 | break 40 | case "favs": 41 | r.SyncFavs() 42 | break 43 | case "unread": 44 | r.SyncUnread() 45 | break 46 | case "sync": 47 | log.Printf("Full sync") 48 | log.Printf("First push deletions..") 49 | r.SyncDeletions() 50 | log.Printf("Then push archived..") 51 | r.SyncArchived() 52 | log.Printf("Then push favs..") 53 | r.SyncFavs() 54 | log.Printf("Then pull unreads..") 55 | r.SyncUnread() 56 | log.Printf("All done!") 57 | break 58 | default: 59 | usageAndExit() 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/srvaroa/repocket 2 | 3 | go 1.12 4 | 5 | require gopkg.in/yaml.v3 v3.0.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 2 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 3 | gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= 4 | gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 5 | -------------------------------------------------------------------------------- /pkg/pocket/pocket.go: -------------------------------------------------------------------------------- 1 | package pocket 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | // https://getpocket.com/developer/docs/v3/retrieve 16 | type retrieveResponse struct { 17 | Status int 18 | List map[string]Article 19 | } 20 | 21 | // https://getpocket.com/developer/docs/v3/send 22 | type sendResponse struct { 23 | ActionResults bool 24 | Status int 25 | } 26 | 27 | // https://getpocket.com/developer/docs/v3/retrieve 28 | type Article struct { 29 | ItemId string `json:"item_id"` 30 | ResolvedId string `json:"resolved_id"` 31 | GivenUrl string `json:"given_url"` 32 | GivenTitle string `json:"given_title"` 33 | Favorite bool `json:"favorite"` 34 | Status int `json:"status"` // 0, 1, 2 where 1 = archived, 2 = to delete 35 | ResolvedTitle string `json:"resolved_title"` 36 | ResolvedUrl string `json:"resolved_url"` 37 | Excerpt string `json:"excerpt"` 38 | IsArticle bool `json:"is_article"` 39 | HasVideo int `json:"has_video"` // 0, 1, 2 where 1 = has videos, 2 = is a video 40 | HasImage int `json:"has_image"` // 0, 1, 2 where 1 = has images, 2 = is an image 41 | WordCount int `json:"word_count"` 42 | Tags string `json:"tags"` // actually another object but I care not now 43 | Authors string `json:"authors"` // actually another object but I care not now 44 | Images string `json:"images"` // actually another object but I care not now 45 | Videos string `json:"videos"` // actually another object but I care not now 46 | } 47 | 48 | const apiUrl = "https://getpocket.com/v3" 49 | 50 | const ( 51 | STATE_ALL = "all" 52 | STATE_UNREAD = "unread" 53 | STATE_ARCHIVE = "archive" 54 | ) 55 | 56 | const ( 57 | ACTION_DELETE = "delete" 58 | ACTION_FAVORITE = "favorite" 59 | ACTION_ARCHIVE = "archive" 60 | ) 61 | 62 | func QueryFavourites(accessToken, consumerKey string) map[string]Article { 63 | return query(accessToken, consumerKey, STATE_ALL, 1, 0) 64 | } 65 | 66 | func QueryNewest(accessToken, consumerKey string, count int) map[string]Article { 67 | return query(accessToken, consumerKey, STATE_UNREAD, 0, count) 68 | } 69 | 70 | func QueryUnread(accessToken, consumerKey string) map[string]Article { 71 | return query(accessToken, consumerKey, STATE_UNREAD, 0, 0) 72 | } 73 | 74 | func query(accessToken, consumerKey, state string, favourites, count int) map[string]Article { 75 | 76 | payload := map[string]interface{}{ 77 | "access_token": accessToken, 78 | "consumer_key": consumerKey, 79 | "favorite": favourites, 80 | "detailType": "complete", 81 | "sort": "newest", 82 | "state": state, 83 | } 84 | 85 | if count > 0 { 86 | payload["count"] = count 87 | } 88 | 89 | data, _ := json.Marshal(payload) 90 | 91 | res, err := http.Post(apiUrl+"/get", 92 | "application/json", 93 | strings.NewReader(string(data))) 94 | 95 | defer res.Body.Close() 96 | body, err := ioutil.ReadAll(res.Body) 97 | if err != nil { 98 | log.Fatal("Unable to retrieve items", err) 99 | } 100 | 101 | var retrieved retrieveResponse 102 | 103 | json.Unmarshal(body, &retrieved) 104 | 105 | return retrieved.List 106 | 107 | } 108 | 109 | func Archive(accessToken, consumerKey string, itemIds []string) bool { 110 | return action(ACTION_ARCHIVE, accessToken, consumerKey, itemIds) 111 | } 112 | 113 | func Delete(accessToken, consumerKey string, itemIds []string) bool { 114 | return action(ACTION_DELETE, accessToken, consumerKey, itemIds) 115 | } 116 | 117 | func Fav(accessToken, consumerKey string, itemIds []string) bool { 118 | return action(ACTION_FAVORITE, accessToken, consumerKey, itemIds) 119 | } 120 | 121 | func action(action, accessToken, consumerKey string, itemIds []string) bool { 122 | 123 | timestamp := time.Now().UTC() 124 | var actions []map[string]interface{} 125 | 126 | if len(itemIds) <= 0 { 127 | return true 128 | } 129 | 130 | for _, itemId := range itemIds { 131 | actions = append(actions, map[string]interface{}{ 132 | "action": action, 133 | "item_id": itemId, 134 | "timestamp": timestamp, 135 | }) 136 | } 137 | 138 | payload := map[string]interface{}{ 139 | "access_token": accessToken, 140 | "consumer_key": consumerKey, 141 | "actions": actions, 142 | } 143 | 144 | data, _ := json.Marshal(payload) 145 | 146 | res, err := http.Post(apiUrl+"/send", 147 | "application/json", 148 | strings.NewReader(string(data))) 149 | 150 | defer res.Body.Close() 151 | body, err := ioutil.ReadAll(res.Body) 152 | if err != nil { 153 | log.Fatal("Unable to delete items", err) 154 | } 155 | 156 | if res.StatusCode != 200 { 157 | log.Fatalf("Error deleting items %v \n", res.Header) 158 | } 159 | 160 | var deleted sendResponse 161 | 162 | json.Unmarshal(body, &deleted) 163 | 164 | return deleted.ActionResults 165 | 166 | } 167 | 168 | // Authorize returns the token for the given consumer key by firing 169 | // GetPocket's auth process. 170 | func Authorize(consumerKey string) (string, error) { 171 | 172 | log.Printf("Fetching token for consumer key: %s", consumerKey) 173 | 174 | log.Print("Initiating OAuth process..") 175 | res, err := http.PostForm(apiUrl+"/oauth/request", url.Values{ 176 | "consumer_key": {consumerKey}, 177 | "redirect_uri": {"localhost"}, 178 | }) 179 | 180 | if err != nil { 181 | log.Fatal("Unable to authorise", err) 182 | return "", err 183 | } 184 | 185 | defer res.Body.Close() 186 | body, err := ioutil.ReadAll(res.Body) 187 | if err != nil { 188 | log.Fatal("Unable to retrieve code", err) 189 | return "", err 190 | } 191 | 192 | code := strings.Split(string(body), "=")[1] 193 | 194 | log.Print("Authorizing application...") 195 | log.Printf("Browse to this URL, you may ignore errors: "+ 196 | "https://getpocket.com/auth/authorize?request_token=%s&redirect_uri=localhost"+ 197 | "\n\nPress enter when done", 198 | code) 199 | 200 | input := bufio.NewScanner(os.Stdin) 201 | input.Scan() 202 | 203 | res, err = http.PostForm(apiUrl+"/oauth/authorize", url.Values{ 204 | "consumer_key": {consumerKey}, 205 | "code": {code}, 206 | }) 207 | 208 | defer res.Body.Close() 209 | body, err = ioutil.ReadAll(res.Body) 210 | if err != nil { 211 | log.Fatal("Unable to retrieve token", err) 212 | return "", err 213 | } 214 | 215 | theBody := string(body) 216 | parts := strings.Split(theBody, "&") 217 | if len(parts) != 2 { 218 | log.Fatalf("Unexpected final autorization response "+ 219 | "expecting access_token=&username= but got %s", 220 | theBody) 221 | } 222 | log.Printf("Authorized as %s", strings.Split(string(parts[1]), "=")[1]) 223 | return strings.Split(string(parts[0]), "=")[1], nil 224 | } 225 | -------------------------------------------------------------------------------- /pkg/repocket/config.go: -------------------------------------------------------------------------------- 1 | package repocket 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "os/user" 9 | 10 | "gopkg.in/yaml.v3" 11 | 12 | "github.com/srvaroa/repocket/pkg/pocket" 13 | ) 14 | 15 | const RepocketConfigFile = ".config/repocket/config" 16 | 17 | type Repocket struct { 18 | ConsumerKey string `yaml:"consumer_key"` 19 | AccessToken string `yaml:"access_token"` 20 | FavsDir string `yaml:"favs_dir"` 21 | UnreadDir string `yaml:"unread_dir"` 22 | DeletedDir string `yaml:"deleted_dir"` 23 | ArchivedDir string `yaml:"archived_dir"` 24 | } 25 | 26 | func (cfg *Repocket) Load() error { 27 | 28 | usr, err := user.Current() 29 | if err != nil { 30 | log.Println("Could not determine user home %s", err) 31 | } 32 | 33 | yamlFile, err := ioutil.ReadFile(usr.HomeDir + "/" + RepocketConfigFile) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | err = yaml.Unmarshal(yamlFile, cfg) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return err 44 | } 45 | 46 | func (cfg *Repocket) Save() error { 47 | 48 | usr, err := user.Current() 49 | if err != nil { 50 | log.Println("Could not determine user home %s", err) 51 | } 52 | 53 | file, err := os.Create(usr.HomeDir + "/" + RepocketConfigFile) 54 | if err != nil { 55 | log.Printf("Failed to create config file at %s: %s", file, err) 56 | return err 57 | } 58 | defer file.Close() 59 | 60 | outBytes, err := yaml.Marshal(cfg) 61 | _, err = io.WriteString(file, string(outBytes)) // TODO: use straight bytes 62 | if err != nil { 63 | log.Printf("Failed to write config file %s: %s", file, err) 64 | } 65 | 66 | return err 67 | } 68 | 69 | // Authenticate will ensure that a given Repocket object is autheticated, 70 | // either by providing a ConsumerKey and AccessToken, or running the 71 | // oauth auth process. In the latter case, the token is persisted in 72 | // the config file. 73 | func (r *Repocket) Authenticate() { 74 | if len(r.AccessToken) > 0 { 75 | return 76 | } 77 | if len(r.ConsumerKey) <= 0 { 78 | log.Fatalf("Your config file seems empty. It should contain " + 79 | "at least an entry with the consumer_key. Please check the " + 80 | "README.md for details") 81 | } 82 | 83 | log.Printf("Loading access token..") 84 | 85 | var err error 86 | r.AccessToken, err = pocket.Authorize(r.ConsumerKey) 87 | if err != nil { 88 | log.Fatal("Failed to authorize against Pocket: %s", err) 89 | } 90 | 91 | err = r.Save() 92 | if err != nil { 93 | log.Printf("Failed to persist user token: %s", err) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /pkg/repocket/repocket.go: -------------------------------------------------------------------------------- 1 | package repocket 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "regexp" 9 | "strings" 10 | 11 | "gopkg.in/yaml.v3" 12 | 13 | "github.com/srvaroa/repocket/pkg/pocket" 14 | "github.com/srvaroa/repocket/pkg/util" 15 | ) 16 | 17 | // GetArticleIds finds all the ids of articles cached in a given dir 18 | func GetArticleIds(dir string) []string { 19 | files, err := ioutil.ReadDir(dir) 20 | if err != nil { 21 | log.Fatalf("Failed to list files in %s %s", dir, err) 22 | } 23 | 24 | var ids []string 25 | for _, f := range files { 26 | if !f.IsDir() { 27 | pieces := strings.Split(f.Name(), "_") 28 | ids = append(ids, pieces[0]) 29 | } 30 | } 31 | 32 | return ids 33 | } 34 | 35 | func DumpArticle(outputDir string, a *pocket.Article) { 36 | 37 | title := a.ResolvedTitle 38 | if len(a.ResolvedTitle) <= 0 && len(a.GivenTitle) > 0 { 39 | title = a.GivenTitle 40 | } 41 | 42 | // Clean up path 43 | re := regexp.MustCompile(`[\.|/\\]+`) 44 | path := outputDir + "/" + 45 | a.ItemId + 46 | "_" + 47 | string(re.ReplaceAll([]byte(title), []byte("-"))) 48 | 49 | // If the article is there, leave it alone 50 | _, err := os.Stat(path) 51 | if !os.IsNotExist(err) { 52 | log.Printf("Skipping (already downloaded): %s", title) 53 | return 54 | } 55 | 56 | log.Printf("Downloading: `%s` to `%s`", title, path) 57 | 58 | // We add the metadata from Pocket to the local copy 59 | metaBytes, err := yaml.Marshal(a) 60 | if err != nil { 61 | log.Printf("Failed to serialize article meta: %s %s", title, err) 62 | } 63 | 64 | // Prepare the contents of the article 65 | txtBytes, err := util.DumpUrl(a.ResolvedUrl) 66 | if err != nil { 67 | log.Print("Failed to download %s, %s", a.ResolvedUrl, err) 68 | return 69 | } 70 | 71 | // Open and write the file 72 | file, err := os.Create(path) 73 | if err != nil { 74 | log.Printf("Failed to create file for %s: %s", title, err) 75 | return 76 | } 77 | defer file.Close() 78 | 79 | _, err = io.WriteString(file, string(metaBytes)+"\n\n---\n\n"+string(txtBytes)) 80 | if err != nil { 81 | log.Printf("Failed to write %s: %s", title, err) 82 | } 83 | } 84 | 85 | // SyncDeleted reads the deleted directory and marks the articles as 86 | // deleted upstream, then removes all the files in the deleted 87 | // directory 88 | func (r *Repocket) SyncDeletions() { 89 | ids := GetArticleIds(r.DeletedDir) 90 | log.Printf("Will delete file: %s", ids) 91 | pocket.Delete(r.AccessToken, r.ConsumerKey, ids) 92 | util.EmptyDir(r.DeletedDir) 93 | } 94 | 95 | // SyncArchived reads the archived directory and marks the articles as 96 | // archived upstream, then removes all the files in the archived 97 | // directory 98 | func (r *Repocket) SyncArchived() { 99 | ids := GetArticleIds(r.ArchivedDir) 100 | log.Printf("Will delete file: %s", ids) 101 | pocket.Archive(r.AccessToken, r.ConsumerKey, ids) 102 | util.EmptyDir(r.ArchivedDir) 103 | } 104 | 105 | // SyncFavs does ths following steps: 106 | // * Keep a local copy of all articles fav'd upstream 107 | // * Ensure that all articles in the local fav dir are fav'd and 108 | // archived upstream 109 | func (r *Repocket) SyncFavs() { 110 | 111 | if len(r.FavsDir) == 0 { 112 | log.Fatalf("No output directory provided") 113 | } 114 | util.EnsureDir(r.FavsDir) 115 | 116 | knownIds := map[string]bool{} 117 | favs := pocket.QueryFavourites(r.AccessToken, r.ConsumerKey) 118 | for _, item := range favs { 119 | DumpArticle(r.FavsDir, &item) 120 | knownIds[item.ItemId] = true 121 | } 122 | 123 | // Find articles that were NOT in the favourited list 124 | var missingIds []string 125 | for _, id := range GetArticleIds(r.FavsDir) { 126 | if !knownIds[id] { 127 | missingIds = append(missingIds, id) 128 | } 129 | } 130 | 131 | log.Printf("Uploading %d new favs %s", len(missingIds), missingIds) 132 | pocket.Fav(r.AccessToken, r.ConsumerKey, missingIds) 133 | pocket.Archive(r.AccessToken, r.ConsumerKey, missingIds) 134 | } 135 | 136 | // SyncUnread reads all articles not archived and stores them in the 137 | // corresponding directory. It does no upstream sync. 138 | func (r *Repocket) SyncUnread() { 139 | favs := pocket.QueryUnread(r.AccessToken, r.ConsumerKey) 140 | if len(r.UnreadDir) == 0 { 141 | log.Fatalf("No output directory provided") 142 | } 143 | util.EnsureDir(r.UnreadDir) 144 | for _, item := range favs { 145 | DumpArticle(r.UnreadDir, &item) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "os/exec" 9 | ) 10 | 11 | type LogWriter struct{} 12 | 13 | func EnsureDir(path string) { 14 | f, err := os.Stat(path) 15 | if os.IsNotExist(err) { 16 | if err != nil { 17 | log.Fatalf("ERROR: expecting directory %s to exist", path) 18 | } 19 | } 20 | if !f.IsDir() { 21 | log.Fatalf("ERROR: expecting path %s to be a directory", path) 22 | } 23 | } 24 | 25 | func (writer LogWriter) Write(bytes []byte) (int, error) { 26 | return fmt.Print(string(bytes)) 27 | } 28 | 29 | func DumpUrl(url string) ([]byte, error) { 30 | return exec.Command("w3m", "-dump", url).Output() 31 | } 32 | 33 | func EmptyDir(dirPath string) { 34 | files, err := ioutil.ReadDir(dirPath) 35 | if err != nil { 36 | log.Fatalf("Failed to list deletion queue: %s", err) 37 | } 38 | for _, f := range files { 39 | filePath := dirPath + "/" + f.Name() 40 | log.Printf("Deleting file %s\n", filePath) 41 | os.Remove(filePath) 42 | } 43 | } 44 | --------------------------------------------------------------------------------