├── .gitignore ├── Makefile ├── README.md ├── api └── router.go ├── build ├── darwin_amd64.tar.gz ├── linux_386.tar.gz ├── linux_amd64.tar.gz ├── linux_arm.tar.gz ├── linux_arm64.tar.gz ├── windows_386.zip └── windows_amd64.zip ├── creds.example.sh ├── docs ├── api.md ├── development.md ├── screenshot.png └── supervisord.md ├── gdrive ├── drive-client.go ├── fixtures │ ├── image-doc.html │ ├── image-doc.md │ ├── sample-doc.html │ └── sample-doc.md ├── gdrive.go ├── parser.go ├── parser_test.go └── parsers │ ├── header.go │ ├── ol.go │ ├── p.go │ ├── parsers.go │ └── ul.go ├── glide.lock ├── glide.yaml ├── install.sh ├── main.go ├── model ├── page-fragment.go ├── page-fragment_test.go └── page.go ├── store ├── postgres │ ├── postgres.go │ └── sql │ │ └── pages.sql └── store.go └── util ├── conf.go ├── interval.go └── slug.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | vendor/ 3 | client_secret.json 4 | creds.sh 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | appname := allwrite-docs 2 | sources := $(wildcard *.go) 3 | 4 | build = GOOS=$(1) GOARCH=$(2) go build -o build/$(appname)$(3) 5 | tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3) 6 | zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3) 7 | 8 | .PHONY: all windows darwin linux clean 9 | 10 | all: windows darwin linux 11 | 12 | clean: 13 | rm -rf build/ 14 | 15 | ##### LINUX BUILDS ##### 16 | linux: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz 17 | 18 | build/linux_386.tar.gz: $(sources) 19 | $(call build,linux,386,) 20 | $(call tar,linux,386) 21 | 22 | build/linux_amd64.tar.gz: $(sources) 23 | $(call build,linux,amd64,) 24 | $(call tar,linux,amd64) 25 | 26 | build/linux_arm.tar.gz: $(sources) 27 | $(call build,linux,arm,) 28 | $(call tar,linux,arm) 29 | 30 | build/linux_arm64.tar.gz: $(sources) 31 | $(call build,linux,arm64,) 32 | $(call tar,linux,arm64) 33 | 34 | ##### DARWIN (MAC) BUILDS ##### 35 | darwin: build/darwin_amd64.tar.gz 36 | 37 | build/darwin_amd64.tar.gz: $(sources) 38 | $(call build,darwin,amd64,) 39 | $(call tar,darwin,amd64) 40 | 41 | ##### WINDOWS BUILDS ##### 42 | windows: build/windows_386.zip build/windows_amd64.zip 43 | 44 | build/windows_386.zip: $(sources) 45 | $(call build,windows,386,.exe) 46 | $(call zip,windows,386,.exe) 47 | 48 | build/windows_amd64.zip: $(sources) 49 | $(call build,windows,amd64,.exe) 50 | $(call zip,windows,amd64,.exe) 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Allwrite Docs 2 | 3 | An incredibly fast documentation API powered by Google Drive written purely in Go. 4 | 5 | This API connects with your Google Drive and provides RESTful endpoints which return the pages within Drive in a organized and usable format. With this API, beautiful (or ugly) user interfaces can be created and reused anywhere you need to display documentation online. 6 | 7 | ![screenshot](/docs/screenshot.png) 8 | 9 | **Features:** 10 | 11 | * Let anyone, technical or not, contribute to documentation. 12 | * Full-text search. 13 | * Auto-generating SSL via [certbot](https://certbot.eff.org/). 14 | * Transforms Google docs to clean markdown and html. 15 | * Images are directly referenced from Google so you don't need to worry about image storage. 16 | * No dependencies other than Postgres. 17 | * URL structure builds itself based on the directory structure. 18 | * Did I mention that it's crazy fast since pages are pre-cached? 19 | 20 | # Table of Contents 21 | 22 | * [Workflow](#workflow) 23 | * [Formatting](#formatting) 24 | * [Examples](#examples) 25 | * [Themes](#themes) 26 | * [Installation](#installation) 27 | * [API](#api) 28 | * [Contributing](#contributing) 29 | * [CLI](#cli) 30 | 31 | ## Workflow 32 | 33 | Give authors access to the activated folder using the typical means of doing so 34 | within Drive. Then authors can create pages and folders within the activated 35 | folder. Pages can have children infinitely deep by using directories. Pages and 36 | folders are both made public and ordered by using a special format in their page 37 | name. 38 | 39 | ``` 40 | |n| Page Title 41 | ``` 42 | 43 | The number between the two pipes (`n`) is what the menu is sorted by. It can be 44 | any number. If a page or directory does not have this format, it will not be 45 | public. This is useful when writing pages that should not yet be public (aka 46 | drafts). 47 | 48 | Lastly, if zero is used (`|0|`), this is considered to be the landing page of 49 | the sub directory. If there is no directory, it is used as the default response, 50 | or "homepage". If a `|0|` is not provided for the root or sub directory, the 51 | return menu item will have a `type` of `dir` instead of `file`. 52 | 53 | ## Formatting 54 | 55 | Formatting within the document is done by using Google's wysiwyg, as usual. Allwrite then translates the content to well formatted html and markdown. Both the html and markdown formats are returned from the API so you may use which ever works best for you. 56 | 57 | Formatting guide: 58 | 59 | * Images just work. Plus they're hosted by Google for free! 60 | * Unordered and ordered lists are treated as such. 61 | * Colors, text alignment, and other frills will have no effect. Create an issue if you have a suggestion. 62 | * Headers will be treated as such (`

`, `

`, and `

`). 63 | * Format code as you normally would (3 backticks followed by the language). 64 | 65 | ## Examples 66 | 67 | * [stackahoy.io](https://stackahoy.io) is going to use it. 68 | 69 | ## Themes 70 | 71 | * [Spartan](https://github.com/LevInteractive/spartan-allwrite/) 72 | * Make your own! 73 | 74 | ## Installation 75 | 76 | First, you should generate a OAuth 2.0 json file [here](https://console.developers.google.com/projectselector/apis/credentials). Select 77 | "other" for Application Type then place the client_secret.json file on the 78 | server you'll be running the API. 79 | 80 | Head to your server and run the following to **install or update** Allwrite: 81 | 82 | ```bash 83 | curl -L https://github.com/LevInteractive/allwrite-docs/blob/master/install.sh?raw=true | sh 84 | ``` 85 | 86 | If it's you're installing for the first time import the postgres schema: 87 | 88 | ```bash 89 | curl -O https://raw.githubusercontent.com/LevInteractive/allwrite-docs/master/store/postgres/sql/pages.sql 90 | psql < pages.sql 91 | ``` 92 | 93 | Finally, try to run the server in the foreground: 94 | 95 | ```bash 96 | # Download the environmental variables. These need to available to the 97 | # user/shell so allwrite can connect. 98 | curl https://raw.githubusercontent.com/LevInteractive/allwrite-docs/master/creds.example.sh > creds 99 | 100 | # Configure. Make sure these variables are correct. 101 | vim creds 102 | 103 | # Load the variables. 104 | source creds 105 | ``` 106 | 107 | Start the server. 108 | 109 | ```bash 110 | # Run the server. You'll eventually want to run this in the background and use 111 | # something like nginx to create a reverse proxy. 112 | allwrite s 113 | ``` 114 | 115 | Once you confirmed that it works, setup something like supervisord to run it for 116 | you. You can see an example configuration file for supervisord [here](/docs/supervisord.md) 117 | 118 | ## API 119 | 120 | See response examples [here](/docs/api.md). 121 | 122 | ## Contributing 123 | 124 | See docs for development [here](/docs/development.md). 125 | 126 | ## CLI 127 | 128 | After installing, you'll have access to the CLI 129 | 130 | ```bash 131 | $ allwrite-docs 132 | ``` 133 | -------------------------------------------------------------------------------- /api/router.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "log" 7 | "net/http" 8 | "regexp" 9 | 10 | "golang.org/x/crypto/acme/autocert" 11 | 12 | "github.com/LevInteractive/allwrite-docs/util" 13 | ) 14 | 15 | type jsonResponse struct { 16 | Code int `json:"code"` 17 | Result interface{} `json:"result"` 18 | Error string `json:"error,omitempty"` 19 | } 20 | 21 | func getPage(env *util.Env, uri string, w http.ResponseWriter, req *http.Request) { 22 | page, err := env.DB.GetPage(uri) 23 | if err != nil { 24 | w.WriteHeader(http.StatusNotFound) 25 | json.NewEncoder(w).Encode(&jsonResponse{ 26 | Code: http.StatusNotFound, 27 | Error: err.Error(), 28 | }) 29 | } else { 30 | w.WriteHeader(http.StatusOK) 31 | json.NewEncoder(w).Encode(jsonResponse{ 32 | Code: http.StatusOK, 33 | Result: page, 34 | }) 35 | } 36 | } 37 | 38 | func getMenu(env *util.Env, uri string, w http.ResponseWriter, req *http.Request) { 39 | menu, err := env.DB.GetMenu() 40 | if err != nil { 41 | w.WriteHeader(http.StatusBadRequest) 42 | json.NewEncoder(w).Encode(&jsonResponse{ 43 | Code: http.StatusBadRequest, 44 | Error: err.Error(), 45 | }) 46 | } else { 47 | w.WriteHeader(http.StatusOK) 48 | json.NewEncoder(w).Encode(jsonResponse{ 49 | Code: http.StatusOK, 50 | Result: menu, 51 | }) 52 | } 53 | } 54 | 55 | func search(env *util.Env, search string, uri string, w http.ResponseWriter, req *http.Request) { 56 | menu, err := env.DB.Search(search) 57 | if err != nil { 58 | w.WriteHeader(http.StatusBadRequest) 59 | json.NewEncoder(w).Encode(&jsonResponse{ 60 | Code: http.StatusBadRequest, 61 | Error: err.Error(), 62 | }) 63 | } else { 64 | w.WriteHeader(http.StatusOK) 65 | json.NewEncoder(w).Encode(jsonResponse{ 66 | Code: http.StatusOK, 67 | Result: menu, 68 | }) 69 | } 70 | } 71 | 72 | // Listen to incoming requests and serve. 73 | func Listen(env *util.Env) { 74 | stripSlashes := regexp.MustCompile("^\\/|\\/$|\\?.*$") 75 | http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { 76 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 77 | w.Header().Set("Access-Control-Allow-Origin", "*") 78 | w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS") 79 | w.Header().Set("Cache-Control", "public, max-age=3600") 80 | 81 | s := req.URL.Query().Get("q") 82 | uri := stripSlashes.ReplaceAllString(req.RequestURI, "") 83 | 84 | if uri == "menu" { 85 | log.Printf("Request: Menu\n") 86 | getMenu(env, uri, w, req) 87 | } else if len(s) > 0 && uri == "" || req.RequestURI == "/?q=" { 88 | log.Printf("Request: Search - %s\n", s) 89 | search(env, s, uri, w, req) 90 | } else { 91 | log.Printf("Request: Get Page - %s\n", uri) 92 | getPage(env, uri, w, req) 93 | } 94 | }) 95 | 96 | log.Printf("\nListening on %s%s\n", env.CFG.Domain, env.CFG.Port) 97 | 98 | if env.CFG.Port == ":443" { 99 | m := autocert.Manager{ 100 | Prompt: autocert.AcceptTOS, 101 | HostPolicy: autocert.HostWhitelist(env.CFG.Domain), 102 | Cache: autocert.DirCache("certs"), 103 | Email: env.CFG.CertbotEmail, 104 | } 105 | server := &http.Server{ 106 | TLSConfig: &tls.Config{ 107 | GetCertificate: m.GetCertificate, 108 | }, 109 | } 110 | if err := server.ListenAndServeTLS("", ""); err != nil { 111 | log.Fatal(err) 112 | } 113 | } else { 114 | if err := http.ListenAndServe(env.CFG.Port, nil); err != nil { 115 | log.Fatal(err) 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /build/darwin_amd64.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/darwin_amd64.tar.gz -------------------------------------------------------------------------------- /build/linux_386.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/linux_386.tar.gz -------------------------------------------------------------------------------- /build/linux_amd64.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/linux_amd64.tar.gz -------------------------------------------------------------------------------- /build/linux_arm.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/linux_arm.tar.gz -------------------------------------------------------------------------------- /build/linux_arm64.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/linux_arm64.tar.gz -------------------------------------------------------------------------------- /build/windows_386.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/windows_386.zip -------------------------------------------------------------------------------- /build/windows_amd64.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/build/windows_amd64.zip -------------------------------------------------------------------------------- /creds.example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # These are the environmental variables you should set before running allwrite. 4 | # You can set these in something like Upstart or in your current shell by doing: 5 | 6 | # source ./creds.sh 7 | # ------------------------------------------------------------------------------ 8 | 9 | # Basically, you just need to make sure these get set somehow, somewhere. 10 | 11 | # The ID of the base directory for the docs (you can grab it from the URL in 12 | # Drive). 13 | export ACTIVE_DIR="xxxxxxxxxxxxxxxxxxx" 14 | 15 | # Path to your Google client secret json file. 16 | export CLIENT_SECRET="$PWD/client_secret.json" 17 | 18 | # The storage system to use - currently postgres is the only option. 19 | export STORAGE="postgres" 20 | export PG_USER="root" 21 | export PG_PASS="root" 22 | export PG_DB="allwrite" 23 | export PG_HOST="localhost" 24 | 25 | # Specify the port to run the application on. 26 | export PORT=":8000" 27 | 28 | # Only needed if listening on port 443. Used for certbot. 29 | export DOMAIN="my-domain.com" 30 | export CERTBOT_EMAIL="engineering@your-company.com" 31 | 32 | # How often Google is queried for updates specified in milliseconds. 33 | export FREQUENCY="300000" 34 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | # API Responses 2 | 3 | ### GET /menu 4 | 5 | Returns a collection of page fragments. 6 | 7 | ```json 8 | { 9 | "code":200, 10 | "result":[ 11 | { 12 | "name":"Homepage", 13 | "type":"file", 14 | "slug":"", 15 | "order":0, 16 | "updated":"2017-09-30T23:35:31.663365Z", 17 | "created":"2017-09-30T23:35:31.663365Z" 18 | }, 19 | { 20 | "name":"Only one deep", 21 | "type":"file", 22 | "slug":"another-sub-directory", 23 | "order":0, 24 | "updated":"2017-09-30T23:35:31.663365Z", 25 | "created":"2017-09-30T23:35:31.663365Z", 26 | "children":[ 27 | { 28 | "name":"This is a deep file", 29 | "type":"file", 30 | "slug":"another-sub-directory/a-deeper-directory", 31 | "order":0, 32 | "updated":"2017-09-30T23:35:31.663365Z", 33 | "created":"2017-09-30T23:35:31.663365Z" 34 | } 35 | ] 36 | }, 37 | { 38 | "name":"A Sub Directory", 39 | "type":"dir", 40 | "slug":"a-sub-directory", 41 | "order":1, 42 | "updated":"2017-09-30T23:35:31.663365Z", 43 | "created":"2017-09-30T23:35:31.663365Z", 44 | "children":[ 45 | { 46 | "name":"How to be a friend", 47 | "type":"file", 48 | "slug":"a-sub-directory/how-to-be-a-friend", 49 | "order":1, 50 | "updated":"2017-09-30T23:35:31.663365Z", 51 | "created":"2017-09-30T23:35:31.663365Z" 52 | } 53 | ] 54 | }, 55 | { 56 | "name":"Images!", 57 | "type":"file", 58 | "slug":"images", 59 | "order":2, 60 | "updated":"2017-09-30T23:35:31.663365Z", 61 | "created":"2017-09-30T23:35:31.663365Z" 62 | } 63 | ] 64 | } 65 | ``` 66 | 67 | ### GET /page(/:slug) 68 | 69 | Pull a page based on its slug. If not provided, `|0|` page will be used. 70 | 71 | ```json 72 | { 73 | "code":200, 74 | "result":{ 75 | "name":"Homepage", 76 | "type":"file", 77 | "slug":"", 78 | "order":0, 79 | "updated":"2017-09-30T23:21:35.044194Z", 80 | "created":"2017-09-30T23:21:35.044194Z", 81 | "doc_id":"1V5G8XmX6ggLVu09QJXqONQkLKfIix-2bMuefFYbmTmE", 82 | "html":"[full clean html]", 83 | "md":"[full clean markdown]" 84 | } 85 | } 86 | ``` 87 | 88 | ### GET /?q=my+search 89 | 90 | This will search for results based on your q parameter. The string needs to be URL encoded. 91 | 92 | ```json 93 | { 94 | "code": 200, 95 | "result": [ 96 | { 97 | "name": "This is a deep file", 98 | "type": "file", 99 | "slug": "another-sub-directory/a-deeper-directory", 100 | "order": 0, 101 | "updated": "2017-09-30T23:35:31.663365Z", 102 | "created": "2017-09-30T23:35:31.663365Z" 103 | } 104 | ] 105 | } 106 | ``` 107 | 108 | ### Page not found 109 | 110 | If a page is not found, an error will be returned with error code `404`. 111 | 112 | ```json 113 | { 114 | "code":400, 115 | "result":null, 116 | "error":"not found" 117 | } 118 | ``` 119 | -------------------------------------------------------------------------------- /docs/development.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | Allwrite Docs uses the Glide dependency manager. 4 | 5 | ```bash 6 | glide install 7 | ``` 8 | 9 | ### Testing 10 | 11 | ```bash 12 | go test github.com/LevInteractive/allwrite-docs/gdrive 13 | ``` 14 | -------------------------------------------------------------------------------- /docs/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaia/allwrite-docs/080c93b7fc34877f65353850a207d7caa313083d/docs/screenshot.png -------------------------------------------------------------------------------- /docs/supervisord.md: -------------------------------------------------------------------------------- 1 | # Supervisord 2 | 3 | Below is an example of a supervisor configuration for running allwrite on 4 | startup. 5 | 6 | ```bash 7 | [program:allwrite] 8 | command=/usr/local/bin/allwrite-docs s 9 | autostart=true 10 | autorestart=true 11 | startretries=10 12 | user=root 13 | directory=/etc/allwrite 14 | environment= 15 | USER="root", 16 | HOME="/root", 17 | ACTIVE_DIR="...", 18 | CLIENT_SECRET="/etc/allwrite/client_secret.json", 19 | STORAGE="postgres", 20 | PG_USER="allwrite", 21 | PG_PASS="allwrite", 22 | PG_DB="allwrite", 23 | PG_HOST="localhost", 24 | PORT=":443", 25 | DOMAIN="my-domain.com", 26 | CERTBOT_EMAIL="my@email.com", 27 | FREQUENCY="300000" 28 | redirect_stderr=true 29 | stdout_logfile=/var/log/supervisor/allwrite.log 30 | stdout_logfile_maxbytes=50MB 31 | stdout_logfile_backups=10 32 | ``` 33 | -------------------------------------------------------------------------------- /gdrive/drive-client.go: -------------------------------------------------------------------------------- 1 | package gdrive 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "os/user" 13 | "path/filepath" 14 | 15 | "golang.org/x/oauth2" 16 | "golang.org/x/oauth2/google" 17 | drive "google.golang.org/api/drive/v3" 18 | ) 19 | 20 | // Client holds the instance to a drive service client. This is returned. 21 | type Client struct { 22 | Service *drive.Service 23 | } 24 | 25 | func getClient(ctx context.Context, config *oauth2.Config) *http.Client { 26 | cacheFile, err := tokenCacheFile() 27 | if err != nil { 28 | log.Fatalf("Unable to get path to cached credential file. %v", err) 29 | } 30 | tok, err := tokenFromFile(cacheFile) 31 | if err != nil { 32 | tok = getTokenFromWeb(config) 33 | saveToken(cacheFile, tok) 34 | } 35 | return config.Client(ctx, tok) 36 | } 37 | 38 | // getTokenFromWeb uses Config to request a Token. 39 | // It returns the retrieved Token. 40 | func getTokenFromWeb(config *oauth2.Config) *oauth2.Token { 41 | authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) 42 | log.Printf("Go to the following link in your browser then type the "+ 43 | "authorization code: \n%v\n", authURL) 44 | 45 | var code string 46 | if _, err := fmt.Scan(&code); err != nil { 47 | log.Fatalf("Unable to read authorization code %v", err) 48 | } 49 | 50 | tok, err := config.Exchange(oauth2.NoContext, code) 51 | if err != nil { 52 | log.Fatalf("Unable to retrieve token from web %v", err) 53 | } 54 | return tok 55 | } 56 | 57 | // tokenCacheFile generates credential file path/filename. 58 | // It returns the generated credential path/filename. 59 | func tokenCacheFile() (string, error) { 60 | usr, err := user.Current() 61 | if err != nil { 62 | return "", err 63 | } 64 | tokenCacheDir := filepath.Join(usr.HomeDir, ".credentials") 65 | os.MkdirAll(tokenCacheDir, 0700) 66 | return filepath.Join(tokenCacheDir, 67 | url.QueryEscape("drive-go-quickstart.json")), err 68 | } 69 | 70 | // RemoveCacheFile will remove the credentials. 71 | func RemoveCacheFile() error { 72 | usr, err := user.Current() 73 | if err != nil { 74 | return err 75 | } 76 | tokenCacheDir := filepath.Join(usr.HomeDir, ".credentials") 77 | return os.RemoveAll(tokenCacheDir) 78 | } 79 | 80 | // tokenFromFile retrieves a Token from a given file path. 81 | // It returns the retrieved Token and any read error encountered. 82 | func tokenFromFile(file string) (*oauth2.Token, error) { 83 | f, err := os.Open(file) 84 | if err != nil { 85 | return nil, err 86 | } 87 | t := &oauth2.Token{} 88 | err = json.NewDecoder(f).Decode(t) 89 | defer f.Close() 90 | return t, err 91 | } 92 | 93 | // saveToken uses a file path to create a file and store the 94 | // token in it. 95 | func saveToken(file string, token *oauth2.Token) { 96 | log.Printf("Saving credential file to: %s\n", file) 97 | f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) 98 | if err != nil { 99 | log.Fatalf("Unable to cache oauth token: %v", err) 100 | } 101 | defer f.Close() 102 | json.NewEncoder(f).Encode(token) 103 | } 104 | 105 | // DriveClient sets up the authentication for drive by retreiving the access 106 | // token. 107 | func DriveClient(clientSecret string) *Client { 108 | ctx := context.Background() 109 | b, err := ioutil.ReadFile(clientSecret) 110 | if err != nil { 111 | log.Fatalf("Unable to read client secret file: '%v': %v", clientSecret, err) 112 | } 113 | 114 | // If modifying these scopes, delete your previously saved credentials 115 | // at ~/.credentials/drive-go-quickstart.json 116 | config, err := google.ConfigFromJSON(b, drive.DriveScope) 117 | if err != nil { 118 | log.Fatalf("Unable to parse client secret file to config: %v", err) 119 | } 120 | client := getClient(ctx, config) 121 | srv, err := drive.New(client) 122 | 123 | if err != nil { 124 | log.Fatalf("Unable to retrieve drive Client %v", err) 125 | } 126 | 127 | return &Client{srv} 128 | } 129 | -------------------------------------------------------------------------------- /gdrive/fixtures/image-doc.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |

Here we go with some images.

9 |

docker.png

10 |

This is right below the image.

11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /gdrive/fixtures/image-doc.md: -------------------------------------------------------------------------------- 1 | Here we go with some images. 2 | 3 | ![docker.png](http://test-image.png) 4 | 5 | This is right below the image. 6 | 7 | -------------------------------------------------------------------------------- /gdrive/fixtures/sample-doc.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 222 | 223 | 224 | 225 |

I am an h1

226 |

I am a paragraph.

227 |
    228 |
  1. I am a ordered list.
    2. 229 |
  2. One
    3. 230 |
  3. Two
    4. 231 |
  4. Three
    5. 232 |
233 |

Sometimes text is bold.

234 | 239 |

I am an h2

240 |

Some general text.

241 |

I am an h3

242 |

Sometimes text has an italic style.

243 |

244 |

Sometimes text has a link.

245 |

I am an h4

246 |

```json

247 |

{

248 |

  “Hello”: 123

249 |

}

250 |

```

251 |

I am a subtitle.

252 |

I am a quote.

253 | 254 | 255 | 256 | -------------------------------------------------------------------------------- /gdrive/fixtures/sample-doc.md: -------------------------------------------------------------------------------- 1 | # I am an h1 2 | 3 | I am a paragraph. 4 | 5 | 1 I am a ordered list. 6 | 2 One 7 | 3 Two 8 | 4 Three 9 | 10 | 11 | Sometimes text is **bold**. 12 | 13 | * I am a unordered lsit. 14 | * Woot 15 | * foobar with a [Link](http://google.com) 16 | 17 | 18 | ## I am an h2 19 | 20 | Some general text. 21 | 22 | ### I am an h3 23 | 24 | Sometimes text has an _italic_ style. 25 | 26 | 27 | 28 | Sometimes text has a [link](https://www.google.com/url?q=https://google.com&sa=D&ust=1505803612304000&usg=AFQjCNFT8g-FU1DmHJ9r4p7SJ6QJebS3VQ). 29 | 30 | #### I am an h4 31 | 32 | ```json 33 | 34 | { 35 | 36 |   "Hello": 123 37 | 38 | } 39 | 40 | ``` 41 | 42 | I am a subtitle. 43 | 44 | I am a quote. 45 | 46 | -------------------------------------------------------------------------------- /gdrive/gdrive.go: -------------------------------------------------------------------------------- 1 | package gdrive 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "log" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | blackfriday "gopkg.in/russross/blackfriday.v2" 13 | 14 | "github.com/LevInteractive/allwrite-docs/model" 15 | "github.com/LevInteractive/allwrite-docs/util" 16 | ) 17 | 18 | const parentDir string = "0B4pmjFk2yyz2NFcwZzQwVHlCRWc" 19 | 20 | type titleParts struct { 21 | Title string 22 | Order int 23 | } 24 | 25 | type pages struct { 26 | collection model.Pages 27 | } 28 | 29 | // Obtain the contents of a google doc by its ID. This essentially pulls the 30 | // nasty html. 31 | func (client *Client) getContents(id string, mimeType string) ([]byte, error) { 32 | res, err := client.Service.Files.Export(id, mimeType).Download() 33 | if err != nil { 34 | return []byte{}, err 35 | } 36 | defer res.Body.Close() 37 | return ioutil.ReadAll(res.Body) 38 | } 39 | 40 | // Write a new page to the collection. 41 | func (s *pages) appendPage(page *model.Page) { 42 | s.collection = append(s.collection, page) 43 | } 44 | 45 | // Responsible for splitting apart the allwrite title format. 46 | // e.g. |n| The Title 47 | func getPartsFromTitle(title string) (*titleParts, error) { 48 | re := regexp.MustCompile("^\\|(\\d+)\\|\\W+?(.+)$") 49 | result := re.FindStringSubmatch(strings.Trim(title, " ")) 50 | 51 | if len(result) == 3 { 52 | order, err := strconv.Atoi(result[1]) 53 | if err != nil { 54 | return &titleParts{}, err 55 | } 56 | 57 | return &titleParts{ 58 | Order: order, 59 | Title: result[2], 60 | }, nil 61 | } 62 | return &titleParts{}, nil 63 | } 64 | 65 | // Determines if the slice already contains a page that isn't a directory with 66 | // the same slug. 67 | func hasLandingPage(collection model.Pages, dir *model.Page) bool { 68 | hasLanding := false 69 | for _, page := range collection { 70 | if page.Type == "file" && page.Slug == dir.Slug { 71 | hasLanding = true 72 | break 73 | } 74 | } 75 | return hasLanding 76 | } 77 | 78 | // Will take in an existing list and remove any directory pages which have the 79 | // same slug as a "file" page. 80 | func consolidate(collection model.Pages) model.Pages { 81 | newSlice := make(model.Pages, 0, len(collection)) 82 | for _, page := range collection { 83 | if page.Type == "dir" { 84 | if hasLandingPage(collection, page) == false { 85 | newSlice = append(newSlice, page) 86 | } 87 | } else { 88 | newSlice = append(newSlice, page) 89 | } 90 | } 91 | return newSlice 92 | } 93 | 94 | // Query google and walk its directory structure pulling out files. 95 | func (client *Client) processDriveFiles(env *util.Env, baseSlug string, parentID string, pages *pages) { 96 | r, err := client.Service.Files.List(). 97 | PageSize(1000). // OK for now. Right? 98 | Q("'" + parentID + "' in parents and trashed=false"). 99 | Do() 100 | 101 | if err != nil { 102 | log.Printf("Unable to retrieve files from google drive: %v\n", err) 103 | return 104 | } 105 | 106 | if len(r.Files) > 0 { 107 | for _, i := range r.Files { 108 | // Grab the sort order and title from formatted title names. 109 | parts, err := getPartsFromTitle(i.Name) 110 | if err != nil { 111 | log.Printf("Skipping document. There was an issue getting parts from title: %s\n", err.Error()) 112 | continue 113 | } 114 | // If the format was incorrect an empty struct will be returned. 115 | if parts.Title == "" { 116 | log.Printf("Skipping document because of format: %s\n", i.Name) 117 | continue 118 | } 119 | 120 | // Define the page that will be saved. 121 | newPage := &model.Page{} 122 | newPage.Name = parts.Title 123 | newPage.DocID = i.Id 124 | newPage.Order = parts.Order 125 | newPage.Created = i.CreatedTime 126 | newPage.Updated = i.ModifiedTime 127 | 128 | // Switch depending on type of ducment. 129 | switch mime := i.MimeType; mime { 130 | case "application/vnd.google-apps.document": 131 | htmlBytes, err := client.getContents(i.Id, "text/html") 132 | 133 | if err != nil { 134 | log.Printf("Skipping. There was an error grabbing the contents for a document: %s", err.Error()) 135 | continue 136 | } 137 | 138 | md, err := MarshalMarkdownFromHTML(bytes.NewReader(htmlBytes)) 139 | if err != nil { 140 | log.Printf("There was a problem parsing html to markdown: %s", err.Error()) 141 | continue 142 | } 143 | 144 | newPage.Md = md 145 | newPage.HTML = string(blackfriday.Run( 146 | []byte(md), 147 | blackfriday.WithExtensions( 148 | blackfriday.Tables|blackfriday.AutoHeadingIDs|blackfriday.FencedCode, 149 | ), 150 | )) 151 | 152 | newPage.Type = "file" 153 | 154 | if parts.Order == 0 { 155 | // If the order is 0, always take on the same path as the directory. 156 | newPage.Slug = baseSlug 157 | } else { 158 | if baseSlug != "" { 159 | newPage.Slug = baseSlug + "/" + util.MarshalSlug(parts.Title) 160 | } else { 161 | newPage.Slug = baseSlug + util.MarshalSlug(parts.Title) 162 | } 163 | } 164 | 165 | log.Printf("Saving page \"%s\" with slug \"%s\".\n", newPage.Name, newPage.Slug) 166 | pages.appendPage(newPage) 167 | 168 | case "application/vnd.google-apps.folder": 169 | var dirBaseSlug string 170 | 171 | if baseSlug != "" { 172 | dirBaseSlug = baseSlug + "/" + util.MarshalSlug(parts.Title) 173 | } else { 174 | dirBaseSlug = util.MarshalSlug(parts.Title) 175 | } 176 | newPage.Type = "dir" 177 | newPage.Slug = dirBaseSlug 178 | log.Printf("Saving directory \"%s\" with slug \"%s\".\n", newPage.Name, newPage.Slug) 179 | pages.appendPage(newPage) 180 | 181 | log.Printf("Submerging deeper into %s\n", i.Name) 182 | client.processDriveFiles(env, dirBaseSlug, i.Id, pages) 183 | default: 184 | log.Printf("Unknown filetype in drive directory: %s\n", mime) 185 | } 186 | } 187 | } else { 188 | log.Println("No files found.") 189 | } 190 | } 191 | 192 | // UpdateMenu triggers the database to sync with the content. 193 | func UpdateMenu(client *Client, env *util.Env) { 194 | log.Printf("Checking for Drive updates.") 195 | p := &pages{} 196 | 197 | // This needs to be syncrounous so we don't hit the rate limit. 198 | client.processDriveFiles( 199 | env, 200 | "", 201 | env.CFG.ActiveDir, 202 | p, 203 | ) 204 | 205 | // Loop through and remove any directories that have parent pages. 206 | p.collection = consolidate(p.collection) 207 | 208 | if err := env.DB.RemoveAll(); err != nil { 209 | log.Fatalf("There was an error removing previous records: %s", err.Error()) 210 | return 211 | } 212 | 213 | if _, err := env.DB.SavePages(p.collection); err != nil { 214 | log.Fatalf("There was an error saving records: %s", err.Error()) 215 | return 216 | } 217 | } 218 | 219 | // WatchDrive will check for updates based on Frequency. 220 | func WatchDrive(client *Client, env *util.Env) { 221 | pull := func() { 222 | UpdateMenu(client, env) 223 | } 224 | util.SetInterval(pull, time.Duration(env.CFG.Frequency)*time.Millisecond) 225 | pull() 226 | } 227 | -------------------------------------------------------------------------------- /gdrive/parser.go: -------------------------------------------------------------------------------- 1 | package gdrive 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | 7 | "github.com/LevInteractive/allwrite-docs/gdrive/parsers" 8 | 9 | "golang.org/x/net/html" 10 | ) 11 | 12 | // MarshalMarkdownFromHTML transforms a nasty google doc to markdown. 13 | func MarshalMarkdownFromHTML(r io.Reader) (string, error) { 14 | doc, err := html.Parse(r) 15 | if err != nil { 16 | return "", err 17 | } 18 | 19 | var buffer bytes.Buffer 20 | parsers.Walk(&buffer, doc) 21 | return buffer.String(), nil 22 | } 23 | -------------------------------------------------------------------------------- /gdrive/parser_test.go: -------------------------------------------------------------------------------- 1 | package gdrive 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/andreyvit/diff" 9 | ) 10 | 11 | func getFixture(filename string) string { 12 | file, err := ioutil.ReadFile(filename) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | return string(file) 18 | } 19 | 20 | func TestMarshalMarkdownFromHTML(t *testing.T) { 21 | mdDoc := getFixture("./fixtures/sample-doc.md") 22 | htmlDoc := getFixture("./fixtures/sample-doc.html") 23 | 24 | r := strings.NewReader(htmlDoc) 25 | transformedMd, err := MarshalMarkdownFromHTML(r) 26 | 27 | if err != nil { 28 | t.Error(err) 29 | } 30 | 31 | if transformedMd != mdDoc { 32 | t.Errorf( 33 | "HTML did not translate to image markdown properly: \n%v", 34 | diff.LineDiff(mdDoc, transformedMd), 35 | ) 36 | } 37 | } 38 | 39 | func TestMarshalMarkdownFromHTMLImages(t *testing.T) { 40 | mdDoc := getFixture("./fixtures/image-doc.md") 41 | htmlDoc := getFixture("./fixtures/image-doc.html") 42 | 43 | r := strings.NewReader(htmlDoc) 44 | transformedMd, err := MarshalMarkdownFromHTML(r) 45 | 46 | if err != nil { 47 | t.Error(err) 48 | } 49 | 50 | if transformedMd != mdDoc { 51 | t.Errorf( 52 | "HTML did not translate to image markdown properly: \n%v", 53 | diff.LineDiff(mdDoc, transformedMd), 54 | ) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /gdrive/parsers/header.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | 6 | "golang.org/x/net/html" 7 | ) 8 | 9 | // Header parses h1, h2, h3, etc, tags. 10 | func Header(b *bytes.Buffer, n *html.Node) { 11 | switch n.DataAtom.String() { 12 | case "h1": 13 | b.WriteString("# ") 14 | case "h2": 15 | b.WriteString("## ") 16 | case "h3": 17 | b.WriteString("### ") 18 | case "h4": 19 | b.WriteString("#### ") 20 | case "h5": 21 | b.WriteString("##### ") 22 | case "h6": 23 | b.WriteString("###### ") 24 | } 25 | 26 | InlineWalker(b, n, GetAttr(n.Attr, "style")) 27 | b.WriteString("\n\n") 28 | } 29 | -------------------------------------------------------------------------------- /gdrive/parsers/ol.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | 7 | "golang.org/x/net/html" 8 | ) 9 | 10 | // Ol is a lot like a Ul but ordered. 11 | func Ol(b *bytes.Buffer, n *html.Node) { 12 | index := 1 13 | for c := n.FirstChild; c != nil; c = c.NextSibling { 14 | if c.Type == html.ElementNode { 15 | b.WriteString(strconv.Itoa(index) + " ") 16 | InlineWalker(b, c, GetAttr(n.Attr, "style")) 17 | b.WriteString("\n") 18 | index++ 19 | } 20 | } 21 | b.WriteString("\n\n") 22 | } 23 | -------------------------------------------------------------------------------- /gdrive/parsers/p.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | 6 | "golang.org/x/net/html" 7 | ) 8 | 9 | // P parses paragraph tags. 10 | func P(b *bytes.Buffer, n *html.Node) { 11 | InlineWalker(b, n, GetAttr(n.Attr, "style")) 12 | b.WriteString("\n\n") 13 | } 14 | -------------------------------------------------------------------------------- /gdrive/parsers/parsers.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "regexp" 8 | "strings" 9 | 10 | "golang.org/x/net/html" 11 | ) 12 | 13 | // TagParsers is a map for handling each type of block element. 14 | var TagParsers = map[string]func(*bytes.Buffer, *html.Node){ 15 | "p": P, 16 | "h1": Header, 17 | "h2": Header, 18 | "h3": Header, 19 | "h4": Header, 20 | "h5": Header, 21 | "h6": Header, 22 | "ul": Ul, 23 | "ol": Ol, 24 | } 25 | 26 | // Pulls a link out of our custom format. 27 | var linkRegexp *regexp.Regexp = regexp.MustCompile( 28 | "___LINKΔΔΔ([^Δ]*)ΔΔΔ", 29 | ) 30 | 31 | // Pulls an image out of our custom format. 32 | var imgRegexp *regexp.Regexp = regexp.MustCompile( 33 | "___IMGΔΔΔ([^Δ]*)ΔΔΔ", 34 | ) 35 | 36 | // FormatStyle an inline style. All text gets procssed through here so we can 37 | // also run some general clean up. Weird characters and things like that. 38 | func FormatStyle(css string, content string) string { 39 | 40 | // Strongs, italics, and strike throughs. 41 | if strings.Contains(css, "font-weight:700") { 42 | content = fmt.Sprintf("**%s**", content) 43 | } 44 | if strings.Contains(css, "font-style:italic") { 45 | content = fmt.Sprintf("_%s_", content) 46 | } 47 | if strings.Contains(css, "text-decoration:line-through") { 48 | content = fmt.Sprintf("~~%s~~", content) 49 | } 50 | 51 | // This is a special case for google's "title" header type. It doesn't 52 | // actually make it an

, but rather adds font-size: 26pt to it. 53 | if strings.Contains(css, "font-size:26pt") { 54 | content = fmt.Sprintf("# %s", content) 55 | } 56 | 57 | // Handle bad quotes. 58 | content = strings.Replace(content, "“", "\"", -1) 59 | content = strings.Replace(content, "”", "\"", -1) 60 | 61 | // Handle images. 62 | if imgRegexp.Match([]byte(css)) { 63 | result := imgRegexp.FindStringSubmatch(css) 64 | if len(result) == 2 { 65 | parts := strings.Split(result[1], "∏") 66 | content = fmt.Sprintf("![%s](%s)", parts[0], parts[1]) 67 | } else { 68 | log.Println("Could not find alt and src from image! " + css) 69 | } 70 | } 71 | 72 | // Handle links. 73 | if linkRegexp.Match([]byte(css)) { 74 | result := linkRegexp.FindStringSubmatch(css) 75 | if len(result) == 2 { 76 | content = fmt.Sprintf("[%s](%s)", content, result[1]) 77 | } else { 78 | log.Println("Could not find link! " + css) 79 | } 80 | } 81 | 82 | return content 83 | } 84 | 85 | // GetAttr will get a attribute out of the html.Attribute slice. 86 | func GetAttr(attrs []html.Attribute, attr string) string { 87 | var style string 88 | for _, a := range attrs { 89 | if a.Key == attr { 90 | style = a.Val 91 | break 92 | } 93 | } 94 | return style 95 | } 96 | 97 | // InlineWalker walks inline. 98 | func InlineWalker(b *bytes.Buffer, n *html.Node, parentCSS string) { 99 | for c := n.FirstChild; c != nil; c = c.NextSibling { 100 | if c.Type == html.ElementNode { 101 | styles := parentCSS + GetAttr(c.Attr, "style") 102 | 103 | switch c.DataAtom.String() { 104 | case "a": 105 | styles += "___LINKΔΔΔ" + GetAttr(c.Attr, "href") + "ΔΔΔ___" 106 | case "img": 107 | // If at an image, there is no reason to go deeper in since nothing can go 108 | // in a image tag. Just write it out. 109 | alt := GetAttr(c.Attr, "alt") 110 | src := GetAttr(c.Attr, "src") 111 | styles += "___IMGΔΔΔ" + alt + "∏" + src + "ΔΔΔ___" 112 | b.WriteString(FormatStyle(styles, c.Data)) 113 | return 114 | case "br": 115 | b.WriteString("\n") 116 | } 117 | 118 | InlineWalker(b, c, styles) 119 | } else if c.Type == html.TextNode { 120 | b.WriteString(FormatStyle(parentCSS, c.Data)) 121 | } else { 122 | InlineWalker(b, c, parentCSS) 123 | } 124 | } 125 | } 126 | 127 | // Walk .. rather, tippy toe through the DOM. 128 | // This is where you should initiate the parsing. 129 | func Walk(b *bytes.Buffer, n *html.Node) { 130 | for c := n.FirstChild; c != nil; c = c.NextSibling { 131 | if fn := TagParsers[c.DataAtom.String()]; fn != nil { 132 | fn(b, c) 133 | } else { 134 | Walk(b, c) 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /gdrive/parsers/ul.go: -------------------------------------------------------------------------------- 1 | package parsers 2 | 3 | import ( 4 | "bytes" 5 | 6 | "golang.org/x/net/html" 7 | ) 8 | 9 | // Ul parses UL types of tags. 10 | func Ul(b *bytes.Buffer, n *html.Node) { 11 | for c := n.FirstChild; c != nil; c = c.NextSibling { 12 | if c.Type == html.ElementNode { 13 | b.WriteString("* ") 14 | InlineWalker(b, c, GetAttr(n.Attr, "style")) 15 | b.WriteString("\n") 16 | } 17 | } 18 | b.WriteString("\n\n") 19 | } 20 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 7fc91718ae228c64bae6727bde8c017f092acb1ab4fc610bb2e26afa0c904a15 2 | updated: 2017-12-25T18:06:42.237648-05:00 3 | imports: 4 | - name: cloud.google.com/go 5 | version: 050b16d2314d5fc3d4c9a51e4cd5c7468e77f162 6 | subpackages: 7 | - compute/metadata 8 | - name: github.com/andreyvit/diff 9 | version: c7f18ee00883bfd3b00e0a2bf7607827e0148ad4 10 | - name: github.com/golang/protobuf 11 | version: 11b8df160996e00fd4b55cbaafb3d84ec6d50fa8 12 | subpackages: 13 | - proto 14 | - name: github.com/joeshaw/envdecode 15 | version: 6326cbed175e32cd5183be1cc1027e0823c91edb 16 | - name: github.com/lib/pq 17 | version: b77235e3890a962fe8a6f8c4c7198679ca7814e7 18 | subpackages: 19 | - oid 20 | - name: github.com/shurcooL/sanitized_anchor_name 21 | version: 86672fcb3f950f35f2e675df2240550f2a50762f 22 | - name: github.com/urfave/cli 23 | version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1 24 | - name: golang.org/x/crypto 25 | version: faadfbdc035307d901e69eea569f5dda451a3ee3 26 | subpackages: 27 | - acme 28 | - acme/autocert 29 | - name: golang.org/x/net 30 | version: 859d1a86bb617c0c20d154590c3c5d3fcb670b07 31 | subpackages: 32 | - context 33 | - context/ctxhttp 34 | - html 35 | - html/atom 36 | - name: golang.org/x/oauth2 37 | version: 13449ad91cb26cb47661c1b080790392170385fd 38 | subpackages: 39 | - '...' 40 | - google 41 | - internal 42 | - jws 43 | - jwt 44 | - name: google.golang.org/api 45 | version: 39c3dd417c5a443607650f18e829ad308da08dd2 46 | subpackages: 47 | - drive/v3 48 | - gensupport 49 | - googleapi 50 | - googleapi/internal/uritemplates 51 | - name: google.golang.org/appengine 52 | version: d9a072cfa7b9736e44311ef77b3e09d804bfa599 53 | subpackages: 54 | - internal 55 | - internal/app_identity 56 | - internal/base 57 | - internal/datastore 58 | - internal/log 59 | - internal/modules 60 | - internal/remote_api 61 | - internal/urlfetch 62 | - urlfetch 63 | - name: gopkg.in/russross/blackfriday.v2 64 | version: 187c33ff049bddf24fc5c5b82facfdd62813a552 65 | testImports: 66 | - name: github.com/sergi/go-diff 67 | version: 1744e2970ca51c86172c8190fadad617561ed6e7 68 | subpackages: 69 | - diffmatchpatch 70 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: allwrite-docs 2 | import: 3 | - package: google.golang.org/api 4 | subpackages: 5 | - drive/v3 6 | - package: golang.org/x/oauth2 7 | subpackages: 8 | - '...' 9 | - package: cloud.google.com/go 10 | version: ^0.13.0 11 | subpackages: 12 | - compute/metadata 13 | - package: github.com/joeshaw/envdecode 14 | - package: github.com/shurcooL/sanitized_anchor_name 15 | - package: gopkg.in/russross/blackfriday.v2 16 | version: ^2.0.0 17 | - package: github.com/lib/pq 18 | - package: github.com/andreyvit/diff 19 | - package: golang.org/x/crypto 20 | subpackages: 21 | - acme/autocert 22 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | KERNEL=$(uname -s) 4 | ARCH=$(uname -m) 5 | 6 | if [ $KERNEL = "Darwin" ] 7 | then 8 | BINARY="https://github.com/LevInteractive/allwrite-docs/blob/master/build/darwin_amd64.tar.gz?raw=true" 9 | elif [ $KERNEL = "Linux" ] 10 | then 11 | if [ $ARCH = "x86_64" ] 12 | then 13 | BINARY="https://github.com/LevInteractive/allwrite-docs/blob/master/build/linux_amd64.tar.gz?raw=true" 14 | elif [ $ARCH = "x86" ] 15 | then 16 | BINARY="https://github.com/LevInteractive/allwrite-docs/blob/master/build/linux_386.tar.gz?raw=true" 17 | elif [ $ARCH = "i386" ] 18 | then 19 | BINARY="https://github.com/LevInteractive/allwrite-docs/blob/master/build/linux_386.tar.gz?raw=true" 20 | elif [ $ARCH = "i686" ] 21 | then 22 | BINARY="https://github.com/LevInteractive/allwrite-docs/blob/master/build/linux_386.tar.gz?raw=true" 23 | else 24 | echo "Unsupported OS: $ARCH" 25 | exit 1 26 | fi 27 | else 28 | echo "Unsupported OS: $ARCH" 29 | exit 1 30 | fi 31 | 32 | curl -L $BINARY | tar xvz 33 | mv -f allwrite-docs /usr/local/bin/ 34 | chmod +x /usr/local/bin/allwrite-docs 35 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/LevInteractive/allwrite-docs/api" 8 | "github.com/LevInteractive/allwrite-docs/gdrive" 9 | "github.com/LevInteractive/allwrite-docs/store/postgres" 10 | "github.com/LevInteractive/allwrite-docs/util" 11 | "github.com/joeshaw/envdecode" 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | var debugInfoMessage = ` 16 | 17 | ------------------------------------------------------ 18 | INFO: 19 | Client Secret: %s 20 | Active Directory: %s 21 | Storage Drive: %s 22 | Address: %s 23 | ------------------------------------------------------ 24 | 25 | ` 26 | 27 | func main() { 28 | app := cli.NewApp() 29 | app.Name = "Allwrite Docs | Publish your documentation with Drive." 30 | app.Version = "0.0.3" 31 | app.Commands = []cli.Command{ 32 | { 33 | Name: "start", 34 | Aliases: []string{"s"}, 35 | Usage: "Start the server in the foreground. This will authenticate with Google if it's the first time you're running.", 36 | Action: func(c *cli.Context) error { 37 | if env := setupConf(); env != nil { 38 | client := gdrive.DriveClient(env.CFG.ClientSecret) 39 | gdrive.WatchDrive(client, env) 40 | api.Listen(env) 41 | } 42 | return nil 43 | }, 44 | }, 45 | { 46 | Name: "setup", 47 | Usage: "Only authenticate with Google and do not run the allwrite server.", 48 | Action: func(c *cli.Context) error { 49 | if env := setupConf(); env != nil { 50 | gdrive.DriveClient(env.CFG.ClientSecret) 51 | } 52 | return nil 53 | }, 54 | }, 55 | { 56 | Name: "pull", 57 | Aliases: []string{"p"}, 58 | Usage: "Pull the latest content from Google Drive.", 59 | Action: func(c *cli.Context) error { 60 | if env := setupConf(); env != nil { 61 | client := gdrive.DriveClient(env.CFG.ClientSecret) 62 | gdrive.UpdateMenu(client, env) 63 | } 64 | return nil 65 | }, 66 | }, 67 | { 68 | Name: "reset", 69 | Aliases: []string{"r"}, 70 | Usage: "Reset any saved authentication credentials for Google. You will need to re-authenticate after doing this.", 71 | Action: func(c *cli.Context) error { 72 | if env := setupConf(); env != nil { 73 | if err := gdrive.RemoveCacheFile(); err != nil { 74 | return err 75 | } 76 | } 77 | return nil 78 | }, 79 | }, 80 | { 81 | Name: "info", 82 | Aliases: []string{"i"}, 83 | Usage: "Display environmental variables. Useful for making sure everything is setup correctly.", 84 | Action: func(c *cli.Context) error { 85 | if env := setupConf(); env != nil { 86 | log.Printf( 87 | debugInfoMessage, 88 | env.CFG.ClientSecret, 89 | env.CFG.ActiveDir, 90 | env.CFG.StoreType, 91 | env.CFG.Port, 92 | ) 93 | } 94 | return nil 95 | }, 96 | }, 97 | } 98 | app.Run(os.Args) 99 | } 100 | 101 | func setupConf() *util.Env { 102 | var cfg util.Conf 103 | if err := envdecode.Decode(&cfg); err != nil { 104 | log.Println("Please make sure the environmental variables are set first.") 105 | return nil 106 | } 107 | env := &util.Env{ 108 | CFG: &cfg, 109 | } 110 | switch cfg.StoreType { 111 | case "postgres": 112 | if db, err := postgres.Init( 113 | cfg.PostgresUser, 114 | cfg.PostgresPassword, 115 | cfg.PostgresHost, 116 | cfg.PostgresDBName, 117 | ); err == nil { 118 | env.DB = db 119 | } else { 120 | log.Printf("Could not connect to postgres: %s", err.Error()) 121 | return nil 122 | } 123 | default: 124 | log.Printf("You must specify a storage system. (postgres)") 125 | return nil 126 | } 127 | 128 | return env 129 | } 130 | -------------------------------------------------------------------------------- /model/page-fragment.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | ) 7 | 8 | // Fragments is a slice of frags. 9 | type Fragments []*PageFragment 10 | 11 | // PageFragment is a simple version of a page. 12 | type PageFragment struct { 13 | Name string `json:"name"` 14 | Type string `json:"type"` 15 | Slug string `json:"slug"` 16 | Order int `json:"order"` 17 | Updated string `json:"updated"` 18 | Created string `json:"created"` 19 | 20 | // Only populated on search results. 21 | MatchingText string `json:"reltext,omitempty"` 22 | 23 | // Children on only populated when querying the menu. 24 | Children Fragments `json:"children,omitempty"` 25 | } 26 | 27 | // ByOrder will sort by order. 28 | type ByOrder Fragments 29 | 30 | func (slice ByOrder) Len() int { 31 | return len(slice) 32 | } 33 | 34 | func (slice ByOrder) Less(i, j int) bool { 35 | return slice[i].Order < slice[j].Order 36 | } 37 | 38 | func (slice ByOrder) Swap(i, j int) { 39 | slice[i], slice[j] = slice[j], slice[i] 40 | } 41 | 42 | func compareSlices(a []string, b []string) int { 43 | n, aLen, bLen := 0, len(a), len(b) 44 | for i := range a { 45 | if i < aLen && i < bLen && a[i] == b[i] { 46 | n++ 47 | } 48 | } 49 | return n 50 | } 51 | 52 | func appendDeep( 53 | master Fragments, 54 | prevSibling *Fragments, 55 | prev *PageFragment, 56 | next *PageFragment, 57 | ) ( 58 | Fragments, 59 | *Fragments, 60 | *PageFragment, 61 | ) { 62 | prevSegs := strings.Split(prev.Slug, "/") 63 | nextSegs := strings.Split(next.Slug, "/") 64 | diff := compareSlices(prevSegs, nextSegs) 65 | var siblings *Fragments 66 | 67 | // These comments are super helpful when debugging. Leaving commented out. 68 | if len(prevSegs) == diff { 69 | // log.Printf("Adding child '%s' to parent '%s' -- diff: %v", next.Name, prev.Name, diff) 70 | prev.Children = append(prev.Children, next) 71 | siblings = &prev.Children 72 | } else if diff > 0 { 73 | // log.Printf("Adding sibling: '%s' along side '%s'", next.Name, prev.Name) 74 | *prevSibling = append(*prevSibling, next) 75 | siblings = prevSibling 76 | } else { 77 | // log.Printf("Adding top level: %s -- diff: %v", next.Name, diff) 78 | master = append(master, next) 79 | siblings = &master 80 | } 81 | 82 | sort.Sort(ByOrder(master)) 83 | sort.Sort(ByOrder(*siblings)) 84 | 85 | return master, siblings, next 86 | } 87 | 88 | // PageTree takes in an ordered collection of pages and sorts them by order and 89 | // populates the children slice. These must be ordered by slug. 90 | // 91 | // For example: 92 | // 93 | // title | slug 94 | // ----------------------+---------------------------------------------- 95 | // Welcome | 96 | // First Page | getting-started 97 | // Code Snippets | getting-started/code-snippets 98 | // Hello World | getting-started/hello-world 99 | // Section One | section-one 100 | // Moderately Deep File | section-one/moderately-deep-file 101 | // Only one deep | section-two 102 | // This is a deep file | section-two/subsection-one 103 | // Subsection Two | section-two/subsection-two 104 | // Deep Page Example | section-two/subsection-two/deep-page-example 105 | func PageTree(pages Fragments) Fragments { 106 | master := make(Fragments, 0, len(pages)) 107 | 108 | // Append the first page to get our master started. Then shift off the first 109 | // item of the main pages array. Singlings gets passed by pointer so siblings 110 | // can be added to it if need be. 111 | master = append(master, pages[0]) 112 | siblings := &master 113 | pages = pages[1:] 114 | 115 | // This is set on each iteration. 116 | prev := master[0] 117 | 118 | // Loop through the rest of the pages adding them to the master where they 119 | // belong. 120 | for _, page := range pages { 121 | master, siblings, prev = appendDeep(master, siblings, prev, page) 122 | } 123 | 124 | return master 125 | } 126 | -------------------------------------------------------------------------------- /model/page-fragment_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var items = Fragments{ 8 | &PageFragment{ // top level 9 | Order: 0, 10 | Name: "Homepage", 11 | Type: "file", 12 | Slug: "", 13 | }, 14 | &PageFragment{ // top level 15 | Order: 3, 16 | Name: "Images!", 17 | Type: "file", 18 | Slug: "images", 19 | }, 20 | &PageFragment{ // top level 21 | Order: 2, 22 | Name: "A Sub Directory", 23 | Type: "dir", 24 | Slug: "a-sub-directory", 25 | }, 26 | &PageFragment{ 27 | Order: 1, 28 | Name: "How to be a friend", 29 | Type: "file", 30 | Slug: "a-sub-directory/how-to-be-a-friend", 31 | }, 32 | &PageFragment{ // top level 33 | Order: 1, 34 | Name: "I go deep", 35 | Type: "file", 36 | Slug: "deep", 37 | }, 38 | &PageFragment{ 39 | Order: 0, 40 | Name: "This is a deep file", 41 | Type: "file", 42 | Slug: "deep/a-deeper-directory", 43 | }, 44 | &PageFragment{ 45 | Order: 0, 46 | Name: "The deepest file", 47 | Type: "file", 48 | Slug: "deep/a-deeper-directory/hey-you", 49 | }, 50 | &PageFragment{ 51 | Order: 1, 52 | Name: "A sibling to the deepest", 53 | Type: "file", 54 | Slug: "deep/a-deeper-directory/foobar", 55 | }, 56 | } 57 | 58 | func TestMenuSorting(t *testing.T) { 59 | sorted := PageTree(items) 60 | 61 | // for _, page := range sorted { 62 | // fmt.Println(page.Name) 63 | // } 64 | if len(sorted) != 4 { 65 | t.Error("There should only be 4.") 66 | } 67 | if sorted[0].Name != "Homepage" { 68 | t.Error("Sorting is out of order: " + sorted[0].Name) 69 | } 70 | if len(sorted[0].Children) != 0 { 71 | t.Error("Index 0 should have zero children.") 72 | } 73 | 74 | if sorted[1].Name != "I go deep" { 75 | t.Error("Sorting is out of order: " + sorted[1].Name) 76 | } 77 | if len(sorted[1].Children) != 1 { 78 | t.Error("Index 1 should have 1 child.") 79 | } 80 | if len(sorted[1].Children[0].Children) != 2 { 81 | t.Error("This sub directory should have 2 deeper children.") 82 | } 83 | 84 | if sorted[2].Name != "A Sub Directory" { 85 | t.Error("Sorting is out of order: " + sorted[2].Name) 86 | } 87 | if len(sorted[2].Children) != 1 { 88 | t.Error("Index 2 should have 1 child.") 89 | } 90 | 91 | if sorted[3].Name != "Images!" { 92 | t.Error("Sorting is out of order: " + sorted[3].Name) 93 | } 94 | if len(sorted[3].Children) != 0 { 95 | t.Error("Index 3 should have no children") 96 | } 97 | if sorted[1].Children[0].Name != "This is a deep file" { 98 | t.Error("Index 1 has the wrong child.") 99 | } 100 | 101 | if sorted[3].Name != "Images!" { 102 | t.Error("Sorting is out of order: " + sorted[3].Name) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /model/page.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | // Page contains everything about a page. This will be stored. 4 | type Page struct { 5 | PageFragment 6 | DocID string `json:"doc_id"` 7 | HTML string `json:"html"` 8 | Md string `json:"md"` 9 | Children []*PageFragment `json:"children,omitempty"` 10 | } 11 | 12 | // Pages is a slide of pages. 13 | type Pages []*Page 14 | -------------------------------------------------------------------------------- /store/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "log" 8 | 9 | "github.com/LevInteractive/allwrite-docs/model" 10 | 11 | // Because postgres 12 | _ "github.com/lib/pq" 13 | ) 14 | 15 | // Store implements the Store interface. 16 | type Store struct { 17 | driver *sql.DB 18 | } 19 | 20 | // SavePages saves a page. This will preform an upsert on the page record. 21 | // It's assumed that the slug is already unique. 22 | // 23 | // The upsert here is also completely unncessary because we are removing all 24 | // rows anytime it's updated. Just keeping it because it doesn't hurt. 25 | func (p *Store) SavePages(pages model.Pages) (model.Pages, error) { 26 | tx, err := p.driver.Begin() 27 | if err != nil { 28 | return pages, err 29 | } 30 | stmt, err := tx.Prepare(` 31 | INSERT INTO pages (doc_id, type, title, slug, md, html, placement) 32 | VALUES ($1, $2, $3, $4, $5, $6, $7) 33 | ON CONFLICT (doc_id) 34 | DO UPDATE set (type, title, slug, md, html, placement) = ($2, $3, $4, $5, $6, $7) 35 | WHERE pages.doc_id = $1 36 | `) 37 | 38 | if err != nil { 39 | return pages, err 40 | } 41 | 42 | defer stmt.Close() 43 | 44 | var statementError error 45 | 46 | for _, page := range pages { 47 | if _, err := stmt.Exec( 48 | page.DocID, 49 | page.Type, 50 | page.Name, 51 | page.Slug, 52 | page.Md, 53 | page.HTML, 54 | page.Order, 55 | ); err != nil { 56 | tx.Rollback() 57 | statementError = err 58 | break 59 | } 60 | } 61 | 62 | if statementError != nil { 63 | return pages, statementError 64 | } 65 | 66 | tx.Commit() 67 | return pages, nil 68 | } 69 | 70 | // RemoveAll removes all pages from postgres. 71 | func (p *Store) RemoveAll() error { 72 | if _, err := p.driver.Exec(`DELETE FROM pages`); err != nil { 73 | return err 74 | } 75 | return nil 76 | } 77 | 78 | // GetPage saves a page. 79 | func (p *Store) GetPage(slug string) (*model.Page, error) { 80 | stmt := ` 81 | SELECT doc_id, created, updated, type, title, slug, md, html, placement 82 | FROM pages 83 | WHERE slug = $1 AND type = 'file'` 84 | page := model.Page{} 85 | 86 | err := p.driver.QueryRow(stmt, slug).Scan( 87 | &page.DocID, 88 | &page.Created, 89 | &page.Updated, 90 | &page.Type, 91 | &page.Name, 92 | &page.Slug, 93 | &page.Md, 94 | &page.HTML, 95 | &page.Order, 96 | ) 97 | 98 | switch err { 99 | case sql.ErrNoRows: 100 | return &page, errors.New("not found") 101 | case nil: 102 | return &page, nil 103 | default: 104 | return &page, err 105 | } 106 | } 107 | 108 | // GetMenu retrieves the full menu tree. 109 | func (p *Store) GetMenu() ([]*model.PageFragment, error) { 110 | rows, err := p.driver.Query(` 111 | SELECT title, type, placement, created, updated, slug 112 | FROM pages 113 | ORDER BY slug ASC 114 | `) 115 | 116 | if err != nil { 117 | return nil, err 118 | } 119 | defer rows.Close() 120 | var records model.Fragments 121 | for rows.Next() { 122 | record := &model.PageFragment{} 123 | err := rows.Scan( 124 | &record.Name, 125 | &record.Type, 126 | &record.Order, 127 | &record.Created, 128 | &record.Updated, 129 | &record.Slug, 130 | ) 131 | if err != nil { 132 | log.Fatal(err) 133 | } 134 | records = append(records, record) 135 | } 136 | err = rows.Err() 137 | if err != nil { 138 | log.Fatal(err) 139 | } 140 | return model.PageTree(records), nil 141 | } 142 | 143 | // Search searches a page. 144 | func (p *Store) Search(q string) ([]*model.PageFragment, error) { 145 | rows, err := p.driver.Query(` 146 | SELECT 147 | title, 148 | type, 149 | placement, 150 | created, 151 | updated, 152 | slug, 153 | ts_headline( 154 | regexp_replace(html, E'[\\n\\r]+', ' ', 'g'), 155 | q 156 | ) as reltext 157 | FROM ( 158 | SELECT title, type, placement, created, updated, slug, q, html 159 | FROM pages, plainto_tsquery('english', $1) q 160 | WHERE to_tsvector(title) || to_tsvector(html) @@ q 161 | ) AS sub; 162 | `, q) 163 | 164 | if err != nil { 165 | return nil, err 166 | } 167 | defer rows.Close() 168 | var records model.Fragments 169 | for rows.Next() { 170 | record := &model.PageFragment{} 171 | err := rows.Scan( 172 | &record.Name, 173 | &record.Type, 174 | &record.Order, 175 | &record.Created, 176 | &record.Updated, 177 | &record.Slug, 178 | &record.MatchingText, 179 | ) 180 | if err != nil { 181 | log.Fatal(err) 182 | } 183 | records = append(records, record) 184 | } 185 | err = rows.Err() 186 | if err != nil { 187 | log.Fatal(err) 188 | } 189 | return records, nil 190 | } 191 | 192 | // Init connection with postgres. 193 | func Init(user string, pass string, host string, db string) (*Store, error) { 194 | driver, err := sql.Open( 195 | "postgres", 196 | fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, pass, host, db), 197 | ) 198 | if err != nil { 199 | return &Store{}, err 200 | } 201 | if err = driver.Ping(); err != nil { 202 | return &Store{}, err 203 | } 204 | 205 | return &Store{driver: driver}, nil 206 | } 207 | -------------------------------------------------------------------------------- /store/postgres/sql/pages.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS pages CASCADE; 2 | CREATE TABLE pages ( 3 | id SERIAL PRIMARY KEY, 4 | type TEXT, 5 | placement SMALLINT, 6 | doc_id TEXT UNIQUE NOT NULL, 7 | created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 8 | updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 9 | title TEXT, 10 | slug TEXT NOT NULL, 11 | md TEXT, 12 | html TEXT 13 | ); 14 | 15 | CREATE INDEX slug_idx ON pages (slug); 16 | -------------------------------------------------------------------------------- /store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "github.com/LevInteractive/allwrite-docs/model" 4 | 5 | // Store interface for any storage client to implement. 6 | // Eventually, we'll probably want to make search return a pagination struct 7 | // with a cursor implementation. Search is also very simple. Could add a few 8 | // more parameters eventually. 9 | type Store interface { 10 | SavePages(model.Pages) (model.Pages, error) 11 | RemoveAll() error 12 | GetPage(slug string) (*model.Page, error) 13 | 14 | // The menu must sort by slug ASC. 15 | // So: 16 | // a 17 | // a/b 18 | // a/c 19 | // b/a 20 | // b/c/d 21 | GetMenu() ([]*model.PageFragment, error) 22 | Search(q string) ([]*model.PageFragment, error) 23 | } 24 | -------------------------------------------------------------------------------- /util/conf.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/LevInteractive/allwrite-docs/store" 5 | ) 6 | 7 | // Conf - Configuration. 8 | type Conf struct { 9 | ActiveDir string `env:"ACTIVE_DIR,required"` 10 | StoreType string `env:"STORAGE,required"` 11 | ClientSecret string `env:"CLIENT_SECRET,required"` 12 | Frequency int `env:"FREQUENCY,required"` 13 | Port string `env:"PORT,required"` 14 | CertbotEmail string `env:"CERTBOT_EMAIL"` 15 | Domain string `env:"DOMAIN"` 16 | PostgresUser string `env:"PG_USER"` 17 | PostgresPassword string `env:"PG_PASS"` 18 | PostgresHost string `env:"PG_HOST"` 19 | PostgresDBName string `env:"PG_DB"` 20 | } 21 | 22 | // Env is a environment specific struct. 23 | type Env struct { 24 | DB store.Store 25 | CFG *Conf 26 | } 27 | -------------------------------------------------------------------------------- /util/interval.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "time" 4 | 5 | // SetInterval calls a function every n. 6 | func SetInterval(action func(), interval time.Duration) func() { 7 | t := time.NewTicker(interval) 8 | q := make(chan struct{}) 9 | go func() { 10 | for { 11 | select { 12 | case <-t.C: 13 | action() 14 | case <-q: 15 | t.Stop() 16 | return 17 | } 18 | } 19 | }() 20 | return func() { 21 | close(q) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /util/slug.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | // Replacement structure 9 | type replacement struct { 10 | re *regexp.Regexp 11 | ch string 12 | } 13 | 14 | // Build regexps and replacements 15 | var ( 16 | rExps = []replacement{ 17 | {re: regexp.MustCompile(`[\xC0-\xC6]`), ch: "A"}, 18 | {re: regexp.MustCompile(`[\xE0-\xE6]`), ch: "a"}, 19 | {re: regexp.MustCompile(`[\xC8-\xCB]`), ch: "E"}, 20 | {re: regexp.MustCompile(`[\xE8-\xEB]`), ch: "e"}, 21 | {re: regexp.MustCompile(`[\xCC-\xCF]`), ch: "I"}, 22 | {re: regexp.MustCompile(`[\xEC-\xEF]`), ch: "i"}, 23 | {re: regexp.MustCompile(`[\xD2-\xD6]`), ch: "O"}, 24 | {re: regexp.MustCompile(`[\xF2-\xF6]`), ch: "o"}, 25 | {re: regexp.MustCompile(`[\xD9-\xDC]`), ch: "U"}, 26 | {re: regexp.MustCompile(`[\xF9-\xFC]`), ch: "u"}, 27 | {re: regexp.MustCompile(`[\xC7-\xE7]`), ch: "c"}, 28 | {re: regexp.MustCompile(`[\xD1]`), ch: "N"}, 29 | {re: regexp.MustCompile(`[\xF1]`), ch: "n"}, 30 | } 31 | spacereg = regexp.MustCompile(`\s+`) 32 | noncharreg = regexp.MustCompile(`[^A-Za-z0-9-]`) 33 | minusrepeatreg = regexp.MustCompile(`\-{2,}`) 34 | ) 35 | 36 | // MarshalSlug function returns slugifies string "s" 37 | func MarshalSlug(s string, lower ...bool) string { 38 | for _, r := range rExps { 39 | s = r.re.ReplaceAllString(s, r.ch) 40 | } 41 | 42 | if len(lower) > 0 && lower[0] { 43 | s = strings.ToLower(s) 44 | } 45 | s = spacereg.ReplaceAllString(s, "-") 46 | s = noncharreg.ReplaceAllString(s, "") 47 | s = minusrepeatreg.ReplaceAllString(s, "-") 48 | 49 | return strings.ToLower(s) 50 | } 51 | --------------------------------------------------------------------------------