├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── main.go └── template.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Takuya Ueda 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # qiitaexporter 2 | 3 | Qiitaから記事をエクスポートするツールです。 4 | 5 | ## インストール 6 | 7 | ```sh 8 | $ go get -u github.com/tenntenn/qiitaexporter 9 | ``` 10 | 11 | ## アクセストークンの取得 12 | 13 | [こちら](https://qiita.com/settings/applications)から取得できます。 14 | 15 | ## 使い方 16 | 17 | 以下のように、環境変数`QIITA`でアクセストークンを指定します。 18 | 19 | ```sh 20 | $ QIITA=xxxx qiitaexporter 21 | ``` 22 | 23 | 出力ファイルを変えたい場合は`-postdir`を指定します。 24 | 画像ファイルは`posts`以下に出力されますが、変えたい場合は`-imgdir`を指定します。 25 | 画像のリンクにプリフィックスをつけたい場合は、`-imgprefix`で指定できます。 26 | 27 | デフォルトではHugo向けに出力しますが、`-template`オプションでテンプレートを変更できます。 28 | `template.go`を参考にすると良いでしょう。 29 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tenntenn/qiitaexporter 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= 4 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 5 | github.com/uetchy/go-qiita v0.0.0-20190620071230-878056c11ee7 h1:PHAku86/Do3YQSxoXt38dU9XyIMxbMe1VP0QneJ4O4E= 6 | github.com/uetchy/go-qiita v0.0.0-20190620071230-878056c11ee7/go.mod h1:jhEiH3PDC4S5X9lSy1QQBVjQlc03PtQ10a91ukIQHNs= 7 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 8 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= 9 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 10 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 11 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 12 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 13 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 14 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 15 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "path" 12 | "path/filepath" 13 | "regexp" 14 | "strconv" 15 | "strings" 16 | "text/template" 17 | "time" 18 | ) 19 | 20 | var ( 21 | flagImgPathPrefix = flag.String("imgprefix", "/images/", "prefix of image path") 22 | flagImgDir = flag.String("imgdir", "images", "image dir") 23 | flagPostDir = flag.String("postdir", "posts", "posts dir") 24 | flagTmplFile = flag.String("template", "", "template file") 25 | 26 | imgRegexp = regexp.MustCompile(`https://qiita-image-store\.s3\.amazonaws\.com/.+\.png`) 27 | ) 28 | 29 | const itemsPerPage = 20 30 | 31 | type Tag struct { 32 | Name string `json:"name"` 33 | Versions []string `json:"versions"` 34 | } 35 | 36 | type Item struct { 37 | ID string `json:"id"` 38 | URL string `json:"url"` 39 | Title string `json:"title"` 40 | Body string `json:"body"` 41 | RenderedBody string `json:"rendered_body"` 42 | CreatedAt time.Time `json:"created_at"` 43 | Tags []*Tag `json:"tags"` 44 | Private bool `json:"private"` 45 | } 46 | 47 | func (item *Item) AllTags() string { 48 | tags := make([]string, len(item.Tags)) 49 | for i := range item.Tags { 50 | tags[i] = strconv.Quote(item.Tags[i].Name) 51 | } 52 | return strings.Join(tags, ",") 53 | } 54 | 55 | func (item *Item) Date() string { 56 | return item.CreatedAt.Format("2006-01-02") 57 | } 58 | 59 | func (item *Item) ImageToLocal(dir string) error { 60 | var ( 61 | rerr error 62 | count int 63 | ) 64 | body := imgRegexp.ReplaceAllStringFunc(item.Body, func(s string) string { 65 | if rerr != nil { 66 | return s 67 | } 68 | 69 | count++ 70 | ext := path.Ext(s) 71 | fname := fmt.Sprintf("qiita-%s-%d%s", item.ID, count, ext) 72 | f, err := os.Create(filepath.Join(dir, fname)) 73 | if err != nil { 74 | rerr = err 75 | return s 76 | } 77 | 78 | resp, err := http.Get(s) 79 | if err != nil { 80 | rerr = err 81 | return s 82 | } 83 | defer resp.Body.Close() 84 | 85 | if _, err := io.Copy(f, resp.Body); err != nil { 86 | rerr = err 87 | return s 88 | } 89 | 90 | if err := f.Close(); err != nil { 91 | rerr = err 92 | return s 93 | } 94 | 95 | return path.Join(*flagImgPathPrefix, fname) 96 | }) 97 | 98 | if rerr != nil { 99 | return rerr 100 | } 101 | 102 | item.Body = body 103 | 104 | return nil 105 | } 106 | 107 | func main() { 108 | flag.Parse() 109 | 110 | if *flagTmplFile != "" { 111 | var err error 112 | tmpl, err = template.ParseFiles(*flagTmplFile) 113 | if err != nil { 114 | fmt.Fprintln(os.Stderr, "Error:", err) 115 | os.Exit(1) 116 | } 117 | } 118 | 119 | for i := 1; ; i++ { 120 | hasNext, err := download100(i) 121 | if err != nil { 122 | fmt.Fprintln(os.Stderr, "Error:", err) 123 | os.Exit(1) 124 | } 125 | 126 | if !hasNext { 127 | break 128 | } 129 | } 130 | } 131 | 132 | func download100(page int) (hasNext bool, rerr error) { 133 | 134 | url := fmt.Sprintf("https://qiita.com/api/v2/authenticated_user/items?page=%d&per_page=%d", page, itemsPerPage) 135 | req, err := http.NewRequest(http.MethodGet, url, nil) 136 | if err != nil { 137 | return false, err 138 | } 139 | 140 | resp, err := do(req) 141 | if err != nil { 142 | return false, err 143 | } 144 | defer resp.Body.Close() 145 | 146 | if resp.StatusCode != http.StatusOK { 147 | return false, errors.New(resp.Status) 148 | } 149 | 150 | var items []*Item 151 | if err := json.NewDecoder(resp.Body).Decode(&items); err != nil { 152 | return false, err 153 | } 154 | 155 | imgdir := filepath.Join(*flagPostDir, *flagImgDir) 156 | if err := os.MkdirAll(imgdir, 0777); err != nil { 157 | return false, err 158 | } 159 | 160 | for i := range items { 161 | if items[i].Private { 162 | continue 163 | } 164 | 165 | if err := items[i].ImageToLocal(imgdir); err != nil { 166 | return false, err 167 | } 168 | 169 | fname := fmt.Sprintf("%s-qiita-%s.ja.md", items[i].Date(), items[i].ID) 170 | fmt.Print(items[i].Title, "....") 171 | f, err := os.Create(filepath.Join(*flagPostDir, fname)) 172 | if err != nil { 173 | return false, err 174 | } 175 | 176 | if err := tmpl.Execute(f, items[i]); err != nil { 177 | return false, err 178 | } 179 | 180 | if err := f.Close(); err != nil { 181 | return false, err 182 | } 183 | 184 | fmt.Println("done") 185 | } 186 | 187 | total, err := strconv.Atoi(resp.Header.Get("Total-Count")) 188 | if err != nil { 189 | return false, err 190 | } 191 | 192 | return itemsPerPage*page < total, nil 193 | } 194 | 195 | func do(req *http.Request) (*http.Response, error) { 196 | token := fmt.Sprintf("Bearer %s", os.Getenv("QIITA")) 197 | req.Header.Set("Authorization", token) 198 | return http.DefaultClient.Do(req) 199 | } 200 | -------------------------------------------------------------------------------- /template.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "text/template" 4 | 5 | var tmpl = template.Must(template.New("template").Parse(`+++ 6 | date = "{{.Date}}" 7 | title = "{{.Title}}" 8 | slug = "qiita-{{.ID}}" 9 | tags = [{{.AllTags}}] 10 | categories = [] 11 | +++ 12 | 13 | *この記事は[Qiita]({{.URL}})の記事をエクスポートしたものです。内容が古くなっている可能性があります。* 14 | 15 | {{.Body}}`)) 16 | --------------------------------------------------------------------------------