├── .github └── workflows │ ├── daily-run.yml │ ├── release.yml │ └── test.yaml ├── .gitignore ├── LICENSE ├── README-zh_CN.md ├── README.md ├── awg ├── client.go ├── client_test.go ├── const.go ├── init.go ├── parser.go ├── parser_test.go ├── reporter.go ├── view.go ├── view_test.go ├── worker.go └── worker_test.go ├── cmd └── awg │ ├── .goreleaser.yml │ ├── errors.go │ ├── logger.go │ ├── logger_test.go │ ├── main.go │ ├── view.go │ ├── view_test.go │ ├── worker.go │ └── worker_test.go ├── configs └── config.yaml ├── exch ├── config │ ├── const.go │ ├── model.go │ ├── model_test.go │ ├── parser.go │ ├── parser_test.go │ └── yaml.go └── github │ ├── client.go │ ├── client_test.go │ ├── const.go │ ├── init.go │ └── model.go ├── go.mod ├── go.sum ├── lib ├── cohttp │ ├── const.go │ ├── http.go │ ├── http_test.go │ ├── init.go │ ├── lib.go │ └── lib_test.go └── errcode │ ├── const.go │ ├── errcode.go │ └── init.go ├── test ├── fake-github │ ├── data.go │ └── server.go └── testdata │ ├── api │ ├── repos │ │ ├── antchfx_xpath.json │ │ ├── goxjs_glfw.json │ │ └── technohippy_go-glmatrix.json │ └── users │ │ └── tester.json │ ├── html │ └── README.html │ └── output │ └── data.json └── web ├── app └── route.go └── static └── js └── view.js /.github/workflows/daily-run.yml: -------------------------------------------------------------------------------- 1 | name: Daily publish 2 | on: 3 | schedule: 4 | - cron: '50 1 * * *' 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v2 11 | with: 12 | fetch-depth: 0 13 | - name: Use Go 14 | uses: actions/setup-go@v2 15 | with: 16 | go-version: 1.14 17 | - name: Build 18 | run: go build -o dist/awg ./cmd/awg 19 | - name: Fetch data 20 | run: dist/awg fetch -config configs/config.yaml 21 | env: 22 | GITHUB_ACCESS_TOKEN: ${{ secrets.FETCH_DATA_GITHUB_TOKEN }} 23 | - name: Upload data 24 | uses: actions/upload-artifact@v2 25 | with: 26 | name: awesome-go 27 | path: awg.json 28 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: [ master ] 5 | pull_request: 6 | branches: [ master ] 7 | jobs: 8 | release: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | - name: Use Go 16 | uses: actions/setup-go@v2 17 | with: 18 | go-version: 1.14 19 | - name: Run GoReleaser 20 | uses: goreleaser/goreleaser-action@v2 21 | with: 22 | version: latest 23 | args: release --rm-dist 24 | workdir: ./cmd/awg 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Unit-Tests 2 | on: 3 | push: 4 | branches: [ master ] 5 | pull_request: 6 | branches: [ master ] 7 | jobs: 8 | test: 9 | name: Test on ${{ matrix.os }} go ${{ matrix.go_version }} 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest, macOS-latest] 14 | go_version: [1.14] 15 | 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v2 19 | - name: Use Go ${{ matrix.go_version }} 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go_version }} 23 | - name: Test 24 | run: go test -race -covermode atomic -coverprofile=profile.cov ./... 25 | - name: Send coverage 26 | uses: shogo82148/actions-goveralls@v1 27 | with: 28 | path-to-profile: profile.cov 29 | flag-name: ${{ matrix.os }} go-${{ matrix.go_version }} 30 | parallel: true 31 | finish: 32 | needs: test 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: shogo82148/actions-goveralls@v1 36 | with: 37 | parallel-finished: true 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | dist 3 | 4 | /cmd/awg/awg 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 rydesun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 | # Awesome GitHub 2 | 3 | [![Build](https://github.com/rydesun/awesome-github/workflows/Build/badge.svg)](https://github.com/rydesun/awesome-github/actions?query=workflow%3ABuild) 4 | [![Unit-Tests](https://github.com/rydesun/awesome-github/workflows/Unit-Tests/badge.svg)](https://github.com/rydesun/awesome-github/actions?query=workflow%3AUnit-Tests) 5 | [![Coverage Status](https://coveralls.io/repos/github/rydesun/awesome-github/badge.svg?branch=master)](https://coveralls.io/github/rydesun/awesome-github?branch=master) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/rydesun/awesome-github)](https://goreportcard.com/report/github.com/rydesun/awesome-github) 7 | [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/rydesun/awesome-github/blob/master/LICENSE) 8 | 9 | 通过命令行工具 awg，探索你钟爱的 Awesome GitHub 仓库！ 10 | 11 | ## Awesome Lists 12 | 13 | 什么是 Awesome Lists？ 14 | 15 | 比如著名的 [awesome-go](https://github.com/avelino/awesome-go) 就是 Awesome Lists 的一员， 16 | 我们可以从中快速找到很多和 Go 相关的框架、库、软件等资源。 17 | 18 | 不止是 Go，还可以从 [Awesome Lists](https://github.com/topics/awesome) 19 | 寻找更多你感兴趣的内容。 20 | 21 | ## 命令行工具 awg 22 | 23 | 当前，Awesome List 一般会列举很多 GitHub 仓库， 24 | 就比如 awesome-go 包含了上千个 GitHub 仓库。 25 | 但是，Awesome List 不包含这些仓库的 star 数、 26 | 更新时间 (最后一次 commit 时间) 之类的信息。 27 | 很多时候，我们需要这些信息作为参考，只能手动打开链接以查看这些仓库的信息。 28 | 29 | 命令行工具 awg，帮助我们进一步挖掘 Awesome List 中所有 GitHub 仓库的惊人信息！ 30 | 31 | awg 将一次性获取指定 Awesome List 中的 GitHub 仓库信息， 32 | 并输出获取到的数据到指定文件中。 33 | 稍后可以用 awg 生成一个 [浏览器页面](#浏览器中查看) 用于查看， 34 | 或者你也可以使用喜欢的数据处理工具，比如 jq 和 python，对这些数据进行分析。 35 | 36 | ![Screenshot](https://user-images.githubusercontent.com/19602440/88459895-f3897480-ce87-11ea-8fe7-13773037c56d.gif) 37 | 38 | 最终输出的文件内容： 39 | 40 | ```javascript 41 | { 42 | "data": { 43 | "Command Line": [ 44 | { 45 | "id": { 46 | "owner": "urfave", 47 | "name": "cli" 48 | }, 49 | "owner": "urfave", 50 | "awesome_name": "urfave/cli", 51 | "link": "https://github.com/urfave/cli", 52 | "watch": 295, 53 | "star": 14171, 54 | "fork": 1134, 55 | "last_commit": "2020-07-12T13:32:01Z", 56 | "description": "A simple, fast, and fun package for building command line apps in Go", 57 | "awesome_description": "urfave/cli - Simple, fast, and fun package for building command line apps in Go (formerly codegangsta/cli)." 58 | }, 59 | // ... 60 | ] 61 | // ... 62 | } 63 | } 64 | ``` 65 | 66 | ### 数据分析 67 | 68 | 可以使用任何工具去分析获得的数据文件。 69 | 70 | 比如在获取 awesome-go 的数据文件`awg.json`后， 71 | 72 | #### 浏览器中查看 73 | 74 | ![Screenshot](https://user-images.githubusercontent.com/19602440/89290996-3fd37200-d649-11ea-8807-a6a117d016f0.png) 75 | 76 | ```bash 77 | # 获取用于处理数据的JS脚本：view.js 78 | curl -fLO https://raw.githubusercontent.com/rydesun/awesome-github/master/web/static/js/view.js 79 | # 启动服务 80 | awg view --script view.js --data awg.json avelino/awesome-go 81 | ``` 82 | 83 | 向 awg 提供 view.js 和数据文件， 84 | 并且表明数据文件指向的 awesome list 是 avelino/awesome-go， 85 | 这将在本地运行一个简单的 Web 服务器，默认监听在`127.0.0.1:3000`， 86 | 用浏览器打开此页面。可以使用`--listen`指定其它地址。 87 | 88 | 注意：这不代表着可以离线查看。互联网的连接是必要的。 89 | 90 | 甚至可以不需要自己获取数据，直接使用他人提供的远程数据文件！ 91 | 用`--data`指定一个 URL，例如`https://example.com/awesome-go.json`(这是一个无效的例子) 92 | 93 | ```bash 94 | # 获取用于处理数据的JS脚本：view.js 95 | curl -fLO https://raw.githubusercontent.com/rydesun/awesome-github/master/web/static/js/view.js 96 | # 启动服务 97 | awg view --script view.js --data https://example.com/awesome-go.json avelino/awesome-go 98 | ``` 99 | 100 | 在指定远程在线的数据文件时，需要在 URL 中加上`http`或者`https`协议，表明来自网络而不是本地。 101 | 102 | #### 虚拟终端中查看 103 | 104 | 通过使用流行的命令行工具 jq，你可以： 105 | 106 | 查看 awesome-go 列表中 [Command Line](https://github.com/avelino/awesome-go#command-line) 107 | 一节的内容，并按照仓库的 star 数进行排序 108 | 109 | ```bash 110 | cat awg.json | jq '.data | ."Command Line" | sort_by(.star)' 111 | ``` 112 | 113 | ## 安装 114 | 115 | 获取命令行工具 awg 116 | 117 | ```bash 118 | go get github.com/rydesun/awesome-github/cmd/awg 119 | ``` 120 | 121 | ## 获取数据 122 | 123 | 在运行 awg 之前，先准备好： 124 | 125 | - GitHub personal access token 126 | - awg 配置文件 127 | 128 | ### Access Token 129 | 130 | awg 通过调用 GitHub GraphQL API 获取 GitHub 仓库信息， 131 | 该官方 API 需要验证你的 personal access token 后才能使用。 132 | 所以，需要向 awg 提供一个 GitHub personal access token。 133 | 134 | 如果没有该 token，请先 135 | [创建 personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)。 136 | 137 | 注意！awg 不需要该 token 有任何作用域或权限。所以，不要授予该 token 任何作用域或权限。 138 | 139 | ### 配置 140 | 141 | 必须准备一个配置文件。可以参考目录`configs`中的 142 | [配置文件模板](https://github.com/rydesun/awesome-github/blob/master/configs/config.yaml)。 143 | 144 | awg 会优先从环境变量`GITHUB_ACCESS_TOKEN`中读取 personal access token， 145 | 所以可以不用将该值储存在配置文件中。 146 | 147 | 提升并发查询请求数可以提升查询速度，但数量不要过大 (当前推荐值为 3)， 148 | 否则会被 GitHub 视作滥用 API 的行为而遭到临时封禁。 149 | 150 | 所有的在配置文件中的相对路径，均相对于 awg 的当前工作目录，而不是配置文件所在的目录。 151 | 也可以使用绝对路径。 152 | 153 | ### 运行 154 | 155 | 获取 JSON 数据文件 156 | 157 | ```bash 158 | awg fetch --config path/to/config.yaml 159 | ``` 160 | 161 | (推荐) 从环境变量中指定 GitHub Personal Access Token 的形式运行 162 | 163 | ```bash 164 | GITHUB_ACCESS_TOKEN= awg fetch --config path/to/config.yaml 165 | ``` 166 | 167 | 请注意速率限制 (RateLimit)，该值不是并发请求数。 168 | 当前 awg 每小时最多查询 5000 个 GitHub 仓库。 169 | 如果查询次数过多，会受到 GitHub 的限制从而导致失败。 170 | 具体信息请参考 [GitHub Resource limitations](https://docs.github.com/en/graphql/overview/resource-limitations#rate-limit)。 171 | 172 | ## 注意事项 173 | 174 | 当前该项目仅测试了 awesome-go 的列表，其他 Awesome List 的结果待检验。 175 | 176 | awg 不支持 Windows 平台。 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Awesome GitHub 2 | 3 | [![Build](https://github.com/rydesun/awesome-github/workflows/Build/badge.svg)](https://github.com/rydesun/awesome-github/actions?query=workflow%3ABuild) 4 | [![Unit-Tests](https://github.com/rydesun/awesome-github/workflows/Unit-Tests/badge.svg)](https://github.com/rydesun/awesome-github/actions?query=workflow%3AUnit-Tests) 5 | [![Coverage Status](https://coveralls.io/repos/github/rydesun/awesome-github/badge.svg?branch=master)](https://coveralls.io/github/rydesun/awesome-github?branch=master) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/rydesun/awesome-github)](https://goreportcard.com/report/github.com/rydesun/awesome-github) 7 | [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/rydesun/awesome-github/blob/master/LICENSE) 8 | 9 | Explore your favorite Awesome GitHub repositories via the command-line tool awg! 10 | 11 | [\[ Chinese | 中文 \]](https://github.com/rydesun/awesome-github/blob/master/README-zh_CN.md) 12 | 13 | ## Awesome Lists 14 | 15 | What is Awesome Lists？ 16 | 17 | Such as famous [awesome-go](https://github.com/avelino/awesome-go), which is a member 18 | of Awesome Lists, where we can quickly find frameworks, libraries, software, 19 | and other resources related to Go. 20 | 21 | Furthermore, we can find more awesome things from 22 | [Awesome Lists](https://github.com/topics/awesome). 23 | 24 | ## Command-line Tool awg 25 | 26 | At the moment, Awesome List usually exhibit lots of GitHub repositories. 27 | For example, awesome-go contains thousands of GitHub repositories. 28 | However, an Awesome List doesn't include information like the count of stars 29 | or the last commit date for those repositories. 30 | In many cases, we demand that information and have to manually open links 31 | to see the them. 32 | 33 | The command-line tool, awg, helps us dig a little deeper into the Awesome List 34 | to find out more about all the GitHub repositories on it! 35 | 36 | awg will fetch information of GitHub repositories listed on an Awesome List, 37 | and output the data to a file of your choice. 38 | You can use awg to generate a [browser page](#view-in-browser) to view the data later, 39 | or use your favorite tools like jq or python to analyze. 40 | 41 | ![Screenshot](https://user-images.githubusercontent.com/19602440/88459895-f3897480-ce87-11ea-8fe7-13773037c56d.gif) 42 | 43 | The final output: 44 | 45 | ```javascript 46 | { 47 | "data": { 48 | "Command Line": [ 49 | { 50 | "id": { 51 | "owner": "urfave", 52 | "name": "cli" 53 | }, 54 | "owner": "urfave", 55 | "awesome_name": "urfave/cli", 56 | "link": "https://github.com/urfave/cli", 57 | "watch": 295, 58 | "star": 14171, 59 | "fork": 1134, 60 | "last_commit": "2020-07-12T13:32:01Z", 61 | "description": "A simple, fast, and fun package for building command line apps in Go", 62 | "awesome_description": "urfave/cli - Simple, fast, and fun package for building command line apps in Go (formerly codegangsta/cli)." 63 | }, 64 | // ... 65 | ] 66 | // ... 67 | } 68 | } 69 | ``` 70 | 71 | ### Data Analysis 72 | 73 | We can use any tool to analyze the obtained data file. 74 | 75 | For example, after getting data file `awg.json`, which is concerned with awesome-go 76 | 77 | #### View in Browser 78 | 79 | The page effect is shown below 80 | 81 | ![Screenshot](https://user-images.githubusercontent.com/19602440/89290996-3fd37200-d649-11ea-8807-a6a117d016f0.png) 82 | 83 | By runing this command 84 | 85 | ```bash 86 | 87 | awg view awg.json 88 | ``` 89 | 90 | It will be running a simple web server locally, 91 | listening at `127.0.0.1:3000` by default. 92 | Open this page with your browser to view. 93 | Can use `--listen` to specify other address. 94 | 95 | Note: This does not mean that it can be viewed offline. 96 | An Internet connection is necessary. 97 | 98 | To replace the implementation for viewing, 99 | you can replace the embedded JS script address via `--script`, 100 | which supports local path. 101 | 102 | #### View in Terminal 103 | 104 | By using the popular command line tool jq, we can: 105 | 106 | View [Command Line section](https://github.com/avelino/awesome-go#command-line) 107 | of awesome-go, sorted by the count of stars 108 | 109 | ```bash 110 | cat awg.json | jq '.data | ."Command Line" | sort_by(.star)' 111 | ``` 112 | 113 | ## Installation 114 | 115 | Get command-line tool awg 116 | 117 | ```bash 118 | go get github.com/rydesun/awesome-github/cmd/awg 119 | ``` 120 | 121 | ## Fetch Data 122 | 123 | First prepare the following before running awg: 124 | 125 | - GitHub personal access token 126 | - awg configuration file 127 | 128 | ### Access Token 129 | 130 | awg fetch information of GitHub repository by calling the GitHub GraphQL API. 131 | This official API requires your personal access token to be verified 132 | before you can use it. 133 | So, you need to provide awg with a GitHub personal access token. 134 | 135 | If you do not have the token, please view the article 136 | [Creating a personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)。 137 | 138 | awg does not need the token to have any scopes or permissions. 139 | Therefore, do not grant the token any scopes or permissions. 140 | 141 | ### Configuration 142 | 143 | A configuration file is necessary. 144 | See [configuration file template](https://github.com/rydesun/awesome-github/blob/master/configs/config.yaml) 145 | in the directory `configs` as a reference. 146 | 147 | awg will read the personal access token from the 148 | environment variable `GITHUB_ACCESS_TOKEN` first. 149 | So it is not necessary to store this value in the configuration file. 150 | 151 | Increasing the number of concurrent query requests will 152 | increase the speed of the query, 153 | but the number should not be too large (the current recommended value is 3), 154 | otherwise this behavior will be viewed as abusing, and then blocked. 155 | 156 | All relative paths in a configuration file are relative to 157 | awg's current working directory, 158 | not to the directory where the configuration file is located. 159 | Absolute paths are encouraged. 160 | 161 | ### Run 162 | 163 | Fetch JSON data 164 | 165 | ```bash 166 | awg fetch --config path/to/config.yaml 167 | ``` 168 | 169 | (Recommended) Specify the GitHub Personal Access Token from an environment variable 170 | 171 | ```bash 172 | GITHUB_ACCESS_TOKEN= awg fetch --config path/to/config.yaml 173 | ``` 174 | 175 | Note the rate limit, which is not identical to the number of concurrent requests. 176 | Currently awg can query up to 5000 GitHub repositories per hour. 177 | If there are too many queries, GitHub will impose limits 178 | on requests and responses. 179 | See [GitHub Resource limitations](https://docs.github.com/en/graphql/overview/resource-limitations#rate-limit) for more. 180 | 181 | ## Notes 182 | 183 | This project is currently only testing awesome-go's lists, 184 | other testing for Awesome List results are pending. 185 | 186 | awg does not support Windows platforms. 187 | -------------------------------------------------------------------------------- /awg/client.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "context" 5 | 6 | "go.uber.org/zap" 7 | 8 | "github.com/rydesun/awesome-github/exch/github" 9 | "github.com/rydesun/awesome-github/lib/errcode" 10 | ) 11 | 12 | type Client struct { 13 | gc *github.Client 14 | } 15 | 16 | // New awg client. 17 | func NewClient(client *github.Client) (*Client, error) { 18 | return &Client{ 19 | gc: client, 20 | }, nil 21 | } 22 | 23 | func (c *Client) GetUser() (*User, error) { 24 | user, err := c.gc.GetUser() 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &User{ 29 | Name: user.Data.Viewer.Login, 30 | RateLimit: RateLimit{ 31 | Total: user.Data.Ratelimit.Limit, 32 | Remaining: user.Data.Ratelimit.Remaining, 33 | ResetAt: user.Data.Ratelimit.ResetAt, 34 | }, 35 | }, nil 36 | } 37 | 38 | // Get Readme html page. 39 | func (c *Client) GetHTMLReadme(id github.RepoID) (string, error) { 40 | const funcIntent = "get readme html page" 41 | const funcErrMsg = "failed to " + funcIntent 42 | return c.gc.GetHTMLReadme(id) 43 | } 44 | 45 | // Fill struct repo with more info. 46 | func (c *Client) Fill(ctx context.Context, repo *AwesomeRepo) error { 47 | const funcIntent = "fill struct repo with more info" 48 | const funcErrMsg = "failed to " + funcIntent 49 | logger := getLogger() 50 | defer logger.Sync() 51 | 52 | id := repo.ID 53 | idStr := repo.ID.String() 54 | 55 | logger.Debug(funcIntent, zap.String("repo", idStr)) 56 | 57 | rawRepo, err := c.gc.GetRepo(ctx, id) 58 | if err != nil { 59 | logger.Error(funcErrMsg, zap.Error(err), 60 | zap.String("repo", idStr)) 61 | return errcode.Wrap(err, funcErrMsg) 62 | } 63 | repo.Aggregate(rawRepo) 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /awg/client_test.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | 16 | "github.com/rydesun/awesome-github/exch/github" 17 | "github.com/rydesun/awesome-github/lib/cohttp" 18 | "github.com/rydesun/awesome-github/test/fake-github" 19 | ) 20 | 21 | var realSrc bool 22 | var accessToken string 23 | 24 | func init() { 25 | flag.BoolVar(&realSrc, "real", false, "fetch data from real github") 26 | flag.StringVar(&accessToken, "token", "", "your github access token") 27 | } 28 | 29 | type ClientTestEnv struct { 30 | awgClient *Client 31 | testdataHolder fakeg.DataHolder 32 | testServer *httptest.Server 33 | } 34 | 35 | func (t *ClientTestEnv) Setup() error { 36 | wd, err := os.Getwd() 37 | if err != nil { 38 | return err 39 | } 40 | testdataDir := filepath.Join(wd, "../test/testdata") 41 | testdataHolder := fakeg.NewDataHolder(testdataDir) 42 | var gbClient *github.Client 43 | if !realSrc { 44 | testServer, err := fakeg.ApiServer(testdataHolder) 45 | if err != nil { 46 | return err 47 | } 48 | t.testServer = testServer 49 | gbClient, err = github.NewClient( 50 | nil, 51 | cohttp.NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil), 52 | github.ClientOption{ 53 | APIHost: testServer.URL, 54 | ApiPathPre: github.APIPathPre, 55 | AccessToken: "123456", 56 | }) 57 | if err != nil { 58 | return err 59 | } 60 | } else { 61 | gbClient, err = github.NewClient( 62 | nil, 63 | cohttp.NewClient(http.Client{}, 16, 0, time.Second, 20, nil), 64 | github.ClientOption{ 65 | APIHost: github.APIHost, 66 | ApiPathPre: github.APIPathPre, 67 | AccessToken: accessToken, 68 | }) 69 | } 70 | if err != nil { 71 | return err 72 | } 73 | awgClient, err := NewClient(gbClient) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | t.testdataHolder = testdataHolder 79 | t.awgClient = awgClient 80 | return nil 81 | } 82 | 83 | func TestClient_GetUser(t *testing.T) { 84 | require := require.New(t) 85 | testEnv := ClientTestEnv{} 86 | err := testEnv.Setup() 87 | require.Nil(err) 88 | 89 | if !realSrc { 90 | user, err := testEnv.awgClient.GetUser() 91 | require.Nil(err) 92 | require.Equal("tester", user.Name) 93 | require.Equal(5000, user.RateLimit.Total) 94 | require.Equal(4999, user.RateLimit.Remaining) 95 | require.False(user.RateLimit.ResetAt.IsZero()) 96 | 97 | testEnv.testServer.Close() 98 | _, err = testEnv.awgClient.GetUser() 99 | require.NotNil(err) 100 | } else { 101 | user, err := testEnv.awgClient.GetUser() 102 | require.Nil(err) 103 | require.NotNil(user.Name) 104 | require.Greater(0, user.RateLimit.Total) 105 | require.False(user.RateLimit.ResetAt.IsZero()) 106 | } 107 | } 108 | 109 | func TestClient_Fill(t *testing.T) { 110 | require := require.New(t) 111 | testEnv := ClientTestEnv{} 112 | err := testEnv.Setup() 113 | require.Nil(err) 114 | 115 | testCases := []struct { 116 | user string 117 | name string 118 | hasErr bool 119 | }{ 120 | { 121 | user: "antchfx", 122 | name: "xpath", 123 | }, 124 | { 125 | user: "invalidUser", 126 | name: "invalidName", 127 | hasErr: true, 128 | }, 129 | } 130 | for _, tc := range testCases { 131 | t.Run(tc.user+"/"+tc.name, func(t *testing.T) { 132 | awesomeRepo := AwesomeRepo{ 133 | Repo: Repo{ 134 | ID: github.RepoID{ 135 | Owner: tc.user, 136 | Name: tc.name, 137 | }, 138 | Owner: tc.user, 139 | AwesomeName: tc.name, 140 | }, 141 | } 142 | err = testEnv.awgClient.Fill(context.Background(), &awesomeRepo) 143 | if err != nil { 144 | if tc.hasErr { 145 | // expected error 146 | return 147 | } 148 | t.Fatal(err) 149 | } 150 | if !realSrc { 151 | content, err := testEnv.testdataHolder.GetJsonRepo(tc.user, tc.name) 152 | require.Nil(err) 153 | expectedRepo := github.Repo{} 154 | _ = json.Unmarshal(content, &expectedRepo) 155 | require.Equal(expectedRepo.Data.Repository.Stargazers.TotalCount, 156 | awesomeRepo.Star) 157 | require.Equal(expectedRepo.Data.Repository.DefaultBranchRef.Target.History.Edges[0].Node.CommittedDate, 158 | awesomeRepo.LastCommit) 159 | require.Equal(expectedRepo.Data.Repository.Description, 160 | awesomeRepo.Description) 161 | } else { 162 | require.Less(0, awesomeRepo.Star) 163 | require.NotEmpty(awesomeRepo.LastCommit) 164 | require.NotEmpty(awesomeRepo.Description) 165 | } 166 | }) 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /awg/const.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import "regexp" 4 | 5 | const ( 6 | ErrScope = "awg" 7 | 8 | ErrCodeContent = 10 9 | ErrCodeRatelimit = 11 10 | ErrCodeNetwork = 12 11 | ) 12 | 13 | const xpathSection = `//div[@id='readme'] 14 | //article[contains(@class,'markdown-body')] 15 | //h2` 16 | const xpathItem = "//following-sibling::ul[count(preceding-sibling::h2)=%v]//li" 17 | const urlMust = "github.com" 18 | 19 | var reLink = regexp.MustCompile("(?U)^https?://github.com/(.+)/(.+)/?$") 20 | -------------------------------------------------------------------------------- /awg/init.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "sync" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | var defaultLogger *zap.Logger 10 | 11 | func SetDefaultLogger(logger *zap.Logger) { 12 | defaultLogger = logger 13 | } 14 | 15 | func getLogger() *zap.Logger { 16 | if defaultLogger != nil { 17 | return defaultLogger 18 | } 19 | var once sync.Once 20 | once.Do(func() { 21 | defaultLogger, _ = zap.NewProduction() 22 | }) 23 | return defaultLogger 24 | } 25 | -------------------------------------------------------------------------------- /awg/parser.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "regexp" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/antchfx/htmlquery" 12 | "go.uber.org/zap" 13 | "golang.org/x/net/html" 14 | 15 | "github.com/rydesun/awesome-github/exch/github" 16 | "github.com/rydesun/awesome-github/lib/cohttp" 17 | "github.com/rydesun/awesome-github/lib/errcode" 18 | ) 19 | 20 | type Parser struct { 21 | client *Client 22 | node *html.Node 23 | xpathSection string 24 | xpathItem string 25 | urlMust string 26 | reLink *regexp.Regexp 27 | reporter *Reporter 28 | ratelimit RateLimit 29 | } 30 | 31 | func NewParser(readme string, client *Client, reporter *Reporter, 32 | rateLimit RateLimit) *Parser { 33 | // Don't worry about this error. 34 | node, _ := htmlquery.Parse(strings.NewReader(readme)) 35 | return &Parser{ 36 | client: client, 37 | node: node, 38 | xpathSection: xpathSection, 39 | xpathItem: xpathItem, 40 | urlMust: urlMust, 41 | reLink: reLink, 42 | reporter: reporter, 43 | ratelimit: rateLimit, 44 | } 45 | } 46 | 47 | // Gather repositories from awesome README.md. 48 | func (p *Parser) Gather() (map[string][]*AwesomeRepo, error) { 49 | const funcIntent = "gather repositories" 50 | const funcErrMsg = "failed to " + funcIntent 51 | logger := getLogger() 52 | defer logger.Sync() 53 | logger.Debug(funcIntent) 54 | 55 | sectionItemsMap, err := p.Parse() 56 | if err != nil { 57 | return nil, err 58 | } 59 | if len(sectionItemsMap) == 0 { 60 | errMsg := "failed to find any valid sections" 61 | logger.Error(errMsg) 62 | return nil, errcode.New(errMsg, ErrCodeContent, ErrScope, nil) 63 | } 64 | total, idxReposMap := p.convert(sectionItemsMap) 65 | // TODO: may be different with graphQL node number 66 | if total > p.ratelimit.Remaining { 67 | errMsg := "Exceed GitHub API ratelimit" 68 | logger.Warn(errMsg, zap.Error(err)) 69 | return nil, errcode.New(errMsg, ErrCodeRatelimit, ErrScope, 70 | []string{strconv.Itoa(total)}) 71 | } 72 | if p.reporter != nil { 73 | p.reporter.TotalRepoNum(total) 74 | } 75 | 76 | err = p.FetchRepos(idxReposMap) 77 | if err != nil { 78 | return nil, err 79 | } 80 | idxReposMap = p.clean(idxReposMap) 81 | return idxReposMap, nil 82 | } 83 | 84 | // Get awesome section nodes from awesome README.md 85 | func (p *Parser) Parse() (map[string][]*html.Node, error) { 86 | logger := getLogger() 87 | defer logger.Sync() 88 | 89 | sectionNodes, err := htmlquery.QueryAll(p.node, p.xpathSection) 90 | if len(sectionNodes) == 0 { 91 | errMsg := "awesome html page does not contain any sections" 92 | logger.Error(errMsg, zap.Error(err)) 93 | return nil, errcode.New(errMsg, ErrCodeContent, 94 | ErrScope, []string{"section"}) 95 | } 96 | logger.Info("get some section nodes", zap.Int("len", len(sectionNodes))) 97 | 98 | sectionItemsMap := make(map[string][]*html.Node, 0) 99 | for i, sectionNode := range sectionNodes { 100 | sectionName := htmlquery.InnerText(sectionNode) 101 | itemNodes, err := p.getItemsFromSection(sectionNode, i, p.xpathItem) 102 | if err != nil || len(itemNodes) == 0 { 103 | errMsg := "wired section has no items" 104 | logger.Warn(errMsg, zap.Error(err), 105 | zap.String("section", sectionName)) 106 | continue 107 | } 108 | sectionItemsMap[sectionName] = itemNodes 109 | } 110 | return sectionItemsMap, nil 111 | } 112 | 113 | // Section -> Index 114 | // Item -> AwesomeRepo 115 | func (p *Parser) convert(sectionItemsMap map[string][]*html.Node) ( 116 | total int, idxReposMap map[string][]*AwesomeRepo) { 117 | logger := getLogger() 118 | defer logger.Sync() 119 | idxReposMap = make(map[string][]*AwesomeRepo, len(sectionItemsMap)) 120 | 121 | for sectionName, itemNodes := range sectionItemsMap { 122 | repos := make([]*AwesomeRepo, 0) 123 | for _, itemNode := range itemNodes { 124 | repo, err := p.parseItem(itemNode) 125 | if err != nil { 126 | logger.Warn("skip invalid item", zap.Error(err)) 127 | continue 128 | } 129 | repos = append(repos, &AwesomeRepo{ 130 | Repo: *repo, 131 | AwesomeDesc: p.getDesc(itemNode), 132 | }) 133 | total++ 134 | } 135 | idxReposMap[sectionName] = repos 136 | } 137 | return total, idxReposMap 138 | } 139 | 140 | // Fetch repositories from remote. 141 | func (p *Parser) FetchRepos(idxReposMap map[string][]*AwesomeRepo) error { 142 | logger := getLogger() 143 | defer logger.Sync() 144 | 145 | var wg sync.WaitGroup 146 | ctx, cancel := context.WithCancel(context.Background()) 147 | defer cancel() 148 | unacceptedError := make(chan error) 149 | for idx, repos := range idxReposMap { 150 | for cnt, repo := range repos { 151 | wg.Add(1) 152 | go func(repo *AwesomeRepo, idx string, cnt int) { 153 | defer wg.Done() 154 | err := p.client.Fill(ctx, repo) 155 | 156 | if p.reporter != nil { 157 | p.reporter.Done() 158 | } 159 | if err != nil { 160 | if cohttp.IsNetowrkError(err) { 161 | errMsg := "Network error occurs" 162 | err = errcode.Wrap(err, errMsg) 163 | unacceptedError <- err 164 | return 165 | } else if github.IsAbuseError(err) { 166 | errMsg := "The frequency of requests is too high. Check max_concurrent" 167 | err = errcode.Wrap(err, errMsg) 168 | unacceptedError <- err 169 | return 170 | } 171 | // accepted error 172 | errMsg := "failed to fill repository info" 173 | logger.Error(errMsg, zap.Error(err)) 174 | if p.reporter != nil { 175 | p.reporter.InvalidRepo(repo.ID) 176 | } 177 | idxReposMap[idx][cnt] = nil 178 | } 179 | }(repo, idx, cnt) 180 | } 181 | } 182 | jobsCompleted := make(chan struct{}) 183 | go func() { 184 | wg.Wait() 185 | close(jobsCompleted) 186 | }() 187 | select { 188 | case err := <-unacceptedError: 189 | return err 190 | case <-jobsCompleted: 191 | return nil 192 | } 193 | } 194 | 195 | // Remove invalid nil from map. 196 | func (p *Parser) clean(raw map[string][]*AwesomeRepo) map[string][]*AwesomeRepo { 197 | result := make(map[string][]*AwesomeRepo, len(raw)) 198 | 199 | for idx, repos := range raw { 200 | for _, repo := range repos { 201 | if repo != nil { 202 | result[idx] = append(result[idx], repo) 203 | } 204 | } 205 | } 206 | return result 207 | } 208 | 209 | // Get awesome item nodes from awesome section. 210 | func (p *Parser) getItemsFromSection(section *html.Node, idx int, xpath string) ( 211 | []*html.Node, error) { 212 | xpath = fmt.Sprintf(xpath, idx+1) 213 | return htmlquery.QueryAll(section, xpath) 214 | } 215 | 216 | // Get awesome link node from a item node. 217 | func (p *Parser) getLinks(itemNode *html.Node) (*html.Node, error) { 218 | const funcIntent = "get awesome links from a section node" 219 | const funcErrMsg = "failed to " + funcIntent 220 | logger := getLogger() 221 | defer logger.Sync() 222 | logger.Debug(funcIntent) 223 | 224 | return htmlquery.Query(itemNode, "//a") 225 | } 226 | 227 | // Get awesome description from a item node. 228 | func (p *Parser) getDesc(itemNode *html.Node) string { 229 | return htmlquery.InnerText(itemNode) 230 | } 231 | 232 | // Get repository from a item node. 233 | // One item one repo. 234 | func (p *Parser) parseItem(itemNode *html.Node) (*Repo, error) { 235 | const funcIntent = "get repo from a item node" 236 | const funcErrMsg = "failed to " + funcIntent 237 | logger := getLogger() 238 | defer logger.Sync() 239 | logger.Debug(funcIntent) 240 | 241 | linkNode, err := p.getLinks(itemNode) 242 | if err != nil { 243 | const blockErrMsg = "failed to get first link node from item node" 244 | logger.Error(blockErrMsg, zap.Error(err)) 245 | return nil, errcode.Wrap(err, blockErrMsg) 246 | } 247 | 248 | name, link := 249 | htmlquery.InnerText(linkNode), 250 | htmlquery.SelectAttr(linkNode, "href") 251 | if name == "" || link == "" { 252 | logger.Error(funcErrMsg) 253 | return nil, errcode.New(funcErrMsg, ErrCodeContent, 254 | ErrScope, nil) 255 | } 256 | 257 | linkSplit := p.reLink.FindStringSubmatch(link) 258 | if len(linkSplit) != 3 || strings.Contains(linkSplit[2], "/") { 259 | var errMsg string 260 | if strings.Contains(link, p.urlMust) { 261 | errMsg = "strange github repository url" 262 | logger.Warn(errMsg, 263 | zap.String("link", link), 264 | zap.String("name", name)) 265 | } else { 266 | errMsg = "discard unrecognized url" 267 | logger.Info(errMsg, 268 | zap.String("link", link), 269 | zap.String("name", name)) 270 | } 271 | return nil, errcode.New(errMsg, ErrCodeContent, ErrScope, nil) 272 | } 273 | 274 | id := github.RepoID{ 275 | Owner: linkSplit[1], 276 | Name: linkSplit[2], 277 | } 278 | return &Repo{ 279 | ID: id, 280 | AwesomeName: name, 281 | Owner: id.Owner, 282 | Link: link, 283 | }, nil 284 | } 285 | -------------------------------------------------------------------------------- /awg/parser_test.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/rydesun/awesome-github/exch/github" 13 | "github.com/rydesun/awesome-github/lib/cohttp" 14 | "github.com/rydesun/awesome-github/test/fake-github" 15 | ) 16 | 17 | func TestParser_Gather(t *testing.T) { 18 | require := require.New(t) 19 | wd, err := os.Getwd() 20 | require.Nil(err) 21 | testdataDir := filepath.Join(wd, "../test/testdata") 22 | testdataHolder := fakeg.NewDataHolder(testdataDir) 23 | apiTestServer, err := fakeg.ApiServer(testdataHolder) 24 | require.Nil(err) 25 | htmlReadme, err := testdataHolder.GetHtmlAwesomeReadme() 26 | require.Nil(err) 27 | apiClient := cohttp.NewClient(*apiTestServer.Client(), 16, 0, time.Second, 20, nil) 28 | gbClient, err := github.NewClient(nil, apiClient, 29 | github.ClientOption{ 30 | APIHost: apiTestServer.URL, 31 | ApiPathPre: github.APIPathPre, 32 | }) 33 | require.Nil(err) 34 | client, err := NewClient(gbClient) 35 | require.Nil(err) 36 | t.Run("", func(t *testing.T) { 37 | reporter := &Reporter{} 38 | awesomeParser := NewParser(string(htmlReadme), client, reporter, RateLimit{ 39 | Total: 100000, 40 | Remaining: 100000, 41 | }) 42 | awesomeRepos, err := awesomeParser.Gather() 43 | require.Nil(err) 44 | 45 | lastCommitTime, err := time.Parse(time.RFC3339, "2019-07-15T19:40:41Z") 46 | require.Nil(err) 47 | expect := map[string][]*AwesomeRepo{ 48 | "XML": { 49 | &AwesomeRepo{ 50 | Repo: Repo{ 51 | ID: github.RepoID{ 52 | Owner: "antchfx", 53 | Name: "xpath", 54 | }, 55 | Owner: "antchfx", 56 | AwesomeName: "xpath", 57 | Link: "https://github.com/antchfx/xpath", 58 | Watch: 8, 59 | Star: 319, 60 | Fork: 36, 61 | LastCommit: lastCommitTime, 62 | Description: "XPath package for Golang, supports HTML, XML, JSON document query.", 63 | }, 64 | AwesomeDesc: "xpath - XPath package for Go.", 65 | }, 66 | }, 67 | "OpenGL": { 68 | { 69 | Repo: Repo{ 70 | ID: github.RepoID{ 71 | Owner: "technohippy", 72 | Name: "go-glmatrix", 73 | }, 74 | Owner: "technohippy", 75 | AwesomeName: "go-glmatrix", 76 | Link: "https://github.com/technohippy/go-glmatrix", 77 | Watch: 1, 78 | Star: 1, 79 | Fork: 0, 80 | LastCommit: lastCommitTime, 81 | Description: "go-glmatrix is a golang version of glMatrix, which is \"designed to perform vector and matrix operations stupidly fast\".", 82 | }, 83 | AwesomeDesc: `go-glmatrix - Go port of glMatrix library.`, 84 | }, 85 | { 86 | Repo: Repo{ 87 | ID: github.RepoID{ 88 | Owner: "goxjs", 89 | Name: "glfw", 90 | }, 91 | Owner: "goxjs", 92 | AwesomeName: "goxjs/glfw", 93 | Link: "https://github.com/goxjs/glfw", 94 | Watch: 6, 95 | Star: 65, 96 | Fork: 14, 97 | LastCommit: lastCommitTime, 98 | Description: "Go cross-platform glfw library for creating an OpenGL context and receiving events.", 99 | }, 100 | AwesomeDesc: "goxjs/glfw - Go cross-platform glfw library for creating an OpenGL context and receiving events.", 101 | }, 102 | }, 103 | } 104 | require.Equal(expect, awesomeRepos) 105 | require.Equal([]github.RepoID{ 106 | { 107 | Owner: "randominvaliduser", 108 | Name: "repo", 109 | }}, reporter.GetInvalidRepo()) 110 | }) 111 | 112 | testcases := []struct { 113 | invalidReadme string 114 | }{ 115 | {invalidReadme: ""}, 116 | {invalidReadme: "

  • "}, 117 | {invalidReadme: ` 118 |
    119 |
    120 |

    invalid

    121 |
  • invalid
    • `}, 122 | } 123 | for i, tc := range testcases { 124 | t.Run("invalid_readme_"+strconv.Itoa(i), func(t *testing.T) { 125 | reporter := &Reporter{} 126 | awesomeParser := NewParser(tc.invalidReadme, client, reporter, RateLimit{ 127 | Total: 100000, 128 | Remaining: 100000, 129 | }) 130 | _, err = awesomeParser.Gather() 131 | require.NotEqual(nil, err) 132 | }) 133 | } 134 | // Test ratelimit 135 | t.Run("invalid_ratelimit", func(t *testing.T) { 136 | reporter := &Reporter{} 137 | awesomeParser := NewParser(string(htmlReadme), client, reporter, RateLimit{ 138 | Total: 100000, 139 | Remaining: 0, 140 | }) 141 | _, err = awesomeParser.Gather() 142 | require.NotNil(err) 143 | }) 144 | // Test invalid network 145 | invalidGbClient, _ := github.NewClient(nil, apiClient, 146 | github.ClientOption{ 147 | // Invalid network for GitHub API 148 | APIHost: "https://127.127.127.127:12345", 149 | ApiPathPre: github.APIPathPre, 150 | }) 151 | invalidClient, _ := NewClient(invalidGbClient) 152 | t.Run("invalid_network", func(t *testing.T) { 153 | reporter := &Reporter{} 154 | awesomeParser := NewParser(string(htmlReadme), invalidClient, reporter, RateLimit{ 155 | Total: 100000, 156 | Remaining: 100000, 157 | }) 158 | _, err = awesomeParser.Gather() 159 | require.NotNil(err) 160 | require.True(cohttp.IsNetowrkError(err)) 161 | }) 162 | } 163 | -------------------------------------------------------------------------------- /awg/reporter.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | 7 | "github.com/rydesun/awesome-github/exch/github" 8 | ) 9 | 10 | type Reporter struct { 11 | con int64 12 | total int64 13 | finished int64 14 | waiting int64 15 | invalid []github.RepoID 16 | invalidLck sync.Mutex 17 | } 18 | 19 | func (r *Reporter) ConReqNum(num int) { 20 | atomic.StoreInt64(&r.con, int64(num)) 21 | } 22 | 23 | func (r *Reporter) GetConReqNum() int { 24 | return int(atomic.LoadInt64(&r.con)) 25 | } 26 | 27 | func (r *Reporter) TotalRepoNum(num int) { 28 | atomic.StoreInt64(&r.total, int64(num)) 29 | } 30 | 31 | func (r *Reporter) GetTotalRepoNum() int { 32 | return int(atomic.LoadInt64(&r.total)) 33 | } 34 | 35 | func (r *Reporter) Done() { 36 | atomic.AddInt64(&r.finished, 1) 37 | } 38 | 39 | func (r *Reporter) GetFinishedRepoNum() int { 40 | return int(atomic.LoadInt64(&r.finished)) 41 | } 42 | 43 | func (r *Reporter) RepoWaiting() { 44 | atomic.AddInt64(&r.waiting, -1) 45 | } 46 | 47 | func (r *Reporter) GetWaitingRepo() int { 48 | return int(atomic.LoadInt64(&r.waiting)) 49 | } 50 | 51 | func (r *Reporter) InvalidRepo(id github.RepoID) { 52 | r.invalidLck.Lock() 53 | r.invalid = append(r.invalid, id) 54 | r.invalidLck.Unlock() 55 | } 56 | 57 | func (r *Reporter) GetInvalidRepo() []github.RepoID { 58 | return r.invalid 59 | } 60 | -------------------------------------------------------------------------------- /awg/view.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/rydesun/awesome-github/exch/github" 7 | "github.com/rydesun/awesome-github/lib/errcode" 8 | ) 9 | 10 | type Repo struct { 11 | ID github.RepoID `json:"id"` 12 | Owner string `json:"owner"` 13 | AwesomeName string `json:"awesome_name"` 14 | Link string `json:"link"` 15 | Watch int `json:"watch"` 16 | Star int `json:"star"` 17 | Fork int `json:"fork"` 18 | LastCommit time.Time `json:"last_commit"` 19 | Description string `json:"description"` 20 | } 21 | 22 | type AwesomeRepo struct { 23 | Repo 24 | AwesomeDesc string `json:"awesome_description"` 25 | } 26 | 27 | func (r *Repo) Aggregate(repo *github.Repo) error { 28 | commitEdges := repo.Data.Repository.DefaultBranchRef.Target.History.Edges 29 | if len(commitEdges) == 0 { 30 | errMsg := "malformed repo struct" 31 | return errcode.New(errMsg, ErrCodeContent, 32 | ErrScope, []string{"commit"}) 33 | } 34 | r.LastCommit = commitEdges[0].Node.CommittedDate 35 | r.Watch = repo.Data.Repository.Watchers.TotalCount 36 | r.Star = repo.Data.Repository.Stargazers.TotalCount 37 | r.Fork = repo.Data.Repository.Forks.TotalCount 38 | r.Description = repo.Data.Repository.Description 39 | return nil 40 | } 41 | 42 | type RateLimit struct { 43 | Total int 44 | Remaining int 45 | ResetAt time.Time 46 | } 47 | 48 | type User struct { 49 | Name string 50 | RateLimit 51 | } 52 | -------------------------------------------------------------------------------- /awg/view_test.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/rydesun/awesome-github/exch/github" 13 | "github.com/rydesun/awesome-github/test/fake-github" 14 | ) 15 | 16 | func TestRepo_Aggregate(t *testing.T) { 17 | require := require.New(t) 18 | wd, err := os.Getwd() 19 | require.Nil(err) 20 | testdataDir := filepath.Join(wd, "../test/testdata") 21 | testdataHolder := fakeg.NewDataHolder(testdataDir) 22 | raw, err := testdataHolder.GetJsonRepo("goxjs", "glfw") 23 | require.Nil(err) 24 | gbRepo := &github.Repo{} 25 | err = json.Unmarshal(raw, gbRepo) 26 | require.Nil(err) 27 | 28 | repo := Repo{} 29 | err = repo.Aggregate(gbRepo) 30 | require.Nil(err) 31 | 32 | lastCommitTime, err := time.Parse(time.RFC3339, "2019-07-15T19:40:41Z") 33 | require.Nil(err) 34 | require.Equal(Repo{ 35 | LastCommit: lastCommitTime, 36 | Watch: 6, 37 | Star: 65, 38 | Fork: 14, 39 | Description: "Go cross-platform glfw library for creating an OpenGL context and receiving events.", 40 | }, repo) 41 | } 42 | -------------------------------------------------------------------------------- /awg/worker.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "github.com/rydesun/awesome-github/exch/github" 5 | ) 6 | 7 | func Workflow(client *Client, reporter *Reporter, awesomeID github.RepoID, 8 | ratelimit RateLimit) ( 9 | awesomeRepos map[string][]*AwesomeRepo, err error) { 10 | logger := getLogger() 11 | defer logger.Sync() 12 | 13 | readme, err := client.GetHTMLReadme(awesomeID) 14 | if err != nil { 15 | return nil, err 16 | } 17 | readmeParser := NewParser(readme, client, reporter, ratelimit) 18 | awesomeRepos, err = readmeParser.Gather() 19 | if err != nil { 20 | return nil, err 21 | } 22 | return awesomeRepos, nil 23 | } 24 | -------------------------------------------------------------------------------- /awg/worker_test.go: -------------------------------------------------------------------------------- 1 | package awg 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/rydesun/awesome-github/exch/github" 12 | "github.com/rydesun/awesome-github/lib/cohttp" 13 | "github.com/rydesun/awesome-github/test/fake-github" 14 | ) 15 | 16 | func TestWorkflow(t *testing.T) { 17 | require := require.New(t) 18 | wd, err := os.Getwd() 19 | require.Nil(err) 20 | testdataDir := filepath.Join(wd, "../test/testdata") 21 | testdataHolder := fakeg.NewDataHolder(testdataDir) 22 | htmlServer, err := fakeg.HtmlServer(testdataHolder) 23 | require.Nil(err) 24 | apiServer, err := fakeg.ApiServer(testdataHolder) 25 | require.Nil(err) 26 | gbClient, err := github.NewClient( 27 | cohttp.NewClient(*htmlServer.Client(), 16, 0, time.Second, 20, nil), 28 | cohttp.NewClient(*apiServer.Client(), 16, 0, time.Second, 20, nil), 29 | github.ClientOption{ 30 | HTMLHost: htmlServer.URL, 31 | HTMLPathPre: github.HTMLPathPre, 32 | APIHost: apiServer.URL, 33 | ApiPathPre: github.APIPathPre, 34 | }) 35 | require.Nil(err) 36 | client, err := NewClient(gbClient) 37 | require.Nil(err) 38 | result, err := Workflow(client, nil, github.RepoID{Owner: "tester", Name: "awesome-test"}, 39 | RateLimit{ 40 | Total: 100000, 41 | Remaining: 100000, 42 | }) 43 | require.Nil(err) 44 | require.Less(0, len(result)) 45 | 46 | // Test invalid, should have a error. 47 | _, err = Workflow(client, nil, github.RepoID{Owner: "invalid", Name: "invalid"}, 48 | RateLimit{ 49 | Total: 100000, 50 | Remaining: 100000, 51 | }) 52 | require.NotNil(err) 53 | // Test network error 54 | apiServer.Close() 55 | _, err = Workflow(client, nil, github.RepoID{Owner: "tester", Name: "awesome-test"}, 56 | RateLimit{ 57 | Total: 100000, 58 | Remaining: 100000, 59 | }) 60 | require.NotNil(err) 61 | } 62 | -------------------------------------------------------------------------------- /cmd/awg/.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: awg 2 | before: 3 | hooks: 4 | - go mod download 5 | builds: 6 | - goos: 7 | - linux 8 | - darwin 9 | archives: 10 | - replacements: 11 | darwin: Darwin 12 | linux: Linux 13 | 386: i386 14 | amd64: x86_64 15 | checksum: 16 | name_template: 'checksums.txt' 17 | snapshot: 18 | name_template: "nightly" 19 | changelog: 20 | sort: asc 21 | filters: 22 | exclude: 23 | - '^style' 24 | - '^refactor' 25 | - '^test' 26 | - '^docs' 27 | - '^chore' 28 | -------------------------------------------------------------------------------- /cmd/awg/errors.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/rydesun/awesome-github/awg" 7 | "github.com/rydesun/awesome-github/exch/config" 8 | "github.com/rydesun/awesome-github/exch/github" 9 | "github.com/rydesun/awesome-github/lib/cohttp" 10 | "github.com/rydesun/awesome-github/lib/errcode" 11 | ) 12 | 13 | func strerr(err error) string { 14 | errc, ok := err.(errcode.Error) 15 | if !ok { 16 | return err.Error() 17 | } 18 | switch errc.Scope { 19 | case awg.ErrScope: 20 | switch errc.Code { 21 | case awg.ErrCodeRatelimit: 22 | msg := "Fetching %s repositories will cause exceeding GitHub API ratelimit." 23 | return fmt.Sprintf(msg, errc.Objects[0]) 24 | } 25 | case github.ErrScope: 26 | switch errc.Code { 27 | case github.ErrCodeAccessToken: 28 | return "Invalid github personal access token." 29 | } 30 | case config.ErrScope: 31 | switch errc.Code { 32 | case config.ErrCodeParameter: 33 | return fmt.Sprintf("Invalid config: %v", errc.Msg) 34 | } 35 | case cohttp.ErrScope: 36 | switch errc.Code { 37 | case cohttp.ErrCodeNetwork: 38 | msg := "Network error occurs. Check your network connection." 39 | if len(errc.Objects) == 0 { 40 | return msg 41 | } 42 | msg = msg + "\n" + errc.Objects[0] 43 | return msg 44 | } 45 | } 46 | return err.Error() 47 | } 48 | -------------------------------------------------------------------------------- /cmd/awg/logger.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | "gopkg.in/natefinch/lumberjack.v2" 9 | 10 | "github.com/rydesun/awesome-github/awg" 11 | "github.com/rydesun/awesome-github/exch/config" 12 | "github.com/rydesun/awesome-github/exch/github" 13 | "github.com/rydesun/awesome-github/lib/cohttp" 14 | "github.com/rydesun/awesome-github/lib/errcode" 15 | ) 16 | 17 | type LoggerConfig struct { 18 | Level zapcore.Level 19 | Path string 20 | Encoding string 21 | } 22 | 23 | func depressLoggers() { 24 | nop := zap.NewNop() 25 | awg.SetDefaultLogger(nop) 26 | errcode.SetDefaultLogger(nop) 27 | github.SetDefaultLogger(nop) 28 | cohttp.SetDefaultLogger(nop) 29 | } 30 | 31 | func setLoggers(config config.Loggers) (*zap.Logger, error) { 32 | defaultLoggerConfig := getLoggerConfig(config.Main) 33 | httpLoggerConfig := getLoggerConfig(config.Http) 34 | 35 | enc := zapcore.NewJSONEncoder(zapcore.EncoderConfig{ 36 | TimeKey: "T", 37 | LevelKey: "L", 38 | NameKey: "N", 39 | CallerKey: "C", 40 | MessageKey: "M", 41 | StacktraceKey: "S", 42 | LineEnding: zapcore.DefaultLineEnding, 43 | EncodeLevel: zapcore.LowercaseLevelEncoder, 44 | EncodeTime: zapcore.ISO8601TimeEncoder, 45 | EncodeDuration: zapcore.SecondsDurationEncoder, 46 | EncodeCaller: zapcore.ShortCallerEncoder, 47 | }) 48 | 49 | defaultWriter := zapcore.AddSync(&lumberjack.Logger{ 50 | Filename: defaultLoggerConfig.Path, 51 | MaxSize: 20, 52 | MaxBackups: 1, 53 | MaxAge: 28, 54 | }) 55 | defaultLogger := zap.New( 56 | zapcore.NewCore(enc, defaultWriter, zap.NewAtomicLevelAt( 57 | defaultLoggerConfig.Level)), zap.AddCaller()) 58 | 59 | var httpLogger *zap.Logger 60 | if httpLoggerConfig.Path == defaultLoggerConfig.Path { 61 | httpLogger = zap.New( 62 | zapcore.NewCore(enc, defaultWriter, zap.NewAtomicLevelAt( 63 | httpLoggerConfig.Level)), zap.AddCaller()) 64 | } else { 65 | httpWriter := zapcore.AddSync(&lumberjack.Logger{ 66 | Filename: httpLoggerConfig.Path, 67 | MaxSize: 5, 68 | MaxBackups: 1, 69 | MaxAge: 28, 70 | }) 71 | httpLogger = zap.New( 72 | zapcore.NewCore(enc, httpWriter, zap.NewAtomicLevelAt( 73 | httpLoggerConfig.Level)), zap.AddCaller()) 74 | } 75 | 76 | awg.SetDefaultLogger(defaultLogger) 77 | errcode.SetDefaultLogger(defaultLogger) 78 | github.SetDefaultLogger(defaultLogger) 79 | 80 | cohttp.SetDefaultLogger(httpLogger) 81 | return defaultLogger, nil 82 | } 83 | 84 | func getLoggerConfig(config config.Logger) LoggerConfig { 85 | level, ok := map[string]zapcore.Level{ 86 | "debug": zap.DebugLevel, 87 | "info": zap.InfoLevel, 88 | "warn": zap.WarnLevel, 89 | "error": zap.ErrorLevel, 90 | "panic": zap.PanicLevel, 91 | }[strings.ToLower(strings.TrimSpace(config.Level))] 92 | if !ok { 93 | level = zap.InfoLevel 94 | } 95 | return LoggerConfig{ 96 | Level: level, 97 | Path: config.Path, 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /cmd/awg/logger_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/rydesun/awesome-github/exch/config" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSetLoggers(t *testing.T) { 13 | require := require.New(t) 14 | tempFile, err := ioutil.TempFile("", "awg_fake_log_*") 15 | require.Nil(err) 16 | defer func() { 17 | tempFile.Close() 18 | os.Remove(tempFile.Name()) 19 | }() 20 | mainLogger, err := setLoggers(config.Loggers{ 21 | Main: config.Logger{ 22 | Path: tempFile.Name(), 23 | }, 24 | Http: config.Logger{ 25 | Path: tempFile.Name(), 26 | }, 27 | }) 28 | require.Nil(err) 29 | mainLogger.Info("test log") 30 | err = mainLogger.Sync() 31 | require.Nil(err) 32 | } 33 | -------------------------------------------------------------------------------- /cmd/awg/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/urfave/cli/v2" 8 | 9 | "github.com/rydesun/awesome-github/exch/config" 10 | "github.com/rydesun/awesome-github/web/app" 11 | ) 12 | 13 | func main() { 14 | app := &cli.App{ 15 | Name: "awg", 16 | Usage: "Awesome GitHub repositories", 17 | Commands: []*cli.Command{ 18 | { 19 | Name: "fetch", 20 | Usage: "Fetch data about awesome repositories", 21 | Action: fetch, 22 | Flags: []cli.Flag{ 23 | &cli.StringFlag{ 24 | Name: "config", 25 | Aliases: []string{"c"}, 26 | Usage: "YAML config", 27 | Required: true, 28 | TakesFile: true, 29 | }, 30 | }, 31 | }, 32 | { 33 | Name: "view", 34 | Usage: "View awesome readme in browser", 35 | Action: view, 36 | Flags: []cli.Flag{ 37 | &cli.StringFlag{ 38 | Name: "script", 39 | Usage: "Embedded script", 40 | Value: "https://cdn.jsdelivr.net/gh/rydesun/awesome-github/web/static/js/view.js", 41 | }, 42 | &cli.StringFlag{ 43 | Name: "listen", 44 | Usage: "Listen address", 45 | Value: "127.0.0.1:3000", 46 | }, 47 | }, 48 | }, 49 | }, 50 | EnableBashCompletion: true, 51 | } 52 | app.Run(os.Args) 53 | } 54 | 55 | func fetch(c *cli.Context) error { 56 | // CLI 57 | writer := os.Stdout 58 | configPath := c.String("config") 59 | 60 | // Always parse config first. 61 | config, err := parseConfig(configPath) 62 | if err != nil { 63 | fmt.Fprintln(writer, err) 64 | cli.OsExiter(1) 65 | } 66 | 67 | // Set loggers right now. 68 | logger, err := setLoggers(config.Log) 69 | if err != nil { 70 | fmt.Fprintln(writer, err) 71 | cli.OsExiter(1) 72 | } 73 | 74 | // The worker takes over all writers, 75 | // do not write again. 76 | worker := NewWorker(writer, logger) 77 | // Must init first. 78 | err = worker.Init(config) 79 | if err != nil { 80 | cli.OsExiter(1) 81 | } 82 | err = worker.Work() 83 | if err != nil { 84 | cli.OsExiter(1) 85 | } 86 | return nil 87 | } 88 | 89 | func view(c *cli.Context) error { 90 | // CLI 91 | writer := os.Stdout 92 | depressLoggers() 93 | 94 | if c.Args().Len() == 0 { 95 | fmt.Fprintln(writer, "Awesome list name is missing") 96 | cli.OsExiter(1) 97 | } 98 | 99 | datafile := c.Args().Get(0) 100 | data, err := LoadOutputFile(datafile) 101 | if err != nil { 102 | fmt.Fprintln(writer, strerr(err)) 103 | cli.OsExiter(1) 104 | } 105 | if !data.IsValid() { 106 | fmt.Fprintln(writer, "Invalid data") 107 | cli.OsExiter(1) 108 | } 109 | 110 | router, err := app.NewRouter(c.String("listen")) 111 | if err != nil { 112 | fmt.Fprintln(writer, strerr(err)) 113 | cli.OsExiter(1) 114 | } 115 | scriptPath := c.String("script") 116 | dataPath := datafile 117 | fmt.Fprintln(writer, "[1/2] Fetching remote readme page...") 118 | err = router.Init(data.AwesomeList, scriptPath, dataPath) 119 | if err != nil { 120 | fmt.Fprintln(writer, strerr(err)) 121 | cli.OsExiter(1) 122 | } 123 | fmt.Fprintf(writer, "[2/2] Serve at http://%s\n", c.String("listen")) 124 | err = router.Route() 125 | if err != nil { 126 | fmt.Fprintln(writer, strerr(err)) 127 | cli.OsExiter(1) 128 | } 129 | return nil 130 | } 131 | 132 | func parseConfig(configPath string) (config.Config, error) { 133 | yamlParser, err := config.NewYAMLParser(configPath) 134 | if err != nil { 135 | err = fmt.Errorf("Failed to parse config files. %v", err) 136 | return config.Config{}, err 137 | } 138 | 139 | conf, err := config.GetConfig(yamlParser) 140 | if err != nil { 141 | err = fmt.Errorf("Failed to parse config files. %v", strerr(err)) 142 | return config.Config{}, err 143 | } 144 | return conf, nil 145 | } 146 | -------------------------------------------------------------------------------- /cmd/awg/view.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "time" 7 | 8 | "github.com/rydesun/awesome-github/awg" 9 | "github.com/rydesun/awesome-github/exch/github" 10 | ) 11 | 12 | type Output struct { 13 | Time time.Time `json:"time"` 14 | AwesomeList github.RepoID `json:"awesome_list"` 15 | Data map[string][]*awg.AwesomeRepo `json:"data"` 16 | Invalid []github.RepoID `json:"invalid"` 17 | } 18 | 19 | func LoadOutputFile(path string) (Output, error) { 20 | raw, err := ioutil.ReadFile(path) 21 | if err != nil { 22 | return Output{}, err 23 | } 24 | data := Output{} 25 | err = json.Unmarshal(raw, &data) 26 | return data, err 27 | } 28 | 29 | func (o *Output) IsValid() bool { 30 | return len(o.Data) > 0 && 31 | !o.Time.IsZero() && 32 | len(o.AwesomeList.Name) > 0 && 33 | len(o.AwesomeList.Owner) > 0 34 | } 35 | -------------------------------------------------------------------------------- /cmd/awg/view_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestOutput(t *testing.T) { 10 | require := require.New(t) 11 | datafile := "../../test/testdata/output/data.json" 12 | data, err := LoadOutputFile(datafile) 13 | require.Nil(err) 14 | require.True(data.IsValid()) 15 | } 16 | -------------------------------------------------------------------------------- /cmd/awg/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/k0kubun/go-ansi" 12 | "github.com/schollz/progressbar/v3" 13 | "go.uber.org/zap" 14 | "golang.org/x/net/context" 15 | 16 | "github.com/rydesun/awesome-github/awg" 17 | "github.com/rydesun/awesome-github/exch/config" 18 | "github.com/rydesun/awesome-github/exch/github" 19 | "github.com/rydesun/awesome-github/lib/cohttp" 20 | ) 21 | 22 | type Worker struct { 23 | repoID github.RepoID 24 | outputPath string 25 | writer io.Writer 26 | reporter *awg.Reporter 27 | awgClient *awg.Client 28 | logger *zap.Logger 29 | 30 | // CLI settings 31 | disableProgressBar bool 32 | } 33 | 34 | func NewWorker(writer io.Writer, logger *zap.Logger) *Worker { 35 | return &Worker{ 36 | writer: writer, 37 | logger: logger, 38 | } 39 | } 40 | 41 | func (w *Worker) Init(config config.Config) error { 42 | writer := w.writer 43 | 44 | // Introdution. 45 | fmt.Fprintln(writer, "=== Awesome GitHub ===") 46 | 47 | // Show config. 48 | fmt.Fprintf(writer, "config file: %s\n", config.ConfigPath) 49 | fmt.Fprintf(writer, "main log files: %s\n", config.Log.Main.Path) 50 | fmt.Fprintf(writer, "http log files: %s\n", config.Log.Http.Path) 51 | fmt.Fprintf(writer, "output file: %s\n", config.Output.Path) 52 | 53 | // Create awg client. 54 | w.reporter = w.newReporter() 55 | awgClient, err := w.newAwgClient(config) 56 | if err != nil { 57 | fmt.Fprintln(writer, err) 58 | return err 59 | } 60 | 61 | w.repoID = config.ID 62 | w.outputPath = config.Output.Path 63 | w.awgClient = awgClient 64 | w.disableProgressBar = config.Cli.DisableProgressBar 65 | return nil 66 | } 67 | 68 | func (w *Worker) Work() error { 69 | writer := w.writer 70 | logger := w.logger 71 | defer logger.Sync() 72 | 73 | // Check access token. 74 | fmt.Fprintf(w.writer, "[1/3] Checking github access token...\n") 75 | user, err := w.awgClient.GetUser() 76 | if err != nil { 77 | errMsg := "Failed to get information about access token." 78 | fmt.Fprintln(w.writer, errMsg, strerr(err)) 79 | logger.Error(errMsg, zap.Error(err)) 80 | return err 81 | } 82 | fmt.Fprintf(w.writer, "Use user(%s) access token.\n", user.Name) 83 | fmt.Fprintf(w.writer, "RateLimit: total %d, remaining %d, reset at %s\n", 84 | user.RateLimit.Total, user.RateLimit.Remaining, user.RateLimit.ResetAt) 85 | 86 | fmt.Fprintln(writer, "[2/3] Fetch and parse awesome README.md...") 87 | 88 | // Progress bar. 89 | var pbCompleted <-chan struct{} 90 | ctx, cancel := context.WithCancel(context.Background()) 91 | defer cancel() 92 | if !w.disableProgressBar { 93 | pbCompleted = w.progressBar(ctx, "[3/3] Fetch repositories from github...") 94 | } else { 95 | fmt.Fprintln(writer, "[3/3] Fetch repositories from github...") 96 | } 97 | 98 | // Actual work. 99 | awesomeRepos, err := awg.Workflow(w.awgClient, w.reporter, w.repoID, user.RateLimit) 100 | if err != nil { 101 | errMsg := "\nFailed to fetch some repositories." 102 | fmt.Fprintln(writer, errMsg, strerr(err)) 103 | logger.Error(errMsg, zap.Error(err)) 104 | return err 105 | } 106 | if !w.disableProgressBar { 107 | logger.Info("Wait for the progress bar to complete.") 108 | <-pbCompleted 109 | logger.Info("Progress bar finished.") 110 | } 111 | invalidRepos := w.reporter.GetInvalidRepo() 112 | 113 | // Format data. 114 | output := Output{ 115 | Time: time.Now(), 116 | AwesomeList: w.repoID, 117 | Data: awesomeRepos, 118 | Invalid: invalidRepos, 119 | } 120 | outputBytes, err := json.Marshal(output) 121 | if err != nil { 122 | errMsg := "Failed to format data." 123 | fmt.Fprintln(writer, errMsg, strerr(err)) 124 | logger.Error(errMsg, zap.Error(err)) 125 | return err 126 | } 127 | 128 | // Output data. 129 | if len(w.outputPath) != 0 { 130 | err := ioutil.WriteFile(w.outputPath, outputBytes, 0644) 131 | if err != nil { 132 | errMsg := "Failed to output data." 133 | fmt.Fprintln(writer, errMsg, strerr(err)) 134 | logger.Error(errMsg, zap.Error(err)) 135 | } 136 | } else { 137 | fmt.Fprintln(writer, string(outputBytes)) 138 | } 139 | 140 | // Warning message. 141 | if len(invalidRepos) > 0 { 142 | fmt.Fprintf(writer, "\nCatch some invalid repositories: %v\n", invalidRepos) 143 | } 144 | 145 | // The last message. 146 | if len(w.outputPath) > 0 { 147 | fmt.Fprintf(writer, "Done. Output file: %s\n", w.outputPath) 148 | } else { 149 | fmt.Fprintln(writer, "Done.") 150 | } 151 | return nil 152 | } 153 | 154 | func (w *Worker) newReporter() *awg.Reporter { 155 | return &awg.Reporter{} 156 | } 157 | 158 | func (w *Worker) newAwgClient(config config.Config) (*awg.Client, error) { 159 | logger := w.logger 160 | defer logger.Sync() 161 | 162 | transport := http.DefaultTransport.(*http.Transport).Clone() 163 | transport.ResponseHeaderTimeout = 20 * time.Second 164 | htmlClient := cohttp.NewClient(http.Client{Transport: transport}, 165 | config.MaxConcurrent, config.Network.RetryTime, 166 | config.Network.RetryInterval, config.LogRespHead, nil) 167 | apiClient := cohttp.NewClient(http.Client{Transport: transport}, 168 | config.MaxConcurrent, config.Network.RetryTime, 169 | config.Network.RetryInterval, config.LogRespHead, w.reporter) 170 | 171 | options := github.NewDefaultClientOption() 172 | options.AccessToken = config.AccessToken 173 | if len(config.Github.HTMLHost) > 0 { 174 | options.HTMLHost = config.Github.HTMLHost 175 | } 176 | if len(config.Github.ApiHost) > 0 { 177 | options.APIHost = config.Github.ApiHost 178 | } 179 | 180 | gc, err := github.NewClient(htmlClient, apiClient, options) 181 | if err != nil { 182 | errMsg := "Failed to create github client." 183 | fmt.Fprintln(w.writer, errMsg) 184 | logger.Error(errMsg, zap.Error(err)) 185 | return nil, err 186 | } 187 | client, err := awg.NewClient(gc) 188 | if err != nil { 189 | errMsg := "Failed to create awg client." 190 | fmt.Fprintln(w.writer, errMsg) 191 | logger.Error(errMsg, zap.Error(err)) 192 | return nil, err 193 | } 194 | return client, nil 195 | } 196 | 197 | func (w *Worker) progressBar(ctx context.Context, prefix string) <-chan struct{} { 198 | pbCompleted := make(chan struct{}) 199 | defer close(pbCompleted) 200 | 201 | // TODO: refactor later 202 | getTotalNum := func() (numTotal int, canceled bool) { 203 | ticker := time.NewTicker(time.Second) 204 | for { 205 | select { 206 | case <-ticker.C: 207 | numTotal = w.reporter.GetTotalRepoNum() 208 | if numTotal > 0 { 209 | return numTotal, false 210 | } 211 | case <-ctx.Done(): 212 | return 0, true 213 | } 214 | } 215 | } 216 | go func() { 217 | numTotal, canceled := getTotalNum() 218 | if canceled { 219 | return 220 | } 221 | bar := progressbar.NewOptions(numTotal, 222 | progressbar.OptionSetWriter(ansi.NewAnsiStdout()), 223 | progressbar.OptionEnableColorCodes(true), 224 | progressbar.OptionSetWidth(15), 225 | progressbar.OptionSetDescription(prefix), 226 | progressbar.OptionShowIts(), 227 | progressbar.OptionShowCount(), 228 | ) 229 | ticker := time.NewTicker(time.Second) 230 | for { 231 | select { 232 | case <-ctx.Done(): 233 | return 234 | case <-ticker.C: 235 | numCompleted := w.reporter.GetFinishedRepoNum() 236 | if numCompleted >= numTotal { 237 | bar.Finish() 238 | return 239 | } 240 | bar.Set(numCompleted) 241 | } 242 | } 243 | }() 244 | return pbCompleted 245 | } 246 | -------------------------------------------------------------------------------- /cmd/awg/worker_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http/httptest" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | "go.uber.org/zap" 13 | 14 | "github.com/rydesun/awesome-github/exch/config" 15 | "github.com/rydesun/awesome-github/exch/github" 16 | "github.com/rydesun/awesome-github/test/fake-github" 17 | ) 18 | 19 | type TestEnv struct { 20 | testdataHolder fakeg.DataHolder 21 | htmlTestServer *httptest.Server 22 | apiTestServer *httptest.Server 23 | } 24 | 25 | func (t *TestEnv) Setup() error { 26 | wd, err := os.Getwd() 27 | if err != nil { 28 | return err 29 | } 30 | testdataDir := filepath.Join(wd, "../../test/testdata") 31 | t.testdataHolder = fakeg.NewDataHolder(testdataDir) 32 | 33 | htmlTestServer, err := fakeg.HtmlServer(t.testdataHolder) 34 | if err != nil { 35 | return err 36 | } 37 | apiTestServer, err := fakeg.ApiServer(t.testdataHolder) 38 | if err != nil { 39 | return err 40 | } 41 | t.htmlTestServer = htmlTestServer 42 | t.apiTestServer = apiTestServer 43 | return nil 44 | } 45 | 46 | func TestWorker_work(t *testing.T) { 47 | require := require.New(t) 48 | testEnv := TestEnv{} 49 | err := testEnv.Setup() 50 | require.Nil(err) 51 | 52 | testCases := []struct { 53 | config config.Config 54 | hasErr bool 55 | }{ 56 | { 57 | config: config.Config{ 58 | AccessToken: "123456", 59 | StartPoint: config.StartPoint{ 60 | ID: github.RepoID{ 61 | Owner: "tester", 62 | Name: "awesome-test", 63 | }, 64 | }, 65 | Github: config.Github{ 66 | HTMLHost: testEnv.htmlTestServer.URL, 67 | ApiHost: testEnv.apiTestServer.URL, 68 | }, 69 | Cli: config.Cli{ 70 | DisableProgressBar: true, 71 | }, 72 | }, 73 | hasErr: false, 74 | }, 75 | { 76 | config: config.Config{ 77 | AccessToken: "123456", 78 | StartPoint: config.StartPoint{ 79 | ID: github.RepoID{ 80 | Owner: "tester", 81 | Name: "awesome-test", 82 | }, 83 | }, 84 | Github: config.Github{ 85 | HTMLHost: testEnv.htmlTestServer.URL, 86 | ApiHost: testEnv.apiTestServer.URL, 87 | }, 88 | Cli: config.Cli{ 89 | // Enable progress bar 90 | DisableProgressBar: false, 91 | }, 92 | }, 93 | hasErr: false, 94 | }, 95 | { 96 | config: config.Config{ 97 | // Invalid 98 | AccessToken: "invalid", 99 | StartPoint: config.StartPoint{ 100 | ID: github.RepoID{ 101 | Owner: "tester", 102 | Name: "awesome-test", 103 | }, 104 | }, 105 | Network: config.Net{ 106 | RetryTime: 2, 107 | RetryInterval: time.Second, 108 | }, 109 | Github: config.Github{ 110 | HTMLHost: testEnv.htmlTestServer.URL, 111 | ApiHost: testEnv.apiTestServer.URL, 112 | }, 113 | Cli: config.Cli{ 114 | DisableProgressBar: true, 115 | }, 116 | }, 117 | hasErr: true, 118 | }, 119 | { 120 | config: config.Config{ 121 | AccessToken: "123456", 122 | StartPoint: config.StartPoint{ 123 | // Invalid 124 | ID: github.RepoID{ 125 | Owner: "invalid", 126 | Name: "invalid", 127 | }, 128 | }, 129 | Github: config.Github{ 130 | HTMLHost: testEnv.htmlTestServer.URL, 131 | ApiHost: testEnv.apiTestServer.URL, 132 | }, 133 | Cli: config.Cli{ 134 | DisableProgressBar: true, 135 | }, 136 | }, 137 | hasErr: true, 138 | }, 139 | { 140 | config: config.Config{ 141 | AccessToken: "123456", 142 | StartPoint: config.StartPoint{ 143 | ID: github.RepoID{ 144 | Owner: "tester", 145 | Name: "awesome-test", 146 | }, 147 | }, 148 | // Invalid 149 | Github: config.Github{ 150 | HTMLHost: "invalid", 151 | ApiHost: "invalid", 152 | }, 153 | Cli: config.Cli{ 154 | DisableProgressBar: true, 155 | }, 156 | }, 157 | hasErr: true, 158 | }, 159 | } 160 | 161 | for _, tc := range testCases { 162 | t.Run("NONAME", func(*testing.T) { 163 | worker := NewWorker(ioutil.Discard, zap.NewNop()) 164 | err = worker.Init(tc.config) 165 | require.Nil(err) 166 | err = worker.Work() 167 | if tc.hasErr { 168 | require.NotNil(err) 169 | } else { 170 | require.Nil(err) 171 | } 172 | }) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /configs/config.yaml: -------------------------------------------------------------------------------- 1 | ## Your GitHub personal access token, used to request GitHub API. 2 | ## Prefer ${GITHUB_ACCESS_TOKEN} env var. 3 | access_token: 4 | ## Max concurrent requests to GitHub API. 5 | max_concurrent: 3 6 | ## Crawler start point. 7 | start_point: 8 | ## Awesome-XXX repostiory path, can be found in URL. 9 | path: avelino/awesome-go 10 | 11 | network: 12 | ## Retry request if network error occurs. 13 | retry_time: 2 14 | retry_interval: 1s 15 | 16 | ## Output final JSON data. 17 | output: 18 | ## default: stdout 19 | path: ./awg.json 20 | 21 | ## Command line interface 22 | cli: 23 | disable_progress_bar: false 24 | 25 | ## GitHub server settings. 26 | github: 27 | # html_host: "" 28 | # api_host: "" 29 | 30 | log: 31 | ## Main log. 32 | main: 33 | path: /tmp/awesome-github.log 34 | 35 | ## HTTP log. 36 | http: 37 | path: /tmp/awesome-github.log 38 | -------------------------------------------------------------------------------- /exch/config/const.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | ErrScope = "config" 5 | 6 | ErrCodeParameter = 1 7 | ) 8 | -------------------------------------------------------------------------------- /exch/config/model.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/go-playground/validator/v10" 8 | 9 | "github.com/rydesun/awesome-github/exch/github" 10 | "github.com/rydesun/awesome-github/lib/errcode" 11 | ) 12 | 13 | type Config struct { 14 | ConfigPath string `yaml:"config"` 15 | AccessToken string `yaml:"access_token" validate:"required"` 16 | MaxConcurrent int `yaml:"max_concurrent"` 17 | LogRespHead int `yaml:"log_resp_head"` 18 | StartPoint `yaml:"start_point"` 19 | Network Net `yaml:"network"` 20 | Output Output `yaml:"output"` 21 | Github Github `yaml:"github"` 22 | Cli Cli `yaml:"cli"` 23 | Log Loggers `yaml:"log"` 24 | } 25 | 26 | type StartPoint struct { 27 | Path string `yaml:"path" validate:"required"` 28 | ID github.RepoID 29 | SectionFilter []string 30 | } 31 | 32 | type Net struct { 33 | RetryTime int `yaml:"retry_time"` 34 | RetryInterval time.Duration `yaml:"retry_interval"` 35 | } 36 | 37 | type Output struct { 38 | Path string `yaml:"path"` 39 | } 40 | 41 | func NewProtectedConfig(config Config) Config { 42 | config.AccessToken = "" 43 | return config 44 | } 45 | 46 | type Github struct { 47 | HTMLHost string `yaml:"html_host"` 48 | ApiHost string `yaml:"api_host"` 49 | } 50 | 51 | type Cli struct { 52 | DisableProgressBar bool `yaml:"disable_progress_bar"` 53 | } 54 | 55 | type Loggers struct { 56 | Main Logger `yaml:"main"` 57 | Http Logger `yaml:"http"` 58 | } 59 | 60 | type Logger struct { 61 | Level string `yaml:"level"` 62 | Path string `yaml:"path"` 63 | Console bool `yaml:"console"` 64 | } 65 | 66 | func (c *Config) Validate() error { 67 | validate := validator.New() 68 | err := validate.Struct(c) 69 | if err != nil { 70 | return err 71 | } 72 | if len(c.Github.HTMLHost) > 0 { 73 | err := validate.Var(c.Github.HTMLHost, "url") 74 | if err != nil { 75 | errMsg := "Invalid github html host" 76 | return errcode.New(errMsg, ErrCodeParameter, ErrScope, 77 | []string{"githubHTMLHost"}) 78 | } 79 | } 80 | if len(c.Github.ApiHost) > 0 { 81 | err := validate.Var(c.Github.ApiHost, "url") 82 | if err != nil { 83 | errMsg := "Invalid github api host" 84 | return errcode.New(errMsg, ErrCodeParameter, ErrScope, 85 | []string{"githubAPIHost"}) 86 | } 87 | } 88 | return nil 89 | } 90 | 91 | func SplitID(id string) (owner, name string, err error) { 92 | sliceStr := strings.Split(id, "/") 93 | if len(sliceStr) != 2 { 94 | return "", "", errcode.New("Invaild path", 95 | ErrCodeParameter, ErrScope, []string{"path"}) 96 | } 97 | return sliceStr[0], sliceStr[1], nil 98 | } 99 | -------------------------------------------------------------------------------- /exch/config/model_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestConfig_Validate(t *testing.T) { 8 | config := Config{ 9 | AccessToken: "123456", 10 | StartPoint: StartPoint{ 11 | Path: "tester/awesome-test", 12 | }, 13 | Github: Github{ 14 | HTMLHost: "https://github.com", 15 | ApiHost: "https://api.github.com", 16 | }, 17 | } 18 | 19 | err := config.Validate() 20 | if err != nil { 21 | t.Errorf("expect no error, but got a error: %v", err) 22 | } 23 | 24 | config.AccessToken = "" 25 | err = config.Validate() 26 | if err == nil { 27 | t.Errorf("expect a error") 28 | } 29 | config.Github.ApiHost = "invalid" 30 | err = config.Validate() 31 | if err == nil { 32 | t.Errorf("expect a error") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /exch/config/parser.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "golang.org/x/sys/unix" 9 | 10 | "github.com/rydesun/awesome-github/lib/errcode" 11 | ) 12 | 13 | type ConfigParser interface { 14 | Parse() (Config, error) 15 | } 16 | 17 | func GetConfig(cp ConfigParser) (Config, error) { 18 | config, err := cp.Parse() 19 | if err != nil { 20 | return Config{}, err 21 | } 22 | accessToken := os.Getenv("GITHUB_ACCESS_TOKEN") 23 | if len(accessToken) > 0 { 24 | config.AccessToken = accessToken 25 | } 26 | config.Output.Path, err = filepath.Abs(config.Output.Path) 27 | if err != nil { 28 | return Config{}, err 29 | } 30 | if !CheckFileWritable(config.Output.Path) { 31 | errMsg := fmt.Sprintf("Failed to Write file. Invalid output path: %s", config.Output.Path) 32 | return Config{}, errcode.New(errMsg, ErrCodeParameter, ErrScope, nil) 33 | } 34 | config.Log.Main.Path, err = filepath.Abs(config.Log.Main.Path) 35 | if err != nil { 36 | return Config{}, err 37 | } 38 | if !CheckFileWritable(config.Log.Main.Path) { 39 | errMsg := fmt.Sprintf("Failed to Write file. Invalid output path: %s", config.Log.Main.Path) 40 | return Config{}, errcode.New(errMsg, 41 | ErrCodeParameter, ErrScope, nil) 42 | } 43 | config.Log.Http.Path, err = filepath.Abs(config.Log.Http.Path) 44 | if err != nil { 45 | return Config{}, err 46 | } 47 | if !CheckFileWritable(config.Log.Http.Path) { 48 | errMsg := fmt.Sprintf("Failed to Write file. Invalid output path: %s", config.Log.Http.Path) 49 | return Config{}, errcode.New(errMsg, 50 | ErrCodeParameter, ErrScope, nil) 51 | } 52 | err = config.Validate() 53 | return config, err 54 | } 55 | 56 | // Check this file path is writable. 57 | // Will create directories first. 58 | func CheckFileWritable(path string) bool { 59 | dir := filepath.Dir(path) 60 | err := os.MkdirAll(dir, 0755) 61 | if err != nil { 62 | return false 63 | } 64 | return unix.Access(dir, unix.W_OK) == nil 65 | } 66 | -------------------------------------------------------------------------------- /exch/config/parser_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/rydesun/awesome-github/exch/github" 14 | ) 15 | 16 | const testAccessToken = "123456" 17 | 18 | func TestMain(m *testing.M) { 19 | oldAccessToken := os.Getenv("GITHUB_ACCESS_TOKEN") 20 | err := os.Setenv("GITHUB_ACCESS_TOKEN", testAccessToken) 21 | if err != nil { 22 | log.Fatalln(err) 23 | } 24 | code := m.Run() 25 | os.Setenv("GITHUB_ACCESS_TOKEN", oldAccessToken) 26 | os.Exit(code) 27 | } 28 | 29 | func TestGetConfig(t *testing.T) { 30 | require := require.New(t) 31 | 32 | path := "../../configs/config.yaml" 33 | yamlParser, err := NewYAMLParser(path) 34 | require.Nil(err) 35 | actual, err := GetConfig(yamlParser) 36 | require.Nil(err) 37 | 38 | raw, _ := json.Marshal(actual) 39 | t.Logf("get config: %s", raw) 40 | 41 | expected := Config{ 42 | ConfigPath: path, 43 | AccessToken: testAccessToken, 44 | MaxConcurrent: 3, 45 | StartPoint: StartPoint{ 46 | Path: "avelino/awesome-go", 47 | ID: github.RepoID{ 48 | Owner: "avelino", 49 | Name: "awesome-go", 50 | }, 51 | }, 52 | Network: Net{ 53 | RetryTime: 2, 54 | RetryInterval: time.Second, 55 | }, 56 | Output: Output{ 57 | Path: "awg.json", 58 | }, 59 | Log: Loggers{ 60 | Main: Logger{ 61 | Path: "/tmp/awesome-github.log", 62 | }, 63 | Http: Logger{ 64 | Path: "/tmp/awesome-github.log", 65 | }, 66 | }, 67 | } 68 | // Check relative path 69 | if strings.HasSuffix(actual.ConfigPath, "config.yaml") { 70 | expected.ConfigPath = actual.ConfigPath 71 | } 72 | if strings.HasSuffix(actual.Output.Path, expected.Output.Path) { 73 | expected.Output.Path = actual.Output.Path 74 | } 75 | require.Equal(expected, actual) 76 | 77 | // Invalid config path 78 | _, err = NewYAMLParser("") 79 | require.NotNil(err) 80 | _, err = NewYAMLParser("a/b/c/d") 81 | require.NotNil(err) 82 | } 83 | -------------------------------------------------------------------------------- /exch/config/yaml.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | 8 | "gopkg.in/yaml.v2" 9 | 10 | "github.com/rydesun/awesome-github/lib/errcode" 11 | ) 12 | 13 | type YAMLParser struct { 14 | fpath string 15 | } 16 | 17 | func NewYAMLParser(fpath string) (*YAMLParser, error) { 18 | if len(fpath) == 0 { 19 | return nil, errcode.New("Invalid config path", 20 | ErrCodeParameter, ErrScope, nil) 21 | } 22 | fpath, err := filepath.Abs(fpath) 23 | if err != nil { 24 | return nil, err 25 | } 26 | _, err = os.Stat(fpath) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return &YAMLParser{ 31 | fpath: fpath, 32 | }, nil 33 | } 34 | 35 | func (p *YAMLParser) Parse() (Config, error) { 36 | config := Config{} 37 | raw, err := ioutil.ReadFile(p.fpath) 38 | if err != nil { 39 | return config, err 40 | } 41 | err = yaml.UnmarshalStrict(raw, &config) 42 | if err != nil { 43 | return config, err 44 | } 45 | config.ConfigPath = p.fpath 46 | owner, name, err := SplitID(config.StartPoint.Path) 47 | if err != nil { 48 | return config, err 49 | } 50 | config.StartPoint.ID.Owner = owner 51 | config.StartPoint.ID.Name = name 52 | return config, nil 53 | } 54 | -------------------------------------------------------------------------------- /exch/github/client.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "path/filepath" 10 | 11 | "go.uber.org/zap" 12 | 13 | "github.com/rydesun/awesome-github/lib/cohttp" 14 | "github.com/rydesun/awesome-github/lib/errcode" 15 | ) 16 | 17 | type HTTPClient interface { 18 | Json(req *http.Request, resp interface{}) (err error) 19 | Text(req *http.Request) (text string, err error) 20 | } 21 | 22 | type ClientOption struct { 23 | HTMLHost string // GitHub HTML host 24 | HTMLPathPre string // GitHub HTML path prefix 25 | APIHost string // GitHub API host 26 | ApiPathPre string // GitHub API path prefix 27 | AccessToken string // GitHub personal access token 28 | } 29 | 30 | type Client struct { 31 | option ClientOption 32 | htmlClient HTTPClient 33 | apiClient HTTPClient 34 | htmlHost url.URL 35 | apiHost url.URL 36 | bearer string 37 | } 38 | 39 | func NewClient(htmlClient HTTPClient, apiClient HTTPClient, option ClientOption) (*Client, error) { 40 | const funcIntent = "create new github client" 41 | const funcErrMsg = "failed to " + funcIntent 42 | logger := getLogger() 43 | defer logger.Sync() 44 | logger.Debug(funcIntent, zap.Any("option", option)) 45 | 46 | htmlHost, err := url.Parse(option.HTMLHost) 47 | if err != nil { 48 | const errMsg = "failed to parse html host" 49 | logger.Error(errMsg, zap.Error(err), 50 | zap.String("url", option.HTMLHost)) 51 | err = errcode.New(errMsg, ErrCodeClientOption, 52 | ErrScope, []string{"htmlHost"}) 53 | return nil, err 54 | } 55 | apiHost, err := url.Parse(option.APIHost) 56 | if err != nil { 57 | const errMsg = "failed to parse api host" 58 | logger.Error(errMsg, zap.Error(err), 59 | zap.String("url", option.APIHost)) 60 | err = errcode.New(errMsg, ErrCodeClientOption, 61 | ErrScope, []string{"apiHost"}) 62 | return nil, err 63 | } 64 | return &Client{ 65 | option: option, 66 | htmlClient: htmlClient, 67 | apiClient: apiClient, 68 | htmlHost: *htmlHost, 69 | apiHost: *apiHost, 70 | bearer: "Bearer " + option.AccessToken, 71 | }, nil 72 | } 73 | 74 | // Get current user information. 75 | func (c *Client) GetUser() (*User, error) { 76 | const funcIntent = "get current user info" 77 | const funcErrMsg = "failed to " + funcIntent 78 | logger := getLogger() 79 | defer logger.Sync() 80 | 81 | logger.Debug(funcIntent) 82 | 83 | url := c.apiHost 84 | url.Path = c.option.ApiPathPre 85 | req, err := http.NewRequest(http.MethodPost, url.String(), 86 | bytes.NewBufferString(QueryUser)) 87 | if err != nil { 88 | errMsg := "failed to create request" 89 | logger.Error(funcErrMsg, zap.Error(err)) 90 | err = errcode.New(errMsg, ErrCodeMakeRequest, 91 | ErrScope, nil) 92 | return nil, err 93 | } 94 | req.Header.Set("Authorization", c.bearer) 95 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 96 | 97 | user := &User{} 98 | err = c.apiClient.Json(req, user) 99 | if err != nil { 100 | logger.Error(funcErrMsg, zap.Error(err)) 101 | err, ok := err.(errcode.Error) 102 | if ok && err.Scope == cohttp.ErrScope && err.Code == 401 { 103 | errMsg := "Invalid access token" 104 | return nil, errcode.New(errMsg, 105 | ErrCodeAccessToken, ErrScope, nil) 106 | } 107 | return nil, errcode.Wrap(err, funcErrMsg) 108 | } 109 | if len(user.Errors) > 0 { 110 | errMsg := "remote server return errors" 111 | logger.Error(errMsg, zap.Any("serverErrors", user.Errors)) 112 | err = errcode.New(errMsg, errcode.CodeUnknown, ErrScope, nil) 113 | return nil, err 114 | } 115 | return user, nil 116 | } 117 | 118 | // Get repository information. 119 | func (c Client) GetRepo(ctx context.Context, id RepoID) (*Repo, error) { 120 | const funcIntent = "get repo info" 121 | const funcErrMsg = "failed to " + funcIntent 122 | logger := getLogger() 123 | defer logger.Sync() 124 | 125 | idStr := id.String() 126 | logger.Debug(funcIntent, zap.String("repo", idStr)) 127 | 128 | url := c.apiHost 129 | url.Path = c.option.ApiPathPre 130 | 131 | query := fmt.Sprintf(QueryRepo, id.Owner, id.Name) 132 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, url.String(), 133 | bytes.NewBufferString(query)) 134 | if err != nil { 135 | errMsg := "failed to create request" 136 | logger.Error(funcErrMsg, zap.Error(err), 137 | zap.String("repo", idStr)) 138 | err = errcode.New(errMsg, ErrCodeMakeRequest, 139 | ErrScope, nil) 140 | return nil, err 141 | } 142 | req.Header.Set("Authorization", c.bearer) 143 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 144 | 145 | repo := &Repo{} 146 | err = c.apiClient.Json(req, repo) 147 | if err != nil { 148 | logger.Error(funcErrMsg, zap.Error(err), 149 | zap.String("repo", idStr)) 150 | err = errcode.Wrap(err, funcErrMsg) 151 | return nil, err 152 | } 153 | if len(repo.Errors) > 0 { 154 | errMsg := "remote server return errors" 155 | logger.Error(errMsg, zap.String("repo", idStr), 156 | zap.Any("serverErrors", repo.Errors)) 157 | err = errcode.New(errMsg, errcode.CodeUnknown, ErrScope, nil) 158 | return nil, err 159 | } 160 | return repo, nil 161 | } 162 | 163 | // Get README.md wrapped in html page. 164 | func (c Client) GetHTMLReadme(id RepoID) (string, error) { 165 | const funcIntent = "get repo readme wrapped in html page" 166 | const funcErrMsg = "failed to " + funcIntent 167 | logger := getLogger() 168 | defer logger.Sync() 169 | 170 | idStr := id.String() 171 | logger.Debug(funcIntent, zap.String("repo", idStr)) 172 | 173 | readme, err := c.GetHTML(id, "README.md") 174 | if err != nil { 175 | logger.Error(funcErrMsg, zap.Error(err), 176 | zap.String("repo", idStr)) 177 | err = errcode.Wrap(err, funcErrMsg) 178 | return "", err 179 | } 180 | return readme, nil 181 | } 182 | 183 | // Get file wrapped in html page. 184 | func (c *Client) GetHTML(id RepoID, path string) (string, error) { 185 | const funcIntent = "get file wrapped in html page" 186 | const funcErrMsg = "failed to " + funcIntent 187 | logger := getLogger() 188 | defer logger.Sync() 189 | 190 | idStr := id.String() 191 | logger.Debug(funcIntent, 192 | zap.String("repo", idStr), 193 | zap.String("path", path)) 194 | 195 | url := c.htmlHost 196 | url.Path = filepath.Join(idStr, c.option.HTMLPathPre, path) 197 | req, err := http.NewRequest(http.MethodGet, url.String(), nil) 198 | if err != nil { 199 | errMsg := "failed to create request" 200 | logger.Error(errMsg, zap.Error(err), 201 | zap.String("repo", idStr), 202 | zap.String("path", path)) 203 | err = errcode.New(errMsg, ErrCodeMakeRequest, 204 | ErrScope, nil) 205 | return "", err 206 | } 207 | content, err := c.htmlClient.Text(req) 208 | if err != nil { 209 | logger.Error(funcErrMsg, zap.Error(err), 210 | zap.String("repo", idStr), 211 | zap.String("path", path)) 212 | err = errcode.New(funcErrMsg, ErrCodeNetwork, 213 | ErrScope, []string{"github"}) 214 | return "", err 215 | } 216 | return content, nil 217 | } 218 | 219 | func IsAbuseError(err error) bool { 220 | // TODO: check "abuse" substring 221 | if err, ok := err.(errcode.Error); ok { 222 | return err.Code == 403 223 | } 224 | return false 225 | } 226 | -------------------------------------------------------------------------------- /exch/github/client_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "net/http/httptest" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/rydesun/awesome-github/lib/cohttp" 14 | "github.com/rydesun/awesome-github/test/fake-github" 15 | ) 16 | 17 | type ClientTestEnv struct { 18 | apiTestServer *httptest.Server 19 | htmlTestServer *httptest.Server 20 | testdataHolder fakeg.DataHolder 21 | } 22 | 23 | func (t *ClientTestEnv) Setup() error { 24 | wd, err := os.Getwd() 25 | if err != nil { 26 | return err 27 | } 28 | testdataDir := filepath.Join(wd, "../../test/testdata") 29 | t.testdataHolder = fakeg.NewDataHolder(testdataDir) 30 | 31 | t.apiTestServer, err = fakeg.ApiServer(t.testdataHolder) 32 | if err != nil { 33 | return err 34 | } 35 | t.htmlTestServer, err = fakeg.HtmlServer(t.testdataHolder) 36 | return err 37 | } 38 | 39 | func TestClient_GetUser(t *testing.T) { 40 | require := require.New(t) 41 | testEnv := ClientTestEnv{} 42 | err := testEnv.Setup() 43 | require.Nil(err) 44 | 45 | testServer := testEnv.apiTestServer 46 | apiClient := cohttp.NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil) 47 | 48 | testCases := []struct { 49 | token string 50 | hasErr bool 51 | }{ 52 | { 53 | token: "123456", 54 | hasErr: false, 55 | }, 56 | { 57 | token: "invalid", 58 | hasErr: true, 59 | }, 60 | } 61 | 62 | for _, tc := range testCases { 63 | t.Run(tc.token, func(t *testing.T) { 64 | client, err := NewClient(nil, apiClient, 65 | ClientOption{ 66 | APIHost: testServer.URL, 67 | ApiPathPre: APIPathPre, 68 | AccessToken: tc.token, 69 | }) 70 | require.Nil(err) 71 | user, err := client.GetUser() 72 | if tc.hasErr { 73 | require.NotNil(err) 74 | } else { 75 | require.Nil(err) 76 | require.NotNil(user) 77 | } 78 | }) 79 | } 80 | } 81 | 82 | func TestClient_GetRepo(t *testing.T) { 83 | require := require.New(t) 84 | testEnv := ClientTestEnv{} 85 | err := testEnv.Setup() 86 | require.Nil(err) 87 | 88 | testServer := testEnv.apiTestServer 89 | apiClient := cohttp.NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil) 90 | 91 | testCases := []struct { 92 | repoID RepoID 93 | hasErr bool 94 | }{ 95 | { 96 | repoID: RepoID{ 97 | Owner: "antchfx", 98 | Name: "xpath", 99 | }, 100 | hasErr: false, 101 | }, 102 | { 103 | repoID: RepoID{ 104 | Owner: "invalid", 105 | Name: "invalid", 106 | }, 107 | hasErr: true, 108 | }, 109 | } 110 | for _, tc := range testCases { 111 | t.Run(tc.repoID.String(), func(t *testing.T) { 112 | client, err := NewClient(nil, apiClient, 113 | ClientOption{ 114 | APIHost: testServer.URL, 115 | ApiPathPre: APIPathPre, 116 | }) 117 | require.Nil(err) 118 | user, err := client.GetRepo(context.Background(), tc.repoID) 119 | if tc.hasErr { 120 | require.NotNil(err) 121 | } else { 122 | require.Nil(err) 123 | require.NotNil(user) 124 | } 125 | }) 126 | } 127 | } 128 | 129 | func TestClient_GetHTMLReadme(t *testing.T) { 130 | require := require.New(t) 131 | testEnv := ClientTestEnv{} 132 | err := testEnv.Setup() 133 | require.Nil(err) 134 | 135 | testServer := testEnv.htmlTestServer 136 | htmlClient := cohttp.NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil) 137 | 138 | testCases := []struct { 139 | repoID RepoID 140 | hasErr bool 141 | }{ 142 | { 143 | repoID: RepoID{ 144 | Owner: "tester", 145 | Name: "awesome-test", 146 | }, 147 | hasErr: false, 148 | }, 149 | { 150 | repoID: RepoID{ 151 | Owner: "invalid", 152 | Name: "invalid", 153 | }, 154 | hasErr: true, 155 | }, 156 | } 157 | for _, tc := range testCases { 158 | t.Run(tc.repoID.String(), func(t *testing.T) { 159 | client, err := NewClient(htmlClient, nil, 160 | ClientOption{ 161 | HTMLHost: testServer.URL, 162 | HTMLPathPre: HTMLPathPre, 163 | }) 164 | require.Nil(err) 165 | user, err := client.GetHTMLReadme(tc.repoID) 166 | if tc.hasErr { 167 | require.NotNil(err) 168 | } else { 169 | require.Nil(err) 170 | require.NotNil(user) 171 | } 172 | }) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /exch/github/const.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | const ( 4 | ErrScope = "github" 5 | 6 | ErrCodeParameter = 10 7 | ErrCodeClientOption = 11 8 | ErrCodeMakeRequest = 20 9 | ErrCodeNetwork = 30 10 | ErrCodeAccessToken = 40 11 | ) 12 | 13 | const ( 14 | HTMLHost = "https://github.com/" 15 | HTMLPathPre = "blob/master" 16 | APIHost = "https://api.github.com/" 17 | APIPathPre = "graphql" 18 | ) 19 | 20 | func NewDefaultClientOption() ClientOption { 21 | return ClientOption{ 22 | HTMLHost: HTMLHost, 23 | HTMLPathPre: HTMLPathPre, 24 | APIHost: APIHost, 25 | ApiPathPre: APIPathPre, 26 | } 27 | } 28 | 29 | const QueryRepo = `{ "query": "query { repository(owner: \"%s\", name: \"%s\") { description forks { totalCount } stargazers { totalCount } watchers { totalCount } defaultBranchRef { target { ... on Commit { history(first: 1) { edges { node { committedDate } } } } } } } }" }` 30 | 31 | const QueryUser = `{ "query": "query { viewer { login } rateLimit { limit, remaining, resetAt }}"` 32 | -------------------------------------------------------------------------------- /exch/github/init.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "sync" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | var defaultLogger *zap.Logger 10 | 11 | func SetDefaultLogger(logger *zap.Logger) { 12 | defaultLogger = logger 13 | } 14 | 15 | func getLogger() *zap.Logger { 16 | if defaultLogger != nil { 17 | return defaultLogger 18 | } 19 | var once sync.Once 20 | once.Do(func() { 21 | defaultLogger, _ = zap.NewProduction() 22 | }) 23 | return defaultLogger 24 | } 25 | -------------------------------------------------------------------------------- /exch/github/model.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type RepoID struct { 9 | Owner string `json:"owner"` 10 | Name string `json:"name"` 11 | } 12 | 13 | func (r RepoID) String() string { 14 | return fmt.Sprintf("%s/%s", r.Owner, r.Name) 15 | } 16 | 17 | type Repo struct { 18 | Data struct { 19 | Repository struct { 20 | Forks struct { 21 | TotalCount int 22 | } 23 | Stargazers struct { 24 | TotalCount int 25 | } 26 | Watchers struct { 27 | TotalCount int 28 | } 29 | DefaultBranchRef struct { 30 | Target struct { 31 | History struct { 32 | Edges []struct { 33 | Node struct { 34 | CommittedDate time.Time 35 | } 36 | } 37 | } 38 | } 39 | } 40 | Description string 41 | } 42 | } 43 | Errors []struct { 44 | Message string 45 | } 46 | } 47 | 48 | type User struct { 49 | Data struct { 50 | Viewer struct { 51 | Login string 52 | } 53 | Ratelimit struct { 54 | Limit int 55 | Remaining int 56 | ResetAt time.Time 57 | } 58 | } 59 | Errors []struct { 60 | Message string 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rydesun/awesome-github 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/antchfx/htmlquery v1.2.3 7 | github.com/go-playground/validator/v10 v10.3.0 8 | github.com/gofiber/cors v0.2.2 9 | github.com/gofiber/fiber v1.13.3 10 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 11 | github.com/schollz/progressbar/v3 v3.3.4 12 | github.com/stretchr/testify v1.4.0 13 | github.com/urfave/cli/v2 v2.2.0 14 | go.uber.org/zap v1.15.0 15 | golang.org/x/net v0.0.0-20200602114024-627f9648deb9 16 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 17 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 18 | gopkg.in/yaml.v2 v2.2.2 19 | ) 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= 4 | github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= 5 | github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= 6 | github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= 7 | github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= 8 | github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= 9 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 10 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 15 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 16 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 17 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 18 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 19 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 20 | github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o= 21 | github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= 22 | github.com/gofiber/cors v0.2.2 h1:NQgLeNq8SWCKsdGotodyFCqLdSnxGLISsp9OU01k/cs= 23 | github.com/gofiber/cors v0.2.2/go.mod h1:lAXoymRHZKASLfydSAtsRGVrukWi3KefFnfxmCEAH5o= 24 | github.com/gofiber/fiber v1.13.3 h1:14kBTW1+n5mNIJZqibsbIdb+yQdC5argcbe9vE7Nz+o= 25 | github.com/gofiber/fiber v1.13.3/go.mod h1:KxRvVkqzfZOO6A7mBu+j7ncX2AcT6Sm6F7oeGR3Kgmw= 26 | github.com/gofiber/utils v0.0.9 h1:Bu4grjEB4zof1TtpmPCG6MeX5nGv8SaQfzaUgjkf3H8= 27 | github.com/gofiber/utils v0.0.9/go.mod h1:9J5aHFUIjq0XfknT4+hdSMG6/jzfaAgCu4HEbWDeBlo= 28 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= 29 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 30 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 31 | github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= 32 | github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= 33 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= 34 | github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= 35 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 36 | github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= 37 | github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= 38 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 39 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 40 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 41 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 42 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 43 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 44 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 45 | github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= 46 | github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 47 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 48 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 49 | github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= 50 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 51 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 52 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 53 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 54 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 55 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 56 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 57 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 58 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 59 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 60 | github.com/schollz/progressbar/v3 v3.3.4 h1:nMinx+JaEm/zJz4cEyClQeAw5rsYSB5th3xv+5lV6Vg= 61 | github.com/schollz/progressbar/v3 v3.3.4/go.mod h1:Rp5lZwpgtYmlvmGo1FyDwXMqagyRBQYSDwzlP9QDu84= 62 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 63 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 64 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 65 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 66 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 67 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 68 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= 69 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 70 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 71 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 72 | github.com/valyala/fasthttp v1.15.1 h1:eRb5jzWhbCn/cGu3gNJMcOfPUfXgXCcQIOHjh9ajAS8= 73 | github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= 74 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= 75 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 76 | go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= 77 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 78 | go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= 79 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 80 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= 81 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 82 | go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= 83 | go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= 84 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 85 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 86 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= 87 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 88 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 89 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 90 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 91 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 92 | golang.org/x/net v0.0.0-20200421231249-e086a090c8fd h1:QPwSajcTUrFriMF1nJ3XzgoqakqQEsnZf9LdXdi2nkI= 93 | golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 94 | golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= 95 | golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 96 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 97 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 98 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 99 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 100 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 101 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 102 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 103 | golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 104 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= 105 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 106 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 107 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 108 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 109 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 110 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 111 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 112 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 113 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 114 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= 115 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 116 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 117 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 118 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 119 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 120 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 121 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= 122 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 123 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 124 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 125 | honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= 126 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 127 | -------------------------------------------------------------------------------- /lib/cohttp/const.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | const ( 4 | ErrScope = "http" 5 | 6 | ErrCodeNetwork = 10 7 | ErrCodeJson = 11 8 | ) 9 | -------------------------------------------------------------------------------- /lib/cohttp/http.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "net/http" 7 | "time" 8 | 9 | "go.uber.org/zap" 10 | 11 | "github.com/rydesun/awesome-github/lib/errcode" 12 | ) 13 | 14 | type Reporter interface { 15 | ConReqNum(int) 16 | } 17 | 18 | type Client struct { 19 | c *http.Client 20 | queue chan struct{} 21 | MaxConcurrent int 22 | RetryTime int 23 | RetryInterval time.Duration 24 | logRespHead int 25 | reporter Reporter 26 | } 27 | 28 | func NewClient(client http.Client, maxConcurrent int, 29 | retryTime int, retryInterval time.Duration, 30 | logRespHead int, reporter Reporter) *Client { 31 | var queue chan struct{} 32 | if maxConcurrent > 0 { 33 | queue = make(chan struct{}, maxConcurrent) 34 | } 35 | if retryTime < 0 { 36 | retryTime = 0 37 | } 38 | return &Client{ 39 | c: &client, 40 | queue: queue, 41 | MaxConcurrent: maxConcurrent, 42 | RetryTime: retryTime, 43 | RetryInterval: retryInterval, 44 | logRespHead: logRespHead, 45 | reporter: reporter, 46 | } 47 | } 48 | 49 | func (c *Client) Do(req *http.Request) (resp *http.Response, err error) { 50 | const funcIntent = "try to send http request" 51 | const funcErrMsg = "failed to send http request" 52 | var ( 53 | method = req.Method 54 | url = req.URL.String() 55 | logger = getLogger() 56 | ) 57 | defer logger.Sync() 58 | logger.Debug(funcIntent, 59 | zap.String("method", method), 60 | zap.String("url", url)) 61 | 62 | if c.queue == nil { 63 | if c.reporter != nil { 64 | c.reporter.ConReqNum(1) 65 | } 66 | } else { 67 | c.queue <- struct{}{} 68 | if c.reporter != nil { 69 | c.reporter.ConReqNum(len(c.queue)) 70 | } 71 | } 72 | logger.Debug("request is being sent", 73 | zap.String("method", method), 74 | zap.String("url", url)) 75 | resp, err = c.c.Do(req) 76 | if c.queue != nil { 77 | <-c.queue 78 | } 79 | if err != nil { 80 | logger.Error(funcErrMsg, zap.Error(err), 81 | zap.String("method", method), 82 | zap.String("url", url)) 83 | err = errcode.New(funcErrMsg, ErrCodeNetwork, ErrScope, 84 | []string{err.Error()}) 85 | return 86 | } 87 | 88 | logger.Debug("receive a response", 89 | zap.String("method", method), 90 | zap.Int("status", resp.StatusCode), 91 | zap.String("url", url)) 92 | return 93 | } 94 | 95 | func (c *Client) DoBetter(req *http.Request) ( 96 | rawdata []byte, hasBody bool, err error) { 97 | const funcIntent = "try to send http request" 98 | const funcErrMsg = "failed to " + funcIntent 99 | var ( 100 | method = req.Method 101 | url = req.URL.String() 102 | logger = getLogger() 103 | ) 104 | defer logger.Sync() 105 | logger.Debug(funcIntent, 106 | zap.String("method", method), 107 | zap.String("url", url)) 108 | 109 | var resp *http.Response 110 | for i := 0; i < c.RetryTime+1; i++ { 111 | resp, err = c.Do(req) 112 | if err == nil { 113 | break 114 | } 115 | logger.Warn(funcErrMsg, zap.Error(err), 116 | zap.String("method", method), 117 | zap.String("url", url), 118 | zap.Int("retry", i)) 119 | time.Sleep(c.RetryInterval) 120 | } 121 | if err != nil { 122 | logger.Error(funcErrMsg, zap.Error(err), 123 | zap.String("method", method), 124 | zap.String("url", url)) 125 | err = errcode.Wrap(err, funcErrMsg) 126 | return 127 | } 128 | rawdata, err = ioutil.ReadAll(resp.Body) 129 | defer resp.Body.Close() 130 | if err != nil { 131 | errMsg := "failed to read response" 132 | logger.Error(errMsg, zap.Error(err)) 133 | err = errcode.Wrap(err, errMsg) 134 | return 135 | } 136 | 137 | // Read body successfully 138 | hasBody = true 139 | 140 | statusCode := resp.StatusCode 141 | if statusCode < 200 || statusCode > 299 { 142 | errMsg := "remote server did not return ok" 143 | logger.Error(errMsg, 144 | zap.String("method", method), 145 | zap.String("url", url), 146 | zap.ByteString("recv", rawdata), 147 | zap.Int("status", statusCode)) 148 | err = errcode.New(errMsg, errcode.ErrCode(statusCode), 149 | ErrScope, nil) 150 | return 151 | } 152 | return 153 | } 154 | 155 | func (c *Client) Text(req *http.Request) (string, error) { 156 | const funcIntent = "try to get text from remote server" 157 | const funcErrMsg = "failed to get text from remote server" 158 | var ( 159 | method = req.Method 160 | url = req.URL.String() 161 | logger = getLogger() 162 | ) 163 | defer logger.Sync() 164 | logger.Debug(funcIntent, 165 | zap.String("method", method), 166 | zap.String("url", url)) 167 | 168 | rawdata, hasBody, err := c.DoBetter(req) 169 | var text string 170 | if hasBody { 171 | text = string(rawdata) 172 | } 173 | if err != nil { 174 | logger.Error(funcErrMsg, zap.Error(err), 175 | zap.String("method", method), 176 | zap.String("url", url)) 177 | err = errcode.Wrap(err, funcErrMsg) 178 | return text, err 179 | } 180 | return text, nil 181 | } 182 | 183 | func (c *Client) Json(req *http.Request, respJson interface{}) error { 184 | const funcIntent = "try to get json from remote server" 185 | const funcErrMsg = "failed to get json from remote server" 186 | var ( 187 | method = req.Method 188 | url = req.URL.String() 189 | logger = getLogger() 190 | ) 191 | defer logger.Sync() 192 | logger.Debug(funcIntent, 193 | zap.String("method", method), 194 | zap.String("url", url)) 195 | 196 | rawdata, hasBody, err := c.DoBetter(req) 197 | 198 | // impossible: hasBody == false && err == nil 199 | if !hasBody && err != nil { 200 | logger.Error(funcErrMsg, zap.Error(err), 201 | zap.String("method", method), 202 | zap.String("url", url)) 203 | err = errcode.Wrap(err, funcErrMsg) 204 | return err 205 | } 206 | logger.Debug("receive rawdata from remote", 207 | zap.ByteString("content", rawdata)) 208 | // DO NOT cover err 209 | _err := json.Unmarshal(rawdata, &respJson) 210 | if _err != nil { 211 | errMsg := "failed to parse response" 212 | length := len(rawdata) 213 | logField := []zap.Field{ 214 | zap.Error(err), 215 | zap.Int("length", length), 216 | zap.String("method", method), 217 | zap.String("url", url), 218 | } 219 | if c.logRespHead > 0 { 220 | content := truncate(rawdata, c.logRespHead) 221 | logField = append(logField, 222 | zap.ByteString("content", content)) 223 | } 224 | // 225 | if err == nil { 226 | err = errcode.New(errMsg, ErrCodeJson, ErrScope, nil) 227 | } 228 | logger.Error(errMsg, logField...) 229 | return err 230 | } 231 | // Must return err! 232 | return err 233 | } 234 | -------------------------------------------------------------------------------- /lib/cohttp/http_test.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var testServer *httptest.Server 13 | 14 | func init() { 15 | handler := func(rw http.ResponseWriter, req *http.Request) { 16 | if req.URL.Path == "/text" { 17 | rw.Write([]byte("simple text")) 18 | } 19 | if req.URL.Path == "/text/wrong_path" { 20 | rw.WriteHeader(400) 21 | rw.Write([]byte("permission denied")) 22 | } 23 | if req.URL.Path == "/json" { 24 | rw.Write([]byte(`{"foo": "bar"}`)) 25 | } 26 | // Invalid path 27 | if req.URL.Path == "/json/wrong_path" { 28 | rw.WriteHeader(500) 29 | } 30 | // Invalid json data 31 | if req.URL.Path == "/json/invalid" { 32 | rw.Write([]byte(`{`)) 33 | } 34 | // Unauthorized but has valid json data 35 | if req.URL.Path == "/json/error" { 36 | rw.WriteHeader(401) 37 | rw.Write([]byte(`{"err": "error message"}`)) 38 | } 39 | } 40 | testServer = httptest.NewServer(http.HandlerFunc(handler)) 41 | } 42 | 43 | func TestClient_Text(t *testing.T) { 44 | require := require.New(t) 45 | testCases := []struct { 46 | in string 47 | out string 48 | hasErr bool 49 | netErr bool 50 | }{ 51 | { 52 | in: "/text", 53 | out: "simple text", 54 | }, 55 | { 56 | in: "/text/wrong_path", 57 | out: "permission denied", 58 | hasErr: true, 59 | }, 60 | { 61 | hasErr: true, 62 | netErr: true, 63 | }, 64 | } 65 | client := NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil) 66 | for _, tc := range testCases { 67 | t.Run("server"+tc.in, func(t *testing.T) { 68 | var url string 69 | if tc.netErr { 70 | url = "invalid url" 71 | } else { 72 | url = testServer.URL + tc.in 73 | } 74 | req, err := http.NewRequest( 75 | http.MethodGet, url, nil) 76 | require.Nil(err) 77 | result, err := client.Text(req) 78 | if !tc.hasErr { 79 | require.Nil(err) 80 | } else { 81 | require.NotNil(err) 82 | } 83 | require.Equal(tc.out, result) 84 | }) 85 | } 86 | } 87 | 88 | func TestClient_Json(t *testing.T) { 89 | require := require.New(t) 90 | type Result struct { 91 | Foo string 92 | } 93 | testCases := []struct { 94 | in string 95 | out Result 96 | hasErr bool 97 | netErr bool 98 | }{ 99 | { 100 | in: "/json", 101 | out: Result{ 102 | Foo: "bar", 103 | }, 104 | }, 105 | { 106 | in: "/json/wrong_path", 107 | hasErr: true, 108 | }, 109 | { 110 | in: "/json/invalid", 111 | hasErr: true, 112 | }, 113 | { 114 | in: "/json/error", 115 | hasErr: true, 116 | }, 117 | { 118 | hasErr: true, 119 | netErr: true, 120 | }, 121 | } 122 | 123 | client := NewClient(*testServer.Client(), 16, 0, time.Second, 20, nil) 124 | for _, tc := range testCases { 125 | t.Run("server"+tc.in, func(t *testing.T) { 126 | var url string 127 | if tc.netErr { 128 | url = "invalid url" 129 | } else { 130 | url = testServer.URL + tc.in 131 | } 132 | req, err := http.NewRequest( 133 | http.MethodGet, url, nil) 134 | require.Nil(err) 135 | var result Result 136 | err = client.Json(req, &result) 137 | if !tc.hasErr { 138 | require.Nil(err) 139 | } else { 140 | require.NotNil(err) 141 | } 142 | require.Equal(tc.out, result) 143 | }) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /lib/cohttp/init.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | import ( 4 | "sync" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | var defaultLogger *zap.Logger 10 | 11 | func SetDefaultLogger(logger *zap.Logger) { 12 | defaultLogger = logger 13 | } 14 | 15 | func getLogger() *zap.Logger { 16 | if defaultLogger != nil { 17 | return defaultLogger 18 | } 19 | var once sync.Once 20 | once.Do(func() { 21 | defaultLogger, _ = zap.NewProduction() 22 | }) 23 | return defaultLogger 24 | } 25 | -------------------------------------------------------------------------------- /lib/cohttp/lib.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | import "github.com/rydesun/awesome-github/lib/errcode" 4 | 5 | func truncate(raw []byte, maxLength int) (result []byte) { 6 | if maxLength == 0 { 7 | return []byte{} 8 | } 9 | length := len(raw) 10 | if length > maxLength { 11 | return append(raw[:maxLength], "..."...) 12 | } else { 13 | return raw 14 | } 15 | } 16 | 17 | func IsNetowrkError(err error) bool { 18 | errc, ok := err.(errcode.Error) 19 | return ok && errc.Code == ErrCodeNetwork 20 | } 21 | -------------------------------------------------------------------------------- /lib/cohttp/lib_test.go: -------------------------------------------------------------------------------- 1 | package cohttp 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestTruncate(t *testing.T) { 10 | require := require.New(t) 11 | testCases := []struct { 12 | maxLength int 13 | raw []byte 14 | result []byte 15 | }{ 16 | { 17 | maxLength: 0, 18 | raw: []byte("abcdefghi"), 19 | result: []byte{}, 20 | }, 21 | { 22 | maxLength: 1, 23 | raw: []byte("abcdefghi"), 24 | result: []byte("a..."), 25 | }, 26 | { 27 | maxLength: 8, 28 | raw: []byte("abcdefghi"), 29 | result: []byte("abcdefgh..."), 30 | }, 31 | { 32 | maxLength: 9, 33 | raw: []byte("abcdefghi"), 34 | result: []byte("abcdefghi"), 35 | }, 36 | { 37 | maxLength: 10, 38 | raw: []byte("abcdefghi"), 39 | result: []byte("abcdefghi"), 40 | }, 41 | } 42 | for _, tc := range testCases { 43 | t.Run("NONAME", func(t *testing.T) { 44 | actual := truncate(tc.raw, tc.maxLength) 45 | require.Equal(tc.result, actual) 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /lib/errcode/const.go: -------------------------------------------------------------------------------- 1 | package errcode 2 | 3 | type ErrScope string 4 | 5 | const ( 6 | ScopeUnknown ErrScope = "unknown" 7 | ScopeInternal ErrScope = "internal" 8 | ) 9 | 10 | type ErrCode int 11 | 12 | const ( 13 | CodeUnknown ErrCode = 0 14 | CodeInternal ErrCode = 1 15 | ) 16 | -------------------------------------------------------------------------------- /lib/errcode/errcode.go: -------------------------------------------------------------------------------- 1 | package errcode 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | type Error struct { 10 | Msg string 11 | Code ErrCode 12 | Scope ErrScope 13 | Objects []string 14 | err string 15 | } 16 | 17 | func (e Error) Error() string { 18 | raw, _ := json.Marshal(e) 19 | return string(raw) 20 | } 21 | 22 | func New(msg string, code ErrCode, scope ErrScope, objects []string) error { 23 | if len(scope) == 0 { 24 | scope = ScopeUnknown 25 | } 26 | if objects == nil { 27 | objects = []string{} 28 | } 29 | return Error{ 30 | Msg: msg, 31 | Code: code, 32 | Scope: scope, 33 | Objects: objects, 34 | } 35 | } 36 | 37 | func Wrap(err error, msg string) error { 38 | e, ok := err.(Error) 39 | if ok { 40 | e.Msg = msg 41 | return e 42 | } 43 | logger := getLogger() 44 | defer logger.Sync() 45 | logger.DPanic("generate a unknown error", 46 | zap.Error(err)) 47 | return New(msg, CodeUnknown, ScopeUnknown, []string{}) 48 | } 49 | -------------------------------------------------------------------------------- /lib/errcode/init.go: -------------------------------------------------------------------------------- 1 | package errcode 2 | 3 | import ( 4 | "sync" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | var defaultLogger *zap.Logger 10 | 11 | func SetDefaultLogger(logger *zap.Logger) { 12 | defaultLogger = logger 13 | } 14 | 15 | func getLogger() *zap.Logger { 16 | if defaultLogger != nil { 17 | return defaultLogger 18 | } 19 | var once sync.Once 20 | once.Do(func() { 21 | defaultLogger, _ = zap.NewProduction() 22 | }) 23 | return defaultLogger 24 | } 25 | -------------------------------------------------------------------------------- /test/fake-github/data.go: -------------------------------------------------------------------------------- 1 | package fakeg 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | ) 8 | 9 | const apiJsonUserPath = "./api/users/%s.json" 10 | const htmlAwesomeReadmePath = "./html/README.html" 11 | const apiJsonRepoPath = "./api/repos/%s_%s.json" 12 | const outputPath = "./output/data.json" 13 | 14 | type DataHolder struct { 15 | dir string 16 | } 17 | 18 | func NewDataHolder(dir string) DataHolder { 19 | return DataHolder{ 20 | dir: dir, 21 | } 22 | } 23 | 24 | func (h *DataHolder) GetUser() ([]byte, error) { 25 | fpath := filepath.Join(h.dir, fmt.Sprintf(apiJsonUserPath, "tester")) 26 | return ioutil.ReadFile(fpath) 27 | } 28 | 29 | func (h *DataHolder) GetHtmlAwesomeReadme() ([]byte, error) { 30 | fpath := filepath.Join(h.dir, htmlAwesomeReadmePath) 31 | return ioutil.ReadFile(fpath) 32 | } 33 | 34 | func (h *DataHolder) GetJsonRepo(user string, name string) ([]byte, error) { 35 | fpath := filepath.Join(h.dir, fmt.Sprintf(apiJsonRepoPath, user, name)) 36 | return ioutil.ReadFile(fpath) 37 | } 38 | 39 | func (h *DataHolder) GetOutput() ([]byte, error) { 40 | fpath := filepath.Join(h.dir, outputPath) 41 | return ioutil.ReadFile(fpath) 42 | } 43 | -------------------------------------------------------------------------------- /test/fake-github/server.go: -------------------------------------------------------------------------------- 1 | package fakeg 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | ) 9 | 10 | func HtmlServer(testdataHoler DataHolder) (testServer *httptest.Server, err error) { 11 | awesomeHtmlReadme, err := testdataHoler.GetHtmlAwesomeReadme() 12 | if err != nil { 13 | return 14 | } 15 | handler := func(rw http.ResponseWriter, req *http.Request) { 16 | if req.URL.Path == "/tester/awesome-test/blob/master/README.md" { 17 | rw.Write([]byte(awesomeHtmlReadme)) 18 | } 19 | if req.URL.Path == "/invalid/invalid/blob/master/README.md" { 20 | rw.WriteHeader(400) 21 | } 22 | } 23 | testServer = httptest.NewServer(http.HandlerFunc(handler)) 24 | return 25 | } 26 | 27 | func ApiServer(testdataHolder DataHolder) (testServer *httptest.Server, err error) { 28 | jsonUser, err := testdataHolder.GetUser() 29 | if err != nil { 30 | return 31 | } 32 | jsonRepoXpath, err := testdataHolder.GetJsonRepo("antchfx", "xpath") 33 | if err != nil { 34 | return 35 | } 36 | jsonRepoGlmatrix, err := testdataHolder.GetJsonRepo("technohippy", "go-glmatrix") 37 | if err != nil { 38 | return 39 | } 40 | jsonRepoGlfw, err := testdataHolder.GetJsonRepo("goxjs", "glfw") 41 | if err != nil { 42 | return 43 | } 44 | handler := func(rw http.ResponseWriter, req *http.Request) { 45 | if !strings.HasPrefix(req.URL.Path, "/graphql") { 46 | rw.WriteHeader(400) 47 | return 48 | } 49 | // There is no need to implement a GraphQL server 50 | raw, err := ioutil.ReadAll(req.Body) 51 | defer req.Body.Close() 52 | if err != nil { 53 | rw.WriteHeader(400) 54 | return 55 | } 56 | if strings.Contains(string(raw), "login") { 57 | bear := req.Header.Get("Authorization") 58 | if strings.HasSuffix(bear, "123456") { 59 | rw.Write([]byte(jsonUser)) 60 | } else { 61 | rw.WriteHeader(401) 62 | } 63 | return 64 | } 65 | if strings.Contains(string(raw), "antchfx") { 66 | rw.Write([]byte(jsonRepoXpath)) 67 | return 68 | } 69 | if strings.Contains(string(raw), "technohippy") { 70 | rw.Write([]byte(jsonRepoGlmatrix)) 71 | return 72 | } 73 | if strings.Contains(string(raw), "goxjs") { 74 | rw.Write([]byte(jsonRepoGlfw)) 75 | return 76 | } 77 | rw.WriteHeader(400) 78 | } 79 | testServer = httptest.NewServer(http.HandlerFunc(handler)) 80 | return 81 | } 82 | -------------------------------------------------------------------------------- /test/testdata/api/repos/antchfx_xpath.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "repository": { 4 | "description": "XPath package for Golang, supports HTML, XML, JSON document query.", 5 | "forks": { 6 | "totalCount": 36 7 | }, 8 | "stargazers": { 9 | "totalCount": 319 10 | }, 11 | "watchers": { 12 | "totalCount": 8 13 | }, 14 | "defaultBranchRef": { 15 | "target": { 16 | "history": { 17 | "edges": [ 18 | { 19 | "node": { 20 | "committedDate": "2019-07-15T19:40:41Z" 21 | } 22 | } 23 | ] 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/testdata/api/repos/goxjs_glfw.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "repository": { 4 | "description": "Go cross-platform glfw library for creating an OpenGL context and receiving events.", 5 | "forks": { 6 | "totalCount": 14 7 | }, 8 | "stargazers": { 9 | "totalCount": 65 10 | }, 11 | "watchers": { 12 | "totalCount": 6 13 | }, 14 | "defaultBranchRef": { 15 | "target": { 16 | "history": { 17 | "edges": [ 18 | { 19 | "node": { 20 | "committedDate": "2019-07-15T19:40:41Z" 21 | } 22 | } 23 | ] 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/testdata/api/repos/technohippy_go-glmatrix.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "repository": { 4 | "description": "go-glmatrix is a golang version of glMatrix, which is \"designed to perform vector and matrix operations stupidly fast\".", 5 | "forks": { 6 | "totalCount": 0 7 | }, 8 | "stargazers": { 9 | "totalCount": 1 10 | }, 11 | "watchers": { 12 | "totalCount": 1 13 | }, 14 | "defaultBranchRef": { 15 | "target": { 16 | "history": { 17 | "edges": [ 18 | { 19 | "node": { 20 | "committedDate": "2019-07-15T19:40:41Z" 21 | } 22 | } 23 | ] 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/testdata/api/users/tester.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "rateLimit": { 4 | "limit": 5000, 5 | "remaining": 4999, 6 | "resetAt": "2222-02-22T22:22:22Z" 7 | }, 8 | "viewer": { 9 | "login": "tester" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/testdata/html/README.html: -------------------------------------------------------------------------------- 1 | 2 |
    3 |
    4 |

    Awesome Test

    5 |

    This is a test file

    6 | 7 |

    XML

    8 |

    Libraries and tools for manipulating XML.

    9 |
      10 |
    • xpath - XPath package for Go.
      • 11 |
    • invalid - This is a GitHub repository, but not link home page.
      • 12 |
    • invalid - A non-existent repository.
      • 13 |
    14 | 15 |

    OpenGL

    16 |

    Libraries for using OpenGL in Go.

    17 |
      18 |
    • go-glmatrix - Go port of glMatrix library.
      • 19 |
    • goxjs/glfw - Go cross-platform glfw library for creating an OpenGL context and receiving events.
      • 20 |
    • invalid - Not a valid project, only for test.
      • 21 |
    22 | 23 |

    Empty

    24 |

    Empty list

    25 |
      26 |
    27 |

    Empty

    28 |

    Empty list

    29 |
    30 |
    31 | 32 | -------------------------------------------------------------------------------- /test/testdata/output/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "time": "2020-08-05T18:03:45.368864844+08:00", 3 | "awesome_list": { 4 | "owner": "akullpp", 5 | "name": "awesome-java" 6 | }, 7 | "data": { 8 | "Projects": [ 9 | { 10 | "id": { 11 | "owner": "doov-io", 12 | "name": "doov" 13 | }, 14 | "owner": "doov-io", 15 | "awesome_name": "dOOv", 16 | "link": "https://github.com/doov-io/doov", 17 | "watch": 13, 18 | "star": 54, 19 | "fork": 6, 20 | "last_commit": "2020-07-30T21:38:29Z", 21 | "description": "dOOv (Domain Object Oriented Validation) a fluent API for type-safe bean validation and mapping", 22 | "awesome_description": "dOOv - Provides fluent API for typesafe domain model validation and mapping. It uses annotations, code generation and a type safe DSL to make bean validation and mapping fast and easy." 23 | } 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /web/app/route.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | "time" 8 | 9 | "github.com/gofiber/cors" 10 | "github.com/gofiber/fiber" 11 | 12 | "github.com/rydesun/awesome-github/exch/github" 13 | "github.com/rydesun/awesome-github/lib/cohttp" 14 | ) 15 | 16 | func main() { 17 | } 18 | 19 | type Router struct { 20 | app *fiber.App 21 | listen string 22 | 23 | html string 24 | scriptPath string 25 | dataPath string 26 | } 27 | 28 | func NewRouter(listen string) (*Router, error) { 29 | return &Router{ 30 | app: fiber.New(&fiber.Settings{ 31 | DisableStartupMessage: true, 32 | }), 33 | listen: listen, 34 | }, nil 35 | } 36 | 37 | func (r *Router) Init(repoID github.RepoID, scriptPath, dataPath string) error { 38 | html, err := FetchHTMLReadme(repoID) 39 | if err != nil { 40 | return err 41 | } 42 | r.html = html 43 | r.dataPath = dataPath 44 | r.scriptPath = scriptPath 45 | return nil 46 | } 47 | 48 | func (r *Router) Route() error { 49 | app := r.app 50 | app.Use(cors.New()) 51 | 52 | scriptPath := r.scriptPath 53 | urlScript, _ := url.Parse(scriptPath) 54 | if urlScript.Scheme != "http" && urlScript.Scheme != "https" { 55 | app.Static("/js", scriptPath) 56 | scriptPath = "/js" 57 | } 58 | app.Static("/data", r.dataPath) 59 | app.Get("/", func(c *fiber.Ctx) { 60 | c.Set("content-type", "text/html; charset=utf-8") 61 | c.Send(wrapReadme(r.html, "/data", scriptPath)) 62 | }) 63 | 64 | return app.Listen(r.listen) 65 | } 66 | 67 | func FetchHTMLReadme(id github.RepoID) (string, error) { 68 | gc, err := github.NewClient(cohttp.NewClient( 69 | http.Client{}, 1, 3, time.Second, 70 | 0, nil), nil, github.NewDefaultClientOption()) 71 | if err != nil { 72 | return "", err 73 | } 74 | return gc.GetHTMLReadme(id) 75 | } 76 | 77 | func wrapReadme(readme, data_url, script_url string) string { 78 | return fmt.Sprintf(` 79 | %s 80 | 81 | 82 | 83 | `, readme, data_url, script_url) 84 | } 85 | -------------------------------------------------------------------------------- /web/static/js/view.js: -------------------------------------------------------------------------------- 1 | var DateTime = luxon.DateTime; 2 | 3 | function fill(data) { 4 | let repos = {}; 5 | for (items of Object.values(data.data)) { 6 | for (i of Object.values(items)) { 7 | repos[i.id.owner+'/'+i.id.name] = i; 8 | } 9 | } 10 | 11 | // Use default icons 12 | document.querySelector(".octicon-star").id = 'icon-star'; 13 | 14 | items = document.getElementsByTagName('li'); 15 | for (item of items) { 16 | link = item.firstElementChild; 17 | if (link === undefined || link === null) { 18 | continue 19 | } 20 | res = /https?:\/\/github.com\/([^\/]*)\/([^\/]*)\/?/.exec(link.href); 21 | if ((res === null) || (res.length !== 3)) { 22 | continue 23 | } 24 | let repoID = res[1] + '/' + res[2]; 25 | if (!(repoID in repos)) { 26 | continue 27 | } 28 | let box = document.createElement("div"); 29 | item.prepend(box); 30 | 31 | // Add nodes. 32 | let updated_at = DateTime.fromISO(repos[repoID].last_commit); 33 | box.innerHTML = [ 34 | '
    ', 35 | '', 36 | '', 37 | '', 38 | repos[repoID].star, 39 | '', 40 | '
    ', 41 | '
    ', 42 | repos[repoID].last_commit, 43 | '
    ', 44 | '
    ', 45 | updated_at.toRelativeCalendar(), 46 | '
    ', 47 | ].join('\n'); 48 | } 49 | 50 | let style = document.createElement('style'); 51 | style.innerHTML = ` 52 | .awg { 53 | display: inline-block; 54 | font-size: .8rem; 55 | border-radius: .5rem; 56 | margin-right: .5rem; 57 | padding: .1rem .4rem 0 .4rem; 58 | color: #fff; 59 | background: #78c878; 60 | } 61 | .awg.star { 62 | min-width: 4rem; 63 | } 64 | .awg.star svg { 65 | vertical-align: -.25rem; 66 | } 67 | ` 68 | let ref = document.querySelector('script'); 69 | ref.parentNode.insertBefore(style, ref); 70 | } 71 | 72 | function main() { 73 | fetch(window.data_url) 74 | .then(resp => resp.json()) 75 | .then(data => { 76 | fill(data); 77 | }); 78 | } 79 | 80 | main(); 81 | --------------------------------------------------------------------------------