├── public ├── img │ ├── favicon.png │ ├── digitalocean.png │ ├── badge.svg │ ├── sourcegraph.svg │ └── sourcegraph-logo.svg ├── fonts │ ├── octicons.eot │ ├── octicons.ttf │ └── octicons.woff ├── less │ ├── _link.less │ └── main.less ├── css │ └── gowalker.min.css └── js │ └── gowalker.js ├── internal ├── base │ ├── README │ ├── template.go │ ├── path.go │ ├── tool.go │ └── gen.go ├── route │ ├── apiv1 │ │ └── api.go │ ├── home.go │ ├── search.go │ └── docs.go ├── doc │ ├── http.go │ ├── golang.go │ ├── struct.go │ ├── code.go │ ├── crawl.go │ ├── github.go │ ├── vcs.go │ └── walker.go ├── spaces │ └── spaces.go ├── prometheus │ ├── js_file.go │ └── package.go ├── db │ ├── models.go │ ├── js_file.go │ ├── routine.go │ └── package.go ├── httplib │ ├── README.md │ ├── httplib_test.go │ └── httplib.go ├── setting │ └── setting.go └── context │ └── context.go ├── Makefile ├── .gitignore ├── README.md ├── .github └── workflows │ └── lsif.yml ├── conf ├── app.ini └── locale │ ├── locale_zh-CN.ini │ └── locale_en-US.ini ├── templates ├── docs │ ├── header.html │ ├── imports.html │ ├── docs.html │ └── tpl.html ├── search.html ├── home.html └── base │ └── base.html ├── go.mod ├── gowalker.go └── LICENSE /public/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unknwon/gowalker/HEAD/public/img/favicon.png -------------------------------------------------------------------------------- /public/fonts/octicons.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unknwon/gowalker/HEAD/public/fonts/octicons.eot -------------------------------------------------------------------------------- /public/fonts/octicons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unknwon/gowalker/HEAD/public/fonts/octicons.ttf -------------------------------------------------------------------------------- /public/fonts/octicons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unknwon/gowalker/HEAD/public/fonts/octicons.woff -------------------------------------------------------------------------------- /public/img/digitalocean.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unknwon/gowalker/HEAD/public/img/digitalocean.png -------------------------------------------------------------------------------- /internal/base/README: -------------------------------------------------------------------------------- 1 | Run following code when Go version upgrades: 2 | 3 | go run gen.go -output data.go 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | go build -v -o gowalker 3 | 4 | web: build 5 | ./gowalker 6 | 7 | release: 8 | env GOOS=linux GOARCH=amd64 go build -o gowalker 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | raw 2 | custom 3 | public/.idea 4 | gowalker 5 | 6 | gowalker.sublime-project 7 | gowalker.sublime-workspace 8 | public/sass/.sass-cache 9 | .idea 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /public/img/badge.svg: -------------------------------------------------------------------------------- 1 | Go WalkerAPI Documentation -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Go Walker 2 | ========= 3 | 4 | Go Walker is a server that generates Go projects API documentation on the fly for the projects on **GitHub**. 5 | 6 | ## Credits 7 | 8 | - [github.com/golang/gddo](https://github.com/golang/gddo) 9 | - Contributors: [chenwenli](http://www.lavachen.cn), [slene](https://github.com/slene), [atotto](https://github.com/atotto). 10 | 11 | ## License 12 | 13 | This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text. 14 | -------------------------------------------------------------------------------- /.github/workflows/lsif.yml: -------------------------------------------------------------------------------- 1 | name: LSIF 2 | on: [push] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v1 8 | - name: Generate LSIF data 9 | uses: sourcegraph/lsif-go-action@master 10 | with: 11 | verbose: 'true' 12 | - name: Upload LSIF data 13 | uses: sourcegraph/lsif-upload-action@master 14 | continue-on-error: true 15 | with: 16 | endpoint: https://sourcegraph.com 17 | github_token: ${{ secrets.GITHUB_TOKEN }} 18 | -------------------------------------------------------------------------------- /conf/app.ini: -------------------------------------------------------------------------------- 1 | RUN_MODE = dev 2 | DISABLE_ROUTER_LOG = false 3 | 4 | [server] 5 | HTTP_PORT = 8080 6 | FETCH_TIMEOUT = 60 7 | DOCS_JS_PATH = raw/docs/ 8 | DOCS_GOB_PATH = raw/gob/ 9 | 10 | [database] 11 | USER = root 12 | PASSWD = 13 | HOST = 127.0.0.1:3306 14 | NAME = gowalker 15 | 16 | [i18n] 17 | LANGS = en-US,zh-CN 18 | NAMES = English,简体中文 19 | REDIRECT = true 20 | 21 | [github] 22 | CLIENT_ID = 23 | CLIENT_SECRET = 24 | 25 | [digitalocean.spaces] 26 | ENABLED = false 27 | ENDPOINT = 28 | ACCESS_KEY = 29 | SECRET_KEY = 30 | BUCKET = 31 | BUCKET_URL = 32 | 33 | [maintenance] 34 | JS_RECYCLE_DAYS = 14 35 | 36 | [log.discord] 37 | ENABLED = false 38 | URL = 39 | -------------------------------------------------------------------------------- /public/img/sourcegraph.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /templates/docs/header.html: -------------------------------------------------------------------------------- 1 |
2 | 19 |
-------------------------------------------------------------------------------- /internal/route/apiv1/api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package apiv1 16 | 17 | import ( 18 | "github.com/unknwon/gowalker/internal/context" 19 | ) 20 | 21 | func Badge(ctx *context.Context) { 22 | ctx.Redirect("https://img.shields.io/badge/Go%20Walker-API%20Documentation-green.svg?style=flat-square") 23 | } 24 | -------------------------------------------------------------------------------- /templates/docs/imports.html: -------------------------------------------------------------------------------- 1 | {% extends "base/base.html" %} 2 | {% block body %} 3 |
4 | {% include "docs/header.html" %} 5 | 6 |
7 |

8 | {% if PageIsImports %} 9 | {{Tr(Lang, "docs.imports.title", ProjectName)}} 10 | {% else %} 11 | {{Tr(Lang, "docs.refs.title", ProjectName)}} 12 | {% endif %} 13 |

14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {% for pkg in Packages %} 24 | 25 | 26 | 27 | 28 | {% endfor %} 29 | 30 |
{{Tr(Lang, "docs.path")}}{{Tr(Lang, "docs.synopsis")}}
{{pkg.ImportPath}}{{pkg.Synopsis}}
31 | 32 |
33 |

{{Tr(Lang, "docs.imports.go_back", Link) | safe}}

34 |
35 |
36 | {% endblock %} -------------------------------------------------------------------------------- /public/less/_link.less: -------------------------------------------------------------------------------- 1 | @charset "UTF-8"; 2 | @font-face{ 3 | font-family: 'octicons'; 4 | src: url("/fonts/octicons.eot"); 5 | src: url("/fonts/octicons.eot#iefix") format("embedded-opentype"), url("/fonts/octicons.woff") format("woff"), url("/fonts/octicons.ttf") format("truetype"); 6 | font-weight: normal; 7 | font-style: normal; 8 | } 9 | .markdown { 10 | h1, h2, h3, h4, h5, h6 { 11 | a.anchor { 12 | text-decoration: none; 13 | line-height: 1; 14 | padding-left: 0; 15 | margin-left: 5px; 16 | top: 15% 17 | } 18 | &:hover .octicon-link { 19 | display: inline-block; 20 | } 21 | } 22 | a span { 23 | &.octicon { 24 | font: normal normal 16px octicons; 25 | line-height: 1; 26 | display: inline-block; 27 | text-decoration: none; 28 | -webkit-font-smoothing: antialiased; 29 | } 30 | &.octicon-link { 31 | display: none; 32 | color: #000; 33 | &:before { 34 | content: '\f05c'; 35 | } 36 | } 37 | } 38 | pre.code { 39 | margin-top: -15px !important; 40 | } 41 | } -------------------------------------------------------------------------------- /templates/search.html: -------------------------------------------------------------------------------- 1 | {% extends "base/base.html" %} 2 | {% block title %}{{Tr(Lang, "search.search")}} - Go Walker{% endblock %} 3 | {% block body %} 4 | 37 | {% endblock %} -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/unknwon/gowalker 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.43.0 // indirect 7 | github.com/denisenkom/go-mssqldb v0.0.0-20190724012636-11b2859924c1 // indirect 8 | github.com/fatih/color v1.7.0 // indirect 9 | github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect 10 | github.com/go-ini/ini v1.46.0 // indirect 11 | github.com/go-macaron/i18n v0.0.0-20190805070610-6d779f6a12cf 12 | github.com/go-macaron/pongo2 v0.0.0-20180906102555-6074d2551820 13 | github.com/go-macaron/session v0.0.0-20190805070824-1a3cdc6f5659 14 | github.com/go-sql-driver/mysql v1.4.1 15 | github.com/go-xorm/xorm v0.7.5 16 | github.com/jackc/pgx v3.5.0+incompatible // indirect 17 | github.com/juju/errors v0.0.0-20190207033735-e65537c515d7 // indirect 18 | github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect 19 | github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect 20 | github.com/mattn/go-colorable v0.1.2 // indirect 21 | github.com/minio/minio-go v6.0.14+incompatible 22 | github.com/mitchellh/go-homedir v1.1.0 // indirect 23 | github.com/prometheus/client_golang v1.1.0 24 | github.com/robfig/cron v1.2.0 25 | github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e 26 | github.com/unknwon/i18n v0.0.0-20190805065654-5c6446a380b6 27 | gopkg.in/clog.v1 v1.2.0 28 | gopkg.in/fsnotify.v1 v1.4.7 29 | gopkg.in/ini.v1 v1.46.0 30 | gopkg.in/macaron.v1 v1.3.4 31 | xorm.io/core v0.7.0 32 | ) 33 | -------------------------------------------------------------------------------- /internal/doc/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "flag" 19 | "net" 20 | "net/http" 21 | "time" 22 | 23 | log "gopkg.in/clog.v1" 24 | ) 25 | 26 | var ( 27 | dialTimeout = flag.Duration("packer_dial_timeout", 10*time.Second, "Timeout for dialing an HTTP connection.") 28 | requestTimeout = flag.Duration("packer_request_timeout", 20*time.Second, "Time out for roundtripping an HTTP request.") 29 | ) 30 | 31 | func timeoutDial(network, addr string) (net.Conn, error) { 32 | return net.DialTimeout(network, addr, *dialTimeout) 33 | } 34 | 35 | type transport struct { 36 | t http.Transport 37 | } 38 | 39 | func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { 40 | timer := time.AfterFunc(*requestTimeout, func() { 41 | t.t.CancelRequest(req) 42 | log.Warn("Canceled request for %s", req.URL) 43 | }) 44 | defer timer.Stop() 45 | resp, err := t.t.RoundTrip(req) 46 | return resp, err 47 | } 48 | 49 | var ( 50 | httpTransport = &transport{t: http.Transport{Dial: timeoutDial, ResponseHeaderTimeout: *requestTimeout / 2}} 51 | Client = &http.Client{Transport: httpTransport} 52 | ) 53 | -------------------------------------------------------------------------------- /conf/locale/locale_zh-CN.ini: -------------------------------------------------------------------------------- 1 | app_desc = Go 语言在线 API 文档 2 | 3 | home = 首页 4 | settings = 个人设置 5 | api_service = API 服务 6 | about = 关于 7 | help = 帮助 8 | 9 | import_path = 导入路径 10 | short_intro = 简介 11 | 12 | [help] 13 | title = 控制面板 14 | desc = 快捷键可以极大程度上提供您的工作效率。 15 | shortcut = 快捷键 16 | usage = 说明 17 | 18 | [home] 19 | hero_title = 搜索 %s 个 Go 语言项目 20 | search_holder = 请输入项目路径或关键字 21 | semantic_search_desc = 切换至由以下站点提供的语义搜索 22 | browse_history = 项目浏览记录 23 | view_time = 浏览时间(本地) 24 | standard = 标准库 25 | 26 | more_projects = 更多项目 27 | gorepos = Go 语言标准库 28 | gosubrepos = Go 语言子仓库 29 | gaesdk = GAE 开发工具包 30 | community = 社区站点 31 | 32 | [form] 33 | flash_error_title = 槽糕!大事不好了,快点跑路吧... 34 | click_to_search = 单击此处搜索 %[1]s 35 | 36 | [docs] 37 | turn_into_search = 温馨提示 如果您想要使用与标准库重名的关键字进行搜索,请单击此链接:%[1]s 38 | view_on_github = 到 GitHub 上查看 39 | view_on_sg = 到 Sourcegraph 上查看 40 | display_readme = 显示 README 内容 41 | directories = 目录 42 | path = 路径 43 | synopsis = 简介 44 | 45 | search.title = 搜索导出对象 46 | search.desc = 通过名称来搜索导出对象。 47 | search.holder = 在此处输入导出对象名称 48 | search.button = 发射! 49 | 50 | refresh = 刷新文档 51 | refresh.too_often = 该文档于 5 分钟内生成,暂时无法进行刷新操作。请稍后再试! 52 | 53 | generate_success = 该项目的文档生成成功! 54 | 55 | note.package = 包 56 | note.import = 导入了 %d 个外部包。 57 | note.import_ref = 导入了 %[2]d 个外部包,并被 %[3]d 个包 引用。 58 | note.generated = 该文档生成于 %s。 59 | 60 | imports.title = 被 %s 导入的外部包 61 | imports.go_back = 返回到 上一页。 62 | refs.title = 导入 %s 的包 63 | 64 | [search] 65 | search = 搜搜搜! 66 | search_holder = 请输入关键字进行搜索 67 | search_btn = 砰! 68 | not_found = 您所搜索的对象已经失联。 69 | 70 | [tool] 71 | ago=之前 72 | from_now=之后 73 | now=现在 74 | 1s=1 秒%s 75 | 1m=1 分钟%s 76 | 1h=1 小时%s 77 | 1d=1 天%s 78 | 1w=1 周%s 79 | 1mon=1 月%s 80 | 1y=1 年%s 81 | seconds=%d 秒%s 82 | minutes=%d 分钟%s 83 | hours=%d 小时%s 84 | days=%d 天%s 85 | weeks=%d 周%s 86 | months=%d 月%s 87 | years=%d 年%s -------------------------------------------------------------------------------- /internal/spaces/spaces.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package spaces 16 | 17 | import ( 18 | "os" 19 | "sync" 20 | 21 | "github.com/minio/minio-go" 22 | log "gopkg.in/clog.v1" 23 | 24 | "github.com/unknwon/gowalker/internal/setting" 25 | ) 26 | 27 | var client *minio.Client 28 | var clientOnce sync.Once 29 | 30 | func Client() *minio.Client { 31 | clientOnce.Do(func() { 32 | var err error 33 | client, err = minio.New( 34 | setting.DigitalOcean.Spaces.Endpoint, 35 | setting.DigitalOcean.Spaces.AccessKey, 36 | setting.DigitalOcean.Spaces.SecretKey, 37 | true) 38 | if err != nil { 39 | log.Fatal(2, "Failed to new minio client: %v", err) 40 | } 41 | }) 42 | return client 43 | } 44 | 45 | // PutObject uploads an object with given path from local file to the bucket. 46 | func PutObject(localPath, objectName string) error { 47 | f, err := os.Open(localPath) 48 | if err != nil { 49 | return err 50 | } 51 | defer f.Close() 52 | 53 | fi, err := os.Stat(localPath) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | _, err = Client().PutObject(setting.DigitalOcean.Spaces.Bucket, objectName, f, fi.Size(), minio.PutObjectOptions{ 59 | UserMetadata: map[string]string{ 60 | "x-amz-acl": "public-read", 61 | }, 62 | }) 63 | return err 64 | } 65 | 66 | // RemoveObject deletes an object with given path from the bucket. 67 | func RemoveObject(objectName string) error { 68 | return Client().RemoveObject(setting.DigitalOcean.Spaces.Bucket, objectName) 69 | } 70 | -------------------------------------------------------------------------------- /internal/base/template.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package base 16 | 17 | import ( 18 | "path/filepath" 19 | "strings" 20 | 21 | "github.com/unknwon/i18n" 22 | log "gopkg.in/clog.v1" 23 | "gopkg.in/fsnotify.v1" 24 | 25 | "github.com/unknwon/gowalker/internal/setting" 26 | ) 27 | 28 | func monitorI18nLocale() { 29 | log.Info("Monitor i18n locale files enabled") 30 | 31 | watcher, err := fsnotify.NewWatcher() 32 | if err != nil { 33 | log.Fatal(2, "Failed to init locale watcher: %v", err) 34 | } 35 | 36 | go func() { 37 | for { 38 | select { 39 | case event := <-watcher.Events: 40 | switch filepath.Ext(event.Name) { 41 | case ".ini": 42 | if err := i18n.ReloadLangs(); err != nil { 43 | log.Error(2, "Failed to relaod locale file reloaded: %v", err) 44 | } 45 | log.Trace("Locale file reloaded: %s", strings.TrimPrefix(event.Name, "conf/locale/")) 46 | } 47 | } 48 | } 49 | }() 50 | 51 | if err := watcher.Add("conf/locale"); err != nil { 52 | log.Fatal(2, "Failed to start locale watcher: %v", err) 53 | } 54 | } 55 | 56 | func init() { 57 | if !setting.ProdMode { 58 | monitorI18nLocale() 59 | } 60 | } 61 | 62 | func SubStr(str string, start, length int) string { 63 | if len(str) == 0 { 64 | return "" 65 | } 66 | end := start + length 67 | if len(str) < end { 68 | return str 69 | } 70 | return str[start:end] + "..." 71 | } 72 | 73 | func RearSubStr(str string, length int) string { 74 | if len(str) == 0 { 75 | return "" 76 | } 77 | if len(str) < length { 78 | return str 79 | } 80 | return "..." + str[len(str)-length:] 81 | } 82 | -------------------------------------------------------------------------------- /internal/prometheus/js_file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package prometheus 16 | 17 | import ( 18 | "github.com/prometheus/client_golang/prometheus" 19 | 20 | "github.com/unknwon/gowalker/internal/db" 21 | ) 22 | 23 | var ( 24 | totalJSFilesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 25 | Namespace: "gowalker", 26 | Subsystem: "js_file", 27 | Name: "total", 28 | Help: "Number of total JS files", 29 | }, func() float64 { 30 | return float64(db.NumTotalJSFiles()) 31 | }) 32 | generatedJSFilesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 33 | Namespace: "gowalker", 34 | Subsystem: "js_file", 35 | Name: "generated", 36 | Help: "Number of generated JS files", 37 | }, func() float64 { 38 | return float64(db.NumGeneratedJSFiles()) 39 | }) 40 | distributedJSFilesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 41 | Namespace: "gowalker", 42 | Subsystem: "js_file", 43 | Name: "distributed", 44 | Help: "Number of distributed JS files", 45 | }, func() float64 { 46 | return float64(db.NumDistributedJSFiles()) 47 | }) 48 | recycledJSFilesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 49 | Namespace: "gowalker", 50 | Subsystem: "js_file", 51 | Name: "recycled", 52 | Help: "Number of recycled JS files", 53 | }, func() float64 { 54 | return float64(db.NumRecycledJSFiles()) 55 | }) 56 | ) 57 | 58 | func init() { 59 | prometheus.MustRegister( 60 | totalJSFilesGaugeFunc, 61 | generatedJSFilesGaugeFunc, 62 | distributedJSFilesGaugeFunc, 63 | recycledJSFilesGaugeFunc, 64 | ) 65 | } 66 | -------------------------------------------------------------------------------- /internal/prometheus/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package prometheus 16 | 17 | import ( 18 | "github.com/prometheus/client_golang/prometheus" 19 | 20 | "github.com/unknwon/gowalker/internal/db" 21 | ) 22 | 23 | var ( 24 | totalPackagesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 25 | Namespace: "gowalker", 26 | Subsystem: "package", 27 | Name: "total", 28 | Help: "Number of total packages", 29 | }, func() float64 { 30 | return float64(db.NumTotalPackages()) 31 | }) 32 | monthlyActivePackagesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 33 | Namespace: "gowalker", 34 | Subsystem: "package", 35 | Name: "monthly_active", 36 | Help: "Number of monthly active packages", 37 | }, func() float64 { 38 | return float64(db.NumMonthlyActivePackages()) 39 | }) 40 | weeklyActivePackagesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 41 | Namespace: "gowalker", 42 | Subsystem: "package", 43 | Name: "weekly_active", 44 | Help: "Number of weekly active packages", 45 | }, func() float64 { 46 | return float64(db.NumWeeklyActivePackages()) 47 | }) 48 | dailyActivePackagesGaugeFunc = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ 49 | Namespace: "gowalker", 50 | Subsystem: "package", 51 | Name: "daily_active", 52 | Help: "Number of daily active packages", 53 | }, func() float64 { 54 | return float64(db.NumDailyActivePackages()) 55 | }) 56 | ) 57 | 58 | func init() { 59 | prometheus.MustRegister( 60 | totalPackagesGaugeFunc, 61 | monthlyActivePackagesGaugeFunc, 62 | weeklyActivePackagesGaugeFunc, 63 | dailyActivePackagesGaugeFunc, 64 | ) 65 | } 66 | -------------------------------------------------------------------------------- /public/less/main.less: -------------------------------------------------------------------------------- 1 | @import '_link'; 2 | 3 | body { 4 | background-color: #FFFFFF; 5 | } 6 | tbody { 7 | td.stars { 8 | text-align: center !important; 9 | } 10 | } 11 | 12 | .break-word { 13 | word-break: break-word; 14 | } 15 | .page-docs { 16 | img.srcgraph { 17 | margin-bottom: -2px; 18 | } 19 | .tools { 20 | line-height: 35px; 21 | a:hover { 22 | text-decoration: none; 23 | } 24 | } 25 | ul { 26 | ul { 27 | margin-top: 0; 28 | margin-bottom: 0; 29 | } 30 | li { 31 | margin-top: 0; 32 | } 33 | } 34 | } 35 | .unstyled { 36 | margin-left: 0; 37 | list-style: none outside none; 38 | padding: 0; 39 | } 40 | 41 | .ui.grid.alert { 42 | padding-top: 10px; 43 | >div { 44 | padding-left: 0!important; 45 | padding-right: 0!important; 46 | } 47 | } 48 | .ui.collapse { 49 | margin-bottom: 10px; 50 | div { 51 | &:first-child { 52 | padding: 0; 53 | pre { 54 | margin-bottom: 0; 55 | } 56 | } 57 | &:nth-child(2) { 58 | display: none; 59 | padding-top: 0 !important; 60 | } 61 | } 62 | } 63 | .readme { 64 | padding: 10px 15px; 65 | border: 1px solid #d9d9d9; 66 | background-color:#fafafa; 67 | border-radius: 3px; 68 | -moz-border-radius: 3px; 69 | -webkit-border-radius: 3px; 70 | } 71 | .toast { 72 | margin-bottom: 5px; 73 | } 74 | .form-autocomplete { 75 | .menu { 76 | padding: 0; 77 | font-size: 15px; 78 | overflow-y: auto; 79 | max-height: 350px; 80 | } 81 | .divider { 82 | margin: 0; 83 | } 84 | .tile-subtitle { 85 | font-size: 14px; 86 | white-space: inherit !important; 87 | } 88 | } 89 | 90 | 91 | /* Comment, String, Internal, External, Return/Break, Keyword, Boolean/nil, Builtin */ 92 | pre { 93 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; 94 | background: #f7f8f9; 95 | padding: 10px 15px; 96 | font-size: 14px; 97 | overflow-x: auto; 98 | .com { color: #007500;} 99 | .boo { color: #0080FF;} 100 | .str { color: #796400;} 101 | .int { color: #5A5AAD;} 102 | .ext { color: #6F00D2;} 103 | .ret { color: #db2828;} 104 | .key { color: #e03997;} 105 | .bui { color: #009393;} 106 | 107 | a:hover { 108 | text-decoration: underline; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /internal/db/models.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "fmt" 19 | "sync/atomic" 20 | "time" 21 | 22 | _ "github.com/go-sql-driver/mysql" 23 | "github.com/go-xorm/xorm" 24 | "github.com/robfig/cron" 25 | log "gopkg.in/clog.v1" 26 | "xorm.io/core" 27 | 28 | "github.com/unknwon/gowalker/internal/setting" 29 | ) 30 | 31 | var x *xorm.Engine 32 | 33 | func init() { 34 | sec := setting.Cfg.Section("database") 35 | var err error 36 | x, err = xorm.NewEngine("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8", 37 | sec.Key("USER").String(), 38 | sec.Key("PASSWD").String(), 39 | sec.Key("HOST").String(), 40 | sec.Key("NAME").String())) 41 | if err != nil { 42 | log.Fatal(2, "Failed to init new engine: %v", err) 43 | } 44 | x.SetMapper(core.GonicMapper{}) 45 | 46 | if err = x.Sync(new(PkgInfo), new(PkgRef), new(JSFile)); err != nil { 47 | log.Fatal(2, "Failed to sync database: %v", err) 48 | } 49 | 50 | numTotalPackages, _ = x.Count(new(PkgInfo)) 51 | c := cron.New() 52 | if err = c.AddFunc("@every 1m", RefreshNumTotalPackages); err != nil { 53 | log.Fatal(2, "Failed to add func: %v", err) 54 | } else if err = c.AddFunc("@every 1m", DistributeJSFiles); err != nil { 55 | log.Fatal(2, "Failed to add func: %v", err) 56 | } else if err = c.AddFunc("@every 5m", RecycleJSFiles); err != nil { 57 | log.Fatal(2, "Failed to add func: %v", err) 58 | } 59 | c.Start() 60 | 61 | time.AfterFunc(5*time.Second, DistributeJSFiles) 62 | time.AfterFunc(10*time.Second, RecycleJSFiles) 63 | } 64 | 65 | // NOTE: Must be operated atomically 66 | var numTotalPackages int64 67 | 68 | func NumTotalPackages() int64 { 69 | return atomic.LoadInt64(&numTotalPackages) 70 | } 71 | -------------------------------------------------------------------------------- /internal/httplib/README.md: -------------------------------------------------------------------------------- 1 | # httplib 2 | httplib is an libs help you to curl remote url. 3 | 4 | # How to use? 5 | 6 | ## GET 7 | you can use Get to crawl data. 8 | 9 | import "github.com/astaxie/beego/httplib" 10 | 11 | str, err := httplib.Get("http://beego.me/").String() 12 | if err != nil { 13 | // error 14 | } 15 | fmt.Println(str) 16 | 17 | ## POST 18 | POST data to remote url 19 | 20 | req := httplib.Post("http://beego.me/") 21 | req.Param("username","astaxie") 22 | req.Param("password","123456") 23 | str, err := req.String() 24 | if err != nil { 25 | // error 26 | } 27 | fmt.Println(str) 28 | 29 | ## Set timeout 30 | 31 | The default timeout is `60` seconds, function prototype: 32 | 33 | SetTimeout(connectTimeout, readWriteTimeout time.Duration) 34 | 35 | Exmaple: 36 | 37 | // GET 38 | httplib.Get("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) 39 | 40 | // POST 41 | httplib.Post("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) 42 | 43 | 44 | ## Debug 45 | 46 | If you want to debug the request info, set the debug on 47 | 48 | httplib.Get("http://beego.me/").Debug(true) 49 | 50 | ## Set HTTP Basic Auth 51 | 52 | str, err := Get("http://beego.me/").SetBasicAuth("user", "passwd").String() 53 | if err != nil { 54 | // error 55 | } 56 | fmt.Println(str) 57 | 58 | ## Set HTTPS 59 | 60 | If request url is https, You can set the client support TSL: 61 | 62 | httplib.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) 63 | 64 | More info about the `tls.Config` please visit http://golang.org/pkg/crypto/tls/#Config 65 | 66 | ## Set HTTP Version 67 | 68 | some servers need to specify the protocol version of HTTP 69 | 70 | httplib.Get("http://beego.me/").SetProtocolVersion("HTTP/1.1") 71 | 72 | ## Set Cookie 73 | 74 | some http request need setcookie. So set it like this: 75 | 76 | cookie := &http.Cookie{} 77 | cookie.Name = "username" 78 | cookie.Value = "astaxie" 79 | httplib.Get("http://beego.me/").SetCookie(cookie) 80 | 81 | ## Upload file 82 | 83 | httplib support mutil file upload, use `req.PostFile()` 84 | 85 | req := httplib.Post("http://beego.me/") 86 | req.Param("username","astaxie") 87 | req.PostFile("uploadfile1", "httplib.pdf") 88 | str, err := req.String() 89 | if err != nil { 90 | // error 91 | } 92 | fmt.Println(str) 93 | 94 | 95 | See godoc for further documentation and examples. 96 | 97 | * [godoc.org/github.com/astaxie/beego/httplib](https://godoc.org/github.com/astaxie/beego/httplib) 98 | -------------------------------------------------------------------------------- /templates/home.html: -------------------------------------------------------------------------------- 1 | {% extends "base/base.html" %} 2 | {% block title %}Go Walker - {{Tr(Lang, "app_desc")}}{% endblock %} 3 | {% block body %} 4 |
5 |
6 |
7 | 8 |

{{Tr(Lang, "home.hero_title", NumTotalPackages)}}

9 |

10 | {% if Lang == "zh-CN" %} 11 | Go Walker 是一个可以在线生成并浏览 Go 项目 API 文档的 Web 服务器,目前已支持包括 GitHub 等代码托管平台。 12 | {% else %} 13 | Go Walker is a server that generates Go projects API documentation on the fly for the projects on GitHub. 14 | {% endif %} 15 |

16 | 17 |
18 |
19 |
20 | 21 | 22 | 26 |
27 | 28 |
29 |
30 |
31 | 32 | {% if BrowsingHistory %} 33 |

34 |
35 |

36 | 37 |

{{Tr(Lang, "home.browse_history")}}

38 | 39 | 40 | {% for p in BrowsingHistory %} 41 | 42 | 48 | 49 | 50 | {% endfor %} 51 | 52 |
= 40 %}class="popup" data-content="{{p.ImportPath}}"{% endif %}> 43 | {{RearSubStr(p.ImportPath, 40)}} 44 | {% if p.IsGoRepo %} 45 | {{Tr(Lang, "home.standard")}} 46 | {% endif %} 47 | {{p.LastViewed}}
53 | {% endif %} 54 |
55 |
56 | {% endblock %} -------------------------------------------------------------------------------- /internal/db/js_file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "errors" 19 | ) 20 | 21 | var ErrJSFileNotFound = errors.New("JS file does not exist") 22 | 23 | type JSFileStatus int 24 | 25 | const ( 26 | JSFileStatusNone JSFileStatus = iota 27 | JSFileStatusGenerated 28 | JSFileStatusDistributed 29 | JSFileStatusRecycled 30 | ) 31 | 32 | type JSFile struct { 33 | ID int64 34 | PkgID int64 `xorm:"INDEX UNIQUE(pkg_id_etag)"` 35 | Etag string `xorm:"UNIQUE(pkg_id_etag)"` 36 | Status JSFileStatus 37 | NumExtraFiles int // Indicates the number of extra JS files generated 38 | } 39 | 40 | func GetJSFile(pkgID int64, etag string) (*JSFile, error) { 41 | jsFile := new(JSFile) 42 | has, err := x.Where("pkg_id = ? AND etag = ?", pkgID, etag).Get(jsFile) 43 | if err != nil { 44 | return nil, err 45 | } else if !has { 46 | return nil, ErrJSFileNotFound 47 | } 48 | 49 | return jsFile, nil 50 | } 51 | 52 | // SaveJSFile inserts or updates JSFile record with given etag. 53 | func SaveJSFile(jsFile *JSFile) error { 54 | if jsFile.PkgID == 0 { 55 | return errors.New("PkgID is zero") 56 | } 57 | 58 | oldJSFile, err := GetJSFile(jsFile.PkgID, jsFile.Etag) 59 | if err != nil && err != ErrJSFileNotFound { 60 | return err 61 | } 62 | 63 | if err == ErrJSFileNotFound { 64 | _, err = x.InsertOne(jsFile) 65 | return err 66 | } 67 | 68 | jsFile.ID = oldJSFile.ID 69 | _, err = x.ID(jsFile.ID).AllCols().Update(jsFile) 70 | return err 71 | } 72 | 73 | func NumTotalJSFiles() int64 { 74 | count, _ := x.Count(new(JSFile)) 75 | return count 76 | } 77 | 78 | func NumGeneratedJSFiles() int64 { 79 | count, _ := x.Where("status = ?", JSFileStatusGenerated).Count(new(JSFile)) 80 | return count 81 | } 82 | 83 | func NumDistributedJSFiles() int64 { 84 | count, _ := x.Where("status = ?", JSFileStatusDistributed).Count(new(JSFile)) 85 | return count 86 | } 87 | 88 | func NumRecycledJSFiles() int64 { 89 | count, _ := x.Where("status = ?", JSFileStatusRecycled).Count(new(JSFile)) 90 | return count 91 | } 92 | -------------------------------------------------------------------------------- /conf/locale/locale_en-US.ini: -------------------------------------------------------------------------------- 1 | app_desc = Online Go API Documentation 2 | 3 | home = Home 4 | settings = Settings 5 | api_service = API Service 6 | about = About 7 | help = Help 8 | 9 | import_path = Import Path 10 | short_intro = Synopsis 11 | 12 | [help] 13 | title = Control Panel 14 | desc = Shortcuts can easily maximize your performance. 15 | shortcut = Shortcut 16 | usage = Usage 17 | 18 | [home] 19 | hero_title = Type to search %s Go projects 20 | search_holder = Type import path or keywords to start 21 | semantic_search_desc = Switch to semantic search powered by 22 | browse_history = Browsing History 23 | view_time = Viewed Time (Local) 24 | standard = Standard 25 | 26 | more_projects = More Projects 27 | gorepos = Go Standard Library 28 | gosubrepos = Go Sub-repositories 29 | gaesdk = Google App Engine SDK 30 | community = Community 31 | 32 | [form] 33 | flash_error_title = Oops! Something is going wrong... 34 | click_to_search = Click to search %[1]s 35 | 36 | [docs] 37 | turn_into_search = Uh huh! The following link will redirect to raw search for keyword %[1]s. 38 | view_on_github = View on GitHub 39 | view_on_sg = View on Sourcegraph 40 | display_readme = Display README 41 | directories = Directories 42 | path = Path 43 | synopsis = Synopsis 44 | 45 | search.title = Search Exports 46 | search.desc = Search exported objects by typing their names. 47 | search.holder = Type exported object name here 48 | search.button = Fire! 49 | 50 | refresh = Refresh 51 | refresh.too_often = This documentation was generated within 5 minutes, cannot be refreshed again at the moment. Please try again later! 52 | 53 | generate_success = Documentation of this package have generated successfully! 54 | 55 | note.package = Package 56 | note.import = imports %d packages. 57 | note.import_ref = imports %[2]d packages, and is imported by %[3]d packages. 58 | note.generated = This documentation is generated %s. 59 | 60 | imports.title = Packages imported by %s 61 | imports.go_back = Go back to previous page. 62 | refs.title = Packages import %s 63 | 64 | [search] 65 | search = Search 66 | search_holder = Type keywords to search 67 | search_btn = Boom! 68 | not_found = No results found. 69 | 70 | [tool] 71 | ago = ago 72 | from_now = from now 73 | now = now 74 | 1s = 1 second %s 75 | 1m = 1 minute %s 76 | 1h = 1 hour %s 77 | 1d = 1 day %s 78 | 1w = 1 week %s 79 | 1mon = 1 month %s 80 | 1y = 1 year %s 81 | seconds = %d seconds %s 82 | minutes = %d minutes %s 83 | hours = %d hours %s 84 | days = %d days %s 85 | weeks = %d weeks %s 86 | months = %d months %s 87 | years = %d years %s -------------------------------------------------------------------------------- /internal/route/home.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package route 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/unknwon/com" 22 | "github.com/unknwon/gowalker/internal/base" 23 | "github.com/unknwon/gowalker/internal/context" 24 | "github.com/unknwon/gowalker/internal/db" 25 | ) 26 | 27 | const ( 28 | HOME = "home" 29 | ) 30 | 31 | type pkgInfo struct { 32 | ImportPath string 33 | IsGoRepo bool 34 | LastViewed int64 35 | } 36 | 37 | func getBrowsingHistory(ctx *context.Context) []*pkgInfo { 38 | rawInfos := strings.Split(ctx.GetCookie("user_history"), "|") 39 | pkgIDs := make([]int64, 0, len(rawInfos)) // ID -> Unix 40 | lastViewedTimes := make(map[int64]int64) 41 | for _, rawInfo := range rawInfos { 42 | fields := strings.Split(rawInfo, ":") 43 | if len(fields) != 2 { 44 | continue 45 | } 46 | 47 | pkgID := com.StrTo(fields[0]).MustInt64() 48 | if pkgID == 0 { 49 | continue 50 | } 51 | pkgIDs = append(pkgIDs, pkgID) 52 | 53 | lastViewedTimes[pkgID] = com.StrTo(fields[1]).MustInt64() 54 | } 55 | 56 | // Get all package info in one single query. 57 | pkgInfos, err := db.GetPkgInfosByIDs(pkgIDs) 58 | if err != nil { 59 | ctx.Flash.Error(fmt.Sprintf("Cannot get browsing history: %v", err), true) 60 | return nil 61 | } 62 | pkgInfosSet := make(map[int64]*pkgInfo) 63 | for i := range pkgInfos { 64 | pkgInfosSet[pkgInfos[i].ID] = &pkgInfo{ 65 | ImportPath: pkgInfos[i].ImportPath, 66 | IsGoRepo: pkgInfos[i].IsGoRepo, 67 | } 68 | } 69 | 70 | // Assign package info in the same order they stored in cookie. 71 | localPkgInfos := make([]*pkgInfo, 0, len(pkgIDs)) 72 | for i := range pkgIDs { 73 | if pkgInfosSet[pkgIDs[i]] == nil { 74 | continue 75 | } 76 | 77 | pkgInfosSet[pkgIDs[i]].LastViewed = lastViewedTimes[pkgIDs[i]] 78 | localPkgInfos = append(localPkgInfos, pkgInfosSet[pkgIDs[i]]) 79 | } 80 | 81 | return localPkgInfos 82 | } 83 | 84 | func Home(c *context.Context) { 85 | c.PageIs("Home") 86 | c.Data["NumTotalPackages"] = base.FormatNumString(db.NumTotalPackages()) 87 | c.Data["BrowsingHistory"] = getBrowsingHistory(c) 88 | c.Success(HOME) 89 | } 90 | -------------------------------------------------------------------------------- /internal/base/path.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Copyright 2015 Unknwon 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 5 | // not use this file except in compliance with the License. You may obtain 6 | // a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | // License for the specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package base 17 | 18 | import ( 19 | "path" 20 | "regexp" 21 | "strings" 22 | ) 23 | 24 | var validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`) 25 | var validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+][-A-Za-z0-9_.]*$`) 26 | 27 | func isValidPathElement(s string) bool { 28 | return validPathElement.MatchString(s) && s != "testdata" 29 | } 30 | 31 | // IsValidRemotePath returns true if importPath is structurally valid for "go get". 32 | func IsValidRemotePath(importPath string) bool { 33 | parts := strings.Split(importPath, "/") 34 | if len(parts) <= 1 { 35 | // Import path must contain at least one "/". 36 | return false 37 | } 38 | if !validTLDs[path.Ext(parts[0])] { 39 | return false 40 | } 41 | if !validHost.MatchString(parts[0]) { 42 | return false 43 | } 44 | for _, part := range parts[1:] { 45 | if !isValidPathElement(part) { 46 | return false 47 | } 48 | } 49 | return true 50 | } 51 | 52 | // IsGoRepoPath returns true if path is in $GOROOT/src. 53 | func IsGoRepoPath(path string) bool { 54 | return PathFlag(path)&goRepoPath != 0 55 | } 56 | 57 | // IsGAERepoPath returns true if path is from appengine SDK. 58 | func IsGAERepoPath(path string) bool { 59 | return PathFlag(path)&gaeRepoPath != 0 60 | } 61 | 62 | // IsValidPath returns true if importPath is structurally valid. 63 | func IsValidPath(importPath string) bool { 64 | return PathFlag(importPath)&packagePath != 0 || IsValidRemotePath(importPath) 65 | } 66 | 67 | func IsDocFile(n string) bool { 68 | if strings.HasSuffix(n, ".go") && n[0] != '_' && n[0] != '.' { 69 | return true 70 | } 71 | return strings.HasPrefix(strings.ToLower(n), "readme") 72 | } 73 | 74 | var filterDirNames = []string{ 75 | "static", "docs", "views", "js", "assets", "public", "img", "css"} 76 | 77 | // FilterDirName guess the file or directory is or contains Go source files. 78 | func FilterDirName(name string) bool { 79 | for _, v := range filterDirNames { 80 | if strings.Contains(strings.ToLower(name), "/"+v+"/") { 81 | return false 82 | } 83 | } 84 | return true 85 | } 86 | -------------------------------------------------------------------------------- /internal/route/search.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package route 16 | 17 | import ( 18 | "strings" 19 | "unicode" 20 | 21 | log "gopkg.in/clog.v1" 22 | 23 | "github.com/unknwon/gowalker/internal/base" 24 | "github.com/unknwon/gowalker/internal/context" 25 | "github.com/unknwon/gowalker/internal/db" 26 | ) 27 | 28 | const ( 29 | SEARCH = "search" 30 | ) 31 | 32 | func Search(ctx *context.Context) { 33 | q := ctx.Query("q") 34 | 35 | // Clean up keyword. 36 | q = strings.TrimFunc(q, func(c rune) bool { 37 | return unicode.IsSpace(c) || c == '"' 38 | }) 39 | 40 | if ctx.Query("auto_redirect") == "true" && 41 | (base.IsGoRepoPath(q) || base.IsGAERepoPath(q) || 42 | base.IsValidRemotePath(q)) { 43 | ctx.Redirect("/" + q) 44 | return 45 | } 46 | 47 | var ( 48 | results []*db.PkgInfo 49 | err error 50 | ) 51 | switch q { 52 | case "gorepos": 53 | results, err = db.GetGoRepos() 54 | case "gosubrepos": 55 | results, err = db.GetGoSubepos() 56 | case "gaesdk": 57 | results, err = db.GetGAERepos() 58 | default: 59 | results, err = db.SearchPkgInfo(100, q) 60 | } 61 | if err != nil { 62 | ctx.Flash.Error(err.Error(), true) 63 | } else { 64 | ctx.Data["Results"] = results 65 | } 66 | 67 | ctx.Data["Keyword"] = q 68 | ctx.HTML(200, SEARCH) 69 | } 70 | 71 | type searchResult struct { 72 | Title string `json:"title"` 73 | Description string `json:"description"` 74 | URL string `json:"url"` 75 | } 76 | 77 | func SearchJSON(ctx *context.Context) { 78 | q := ctx.Query("q") 79 | 80 | // Clean up keyword 81 | q = strings.TrimFunc(q, func(c rune) bool { 82 | return unicode.IsSpace(c) || c == '"' 83 | }) 84 | 85 | pinfos, err := db.SearchPkgInfo(7, q) 86 | if err != nil { 87 | log.Error(2, "SearchPkgInfo '%s': %v", q, err) 88 | return 89 | } 90 | 91 | results := make([]*searchResult, len(pinfos)) 92 | for i := range pinfos { 93 | results[i] = &searchResult{ 94 | Title: pinfos[i].ImportPath, 95 | Description: pinfos[i].Synopsis, 96 | URL: "/" + pinfos[i].ImportPath, 97 | } 98 | } 99 | 100 | ctx.JSON(200, map[string]interface{}{ 101 | "results": results, 102 | }) 103 | } 104 | -------------------------------------------------------------------------------- /gowalker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | // Go Walker is a server that generates Go projects API documentation on the fly. 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "net/http" 21 | "strings" 22 | 23 | "github.com/go-macaron/i18n" 24 | "github.com/go-macaron/pongo2" 25 | "github.com/go-macaron/session" 26 | "github.com/prometheus/client_golang/prometheus/promhttp" 27 | log "gopkg.in/clog.v1" 28 | "gopkg.in/macaron.v1" 29 | 30 | "github.com/unknwon/gowalker/internal/context" 31 | _ "github.com/unknwon/gowalker/internal/prometheus" 32 | "github.com/unknwon/gowalker/internal/route" 33 | "github.com/unknwon/gowalker/internal/route/apiv1" 34 | "github.com/unknwon/gowalker/internal/setting" 35 | ) 36 | 37 | const Version = "2.5.3.1020" 38 | 39 | func init() { 40 | setting.AppVer = Version 41 | } 42 | 43 | // newMacaron initializes Macaron instance. 44 | func newMacaron() *macaron.Macaron { 45 | m := macaron.New() 46 | if !setting.DisableRouterLog { 47 | m.Use(macaron.Logger()) 48 | } 49 | m.Use(macaron.Recovery()) 50 | m.Use(macaron.Static("public", 51 | macaron.StaticOptions{ 52 | SkipLogging: setting.ProdMode, 53 | }, 54 | )) 55 | m.Use(macaron.Static("raw", 56 | macaron.StaticOptions{ 57 | Prefix: "raw", 58 | SkipLogging: setting.ProdMode, 59 | })) 60 | m.Use(pongo2.Pongoer(pongo2.Options{ 61 | IndentJSON: !setting.ProdMode, 62 | })) 63 | m.Use(i18n.I18n()) 64 | m.Use(session.Sessioner()) 65 | m.Use(context.Contexter()) 66 | return m 67 | } 68 | 69 | func main() { 70 | log.Info("Go Walker %s", Version) 71 | log.Info("Run Mode: %s", strings.Title(macaron.Env)) 72 | 73 | m := newMacaron() 74 | m.Get("/", route.Home) 75 | m.Get("/search", route.Search) 76 | m.Get("/search/json", route.SearchJSON) 77 | 78 | m.Group("/api", func() { 79 | m.Group("/v1", func() { 80 | m.Get("/badge", apiv1.Badge) 81 | }) 82 | }) 83 | 84 | m.Get("/-/metrics", promhttp.Handler()) 85 | 86 | m.Get("/robots.txt", func() string { 87 | return `User-agent: * 88 | Disallow: /search` 89 | }) 90 | m.Get("/*", route.Docs) 91 | 92 | listenAddr := fmt.Sprintf("0.0.0.0:%d", setting.HTTPPort) 93 | log.Info("Listen: http://%s", listenAddr) 94 | if err := http.ListenAndServe(listenAddr, m); err != nil { 95 | log.Fatal(2, "Failed to start server: %v", err) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /templates/docs/docs.html: -------------------------------------------------------------------------------- 1 | {% extends "base/base.html" %} 2 | {% block body %} 3 |
4 | {% include "docs/header.html" %} 5 | 6 | {% if IsHasReadme %} 7 |
8 |
9 | 10 | {{Tr(Lang, "docs.display_readme")}} 11 |
12 |
13 |
14 |
15 |
16 |
17 | {% else %} 18 |
19 | {% endif %} 20 | 21 |
22 | {% for doc in DocJS %} 23 | 24 | {% endfor %} 25 | 26 | {% if IsHasSubdirs %} 27 |

28 | {{Tr(Lang, "docs.directories")}} 29 |

30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | {% for dir in Subdirs %} 40 | 41 | 42 | 43 | 44 | {% endfor %} 45 | 46 |
{{Tr(Lang, "docs.path")}}{{Tr(Lang, "docs.synopsis")}}
{{dir.Name}}{{dir.Synopsis}}
47 |
48 | {% endif %} 49 |
50 | 51 |

52 | {{Tr(Lang, "docs.note.package")}} {{ProjectName}} {% if RefNum == int64(0) %}{{Tr(Lang, "docs.note.import", Link, ImportNum) | safe}}{% else %}{{Tr(Lang, "docs.note.import_ref", Link, ImportNum, RefNum) | safe}}{% endif %} {{Tr(Lang, "docs.note.generated", TimeDuration)}} 53 | 54 | {% if CanRefresh %} 55 | 56 | {{Tr(Lang, "docs.refresh")}} 57 | 58 | {% endif %} 59 |

60 |
61 | 62 | 84 | {% endblock %} -------------------------------------------------------------------------------- /internal/setting/setting.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package setting 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/unknwon/com" 21 | log "gopkg.in/clog.v1" 22 | "gopkg.in/ini.v1" 23 | "gopkg.in/macaron.v1" 24 | ) 25 | 26 | var ( 27 | // Application settings 28 | AppVer string 29 | ProdMode bool 30 | DisableRouterLog bool 31 | 32 | // Server settings 33 | HTTPPort int 34 | FetchTimeout time.Duration 35 | DocsJSPath string 36 | DocsGobPath string 37 | 38 | DigitalOcean struct { 39 | Spaces struct { 40 | Enabled bool 41 | Endpoint string 42 | AccessKey string 43 | SecretKey string 44 | Bucket string 45 | BucketURL string `ini:"BUCKET_URL"` 46 | } 47 | } 48 | 49 | Maintenance struct { 50 | JSRecycleDays int `ini:"JS_RECYCLE_DAYS"` 51 | } 52 | 53 | // Global settings 54 | Cfg *ini.File 55 | GitHub struct { 56 | ClientID string `ini:"CLIENT_ID"` 57 | ClientSecret string 58 | } 59 | RefreshInterval = 5 * time.Minute 60 | ) 61 | 62 | func init() { 63 | log.New(log.CONSOLE, log.ConsoleConfig{}) 64 | 65 | sources := []interface{}{"conf/app.ini"} 66 | if com.IsFile("custom/app.ini") { 67 | sources = append(sources, "custom/app.ini") 68 | } 69 | 70 | var err error 71 | Cfg, err = macaron.SetConfig(sources[0], sources[1:]...) 72 | if err != nil { 73 | log.Fatal(2, "Failed to set configuration: %v", err) 74 | } 75 | Cfg.NameMapper = ini.AllCapsUnderscore 76 | 77 | if Cfg.Section("").Key("RUN_MODE").String() == "prod" { 78 | ProdMode = true 79 | macaron.Env = macaron.PROD 80 | macaron.ColorLog = false 81 | 82 | log.New(log.CONSOLE, log.ConsoleConfig{ 83 | Level: log.INFO, 84 | BufferSize: 100, 85 | }) 86 | } 87 | 88 | DisableRouterLog = Cfg.Section("").Key("DISABLE_ROUTER_LOG").MustBool() 89 | 90 | sec := Cfg.Section("server") 91 | HTTPPort = sec.Key("HTTP_PORT").MustInt(8080) 92 | FetchTimeout = time.Duration(sec.Key("FETCH_TIMEOUT").MustInt(60)) * time.Second 93 | DocsJSPath = sec.Key("DOCS_JS_PATH").MustString("raw/docs/") 94 | DocsGobPath = sec.Key("DOCS_GOB_PATH").MustString("raw/gob/") 95 | 96 | if err = Cfg.Section("github").MapTo(&GitHub); err != nil { 97 | log.Fatal(2, "Failed to map GitHub settings: %v", err) 98 | } else if err = Cfg.Section("digitalocean.spaces").MapTo(&DigitalOcean.Spaces); err != nil { 99 | log.Fatal(2, "Failed to map DigitalOcean.Spaces settings: %v", err) 100 | } else if err = Cfg.Section("maintenance").MapTo(&Maintenance); err != nil { 101 | log.Fatal(2, "Failed to map Maintenance settings: %v", err) 102 | } 103 | 104 | sec = Cfg.Section("log.discord") 105 | if sec.Key("ENABLED").MustBool() { 106 | log.New(log.DISCORD, log.DiscordConfig{ 107 | Level: log.ERROR, 108 | BufferSize: 100, 109 | URL: sec.Key("URL").MustString(""), 110 | Username: "Go Walker", 111 | }) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /internal/context/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package context 16 | 17 | import ( 18 | "fmt" 19 | "net/http" 20 | "strings" 21 | "time" 22 | 23 | "github.com/go-macaron/session" 24 | log "gopkg.in/clog.v1" 25 | "gopkg.in/macaron.v1" 26 | 27 | "github.com/unknwon/gowalker/internal/base" 28 | "github.com/unknwon/gowalker/internal/setting" 29 | ) 30 | 31 | // Context represents context of a request. 32 | type Context struct { 33 | *macaron.Context 34 | Flash *session.Flash 35 | } 36 | 37 | // Title sets "Title" field in template data. 38 | func (c *Context) Title(locale string) { 39 | c.Data["Title"] = c.Tr(locale) 40 | } 41 | 42 | // PageIs sets "PageIsxxx" field in template data. 43 | func (c *Context) PageIs(name string) { 44 | c.Data["PageIs"+name] = true 45 | } 46 | 47 | // HasError returns true if error occurs in form validation. 48 | func (c *Context) HasError() bool { 49 | hasErr, ok := c.Data["HasError"] 50 | if !ok { 51 | return false 52 | } 53 | c.Flash.ErrorMsg = c.Data["ErrorMsg"].(string) 54 | c.Data["Flash"] = c.Flash 55 | return hasErr.(bool) 56 | } 57 | 58 | // HTML calls Context.HTML and converts template name to string. 59 | func (c *Context) HTML(status int, name string) { 60 | c.Context.HTML(status, name) 61 | } 62 | 63 | // Success responses template with status http.StatusOK. 64 | func (c *Context) Success(name string) { 65 | c.HTML(http.StatusOK, name) 66 | } 67 | 68 | // RenderWithErr used for page has form validation but need to prompt error to users. 69 | func (c *Context) RenderWithErr(msg string, tpl string, form interface{}) { 70 | if form != nil { 71 | // auth.AssignForm(form, c.Data) 72 | } 73 | c.Flash.ErrorMsg = msg 74 | c.Data["Flash"] = c.Flash 75 | c.Success(tpl) 76 | } 77 | 78 | // Handle handles and logs error by given status. 79 | func (c *Context) Handle(status int, title string, err error) { 80 | if err != nil { 81 | log.Error(2, "%s: %v", title, err) 82 | if macaron.Env != macaron.PROD { 83 | c.Data["ErrorMsg"] = err 84 | } 85 | } 86 | 87 | switch status { 88 | case 404: 89 | c.Data["Title"] = "Page Not Found" 90 | case 500: 91 | c.Data["Title"] = "Internal Server Error" 92 | } 93 | c.HTML(status, fmt.Sprintf("status/%d", status)) 94 | } 95 | 96 | // Contexter initializes a classic context for a request. 97 | func Contexter() macaron.Handler { 98 | return func(c *macaron.Context, f *session.Flash) { 99 | ctx := &Context{ 100 | Context: c, 101 | Flash: f, 102 | } 103 | 104 | // Compute current URL for real-time change language. 105 | ctx.Data["Link"] = ctx.Req.URL.Path 106 | 107 | ctx.Data["AppVer"] = setting.AppVer 108 | ctx.Data["ProdMode"] = setting.ProdMode 109 | ctx.Data["SubStr"] = base.SubStr 110 | ctx.Data["RearSubStr"] = base.RearSubStr 111 | ctx.Data["HasPrefix"] = strings.HasPrefix 112 | ctx.Data["int64"] = base.Int64 113 | ctx.Data["Year"] = time.Now().Year() 114 | 115 | c.Map(ctx) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /internal/base/tool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package base 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "time" 21 | 22 | "github.com/unknwon/com" 23 | "github.com/unknwon/i18n" 24 | ) 25 | 26 | // MapToSortedStrings converts a string map to a alphabet sorted slice without duplication. 27 | func MapToSortedStrings(m map[string]bool) []string { 28 | strs := make([]string, 0, len(m)) 29 | for s := range m { 30 | strs = append(strs, s) 31 | } 32 | sort.Strings(strs) 33 | return strs 34 | } 35 | 36 | func Int64(v int) int64 { 37 | return int64(v) 38 | } 39 | 40 | // Seconds-based time units 41 | const ( 42 | Minute = 60 43 | Hour = 60 * Minute 44 | Day = 24 * Hour 45 | Week = 7 * Day 46 | Month = 30 * Day 47 | Year = 12 * Month 48 | ) 49 | 50 | func TimeSince(then time.Time, lang string) string { 51 | now := time.Now() 52 | 53 | lbl := i18n.Tr(lang, "tool.ago") 54 | diff := now.Unix() - then.Unix() 55 | if then.After(now) { 56 | lbl = i18n.Tr(lang, "tool.from_now") 57 | diff = then.Unix() - now.Unix() 58 | } 59 | 60 | switch { 61 | case diff <= 0: 62 | return i18n.Tr(lang, "tool.now") 63 | case diff <= 2: 64 | return i18n.Tr(lang, "tool.1s", lbl) 65 | case diff < 1*Minute: 66 | return i18n.Tr(lang, "tool.seconds", diff, lbl) 67 | 68 | case diff < 2*Minute: 69 | return i18n.Tr(lang, "tool.1m", lbl) 70 | case diff < 1*Hour: 71 | return i18n.Tr(lang, "tool.minutes", diff/Minute, lbl) 72 | 73 | case diff < 2*Hour: 74 | return i18n.Tr(lang, "tool.1h", lbl) 75 | case diff < 1*Day: 76 | return i18n.Tr(lang, "tool.hours", diff/Hour, lbl) 77 | 78 | case diff < 2*Day: 79 | return i18n.Tr(lang, "tool.1d", lbl) 80 | case diff < 1*Week: 81 | return i18n.Tr(lang, "tool.days", diff/Day, lbl) 82 | 83 | case diff < 2*Week: 84 | return i18n.Tr(lang, "tool.1w", lbl) 85 | case diff < 1*Month: 86 | return i18n.Tr(lang, "tool.weeks", diff/Week, lbl) 87 | 88 | case diff < 2*Month: 89 | return i18n.Tr(lang, "tool.1mon", lbl) 90 | case diff < 1*Year: 91 | return i18n.Tr(lang, "tool.months", diff/Month, lbl) 92 | 93 | case diff < 2*Year: 94 | return i18n.Tr(lang, "tool.1y", lbl) 95 | default: 96 | return i18n.Tr(lang, "tool.years", diff/Year, lbl) 97 | } 98 | } 99 | 100 | func FormatNumString(num int64) (s string) { 101 | if num < 1000 { 102 | return com.ToStr(num) 103 | } 104 | 105 | for { 106 | d := num / 1000 107 | r := num % 1000 108 | if d == 0 { 109 | // Don't need leading 0's. 110 | s = "," + com.ToStr(r) + s 111 | break 112 | } 113 | s = fmt.Sprintf(",%03d", r) + s 114 | num = d 115 | } 116 | return s[1:] 117 | } 118 | 119 | // Int64sToStrings converts a slice of int64 to a slice of string. 120 | // CR: copied from github.com/gogits/gogs/modules/base/tool.go 121 | func Int64sToStrings(ints []int64) []string { 122 | strs := make([]string, len(ints)) 123 | for i := range ints { 124 | strs[i] = com.ToStr(ints[i]) 125 | } 126 | return strs 127 | } 128 | -------------------------------------------------------------------------------- /public/css/gowalker.min.css: -------------------------------------------------------------------------------- 1 | @charset "UTF-8"; 2 | @font-face { 3 | font-family: 'octicons'; 4 | src: url("/fonts/octicons.eot"); 5 | src: url("/fonts/octicons.eot#iefix") format("embedded-opentype"), url("/fonts/octicons.woff") format("woff"), url("/fonts/octicons.ttf") format("truetype"); 6 | font-weight: normal; 7 | font-style: normal; 8 | } 9 | .markdown h1 a.anchor, 10 | .markdown h2 a.anchor, 11 | .markdown h3 a.anchor, 12 | .markdown h4 a.anchor, 13 | .markdown h5 a.anchor, 14 | .markdown h6 a.anchor { 15 | text-decoration: none; 16 | line-height: 1; 17 | padding-left: 0; 18 | margin-left: 5px; 19 | top: 15%; 20 | } 21 | .markdown h1:hover .octicon-link, 22 | .markdown h2:hover .octicon-link, 23 | .markdown h3:hover .octicon-link, 24 | .markdown h4:hover .octicon-link, 25 | .markdown h5:hover .octicon-link, 26 | .markdown h6:hover .octicon-link { 27 | display: inline-block; 28 | } 29 | .markdown a span.octicon { 30 | font: normal normal 16px octicons; 31 | line-height: 1; 32 | display: inline-block; 33 | text-decoration: none; 34 | -webkit-font-smoothing: antialiased; 35 | } 36 | .markdown a span.octicon-link { 37 | display: none; 38 | color: #000; 39 | } 40 | .markdown a span.octicon-link:before { 41 | content: '\f05c'; 42 | } 43 | .markdown pre.code { 44 | margin-top: -15px !important; 45 | } 46 | body { 47 | background-color: #FFFFFF; 48 | } 49 | tbody td.stars { 50 | text-align: center !important; 51 | } 52 | .break-word { 53 | word-break: break-word; 54 | } 55 | .page-docs img.srcgraph { 56 | margin-bottom: -2px; 57 | } 58 | .page-docs .tools { 59 | line-height: 35px; 60 | } 61 | .page-docs .tools a:hover { 62 | text-decoration: none; 63 | } 64 | .page-docs ul ul { 65 | margin-top: 0; 66 | margin-bottom: 0; 67 | } 68 | .page-docs ul li { 69 | margin-top: 0; 70 | } 71 | .unstyled { 72 | margin-left: 0; 73 | list-style: none outside none; 74 | padding: 0; 75 | } 76 | .ui.grid.alert { 77 | padding-top: 10px; 78 | } 79 | .ui.grid.alert > div { 80 | padding-left: 0!important; 81 | padding-right: 0!important; 82 | } 83 | .ui.collapse { 84 | margin-bottom: 10px; 85 | } 86 | .ui.collapse div:first-child { 87 | padding: 0; 88 | } 89 | .ui.collapse div:first-child pre { 90 | margin-bottom: 0; 91 | } 92 | .ui.collapse div:nth-child(2) { 93 | display: none; 94 | padding-top: 0 !important; 95 | } 96 | .readme { 97 | padding: 10px 15px; 98 | border: 1px solid #d9d9d9; 99 | background-color: #fafafa; 100 | border-radius: 3px; 101 | -moz-border-radius: 3px; 102 | -webkit-border-radius: 3px; 103 | } 104 | .toast { 105 | margin-bottom: 5px; 106 | } 107 | .form-autocomplete .menu { 108 | padding: 0; 109 | font-size: 15px; 110 | overflow-y: auto; 111 | max-height: 350px; 112 | } 113 | .form-autocomplete .divider { 114 | margin: 0; 115 | } 116 | .form-autocomplete .tile-subtitle { 117 | font-size: 14px; 118 | white-space: inherit !important; 119 | } 120 | /* Comment, String, Internal, External, Return/Break, Keyword, Boolean/nil, Builtin */ 121 | pre { 122 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; 123 | background: #f7f8f9; 124 | padding: 10px 15px; 125 | font-size: 14px; 126 | overflow-x: auto; 127 | } 128 | pre .com { 129 | color: #007500; 130 | } 131 | pre .boo { 132 | color: #0080FF; 133 | } 134 | pre .str { 135 | color: #796400; 136 | } 137 | pre .int { 138 | color: #5A5AAD; 139 | } 140 | pre .ext { 141 | color: #6F00D2; 142 | } 143 | pre .ret { 144 | color: #db2828; 145 | } 146 | pre .key { 147 | color: #e03997; 148 | } 149 | pre .bui { 150 | color: #009393; 151 | } 152 | pre a:hover { 153 | text-decoration: underline; 154 | } 155 | -------------------------------------------------------------------------------- /internal/doc/golang.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "path" 21 | "strings" 22 | 23 | "github.com/unknwon/com" 24 | 25 | "github.com/unknwon/gowalker/internal/base" 26 | "github.com/unknwon/gowalker/internal/db" 27 | "github.com/unknwon/gowalker/internal/httplib" 28 | "github.com/unknwon/gowalker/internal/setting" 29 | ) 30 | 31 | var ( 32 | ErrPackageNotModified = errors.New("Package has not been modified") 33 | ErrPackageNoGoFile = errors.New("Package does not contain Go file") 34 | ) 35 | 36 | func getGolangDoc(importPath, etag string) (*Package, error) { 37 | httpGet := func(url string, v interface{}) error { 38 | return httplib.Get(url). 39 | SetBasicAuth(setting.GitHub.ClientID, setting.GitHub.ClientSecret). 40 | ToJson(v) 41 | } 42 | // Check revision. 43 | commit, err := getGithubRevision("github.com/golang/go", "master") 44 | if err != nil { 45 | return nil, fmt.Errorf("get revision: %v", err) 46 | } 47 | if commit == etag { 48 | return nil, ErrPackageNotModified 49 | } 50 | 51 | // Get files. 52 | var tree struct { 53 | Tree []struct { 54 | Url string 55 | Path string 56 | Type string 57 | } 58 | Url string 59 | } 60 | 61 | if err := httpGet("https://api.github.com/repos/golang/go/git/trees/master?recursive=1", &tree); err != nil { 62 | return nil, fmt.Errorf("get tree: %v", err) 63 | } 64 | 65 | dirPrefix := "src/" + importPath + "/" 66 | dirLevel := len(strings.Split(dirPrefix, "/")) 67 | dirLength := len(dirPrefix) 68 | dirMap := make(map[string]bool) 69 | files := make([]com.RawFile, 0, 10) 70 | 71 | for _, node := range tree.Tree { 72 | // Skip directories and files in irrelevant directories. 73 | if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) { 74 | continue 75 | } 76 | 77 | // Get files and check if directories have acceptable files. 78 | if d, f := path.Split(node.Path); base.IsDocFile(f) { 79 | // Check if file is in the directory that is corresponding to import path. 80 | if d == dirPrefix { 81 | files = append(files, &Source{ 82 | SrcName: f, 83 | BrowseUrl: com.Expand("github.com/golang/go/blob/master/{0}", nil, node.Path), 84 | RawSrcUrl: com.Expand("https://raw.github.com/golang/go/master/{0}", nil, node.Path), 85 | }) 86 | continue 87 | } 88 | 89 | // Otherwise, check if it's a direct sub-directory of import path. 90 | if len(strings.Split(d, "/"))-dirLevel == 1 { 91 | dirMap[d[dirLength:len(d)-1]] = true 92 | continue 93 | } 94 | } 95 | } 96 | 97 | dirs := base.MapToSortedStrings(dirMap) 98 | 99 | if len(files) == 0 && len(dirs) == 0 { 100 | return nil, ErrPackageNoGoFile 101 | } else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil { 102 | return nil, fmt.Errorf("fetch files: %v", err) 103 | } 104 | 105 | // Start generating data. 106 | w := &Walker{ 107 | LineFmt: "#L%d", 108 | Pdoc: &Package{ 109 | PkgInfo: &db.PkgInfo{ 110 | ImportPath: importPath, 111 | ProjectPath: "github.com/golang/go", 112 | ViewDirPath: "github.com/golang/go/tree/master/src/" + importPath, 113 | Etag: commit, 114 | IsGoRepo: true, 115 | Subdirs: strings.Join(dirs, "|"), 116 | }, 117 | }, 118 | } 119 | 120 | srcs := make([]*Source, 0, len(files)) 121 | srcMap := make(map[string]*Source) 122 | for _, f := range files { 123 | s := f.(*Source) 124 | srcs = append(srcs, s) 125 | 126 | if !strings.HasSuffix(f.Name(), "_test.go") { 127 | srcMap[f.Name()] = s 128 | } 129 | } 130 | 131 | pdoc, err := w.Build(&WalkRes{ 132 | WalkDepth: WD_All, 133 | WalkType: WT_Memory, 134 | WalkMode: WM_All, 135 | Srcs: srcs, 136 | }) 137 | if err != nil { 138 | return nil, fmt.Errorf("walk package: %v", err) 139 | } 140 | 141 | return pdoc, nil 142 | } 143 | -------------------------------------------------------------------------------- /internal/doc/struct.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "go/ast" 19 | "go/doc" 20 | "go/token" 21 | "os" 22 | "time" 23 | 24 | "github.com/unknwon/gowalker/internal/db" 25 | ) 26 | 27 | // Source represents a Source code file. 28 | type Source struct { 29 | SrcName string 30 | BrowseUrl string 31 | RawSrcUrl string 32 | SrcData []byte 33 | } 34 | 35 | func (s *Source) Name() string { return s.SrcName } 36 | func (s *Source) Size() int64 { return int64(len(s.RawSrcUrl)) } 37 | func (s *Source) Mode() os.FileMode { return 0 } 38 | func (s *Source) ModTime() time.Time { return time.Time{} } 39 | func (s *Source) IsDir() bool { return false } 40 | func (s *Source) Sys() interface{} { return nil } 41 | func (s *Source) RawUrl() string { return s.RawSrcUrl } 42 | func (s *Source) Data() []byte { return s.SrcData } 43 | func (s *Source) SetData(p []byte) { s.SrcData = p } 44 | 45 | // Example represents function or method examples. 46 | type Example struct { 47 | Name string 48 | Doc string 49 | Code string 50 | //Play string 51 | Output string 52 | IsUsed bool // Indicates if it's used by any kind object. 53 | } 54 | 55 | // Value represents constants and variable 56 | type Value struct { 57 | Name string // Value name. 58 | Doc string 59 | Decl, FmtDecl string // Normal and formatted form of declaration. 60 | URL string // VCS URL. 61 | } 62 | 63 | // Func represents functions 64 | type Func struct { 65 | Name, FullName string 66 | Doc string 67 | Decl, FmtDecl string 68 | URL string // VCS URL. 69 | Code string // Included field 'Decl', formatted. 70 | Examples []*Example 71 | } 72 | 73 | // Type represents structs and interfaces. 74 | type Type struct { 75 | Name string // Type name. 76 | Doc string 77 | Decl, FmtDecl string // Normal and formatted form of declaration. 78 | URL string // VCS URL. 79 | 80 | Consts, Vars []*Value 81 | Funcs []*Func // Exported functions that return this type. 82 | Methods []*Func // Exported methods. 83 | 84 | IFuncs []*Func // Internal functions that return this type. 85 | IMethods []*Func // Internal methods. 86 | 87 | Examples []*Example 88 | } 89 | 90 | // A File describles declaration of file. 91 | type File struct { 92 | // Top-level declarations. 93 | Consts []*Value 94 | Funcs []*Func 95 | Types []*Type 96 | Vars []*Value 97 | 98 | // Internal declarations. 99 | Ifuncs []*Func 100 | Itypes []*Type 101 | } 102 | 103 | // PkgDecl is package declaration in database acceptable form. 104 | type PkgDecl struct { 105 | Tag string // Current tag of project. 106 | Doc string // Package documentation(doc.go). 107 | 108 | File 109 | 110 | Examples []*Example // Function or method example. 111 | Imports, TestImports []string // Imports. 112 | Files, TestFiles []*Source // Source files. 113 | 114 | Notes []string // Source code notes. 115 | Dirs []string // Subdirectories 116 | } 117 | 118 | // Package represents the full documentation and declaration of a project or package. 119 | type Package struct { 120 | *db.PkgInfo 121 | 122 | Readme map[string][]byte 123 | 124 | *PkgDecl 125 | 126 | IsHasExport bool 127 | 128 | // Top-level declarations. 129 | IsHasConst, IsHasVar bool 130 | 131 | IsHasExample bool 132 | 133 | IsHasFile bool 134 | IsHasSubdir bool 135 | } 136 | 137 | // Walker holds the state used when building the documentation. 138 | type Walker struct { 139 | LineFmt string 140 | Pdoc *Package 141 | apkg *ast.Package 142 | Examples []*doc.Example // Function or method example. 143 | Fset *token.FileSet 144 | SrcLines map[string][]string // Source file line slices. 145 | SrcFiles map[string]*Source 146 | Buf []byte // scratch space for printNode method. 147 | } 148 | -------------------------------------------------------------------------------- /internal/base/gen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Go Authors. All rights reserved. 2 | // Copyright 2015 Unknwon 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | // +build ignore 7 | 8 | package main 9 | 10 | import ( 11 | "bytes" 12 | "encoding/json" 13 | "flag" 14 | "go/format" 15 | "log" 16 | "net/http" 17 | "os" 18 | "os/exec" 19 | "strings" 20 | "text/template" 21 | 22 | "github.com/unknwon/com" 23 | ) 24 | 25 | const ( 26 | goRepoPath = 1 << iota 27 | goSubrepoPath 28 | gaeRepoPath 29 | packagePath 30 | ) 31 | 32 | var tmpl = template.Must(template.New("").Parse(`// Created by go generate; DO NOT EDIT 33 | // Copyright 2014 The Go Authors. All rights reserved. 34 | // Use of this source code is governed by a BSD-style 35 | // license that can be found in the LICENSE file. 36 | 37 | package base 38 | 39 | const ( 40 | goRepoPath = {{.goRepoPath}} 41 | goSubrepoPath = {{.goSubrepoPath}} 42 | gaeRepoPath = {{.gaeRepoPath}} 43 | packagePath = {{.packagePath}} 44 | ) 45 | 46 | var pathFlags = map[string]int{ 47 | {{range $k, $v := .pathFlags}}{{printf "%q" $k}}: {{$v}}, 48 | {{end}} } 49 | 50 | func PathFlag(path string) int { 51 | return pathFlags[path] 52 | } 53 | 54 | func NumOfPathFlags() int { 55 | return len(pathFlags) 56 | } 57 | 58 | var paths []string 59 | 60 | func init() { 61 | paths = make([]string, 0, len(pathFlags)) 62 | for k := range pathFlags { 63 | paths = append(paths, k) 64 | } 65 | } 66 | 67 | func Paths() []string { 68 | return paths 69 | } 70 | 71 | var validTLDs = map[string]bool{ 72 | {{range $v := .validTLDs}}{{printf "%q" $v}}: true, 73 | {{end}} } 74 | `)) 75 | 76 | var output = flag.String("output", "data.go", "file name to write") 77 | 78 | type gitTree struct { 79 | Tree []struct { 80 | Path string `json:"path"` 81 | Type string `json:"type"` 82 | } `json:"tree"` 83 | } 84 | 85 | func getGitTree(url string) *gitTree { 86 | p, err := com.HttpGetBytes(&http.Client{}, url, nil) 87 | if err != nil { 88 | log.Fatal(err) 89 | } 90 | var t gitTree 91 | if err = json.Unmarshal(p, &t); err != nil { 92 | log.Fatal(err) 93 | } 94 | return &t 95 | } 96 | 97 | // getPathFlags builds map of standard/core repository path flags. 98 | func getPathFlags() map[string]int { 99 | cmd := exec.Command("go", "list", "std") 100 | p, err := cmd.Output() 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | pathFlags := map[string]int{ 105 | "builtin": packagePath | goRepoPath, 106 | "C": packagePath, 107 | } 108 | for _, path := range strings.Fields(string(p)) { 109 | if strings.HasPrefix(path, "cmd/") || strings.HasPrefix(path, "vendor/") { 110 | continue 111 | } 112 | pathFlags[path] |= packagePath | goRepoPath 113 | for { 114 | i := strings.LastIndex(path, "/") 115 | if i < 0 { 116 | break 117 | } 118 | path = path[:i] 119 | pathFlags[path] |= goRepoPath 120 | } 121 | } 122 | 123 | // Get GAE repository path flags. 124 | t := getGitTree("https://api.github.com/repos/golang/appengine/git/trees/master?recursive=1") 125 | pathFlags["appengine"] |= packagePath | gaeRepoPath 126 | for _, blob := range t.Tree { 127 | if blob.Type != "tree" { 128 | continue 129 | } 130 | pathFlags["appengine/"+blob.Path] |= packagePath | gaeRepoPath 131 | } 132 | return pathFlags 133 | } 134 | 135 | // getValidTLDs gets and returns list of valid TLDs. 136 | func getValidTLDs() (validTLDs []string) { 137 | p, err := com.HttpGetBytes(&http.Client{}, "http://data.iana.org/TLD/tlds-alpha-by-domain.txt", nil) 138 | if err != nil { 139 | log.Fatal(err) 140 | } 141 | 142 | for _, line := range strings.Split(string(p), "\n") { 143 | line = strings.TrimSpace(line) 144 | if len(line) == 0 || line[0] == '#' { 145 | continue 146 | } 147 | validTLDs = append(validTLDs, "."+strings.ToLower(line)) 148 | } 149 | return validTLDs 150 | } 151 | 152 | func main() { 153 | log.SetFlags(0) 154 | log.SetPrefix("gen: ") 155 | flag.Parse() 156 | if flag.NArg() != 0 { 157 | log.Fatal("usage: decgen [--output filename]") 158 | } 159 | 160 | // Generate output. 161 | var buf bytes.Buffer 162 | err := tmpl.Execute(&buf, map[string]interface{}{ 163 | "output": *output, 164 | "goRepoPath": goRepoPath, 165 | "goSubrepoPath": goSubrepoPath, 166 | "gaeRepoPath": gaeRepoPath, 167 | "packagePath": packagePath, 168 | "pathFlags": getPathFlags(), 169 | "validTLDs": getValidTLDs(), 170 | }) 171 | if err != nil { 172 | log.Fatal("template error:", err) 173 | } 174 | source, err := format.Source(buf.Bytes()) 175 | if err != nil { 176 | log.Fatal("source format error:", err) 177 | } 178 | fd, err := os.Create(*output) 179 | _, err = fd.Write(source) 180 | if err != nil { 181 | log.Fatal(err) 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /internal/httplib/httplib_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package httplib 16 | 17 | import ( 18 | "io/ioutil" 19 | "os" 20 | "strings" 21 | "testing" 22 | ) 23 | 24 | func TestResponse(t *testing.T) { 25 | req := Get("http://httpbin.org/get") 26 | resp, err := req.Response() 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | t.Log(resp) 31 | } 32 | 33 | func TestGet(t *testing.T) { 34 | req := Get("http://httpbin.org/get") 35 | b, err := req.Bytes() 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | t.Log(b) 40 | 41 | s, err := req.String() 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | t.Log(s) 46 | 47 | if string(b) != s { 48 | t.Fatal("request data not match") 49 | } 50 | } 51 | 52 | func TestSimplePost(t *testing.T) { 53 | v := "smallfish" 54 | req := Post("http://httpbin.org/post") 55 | req.Param("username", v) 56 | 57 | str, err := req.String() 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | t.Log(str) 62 | 63 | n := strings.Index(str, v) 64 | if n == -1 { 65 | t.Fatal(v + " not found in post") 66 | } 67 | } 68 | 69 | //func TestPostFile(t *testing.T) { 70 | // v := "smallfish" 71 | // req := Post("http://httpbin.org/post") 72 | // req.Debug(true) 73 | // req.Param("username", v) 74 | // req.PostFile("uploadfile", "httplib_test.go") 75 | 76 | // str, err := req.String() 77 | // if err != nil { 78 | // t.Fatal(err) 79 | // } 80 | // t.Log(str) 81 | 82 | // n := strings.Index(str, v) 83 | // if n == -1 { 84 | // t.Fatal(v + " not found in post") 85 | // } 86 | //} 87 | 88 | func TestSimplePut(t *testing.T) { 89 | str, err := Put("http://httpbin.org/put").String() 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | t.Log(str) 94 | } 95 | 96 | func TestSimpleDelete(t *testing.T) { 97 | str, err := Delete("http://httpbin.org/delete").String() 98 | if err != nil { 99 | t.Fatal(err) 100 | } 101 | t.Log(str) 102 | } 103 | 104 | func TestWithCookie(t *testing.T) { 105 | v := "smallfish" 106 | str, err := Get("http://httpbin.org/cookies/set?k1=" + v).SetEnableCookie(true).String() 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | t.Log(str) 111 | 112 | str, err = Get("http://httpbin.org/cookies").SetEnableCookie(true).String() 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | t.Log(str) 117 | 118 | n := strings.Index(str, v) 119 | if n == -1 { 120 | t.Fatal(v + " not found in cookie") 121 | } 122 | } 123 | 124 | func TestWithBasicAuth(t *testing.T) { 125 | str, err := Get("http://httpbin.org/basic-auth/user/passwd").SetBasicAuth("user", "passwd").String() 126 | if err != nil { 127 | t.Fatal(err) 128 | } 129 | t.Log(str) 130 | n := strings.Index(str, "authenticated") 131 | if n == -1 { 132 | t.Fatal("authenticated not found in response") 133 | } 134 | } 135 | 136 | func TestWithUserAgent(t *testing.T) { 137 | v := "beego" 138 | str, err := Get("http://httpbin.org/headers").SetUserAgent(v).String() 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | t.Log(str) 143 | 144 | n := strings.Index(str, v) 145 | if n == -1 { 146 | t.Fatal(v + " not found in user-agent") 147 | } 148 | } 149 | 150 | func TestWithSetting(t *testing.T) { 151 | v := "beego" 152 | var setting BeegoHttpSettings 153 | setting.EnableCookie = true 154 | setting.UserAgent = v 155 | setting.Transport = nil 156 | SetDefaultSetting(setting) 157 | 158 | str, err := Get("http://httpbin.org/get").String() 159 | if err != nil { 160 | t.Fatal(err) 161 | } 162 | t.Log(str) 163 | 164 | n := strings.Index(str, v) 165 | if n == -1 { 166 | t.Fatal(v + " not found in user-agent") 167 | } 168 | } 169 | 170 | func TestToJson(t *testing.T) { 171 | req := Get("http://httpbin.org/ip") 172 | resp, err := req.Response() 173 | if err != nil { 174 | t.Fatal(err) 175 | } 176 | t.Log(resp) 177 | 178 | // httpbin will return http remote addr 179 | type Ip struct { 180 | Origin string `json:"origin"` 181 | } 182 | var ip Ip 183 | err = req.ToJson(&ip) 184 | if err != nil { 185 | t.Fatal(err) 186 | } 187 | t.Log(ip.Origin) 188 | 189 | if n := strings.Count(ip.Origin, "."); n != 3 { 190 | t.Fatal("response is not valid ip") 191 | } 192 | } 193 | 194 | func TestToFile(t *testing.T) { 195 | f := "beego_testfile" 196 | req := Get("http://httpbin.org/ip") 197 | err := req.ToFile(f) 198 | if err != nil { 199 | t.Fatal(err) 200 | } 201 | defer os.Remove(f) 202 | b, err := ioutil.ReadFile(f) 203 | if n := strings.Index(string(b), "origin"); n == -1 { 204 | t.Fatal(err) 205 | } 206 | } 207 | 208 | func TestHeader(t *testing.T) { 209 | req := Get("http://httpbin.org/headers") 210 | req.Header("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36") 211 | str, err := req.String() 212 | if err != nil { 213 | t.Fatal(err) 214 | } 215 | t.Log(str) 216 | } 217 | -------------------------------------------------------------------------------- /public/img/sourcegraph-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /templates/docs/tpl.html: -------------------------------------------------------------------------------- 1 | {{ PkgFullIntro | safe }} 2 | 3 | {# START: Index #} 4 | {% if IsHasExports %} 5 |

6 | Index 7 |

8 | 46 | {% endif %} 47 | 48 | {% macro example_detail(ex) %} 49 |
50 |
51 |
52 | Example({{ex.Name}}) 53 |
54 |
55 |
56 | Code: 57 |
{{ex.Code | safe}}
58 | {% if ex.Output %} 59 | Output: 60 |
{{ex.Output}}
61 | {% endif %} 62 |
63 |
64 | {% endmacro %} 65 | 66 | {% if IsHasExample %} 67 |

Examples

68 | 79 | {% endif %} 80 | 81 | {# END: Index #} 82 | 83 | {# START: Constants #} 84 | {% if IsHasConst %} 85 |

Constants

86 | {% for c in Consts %} 87 |
{{c.FmtDecl | safe}}
88 | {{c.Doc | safe}} 89 | {% endfor %} 90 | {% endif %} 91 | {# END: Constants #} 92 | 93 | {# START: Variables #} 94 | {% if IsHasVar %} 95 |

Variables

96 | {% for v in Vars %} 97 |
{{v.FmtDecl | safe}}
98 | {{v.Doc | safe}} 99 | {% endfor %} 100 | {% endif %} 101 | 102 | {# END: Variables #} 103 | 104 | {# START: Functions #} 105 | {% for fn in Funcs %} 106 |

107 | func 108 | {{fn.Name}} 109 | 110 | 111 | 112 |

113 |
114 |
115 |
{{fn.FmtDecl | safe}}
116 |
117 |
118 |
{{fn.Code | safe}}
119 |
120 |
121 | 122 | {{fn.Doc | safe}} 123 | 124 | {% for ex in fn.Examples %} 125 | {{example_detail(ex)}} 126 | {% endfor %} 127 | {% endfor %} 128 | 129 | {# END: Functions #} 130 | 131 | {# START: Types #} 132 | {% for tp in Types %} 133 |

134 | type 135 | {{tp.Name}} 136 |

137 | 138 |
{{tp.FmtDecl | safe}}
139 | 140 | {{tp.Doc | safe}} 141 | 142 | {% for ex in tp.Examples %} 143 | {{example_detail(ex)}} 144 | {% endfor %} 145 | 146 | {# START: Types.Constants #} 147 | {% for c in tp.Consts %} 148 |
{{c.FmtDecl | safe}}
149 | {{c.Doc | safe}} 150 | {% endfor %} 151 | {# END: Types.Constants #} 152 | 153 | {# START: Types.Variables #} 154 | {% for v in tp.Vars %} 155 |
{{v.FmtDecl | safe}}
156 | {{v.Doc | safe}} 157 | {% endfor %} 158 | 159 | {# END: Types.Variables #} 160 | 161 | {# START: Types.Functions #} 162 | {% for fn in tp.Funcs %} 163 |

164 | func 165 | {{fn.Name}} 166 | 167 | 168 | 169 |

170 |
171 |
172 |
{{fn.FmtDecl | safe}}
173 |
174 |
175 |
{{fn.Code | safe}}
176 |
177 |
178 | 179 | {{fn.Doc | safe}} 180 | 181 | {% for ex in fn.Examples %} 182 | {{example_detail(ex)}} 183 | {% endfor %} 184 | {% endfor %} 185 | 186 | {# END: Types.Functions #} 187 | 188 | {# START: Types.Methods #} 189 | {% for fn in tp.Methods %} 190 |

191 | func 192 | {{fn.Name}} 193 | 194 | 195 | 196 |

197 | 198 |
199 |
200 |
{{fn.FmtDecl | safe}}
201 |
202 |
203 |
{{fn.Code | safe}}
204 |
205 |
206 | 207 | {{fn.Doc | safe}} 208 | 209 | {% for ex in fn.Examples %} 210 | {{example_detail(ex)}} 211 | {% endfor %} 212 | {% endfor %} 213 | {# END: Types.Methods #} 214 | {% endfor %} 215 | 216 | {# END: Types #} 217 | 218 | {% if IsHasFiles and ViewFilePath != "./" %} 219 |

220 | Files 221 |

222 |

223 | {% for f in Files %} 224 | {{f.SrcName}} 225 | {% endfor %} 226 |

227 | {% endif %} 228 | 229 | {{ExportDataSrc|safe}} -------------------------------------------------------------------------------- /internal/db/routine.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "sync/atomic" 21 | "time" 22 | 23 | log "gopkg.in/clog.v1" 24 | 25 | "github.com/unknwon/gowalker/internal/setting" 26 | "github.com/unknwon/gowalker/internal/spaces" 27 | ) 28 | 29 | func RefreshNumTotalPackages() { 30 | count, _ := x.Count(new(PkgInfo)) 31 | atomic.StoreInt64(&numTotalPackages, count) 32 | } 33 | 34 | func ComposeSpacesObjectNames(importPath, etag string, numExtraFiles int) []string { 35 | names := make([]string, numExtraFiles+1) 36 | for i := range names { 37 | if i == 0 { 38 | names[i] = fmt.Sprintf("%s-%s.js", importPath, etag) 39 | } else { 40 | names[i] = fmt.Sprintf("%s-%s-%d.js", importPath, etag, i) 41 | } 42 | } 43 | return names 44 | } 45 | 46 | var distributeJSFilesStatus int32 = 0 47 | 48 | // DistributeJSFiles uploads local JS files to DigitalOcean Spaces. 49 | func DistributeJSFiles() { 50 | if !setting.DigitalOcean.Spaces.Enabled { 51 | return 52 | } 53 | 54 | if !atomic.CompareAndSwapInt32(&distributeJSFilesStatus, 0, 1) { 55 | return 56 | } 57 | defer atomic.StoreInt32(&distributeJSFilesStatus, 0) 58 | 59 | log.Trace("Routine started: DistributeJSFiles") 60 | defer log.Trace("Routine ended: DistributeJSFiles") 61 | 62 | if err := x.Where("status = ?", JSFileStatusGenerated).Iterate(new(JSFile), func(idx int, bean interface{}) error { 63 | jsFile := bean.(*JSFile) 64 | 65 | // Gather package information 66 | pinfo, err := GetPkgInfoByID(jsFile.PkgID) 67 | if err != nil { 68 | if err == ErrPackageVersionTooOld { 69 | return nil 70 | } 71 | log.Error(2, "Failed to get package info by ID[%d]: %v", jsFile.PkgID, err) 72 | return nil 73 | } 74 | log.Trace("DistributeJSFiles[%d]: Distributing %q", jsFile.ID, pinfo.ImportPath) 75 | 76 | // Compose object names 77 | localJSPaths := pinfo.LocalJSPaths() 78 | objectNames := ComposeSpacesObjectNames(pinfo.ImportPath, jsFile.Etag, jsFile.NumExtraFiles) 79 | if len(objectNames) != len(localJSPaths) { 80 | log.Warn("DistributeJSFiles[%d]: Number of object names does not match local JS files: %d != %d", 81 | jsFile.ID, len(objectNames), len(localJSPaths)) 82 | return nil 83 | } 84 | for i, localPath := range localJSPaths { 85 | if err = spaces.PutObject(localPath, objectNames[i]); err != nil { 86 | log.Error(2, "Failed to put object[%s]: %v", objectNames[i], err) 87 | return nil 88 | } 89 | } 90 | 91 | // Update database records and clean up local disk 92 | jsFile.Status = JSFileStatusDistributed 93 | if err = SaveJSFile(jsFile); err != nil { 94 | log.Error(2, "Failed to save JS file[%d]: %v", jsFile.ID, err) 95 | return nil 96 | } 97 | 98 | for i := range localJSPaths { 99 | os.Remove(localJSPaths[i]) 100 | } 101 | 102 | log.Trace("DistributeJSFiles[%d]: Distributed %d files", jsFile.ID, len(objectNames)) 103 | return nil 104 | }); err != nil { 105 | log.Error(2, "Failed to distribute JS files: %v", err) 106 | } 107 | } 108 | 109 | var recycleJSFilesStatus int32 = 0 110 | 111 | // RecycleJSFiles deletes local or distributed JS files due to inactive status. 112 | func RecycleJSFiles() { 113 | if !atomic.CompareAndSwapInt32(&recycleJSFilesStatus, 0, 1) { 114 | return 115 | } 116 | defer atomic.StoreInt32(&recycleJSFilesStatus, 0) 117 | 118 | log.Trace("Routine started: RecycleJSFiles") 119 | defer log.Trace("Routine ended: RecycleJSFiles") 120 | 121 | outdated := time.Now().Add(-1 * time.Duration(setting.Maintenance.JSRecycleDays) * 24 * time.Hour).Unix() 122 | if err := x.Join("INNER", "pkg_info", "pkg_info.id = js_file.pkg_id"). 123 | Where("js_file.status < ? AND pkg_info.last_viewed < ?", JSFileStatusRecycled, outdated). 124 | Iterate(new(JSFile), func(idx int, bean interface{}) error { 125 | jsFile := bean.(*JSFile) 126 | 127 | // Gather package information 128 | pinfo, err := GetPkgInfoByID(jsFile.PkgID) 129 | if err != nil { 130 | if err == ErrPackageVersionTooOld { 131 | return nil 132 | } 133 | log.Error(2, "Failed to get package info by ID[%d]: %v", jsFile.PkgID, err) 134 | return nil 135 | } 136 | 137 | var numFiles int 138 | switch jsFile.Status { 139 | case JSFileStatusGenerated: 140 | localJSPaths := pinfo.LocalJSPaths() 141 | for i := range localJSPaths { 142 | os.Remove(localJSPaths[i]) 143 | } 144 | numFiles = len(localJSPaths) 145 | 146 | case JSFileStatusDistributed: 147 | if !setting.DigitalOcean.Spaces.Enabled { 148 | log.Warn("RecycleJSFiles[%d]: DigitalOcean Spaces is not enabled", jsFile.ID) 149 | return nil 150 | } 151 | 152 | objectNames := ComposeSpacesObjectNames(pinfo.ImportPath, jsFile.Etag, jsFile.NumExtraFiles) 153 | for i := range objectNames { 154 | if err = spaces.RemoveObject(objectNames[i]); err != nil { 155 | log.Error(2, "Failed to remove object[%s]: %v", objectNames[i], err) 156 | return nil 157 | } 158 | } 159 | numFiles = len(objectNames) 160 | 161 | default: 162 | log.Warn("RecycleJSFiles[%d]: Unexpected status %v", jsFile.ID, jsFile.Status) 163 | return nil 164 | } 165 | 166 | // FIXME: Database could be outdated if this operation fails, human must take action! 167 | jsFile.Status = JSFileStatusRecycled 168 | if err = SaveJSFile(jsFile); err != nil { 169 | log.Error(2, "Failed to save JS file[%d]: %v", jsFile.ID, err) 170 | return nil 171 | } 172 | 173 | log.Trace("RecycleJSFiles[%d]: Recycled %d files", jsFile.ID, numFiles) 174 | return nil 175 | }); err != nil { 176 | log.Error(2, "Failed to recycle JS files: %v", err) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /internal/route/docs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package route 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "path" 21 | "strings" 22 | "time" 23 | 24 | "github.com/unknwon/com" 25 | 26 | "github.com/unknwon/gowalker/internal/base" 27 | "github.com/unknwon/gowalker/internal/context" 28 | "github.com/unknwon/gowalker/internal/db" 29 | "github.com/unknwon/gowalker/internal/doc" 30 | "github.com/unknwon/gowalker/internal/setting" 31 | ) 32 | 33 | const ( 34 | DOCS = "docs/docs" 35 | DOCS_IMPORTS = "docs/imports" 36 | ) 37 | 38 | // updateHistory updates browser history. 39 | func updateHistory(ctx *context.Context, id int64) { 40 | pairs := make([]string, 1, 10) 41 | pairs[0] = com.ToStr(id) + ":" + com.ToStr(time.Now().UTC().Unix()) 42 | 43 | count := 0 44 | for _, pair := range strings.Split(ctx.GetCookie("user_history"), "|") { 45 | infos := strings.Split(pair, ":") 46 | if len(infos) != 2 { 47 | continue 48 | } 49 | 50 | pid := com.StrTo(infos[0]).MustInt64() 51 | if pid == 0 || pid == id { 52 | continue 53 | } 54 | 55 | pairs = append(pairs, pair) 56 | 57 | count++ 58 | if count == 9 { 59 | break 60 | } 61 | } 62 | ctx.SetCookie("user_history", strings.Join(pairs, "|"), 9999999) 63 | } 64 | 65 | func handleError(ctx *context.Context, err error) { 66 | importPath := ctx.Params("*") 67 | if err == doc.ErrInvalidRemotePath { 68 | ctx.Redirect("/search?q=" + importPath) 69 | return 70 | } 71 | 72 | if strings.Contains(err.Error(), " not found") || 73 | strings.Contains(err.Error(), "resource not found") { 74 | db.DeletePackageByPath(importPath) 75 | } 76 | 77 | ctx.Flash.Error(importPath+": "+err.Error(), true) 78 | ctx.Flash.Info(ctx.Tr("form.click_to_search", importPath), true) 79 | Home(ctx) 80 | } 81 | 82 | func specialHandles(ctx *context.Context, pinfo *db.PkgInfo) bool { 83 | // Only show imports. 84 | if strings.HasSuffix(ctx.Req.RequestURI, "?imports") { 85 | ctx.Data["PageIsImports"] = true 86 | ctx.Data["Packages"] = db.GetPkgInfosByPaths(strings.Split(pinfo.ImportPaths, "|")) 87 | ctx.HTML(200, DOCS_IMPORTS) 88 | return true 89 | } 90 | 91 | // Only show references. 92 | if strings.HasSuffix(ctx.Req.RequestURI, "?refs") { 93 | ctx.Data["PageIsRefs"] = true 94 | ctx.Data["Packages"] = pinfo.GetRefs() 95 | ctx.HTML(200, DOCS_IMPORTS) 96 | return true 97 | } 98 | 99 | // Refresh documentation. 100 | if strings.HasSuffix(ctx.Req.RequestURI, "?refresh") { 101 | if !pinfo.CanRefresh() { 102 | ctx.Flash.Info(ctx.Tr("docs.refresh.too_often")) 103 | } else { 104 | importPath := ctx.Params("*") 105 | _, err := doc.CheckPackage(importPath, ctx.Render, doc.RequestTypeRefresh) 106 | if err != nil { 107 | handleError(ctx, err) 108 | return true 109 | } 110 | } 111 | ctx.Redirect(ctx.Data["Link"].(string)) 112 | return true 113 | } 114 | 115 | return false 116 | } 117 | 118 | func Docs(c *context.Context) { 119 | importPath := c.Params("*") 120 | 121 | // Check if import path looks like a vendor directory 122 | if strings.Contains(importPath, "/vendor/") { 123 | handleError(c, errors.New("import path looks like is a vendor directory, don't try to fool me! :D")) 124 | return 125 | } 126 | 127 | if base.IsGAERepoPath(importPath) { 128 | c.Redirect("/google.golang.org/" + importPath) 129 | return 130 | } 131 | 132 | pinfo, err := doc.CheckPackage(importPath, c.Render, doc.RequestTypeHuman) 133 | if err != nil { 134 | handleError(c, err) 135 | return 136 | } 137 | 138 | c.PageIs("Docs") 139 | c.Title(pinfo.ImportPath) 140 | c.Data["ParentPath"] = path.Dir(pinfo.ImportPath) 141 | c.Data["ProjectName"] = path.Base(pinfo.ImportPath) 142 | c.Data["ProjectPath"] = pinfo.ProjectPath 143 | c.Data["NumStars"] = pinfo.Stars 144 | 145 | if specialHandles(c, pinfo) { 146 | return 147 | } 148 | 149 | if pinfo.IsGoRepo { 150 | c.Flash.Info(c.Tr("docs.turn_into_search", importPath), true) 151 | } 152 | 153 | c.Data["PkgDesc"] = pinfo.Synopsis 154 | 155 | // README 156 | lang := c.Data["Lang"].(string)[:2] 157 | readmePath := setting.DocsJSPath + pinfo.ImportPath + "_RM_" + lang + ".js" 158 | if com.IsFile(readmePath) { 159 | c.Data["IsHasReadme"] = true 160 | c.Data["ReadmePath"] = readmePath 161 | } else { 162 | readmePath := setting.DocsJSPath + pinfo.ImportPath + "_RM_en.js" 163 | if com.IsFile(readmePath) { 164 | c.Data["IsHasReadme"] = true 165 | c.Data["ReadmePath"] = readmePath 166 | } 167 | } 168 | 169 | // Documentation 170 | if pinfo.JSFile.Status == db.JSFileStatusDistributed { 171 | docJS := db.ComposeSpacesObjectNames(pinfo.ImportPath, pinfo.JSFile.Etag, pinfo.JSFile.NumExtraFiles) 172 | for i := range docJS { 173 | docJS[i] = setting.DigitalOcean.Spaces.BucketURL + docJS[i] 174 | } 175 | c.Data["DocJS"] = docJS 176 | 177 | } else { 178 | docJS := make([]string, 0, pinfo.JSFile.NumExtraFiles+1) 179 | docJS = append(docJS, "/"+setting.DocsJSPath+importPath+".js") 180 | for i := 1; i <= pinfo.JSFile.NumExtraFiles; i++ { 181 | docJS = append(docJS, fmt.Sprintf("/%s%s-%d.js", setting.DocsJSPath, importPath, i)) 182 | } 183 | c.Data["DocJS"] = docJS 184 | } 185 | c.Data["Timestamp"] = pinfo.Created 186 | if time.Now().UTC().Add(-5*time.Second).Unix() < pinfo.Created { 187 | c.Flash.Success(c.Tr("docs.generate_success"), true) 188 | } 189 | 190 | // Subdirs 191 | if len(pinfo.Subdirs) > 0 { 192 | c.Data["IsHasSubdirs"] = true 193 | c.Data["ViewDirPath"] = pinfo.ViewDirPath 194 | c.Data["Subdirs"] = db.GetSubPkgs(pinfo.ImportPath, strings.Split(pinfo.Subdirs, "|")) 195 | } 196 | 197 | // Imports and references 198 | c.Data["ImportNum"] = pinfo.ImportNum 199 | c.Data["RefNum"] = pinfo.RefNum 200 | 201 | // Tools 202 | c.Data["TimeDuration"] = base.TimeSince(time.Unix(pinfo.Created, 0), c.Locale.Language()) 203 | c.Data["CanRefresh"] = pinfo.CanRefresh() 204 | 205 | updateHistory(c, pinfo.ID) 206 | 207 | c.Success(DOCS) 208 | } 209 | -------------------------------------------------------------------------------- /public/js/gowalker.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function () { 2 | $('.ui.accordion .title').click(function () { 3 | var $icon = $(this).find('.fas'); 4 | var $content = $('.ui.accordion .content'); 5 | if ($content.hasClass('d-hide')) { 6 | // Resize images if too large. 7 | if ($('#readme').length) { 8 | $(this).find("img").each(function () { 9 | var w = $(this).width(); 10 | $(this).width(w > 600 ? 600 : w); 11 | }); 12 | } 13 | 14 | $content.removeClass('d-hide'); 15 | $icon.removeClass('fa-caret-right'); 16 | $icon.addClass('fa-caret-down'); 17 | return; 18 | } 19 | 20 | $content.addClass('d-hide'); 21 | $icon.removeClass('fa-caret-down'); 22 | $icon.addClass('fa-caret-right'); 23 | }); 24 | 25 | var delay = (function(){ 26 | var timer = 0; 27 | return function(callback, ms){ 28 | clearTimeout(timer); 29 | timer = setTimeout(callback, ms); 30 | }; 31 | })(); 32 | 33 | $('#import-search').keyup(function() { 34 | delay(function () { 35 | var $this = $('#import-search'); 36 | if ($this.val().length < 3) { 37 | $('#search-results').html(""); 38 | return; 39 | } 40 | 41 | $('.form-autocomplete i.fa-search').addClass('d-hide'); 42 | $('.form-autocomplete .loading').removeClass('d-hide'); 43 | $.get('/search/json?q=' + $this.val(), function (data) { 44 | $('#search-results').html(""); 45 | for (var i = 0; i < data.results.length; i++) { 46 | $('#search-results').append(` 47 |
48 |
49 |

` + data.results[i].title + `

50 |

` + data.results[i].description + `

51 |
52 |
53 |
`) 54 | if (i+1 < data.results.length) { 55 | $('#search-results').append(`
`) 56 | } 57 | } 58 | $('.form-autocomplete .loading').addClass('d-hide'); 59 | $('.form-autocomplete i.fa-search').removeClass('d-hide'); 60 | }); 61 | }, 500) 62 | }); 63 | 64 | var is_page_docs = $('#readme').length > 0; 65 | 66 | if (is_page_docs) { 67 | // Search export objects. 68 | var $searchExportPanel = $('#search-export-panel'); 69 | $searchExportPanel.find('.btn-clear').click(function () { 70 | $searchExportPanel.removeClass('active'); 71 | }) 72 | $('#exports-search').keyup(function () { 73 | if ($(this).val().length < 1) { 74 | $('#search-results').html(""); 75 | return; 76 | } 77 | 78 | $('#search-results').html(""); 79 | for (var i = 0; i < exportDataSrc.length; i++) { 80 | if (exportDataSrc[i].title.toLowerCase().includes($(this).val().toLowerCase())) { 81 | $('#search-results').append(` 82 |
83 |
` + exportDataSrc[i].title + `
84 |
85 |
`) 86 | } 87 | } 88 | 89 | $('#search-results a').click(function () { 90 | $searchExportPanel.removeClass('active'); 91 | }); 92 | }); 93 | } 94 | 95 | // Help panel 96 | var $help_panel = $('#help-panel'); 97 | $help_panel.find('.btn-clear').click(function () { 98 | $help_panel.removeClass('active'); 99 | }) 100 | 101 | var preKeyG = 0; 102 | 103 | function Gkey(callback) { 104 | if (preKeyG === 1) { 105 | callback(); 106 | } 107 | preKeyG = 0; 108 | } 109 | 110 | $(document).keypress(function (event) { 111 | // Check if any input box is focused. 112 | if ($('input:focus').length > 0) { 113 | return true; 114 | } 115 | 116 | var code = event.keyCode ? event.keyCode : event.charCode; 117 | switch (code) { 118 | case 63: // for '?' 63 119 | $help_panel.addClass('active'); 120 | break; 121 | case 98: // for 'g then b' 'b' 98 122 | $help_panel.removeClass('active'); 123 | Gkey(function () { 124 | $('html,body').animate({scrollTop: $(document).height()}, 120); 125 | }); 126 | break; 127 | case 103: // for 'g then g' 'g' 103 128 | $help_panel.removeClass('active'); 129 | 130 | if (preKeyG === 0) { 131 | preKeyG = 1; 132 | setTimeout(function () { 133 | preKeyG = 0; 134 | }, 2000); 135 | return false; 136 | } 137 | Gkey(function () { 138 | $('html,body').animate({scrollTop: 0}, 120); 139 | }); 140 | break; 141 | case 115: // for 's' 105 142 | if (!is_page_docs) return true; 143 | $searchExportPanel.addClass('active'); 144 | break; 145 | default: 146 | preKeyG = 0; 147 | } 148 | }); 149 | 150 | // View code. 151 | $('.show.code').click(function () { 152 | $($(this).data('target')).toggle(); 153 | }); 154 | 155 | // Show example. 156 | $('.ex-link').click(function () { 157 | $($(this).data("name")).show(); 158 | }); 159 | $('.show.example').click(function (event) { 160 | $($(this).attr('href')).toggle(); 161 | event.preventDefault(); 162 | }); 163 | 164 | // Browse history. 165 | if ($('#browse_history').length) { 166 | $(this).each(function () { 167 | $(this).find('.meta-time').each(function () { 168 | $(this).text(new Date(parseInt($(this).text()) * 1000).toLocaleString()); 169 | }); 170 | }); 171 | } 172 | 173 | // Anchor. 174 | if ($('#markdown').length) { 175 | $(this).find('h1, h2, h3, h4, h5, h6').each(function () { 176 | var node = $(this); 177 | var id = node.attr("id"); 178 | if (typeof(id) !== "undefined") { 179 | node = node.wrap('
'); 180 | node.append(''); 181 | } 182 | }); 183 | } 184 | }); -------------------------------------------------------------------------------- /internal/doc/code.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "go/ast" 19 | "go/printer" 20 | "go/scanner" 21 | "go/token" 22 | "math" 23 | "strconv" 24 | ) 25 | 26 | const ( 27 | notPredeclared = iota 28 | predeclaredType 29 | predeclaredConstant 30 | predeclaredFunction 31 | ) 32 | 33 | // predeclared represents the set of all predeclared identifiers. 34 | var predeclared = map[string]int{ 35 | "bool": predeclaredType, 36 | "byte": predeclaredType, 37 | "complex128": predeclaredType, 38 | "complex64": predeclaredType, 39 | "error": predeclaredType, 40 | "float32": predeclaredType, 41 | "float64": predeclaredType, 42 | "int16": predeclaredType, 43 | "int32": predeclaredType, 44 | "int64": predeclaredType, 45 | "int8": predeclaredType, 46 | "int": predeclaredType, 47 | "rune": predeclaredType, 48 | "string": predeclaredType, 49 | "uint16": predeclaredType, 50 | "uint32": predeclaredType, 51 | "uint64": predeclaredType, 52 | "uint8": predeclaredType, 53 | "uint": predeclaredType, 54 | "uintptr": predeclaredType, 55 | 56 | "true": predeclaredConstant, 57 | "false": predeclaredConstant, 58 | "iota": predeclaredConstant, 59 | "nil": predeclaredConstant, 60 | 61 | "append": predeclaredFunction, 62 | "cap": predeclaredFunction, 63 | "close": predeclaredFunction, 64 | "complex": predeclaredFunction, 65 | "copy": predeclaredFunction, 66 | "delete": predeclaredFunction, 67 | "imag": predeclaredFunction, 68 | "len": predeclaredFunction, 69 | "make": predeclaredFunction, 70 | "new": predeclaredFunction, 71 | "panic": predeclaredFunction, 72 | "print": predeclaredFunction, 73 | "println": predeclaredFunction, 74 | "real": predeclaredFunction, 75 | "recover": predeclaredFunction, 76 | } 77 | 78 | const ( 79 | ExportLinkAnnotation AnnotationKind = iota 80 | AnchorAnnotation 81 | CommentAnnotation 82 | PackageLinkAnnotation 83 | BuiltinAnnotation 84 | ) 85 | 86 | // annotationVisitor collects annotations. 87 | type annotationVisitor struct { 88 | annotations []Annotation 89 | } 90 | 91 | func (v *annotationVisitor) add(kind AnnotationKind, importPath string) { 92 | v.annotations = append(v.annotations, Annotation{Kind: kind, ImportPath: importPath}) 93 | } 94 | 95 | func (v *annotationVisitor) ignoreName() { 96 | v.add(-1, "") 97 | } 98 | 99 | func (v *annotationVisitor) Visit(n ast.Node) ast.Visitor { 100 | switch n := n.(type) { 101 | case *ast.TypeSpec: 102 | v.ignoreName() 103 | ast.Walk(v, n.Type) 104 | case *ast.FuncDecl: 105 | if n.Recv != nil { 106 | ast.Walk(v, n.Recv) 107 | } 108 | v.ignoreName() 109 | ast.Walk(v, n.Type) 110 | case *ast.Field: 111 | for _ = range n.Names { 112 | v.ignoreName() 113 | } 114 | ast.Walk(v, n.Type) 115 | case *ast.ValueSpec: 116 | for _ = range n.Names { 117 | v.add(AnchorAnnotation, "") 118 | } 119 | if n.Type != nil { 120 | ast.Walk(v, n.Type) 121 | } 122 | for _, x := range n.Values { 123 | ast.Walk(v, x) 124 | } 125 | case *ast.Ident: 126 | switch { 127 | case n.Obj == nil && predeclared[n.Name] != notPredeclared: 128 | v.add(BuiltinAnnotation, "") 129 | case n.Obj != nil && ast.IsExported(n.Name): 130 | v.add(ExportLinkAnnotation, "") 131 | default: 132 | v.ignoreName() 133 | } 134 | case *ast.SelectorExpr: 135 | if x, _ := n.X.(*ast.Ident); x != nil { 136 | if obj := x.Obj; obj != nil && obj.Kind == ast.Pkg { 137 | if spec, _ := obj.Decl.(*ast.ImportSpec); spec != nil { 138 | if path, err := strconv.Unquote(spec.Path.Value); err == nil { 139 | v.add(PackageLinkAnnotation, path) 140 | if path == "C" { 141 | v.ignoreName() 142 | } else { 143 | v.add(ExportLinkAnnotation, path) 144 | } 145 | return nil 146 | } 147 | } 148 | } 149 | } 150 | ast.Walk(v, n.X) 151 | v.ignoreName() 152 | default: 153 | return v 154 | } 155 | return nil 156 | } 157 | 158 | func printDecl(decl ast.Node, fset *token.FileSet, buf []byte) (Code, []byte) { 159 | v := &annotationVisitor{} 160 | ast.Walk(v, decl) 161 | 162 | buf = buf[:0] 163 | err := (&printer.Config{Mode: printer.UseSpaces, Tabwidth: 4}).Fprint(sliceWriter{&buf}, fset, decl) 164 | if err != nil { 165 | return Code{Text: err.Error()}, buf 166 | } 167 | 168 | var annotations []Annotation 169 | var s scanner.Scanner 170 | fset = token.NewFileSet() 171 | file := fset.AddFile("", fset.Base(), len(buf)) 172 | s.Init(file, buf, nil, scanner.ScanComments) 173 | loop: 174 | for { 175 | pos, tok, lit := s.Scan() 176 | switch tok { 177 | case token.EOF: 178 | break loop 179 | case token.COMMENT: 180 | p := file.Offset(pos) 181 | e := p + len(lit) 182 | if p > math.MaxInt16 || e > math.MaxInt16 { 183 | break loop 184 | } 185 | annotations = append(annotations, Annotation{Kind: CommentAnnotation, Pos: int16(p), End: int16(e)}) 186 | case token.IDENT: 187 | if len(v.annotations) == 0 { 188 | // Oops! 189 | break loop 190 | } 191 | annotation := v.annotations[0] 192 | v.annotations = v.annotations[1:] 193 | if annotation.Kind == -1 { 194 | continue 195 | } 196 | p := file.Offset(pos) 197 | e := p + len(lit) 198 | if p > math.MaxInt16 || e > math.MaxInt16 { 199 | break loop 200 | } 201 | annotation.Pos = int16(p) 202 | annotation.End = int16(e) 203 | if len(annotations) > 0 && annotation.Kind == ExportLinkAnnotation { 204 | prev := annotations[len(annotations)-1] 205 | if prev.Kind == PackageLinkAnnotation && 206 | prev.ImportPath == annotation.ImportPath && 207 | prev.End+1 == annotation.Pos { 208 | // merge with previous 209 | annotation.Pos = prev.Pos 210 | annotations[len(annotations)-1] = annotation 211 | continue loop 212 | } 213 | } 214 | annotations = append(annotations, annotation) 215 | } 216 | } 217 | return Code{Text: string(buf), Annotations: annotations}, buf 218 | } 219 | 220 | type AnnotationKind int16 221 | 222 | type Annotation struct { 223 | Pos, End int16 224 | Kind AnnotationKind 225 | ImportPath string 226 | } 227 | 228 | type Code struct { 229 | Text string 230 | Annotations []Annotation 231 | } 232 | 233 | func commentAnnotations(src string) []Annotation { 234 | var annotations []Annotation 235 | var s scanner.Scanner 236 | fset := token.NewFileSet() 237 | file := fset.AddFile("", fset.Base(), len(src)) 238 | s.Init(file, []byte(src), nil, scanner.ScanComments) 239 | for { 240 | pos, tok, lit := s.Scan() 241 | switch tok { 242 | case token.EOF: 243 | return annotations 244 | case token.COMMENT: 245 | p := file.Offset(pos) 246 | e := p + len(lit) 247 | if p > math.MaxInt16 || e > math.MaxInt16 { 248 | return annotations 249 | } 250 | annotations = append(annotations, Annotation{Kind: CommentAnnotation, Pos: int16(p), End: int16(e)}) 251 | } 252 | } 253 | return nil 254 | } 255 | -------------------------------------------------------------------------------- /templates/base/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | {% block title %}{{Title}} - Go Walker{% endblock %} 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
23 |
24 |
25 | 38 |
39 | 40 |
41 |
42 |
43 | 44 |
45 | {% if Flash.ErrorMsg %} 46 |
47 |
{{Tr(Lang, "form.flash_error_title")}}
48 |

{{Flash.ErrorMsg|safe}}

49 |
50 | {% endif %} 51 | {% if Flash.InfoMsg %} 52 |
53 | {% if InfoHeader %} 54 |
{{InfoHeader}}
55 | {% endif %} 56 | {{Flash.InfoMsg|safe}} 57 |
58 | {% endif %} 59 | {% if Flash.SuccessMsg %} 60 |
61 | {% if InfoHeader %} 62 |
{{SuccessHeader}}
63 | {% endif %} 64 | {{Flash.SuccessMsg|safe}} 65 |
66 | {% endif %} 67 | 68 | {% block body %}{% endblock %} 69 |
70 | 71 | 94 |
95 |
96 | 97 | 188 | 189 | {% if ProdMode %} 190 | 199 | {% endif %} 200 | 201 | -------------------------------------------------------------------------------- /internal/doc/crawl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "encoding/xml" 19 | "errors" 20 | "fmt" 21 | "io" 22 | "path" 23 | "regexp" 24 | "strings" 25 | 26 | "github.com/unknwon/com" 27 | log "gopkg.in/clog.v1" 28 | 29 | "github.com/unknwon/gowalker/internal/base" 30 | "github.com/unknwon/gowalker/internal/httplib" 31 | "github.com/unknwon/gowalker/internal/setting" 32 | ) 33 | 34 | var ( 35 | ErrInvalidRemotePath = errors.New("invalid package remote path") 36 | ErrNoServiceMatch = errors.New("package remote path does not match any service") 37 | ) 38 | 39 | type crawlResult struct { 40 | pdoc *Package 41 | err error 42 | } 43 | 44 | // service represents a source code control service. 45 | type service struct { 46 | pattern *regexp.Regexp 47 | prefix string 48 | get func(map[string]string, string) (*Package, error) 49 | } 50 | 51 | // services is the list of source code control services handled by gowalker. 52 | var services = []*service{ 53 | {githubPattern, "github.com/", getGitHubDoc}, 54 | // {googlePattern, "code.google.com/", getGoogleDoc}, 55 | // {bitbucketPattern, "bitbucket.org/", getBitbucketDoc}, 56 | // {launchpadPattern, "launchpad.net/", getLaunchpadDoc}, 57 | // {oscPattern, "git.oschina.net/", getOSCDoc}, 58 | } 59 | 60 | // getStatic gets a document from a statically known service. 61 | // It returns ErrNoServiceMatch if the import path is not recognized. 62 | func getStatic(importPath, etag string) (pdoc *Package, err error) { 63 | for _, s := range services { 64 | if s.get == nil || !strings.HasPrefix(importPath, s.prefix) { 65 | continue 66 | } 67 | m := s.pattern.FindStringSubmatch(importPath) 68 | if m == nil { 69 | if s.prefix != "" { 70 | log.Trace("Import path prefix matches known service, but regexp does not: %s", importPath) 71 | return nil, ErrInvalidRemotePath 72 | } 73 | continue 74 | } 75 | match := map[string]string{"importPath": importPath} 76 | for i, n := range s.pattern.SubexpNames() { 77 | if n != "" { 78 | match[n] = m[i] 79 | } 80 | } 81 | return s.get(match, etag) 82 | } 83 | return nil, ErrNoServiceMatch 84 | } 85 | 86 | func attrValue(attrs []xml.Attr, name string) string { 87 | for _, a := range attrs { 88 | if strings.EqualFold(a.Name.Local, name) { 89 | return a.Value 90 | } 91 | } 92 | return "" 93 | } 94 | 95 | func parseMeta(scheme, importPath string, r io.Reader) (map[string]string, error) { 96 | var match map[string]string 97 | 98 | d := xml.NewDecoder(r) 99 | d.Strict = false 100 | metaScan: 101 | for { 102 | t, tokenErr := d.Token() 103 | if tokenErr != nil { 104 | break metaScan 105 | } 106 | switch t := t.(type) { 107 | case xml.EndElement: 108 | if strings.EqualFold(t.Name.Local, "head") { 109 | break metaScan 110 | } 111 | case xml.StartElement: 112 | if strings.EqualFold(t.Name.Local, "body") { 113 | break metaScan 114 | } 115 | if !strings.EqualFold(t.Name.Local, "meta") || 116 | attrValue(t.Attr, "name") != "go-import" { 117 | continue metaScan 118 | } 119 | f := strings.Fields(attrValue(t.Attr, "content")) 120 | if len(f) != 3 || 121 | !strings.HasPrefix(importPath, f[0]) || 122 | !(len(importPath) == len(f[0]) || importPath[len(f[0])] == '/') { 123 | continue metaScan 124 | } 125 | if match != nil { 126 | return nil, fmt.Errorf("More than one found at %s://%s", scheme, importPath) 127 | } 128 | 129 | projectRoot, vcs, repo := f[0], f[1], f[2] 130 | 131 | repo = strings.TrimSuffix(repo, "."+vcs) 132 | i := strings.Index(repo, "://") 133 | if i < 0 { 134 | return nil, errors.New("Bad repo URL in ") 135 | } 136 | proto := repo[:i] 137 | repo = repo[i+len("://"):] 138 | 139 | match = map[string]string{ 140 | // Used in getVCSDoc, same as vcsPattern matches. 141 | "importPath": importPath, 142 | "repo": repo, 143 | "vcs": vcs, 144 | "dir": importPath[len(projectRoot):], 145 | 146 | // Used in getVCSDoc 147 | "scheme": proto, 148 | 149 | // Used in getDynamic. 150 | "projectRoot": projectRoot, 151 | "projectName": path.Base(projectRoot), 152 | "projectURL": scheme + "://" + projectRoot, 153 | } 154 | } 155 | } 156 | if match == nil { 157 | return nil, errors.New(" not found") 158 | } 159 | return match, nil 160 | } 161 | 162 | func fetchMeta(importPath string) (map[string]string, error) { 163 | uri := importPath 164 | if !strings.Contains(uri, "/") { 165 | // Add slash for root of domain. 166 | uri = uri + "/" 167 | } 168 | uri = uri + "?go-get=1" 169 | 170 | scheme := "https" 171 | resp, err := Client.Get(scheme + "://" + uri) 172 | if err != nil || resp.StatusCode != 200 { 173 | if err == nil { 174 | resp.Body.Close() 175 | } 176 | scheme = "http" 177 | resp, err = Client.Get(scheme + "://" + uri) 178 | if err != nil { 179 | return nil, err 180 | } 181 | } 182 | defer resp.Body.Close() 183 | return parseMeta(scheme, importPath, resp.Body) 184 | } 185 | 186 | func getDynamic(importPath, etag string) (pdoc *Package, err error) { 187 | match, err := fetchMeta(importPath) 188 | if err != nil { 189 | return nil, err 190 | } 191 | 192 | if match["projectRoot"] != importPath { 193 | rootMatch, err := fetchMeta(match["projectRoot"]) 194 | if err != nil { 195 | return nil, err 196 | } 197 | if rootMatch["projectRoot"] != match["projectRoot"] { 198 | return nil, errors.New("Project root mismatch") 199 | } 200 | } 201 | 202 | isGoSubrepo := false 203 | if strings.HasPrefix(match["repo"], "go.googlesource.com") { 204 | isGoSubrepo = true 205 | match["dir"] = "/" + path.Base(match["repo"]) + match["dir"] 206 | match["repo"] = "github.com/golang" 207 | } 208 | 209 | pdoc, err = getStatic(com.Expand("{repo}{dir}", match), etag) 210 | if err == ErrNoServiceMatch { 211 | pdoc, err = getVCSDoc(match, etag) 212 | } else if pdoc != nil { 213 | pdoc.ImportPath = importPath 214 | pdoc.IsGoSubrepo = isGoSubrepo 215 | } 216 | if err != nil { 217 | return nil, err 218 | } 219 | 220 | return pdoc, err 221 | } 222 | 223 | func crawlDoc(importPath, etag string) (pdoc *Package, err error) { 224 | switch { 225 | case base.IsGoRepoPath(importPath): 226 | pdoc, err = getGolangDoc(importPath, etag) 227 | case base.IsGAERepoPath(strings.TrimPrefix(importPath, "google.golang.org/")): 228 | subPath := strings.TrimPrefix(importPath, "google.golang.org/") 229 | pdoc, err = getStatic("github.com/golang/"+subPath, etag) 230 | if pdoc != nil { 231 | pdoc.ImportPath = importPath 232 | pdoc.IsGaeRepo = true 233 | } 234 | case base.IsValidRemotePath(importPath): 235 | pdoc, err = getStatic(importPath, etag) 236 | if err == ErrNoServiceMatch { 237 | pdoc, err = getDynamic(importPath, etag) 238 | } 239 | default: 240 | err = ErrInvalidRemotePath 241 | } 242 | 243 | if err != nil { 244 | return nil, err 245 | } 246 | 247 | // Render README 248 | for name, content := range pdoc.Readme { 249 | p, err := httplib.Post("https://api.github.com/markdown/raw"). 250 | SetBasicAuth(setting.GitHub.ClientID, setting.GitHub.ClientSecret). 251 | Header("Content-Type", "text/plain"). 252 | Body(content). 253 | Bytes() 254 | if err != nil { 255 | return nil, fmt.Errorf("error rendering README: %v", err) 256 | } 257 | pdoc.Readme[name] = p 258 | } 259 | 260 | return pdoc, nil 261 | } 262 | -------------------------------------------------------------------------------- /internal/doc/github.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package doc 16 | 17 | import ( 18 | "bytes" 19 | "errors" 20 | "fmt" 21 | "net/http" 22 | "path" 23 | "regexp" 24 | "strings" 25 | "time" 26 | 27 | "github.com/unknwon/com" 28 | log "gopkg.in/clog.v1" 29 | 30 | "github.com/unknwon/gowalker/internal/base" 31 | "github.com/unknwon/gowalker/internal/db" 32 | "github.com/unknwon/gowalker/internal/httplib" 33 | "github.com/unknwon/gowalker/internal/setting" 34 | ) 35 | 36 | var ( 37 | githubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}} 38 | githubRevisionPattern = regexp.MustCompile(`value="[a-z0-9A-Z]+"`) 39 | githubPattern = regexp.MustCompile(`^github\.com/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/[a-z0-9A-Z_.\-/]*)?$`) 40 | ) 41 | 42 | func getGithubRevision(importPath, tag string) (string, error) { 43 | data, err := com.HttpGetBytes(Client, fmt.Sprintf("https://%s/commits/%s", importPath, tag), nil) 44 | if err != nil { 45 | return "", fmt.Errorf("fetch commits page for %q: %v", importPath, err) 46 | } 47 | 48 | i := bytes.Index(data, []byte(`class="BtnGroup"`)) 49 | if i == -1 { 50 | return "", fmt.Errorf("find locater in page for %q", importPath) 51 | } 52 | data = data[i+1:] 53 | m := githubRevisionPattern.FindSubmatch(data) 54 | if m == nil { 55 | return "", fmt.Errorf("find revision in page for %q", importPath) 56 | } 57 | return strings.TrimSuffix(strings.TrimPrefix(string(m[0]), `value="`), `"`), nil 58 | } 59 | 60 | type RepoInfo struct { 61 | DefaultBranch string `json:"default_branch"` 62 | Fork bool `json:"fork"` 63 | Parent struct { 64 | FullName string `json:"full_name"` 65 | } `json:"parent"` 66 | } 67 | 68 | type RepoCommit struct { 69 | Commit struct { 70 | Committer struct { 71 | Date time.Time `json:"date"` 72 | } `json:"committer"` 73 | } `json:"commit"` 74 | } 75 | 76 | func getGitHubDoc(match map[string]string, etag string) (*Package, error) { 77 | httpGet := func(url string, v interface{}) error { 78 | return httplib.Get(url). 79 | SetBasicAuth(setting.GitHub.ClientID, setting.GitHub.ClientSecret). 80 | ToJson(v) 81 | } 82 | repoInfo := new(RepoInfo) 83 | err := httpGet(com.Expand("https://api.github.com/repos/{owner}/{repo}", match), repoInfo) 84 | if err != nil { 85 | return nil, fmt.Errorf("get repo default branch: %v", err) 86 | } 87 | 88 | // Set default branch if not presented. 89 | if len(match["tag"]) == 0 { 90 | match["tag"] = repoInfo.DefaultBranch 91 | } 92 | 93 | // Check if last commit time is behind upstream for fork repository. 94 | if repoInfo.Fork { 95 | url := com.Expand("https://api.github.com/repos/{owner}/{repo}/commits?per_page=1&{cred}", match) 96 | forkCommits := make([]*RepoCommit, 0, 1) 97 | if err := httpGet(url, &forkCommits); err != nil { 98 | return nil, fmt.Errorf("get fork repository commits: %v", err) 99 | } 100 | if len(forkCommits) == 0 { 101 | return nil, fmt.Errorf("unexpected zero number of fork repository commits: %s", url) 102 | } 103 | 104 | match["parent"] = repoInfo.Parent.FullName 105 | url = com.Expand("https://api.github.com/repos/{parent}/commits?per_page=1&{cred}", match) 106 | parentCommits := make([]*RepoCommit, 0, 1) 107 | if err := httpGet(url, &parentCommits); err != nil { 108 | return nil, fmt.Errorf("get parent repository commits: %v", err) 109 | } 110 | if len(parentCommits) == 0 { 111 | return nil, fmt.Errorf("unexpected zero number of parent repository commits: %s", url) 112 | } 113 | 114 | if !forkCommits[0].Commit.Committer.Date.After(parentCommits[0].Commit.Committer.Date) { 115 | return nil, fmt.Errorf("commits of this fork repository are behind or equal to its parent: %s", repoInfo.Parent.FullName) 116 | } 117 | } 118 | 119 | // Check revision. 120 | var commit string 121 | if strings.HasPrefix(match["importPath"], "gopkg.in") { 122 | // FIXME: get commit ID of gopkg.in indepdently. 123 | var obj struct { 124 | Sha string `json:"sha"` 125 | } 126 | 127 | if err := com.HttpGetJSON(Client, 128 | com.Expand("https://gopm.io/api/v1/revision?pkgname={importPath}", match), &obj); err != nil { 129 | return nil, fmt.Errorf("get gopkg.in revision: %v", err) 130 | } 131 | 132 | commit = obj.Sha 133 | match["tag"] = commit 134 | log.Trace("Import path %q found commit: %s", match["importPath"], commit) 135 | } else { 136 | commit, err = getGithubRevision(com.Expand("github.com/{owner}/{repo}", match), match["tag"]) 137 | if err != nil { 138 | return nil, fmt.Errorf("get revision: %v", err) 139 | } 140 | if commit == etag { 141 | return nil, ErrPackageNotModified 142 | } 143 | } 144 | 145 | // Get files. 146 | var tree struct { 147 | Tree []struct { 148 | Url string 149 | Path string 150 | Type string 151 | } 152 | Url string 153 | } 154 | 155 | if err := httpGet(com.Expand("https://api.github.com/repos/{owner}/{repo}/git/trees/{tag}?recursive=1", match), &tree); err != nil { 156 | return nil, fmt.Errorf("get tree: %v", err) 157 | } 158 | 159 | // Because Github API URLs are case-insensitive, we need to check that the 160 | // userRepo returned from Github matches the one that we are requesting. 161 | if !strings.HasPrefix(tree.Url, com.Expand("https://api.github.com/repos/{owner}/{repo}/", match)) { 162 | return nil, errors.New("GitHub import path has incorrect case") 163 | } 164 | 165 | // Get source file data and subdirectories. 166 | dirPrefix := match["dir"] 167 | if dirPrefix != "" { 168 | dirPrefix = dirPrefix[1:] + "/" 169 | } 170 | dirLevel := len(strings.Split(dirPrefix, "/")) 171 | dirLength := len(dirPrefix) 172 | dirMap := make(map[string]bool) 173 | files := make([]com.RawFile, 0, 10) 174 | 175 | for _, node := range tree.Tree { 176 | // Skip directories and files in wrong directories, get them later. 177 | if node.Type != "blob" || !strings.HasPrefix(node.Path, dirPrefix) { 178 | continue 179 | } 180 | 181 | // Get files and check if directories have acceptable files. 182 | if d, f := path.Split(node.Path); base.IsDocFile(f) { 183 | // Check if file is in the directory that is corresponding to import path. 184 | if d == dirPrefix { 185 | files = append(files, &Source{ 186 | SrcName: f, 187 | BrowseUrl: com.Expand("github.com/{owner}/{repo}/blob/{tag}/{0}", match, node.Path), 188 | RawSrcUrl: com.Expand("https://raw.github.com/{owner}/{repo}/{tag}/{0}", match, node.Path), 189 | }) 190 | continue 191 | } 192 | 193 | // Otherwise, check if it's a direct sub-directory of import path. 194 | if len(strings.Split(d, "/"))-dirLevel == 1 { 195 | dirMap[d[dirLength:len(d)-1]] = true 196 | continue 197 | } 198 | } 199 | } 200 | 201 | dirs := base.MapToSortedStrings(dirMap) 202 | 203 | if len(files) == 0 && len(dirs) == 0 { 204 | return nil, ErrPackageNoGoFile 205 | } else if err := com.FetchFiles(Client, files, githubRawHeader); err != nil { 206 | return nil, fmt.Errorf("fetch files: %v", err) 207 | } 208 | 209 | // Start generating data. 210 | // IsGoSubrepo check has been placed to crawl.getDynamic. 211 | w := &Walker{ 212 | LineFmt: "#L%d", 213 | Pdoc: &Package{ 214 | PkgInfo: &db.PkgInfo{ 215 | ImportPath: match["importPath"], 216 | ProjectPath: com.Expand("github.com/{owner}/{repo}", match), 217 | ViewDirPath: com.Expand("github.com/{owner}/{repo}/tree/{tag}/{importPath}", match), 218 | Etag: commit, 219 | Subdirs: strings.Join(dirs, "|"), 220 | }, 221 | }, 222 | } 223 | 224 | srcs := make([]*Source, 0, len(files)) 225 | srcMap := make(map[string]*Source) 226 | for _, f := range files { 227 | s, _ := f.(*Source) 228 | srcs = append(srcs, s) 229 | 230 | if !strings.HasSuffix(f.Name(), "_test.go") { 231 | srcMap[f.Name()] = s 232 | } 233 | } 234 | 235 | pdoc, err := w.Build(&WalkRes{ 236 | WalkDepth: WD_All, 237 | WalkType: WT_Memory, 238 | WalkMode: WM_All, 239 | Srcs: srcs, 240 | }) 241 | if err != nil { 242 | return nil, fmt.Errorf("error walking package: %v", err) 243 | } 244 | 245 | // Get stars. 246 | var repoTree struct { 247 | Stars int64 `json:"watchers"` 248 | } 249 | if err := httpGet(com.Expand("https://api.github.com/repos/{owner}/{repo}", match), &repoTree); err != nil { 250 | return nil, fmt.Errorf("get repoTree: %v", err) 251 | } 252 | pdoc.Stars = repoTree.Stars 253 | 254 | return pdoc, nil 255 | } 256 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "[]" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright [yyyy] [name of copyright owner] 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. -------------------------------------------------------------------------------- /internal/doc/vcs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Gary Burd 2 | // Copyright 2013 Unknwon 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 5 | // not use this file except in compliance with the License. You may obtain 6 | // a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | // License for the specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package doc 17 | 18 | import ( 19 | "archive/zip" 20 | "bytes" 21 | "errors" 22 | "fmt" 23 | "io/ioutil" 24 | "log" 25 | "os" 26 | "os/exec" 27 | "path" 28 | "regexp" 29 | "strings" 30 | 31 | "github.com/unknwon/com" 32 | 33 | "github.com/unknwon/gowalker/internal/base" 34 | "github.com/unknwon/gowalker/internal/db" 35 | ) 36 | 37 | // TODO: specify with command line flag 38 | const repoRoot = "/tmp/gw" 39 | 40 | func init() { 41 | os.RemoveAll(repoRoot) 42 | } 43 | 44 | var urlTemplates = []struct { 45 | re *regexp.Regexp 46 | template string 47 | lineFmt string 48 | }{ 49 | { 50 | regexp.MustCompile(`^git\.gitorious\.org/(?P[^/]+/[^/]+)$`), 51 | "https://gitorious.org/{repo}/blobs/{tag}/{dir}{0}", 52 | "#line%d", 53 | }, 54 | { 55 | regexp.MustCompile(`^camlistore\.org/r/p/(?P[^/]+)$`), 56 | "http://camlistore.org/code/?p={repo}.git;hb={tag};f={dir}{0}", 57 | "#l%d", 58 | }, 59 | } 60 | 61 | // lookupURLTemplate finds an expand() template, match map and line number 62 | // format for well known repositories. 63 | func lookupURLTemplate(repo, dir, tag string) (string, map[string]string, string) { 64 | if strings.HasPrefix(dir, "/") { 65 | dir = dir[1:] + "/" 66 | } 67 | for _, t := range urlTemplates { 68 | if m := t.re.FindStringSubmatch(repo); m != nil { 69 | match := map[string]string{ 70 | "dir": dir, 71 | "tag": tag, 72 | } 73 | for i, name := range t.re.SubexpNames() { 74 | if name != "" { 75 | match[name] = m[i] 76 | } 77 | } 78 | return t.template, match, t.lineFmt 79 | } 80 | } 81 | return "", nil, "" 82 | } 83 | 84 | type vcsCmd struct { 85 | schemes []string 86 | download func([]string, string, string) (string, string, error) 87 | } 88 | 89 | var vcsCmds = map[string]*vcsCmd{ 90 | "git": &vcsCmd{ 91 | schemes: []string{"http", "https", "git"}, 92 | download: downloadGit, 93 | }, 94 | } 95 | 96 | var lsremoteRe = regexp.MustCompile(`(?m)^([0-9a-f]{40})\s+refs/(?:tags|heads)/(.+)$`) 97 | 98 | func downloadGit(schemes []string, repo, savedEtag string) (string, string, error) { 99 | var p []byte 100 | var scheme string 101 | for i := range schemes { 102 | cmd := exec.Command("git", "ls-remote", "--heads", "--tags", schemes[i]+"://"+repo+".git") 103 | log.Println(strings.Join(cmd.Args, " ")) 104 | var err error 105 | p, err = cmd.Output() 106 | if err == nil { 107 | scheme = schemes[i] 108 | break 109 | } 110 | } 111 | 112 | if scheme == "" { 113 | return "", "", com.NotFoundError{"VCS not found"} 114 | } 115 | 116 | tags := make(map[string]string) 117 | for _, m := range lsremoteRe.FindAllSubmatch(p, -1) { 118 | tags[string(m[2])] = string(m[1]) 119 | } 120 | 121 | tag, commit, err := bestTag(tags, "master") 122 | if err != nil { 123 | return "", "", err 124 | } 125 | 126 | etag := scheme + "-" + commit 127 | 128 | if etag == savedEtag { 129 | return "", "", ErrPackageNotModified 130 | } 131 | 132 | dir := path.Join(repoRoot, repo+".git") 133 | p, err = ioutil.ReadFile(path.Join(dir, ".git/HEAD")) 134 | switch { 135 | case err != nil: 136 | if err := os.MkdirAll(dir, 0777); err != nil { 137 | return "", "", err 138 | } 139 | cmd := exec.Command("git", "clone", scheme+"://"+repo, dir) 140 | log.Println(strings.Join(cmd.Args, " ")) 141 | if err := cmd.Run(); err != nil { 142 | return "", "", err 143 | } 144 | case string(bytes.TrimRight(p, "\n")) == commit: 145 | return tag, etag, nil 146 | default: 147 | cmd := exec.Command("git", "fetch") 148 | log.Println(strings.Join(cmd.Args, " ")) 149 | cmd.Dir = dir 150 | if err := cmd.Run(); err != nil { 151 | return "", "", err 152 | } 153 | } 154 | 155 | cmd := exec.Command("git", "checkout", "--detach", "--force", commit) 156 | cmd.Dir = dir 157 | if err := cmd.Run(); err != nil { 158 | return "", "", err 159 | } 160 | 161 | return tag, etag, nil 162 | } 163 | 164 | var ( 165 | vcsPattern = regexp.MustCompile(`^(?P(?:[a-z0-9.\-]+\.)+[a-z0-9.\-]+(?::[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?Pbzr|git|hg|svn)(?P/[A-Za-z0-9_.\-/]*)?$`) 166 | gopkgPathPattern = regexp.MustCompile(`^/(?:([a-zA-Z0-9][-a-zA-Z0-9]+)/)?([a-zA-Z][-.a-zA-Z0-9]*)\.((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2})(?:\.git)?((?:/[a-zA-Z0-9][-.a-zA-Z0-9]*)*)$`) 167 | ) 168 | 169 | func getVCSDoc(match map[string]string, etagSaved string) (*Package, error) { 170 | if strings.HasPrefix(match["importPath"], "golang.org/x/") { 171 | match["owner"] = "golang" 172 | match["repo"] = path.Dir(strings.TrimPrefix(match["importPath"], "golang.org/x/")) 173 | return getGitHubDoc(match, etagSaved) 174 | } else if strings.HasPrefix(match["importPath"], "gopkg.in/") { 175 | m := gopkgPathPattern.FindStringSubmatch(strings.TrimPrefix(match["importPath"], "gopkg.in")) 176 | if m == nil { 177 | return nil, fmt.Errorf("unsupported gopkg.in import path: %s", match["importPath"]) 178 | } 179 | user := m[1] 180 | repo := m[2] 181 | if len(user) == 0 { 182 | user = "go-" + repo 183 | } 184 | match["owner"] = user 185 | match["repo"] = repo 186 | match["tag"] = m[3] 187 | return getGitHubDoc(match, etagSaved) 188 | } 189 | 190 | cmd := vcsCmds[match["vcs"]] 191 | if cmd == nil { 192 | return nil, com.NotFoundError{com.Expand("VCS not supported: {vcs}", match)} 193 | } 194 | 195 | scheme := match["scheme"] 196 | if scheme == "" { 197 | i := strings.Index(etagSaved, "-") 198 | if i > 0 { 199 | scheme = etagSaved[:i] 200 | } 201 | } 202 | 203 | schemes := cmd.schemes 204 | if scheme != "" { 205 | for i := range cmd.schemes { 206 | if cmd.schemes[i] == scheme { 207 | schemes = cmd.schemes[i : i+1] 208 | break 209 | } 210 | } 211 | } 212 | 213 | // Download and checkout. 214 | 215 | tag, _, err := cmd.download(schemes, match["repo"], etagSaved) 216 | if err != nil { 217 | return nil, err 218 | } 219 | 220 | // Find source location. 221 | 222 | urlTemplate, urlMatch, lineFmt := lookupURLTemplate(match["repo"], match["dir"], tag) 223 | 224 | // Slurp source files. 225 | 226 | d := path.Join(repoRoot, com.Expand("{repo}.{vcs}", match), match["dir"]) 227 | f, err := os.Open(d) 228 | if err != nil { 229 | if os.IsNotExist(err) { 230 | err = com.NotFoundError{err.Error()} 231 | } 232 | return nil, err 233 | } 234 | fis, err := f.Readdir(-1) 235 | if err != nil { 236 | return nil, err 237 | } 238 | 239 | // Get source file data. 240 | var files []com.RawFile 241 | for _, fi := range fis { 242 | if fi.IsDir() || !base.IsDocFile(fi.Name()) { 243 | continue 244 | } 245 | b, err := ioutil.ReadFile(path.Join(d, fi.Name())) 246 | if err != nil { 247 | return nil, err 248 | } 249 | files = append(files, &Source{ 250 | SrcName: fi.Name(), 251 | BrowseUrl: com.Expand(urlTemplate, urlMatch, fi.Name()), 252 | SrcData: b, 253 | }) 254 | } 255 | 256 | // Start generating data. 257 | w := &Walker{ 258 | LineFmt: lineFmt, 259 | Pdoc: &Package{ 260 | PkgInfo: &db.PkgInfo{ 261 | ImportPath: match["importPath"], 262 | }, 263 | }, 264 | } 265 | 266 | srcs := make([]*Source, 0, len(files)) 267 | for _, f := range files { 268 | s, _ := f.(*Source) 269 | srcs = append(srcs, s) 270 | } 271 | 272 | return w.Build(&WalkRes{ 273 | WalkDepth: WD_All, 274 | WalkType: WT_Memory, 275 | WalkMode: WM_All, 276 | Srcs: srcs, 277 | }) 278 | } 279 | 280 | var defaultTags = map[string]string{"git": "master", "hg": "default", "svn": "trunk"} 281 | 282 | func bestTag(tags map[string]string, defaultTag string) (string, string, error) { 283 | if commit, ok := tags["go1"]; ok { 284 | return "go1", commit, nil 285 | } 286 | if commit, ok := tags[defaultTag]; ok { 287 | return defaultTag, commit, nil 288 | } 289 | return "", "", com.NotFoundError{"Tag or branch not found."} 290 | } 291 | 292 | // checkDir checks if directory has been appended to slice. 293 | func checkDir(dir string, dirs []string) bool { 294 | for _, d := range dirs { 295 | if dir == d { 296 | return true 297 | } 298 | } 299 | return false 300 | } 301 | 302 | // Only support .zip. 303 | func getRepoByArchive(match map[string]string, downloadPath string) (bool, string, []com.RawFile, []string, error) { 304 | stdout, _, err := com.ExecCmd("curl", downloadPath) 305 | if err != nil { 306 | return false, "", nil, nil, err 307 | } 308 | p := []byte(stdout) 309 | 310 | r, err := zip.NewReader(bytes.NewReader(p), int64(len(p))) 311 | if err != nil { 312 | return false, "", nil, nil, errors.New(downloadPath + " -> new zip: " + err.Error()) 313 | } 314 | 315 | if len(r.File) == 0 { 316 | return false, "", nil, nil, nil 317 | } 318 | 319 | nameLen := strings.Index(r.File[0].Name, "/") 320 | dirPrefix := match["dir"] 321 | if len(dirPrefix) != 0 { 322 | dirPrefix = dirPrefix[1:] + "/" 323 | } 324 | preLen := len(dirPrefix) 325 | isGoPro := false 326 | 327 | // for k, v := range match { 328 | // println(k, v) 329 | // } 330 | comment := r.Comment 331 | 332 | files := make([]com.RawFile, 0, 5) 333 | dirs := make([]string, 0, 5) 334 | for _, f := range r.File { 335 | fileName := f.Name[nameLen+1:] 336 | // Skip directories and files in wrong directories, get them later. 337 | if strings.HasSuffix(fileName, "/") || !strings.HasPrefix(fileName, dirPrefix) { 338 | continue 339 | } 340 | //fmt.Println(fileName) 341 | 342 | // Get files and check if directories have acceptable files. 343 | if d, fn := path.Split(fileName); base.IsDocFile(fn) && 344 | base.FilterDirName(d) { 345 | // Check if it's a Go file. 346 | if !isGoPro && strings.HasSuffix(fn, ".go") { 347 | isGoPro = true 348 | } 349 | 350 | // Check if file is in the directory that is corresponding to import path. 351 | if d == dirPrefix { 352 | // Yes. 353 | if !isGoPro && strings.HasSuffix(fn, ".go") { 354 | isGoPro = true 355 | } 356 | // Get file from archive. 357 | rc, err := f.Open() 358 | if err != nil { 359 | return isGoPro, comment, files, dirs, 360 | errors.New(downloadPath + " -> open file: " + err.Error()) 361 | } 362 | 363 | p := make([]byte, f.FileInfo().Size()) 364 | rc.Read(p) 365 | if err != nil { 366 | return isGoPro, comment, files, dirs, 367 | errors.New(downloadPath + " -> read file: " + err.Error()) 368 | } 369 | //fmt.Println(com.Expand(match["browserUrlTpl"], match, fn)) 370 | files = append(files, &Source{ 371 | SrcName: fn, 372 | BrowseUrl: com.Expand(match["browserUrlTpl"], match, fn), 373 | RawSrcUrl: com.Expand(match["rawSrcUrlTpl"], match, fileName[preLen:]), 374 | SrcData: p, 375 | }) 376 | 377 | } else { 378 | sd, _ := path.Split(d[preLen:]) 379 | sd = strings.TrimSuffix(sd, "/") 380 | if !checkDir(sd, dirs) { 381 | dirs = append(dirs, sd) 382 | } 383 | } 384 | } 385 | } 386 | return isGoPro, comment, files, dirs, nil 387 | } 388 | -------------------------------------------------------------------------------- /internal/db/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "bytes" 19 | "errors" 20 | "fmt" 21 | "path" 22 | "strings" 23 | "time" 24 | 25 | "github.com/unknwon/com" 26 | log "gopkg.in/clog.v1" 27 | 28 | "github.com/unknwon/gowalker/internal/base" 29 | "github.com/unknwon/gowalker/internal/setting" 30 | ) 31 | 32 | var ( 33 | ErrEmptyPackagePath = errors.New("package import path is empty") 34 | ErrPackageNotFound = errors.New("package does not found") 35 | ErrPackageVersionTooOld = errors.New("package version is too old") 36 | ) 37 | 38 | // PkgInfo represents the package information. 39 | type PkgInfo struct { 40 | ID int64 41 | Name string `xorm:"-"` 42 | ImportPath string `xorm:"UNIQUE"` 43 | Etag string 44 | 45 | ProjectPath string 46 | ViewDirPath string 47 | Synopsis string 48 | 49 | IsCmd bool 50 | IsCgo bool 51 | IsGoRepo bool 52 | IsGoSubrepo bool 53 | IsGaeRepo bool 54 | 55 | PkgVer int 56 | 57 | Priority int `xorm:" NOT NULL"` 58 | Views int64 59 | Stars int64 60 | 61 | ImportNum int64 62 | ImportIDs string `xorm:"import_ids LONGTEXT"` 63 | // Import num usually is small so save it to reduce a database query. 64 | ImportPaths string `xorm:"LONGTEXT"` 65 | 66 | RefNum int64 67 | RefIDs string `xorm:"ref_ids LONGTEXT"` 68 | 69 | Subdirs string `xorm:"TEXT"` 70 | 71 | LastViewed int64 `xorm:"NOT NULL DEFAULT 0"` 72 | Created int64 73 | 74 | JSFile *JSFile `xorm:"-"` 75 | } 76 | 77 | // HasJSFile returns false if JS file must be regenerated, 78 | // it populates the JSFile field when file exists. 79 | func (p *PkgInfo) HasJSFile() bool { 80 | jsFile, err := GetJSFile(p.ID, p.Etag) 81 | if err != nil { 82 | if err != ErrJSFileNotFound { 83 | log.Error(2, "GetJSFile: %v", err) 84 | } 85 | return false 86 | } 87 | 88 | if jsFile.Status == JSFileStatusDistributed || 89 | (jsFile.Status == JSFileStatusGenerated && com.IsFile(p.LocalJSPath())) { 90 | p.JSFile = jsFile 91 | return true 92 | } 93 | 94 | return false 95 | } 96 | 97 | func (p *PkgInfo) LocalJSPath() string { 98 | return path.Join(setting.DocsJSPath, p.ImportPath) + ".js" 99 | } 100 | 101 | func (p *PkgInfo) LocalJSPaths() []string { 102 | if p.JSFile == nil { 103 | return []string{p.LocalJSPath()} 104 | } 105 | 106 | paths := make([]string, 0, p.JSFile.NumExtraFiles+1) 107 | paths = append(paths, setting.DocsJSPath+p.ImportPath+".js") 108 | for i := 1; i <= p.JSFile.NumExtraFiles; i++ { 109 | paths = append(paths, fmt.Sprintf("%s%s-%d.js", setting.DocsJSPath, p.ImportPath, i)) 110 | } 111 | return paths 112 | } 113 | 114 | // CanRefresh returns true if package is available to refresh. 115 | func (p *PkgInfo) CanRefresh() bool { 116 | return time.Now().UTC().Add(-1*setting.RefreshInterval).Unix() > p.Created 117 | } 118 | 119 | // GetRefs returns a list of packages that import this one. 120 | func (p *PkgInfo) GetRefs() []*PkgInfo { 121 | pinfos := make([]*PkgInfo, 0, p.RefNum) 122 | refIDs := strings.Split(p.RefIDs, "|") 123 | for i := range refIDs { 124 | if len(refIDs[i]) == 0 { 125 | continue 126 | } 127 | 128 | id := com.StrTo(refIDs[i][1:]).MustInt64() 129 | if pinfo, _ := GetPkgInfoByID(id); pinfo != nil { 130 | pinfo.Name = path.Base(pinfo.ImportPath) 131 | pinfos = append(pinfos, pinfo) 132 | } 133 | } 134 | return pinfos 135 | } 136 | 137 | // PackageVersion is modified when previously stored packages are invalid. 138 | const PackageVersion = 1 139 | 140 | // PkgRef represents temporary reference information of a package. 141 | type PkgRef struct { 142 | ID int64 143 | ImportPath string `xorm:"UNIQUE"` 144 | RefNum int64 145 | RefIDs string `xorm:"ref_ids LONGTEXT"` 146 | } 147 | 148 | func updatePkgRef(pid int64, refPath string) error { 149 | if base.IsGoRepoPath(refPath) || 150 | refPath == "C" || 151 | refPath[1] == '.' || 152 | !base.IsValidRemotePath(refPath) { 153 | return nil 154 | } 155 | 156 | ref := new(PkgRef) 157 | has, err := x.Where("import_path=?", refPath).Get(ref) 158 | if err != nil { 159 | return fmt.Errorf("get PkgRef: %v", err) 160 | } 161 | 162 | queryStr := "$" + com.ToStr(pid) + "|" 163 | if !has { 164 | if _, err = x.Insert(&PkgRef{ 165 | ImportPath: refPath, 166 | RefNum: 1, 167 | RefIDs: queryStr, 168 | }); err != nil { 169 | return fmt.Errorf("insert PkgRef: %v", err) 170 | } 171 | return nil 172 | } 173 | 174 | i := strings.Index(ref.RefIDs, queryStr) 175 | if i > -1 { 176 | return nil 177 | } 178 | 179 | ref.RefIDs += queryStr 180 | ref.RefNum++ 181 | _, err = x.Id(ref.ID).AllCols().Update(ref) 182 | return err 183 | } 184 | 185 | // checkRefs checks if given packages are still referencing this one. 186 | func checkRefs(pinfo *PkgInfo) { 187 | var buf bytes.Buffer 188 | pinfo.RefNum = 0 189 | refIDs := strings.Split(pinfo.RefIDs, "|") 190 | for i := range refIDs { 191 | if len(refIDs[i]) == 0 { 192 | continue 193 | } 194 | 195 | pkg, _ := GetPkgInfoByID(com.StrTo(refIDs[i][1:]).MustInt64()) 196 | if pkg == nil { 197 | continue 198 | } 199 | 200 | if strings.Index(pkg.ImportIDs, "$"+com.ToStr(pinfo.ID)+"|") == -1 { 201 | continue 202 | } 203 | 204 | buf.WriteString("$") 205 | buf.WriteString(com.ToStr(pkg.ID)) 206 | buf.WriteString("|") 207 | pinfo.RefNum++ 208 | } 209 | pinfo.RefIDs = buf.String() 210 | } 211 | 212 | // updateRef updates or crates corresponding reference import information. 213 | func updateRef(pid int64, refPath string) (int64, error) { 214 | if len(refPath) == 0 { 215 | return 0, nil 216 | } 217 | 218 | pinfo, err := GetPkgInfo(refPath) 219 | if err != nil && pinfo == nil { 220 | if err == ErrPackageNotFound || 221 | err == ErrPackageVersionTooOld { 222 | // Package hasn't existed yet, save to temporary place. 223 | return 0, updatePkgRef(pid, refPath) 224 | } 225 | return 0, fmt.Errorf("GetPkgInfo(%s): %v", refPath, err) 226 | } 227 | 228 | // Check if reference information has beed recorded. 229 | queryStr := "$" + com.ToStr(pid) + "|" 230 | i := strings.Index(pinfo.RefIDs, queryStr) 231 | if i > -1 { 232 | return pinfo.ID, nil 233 | } 234 | 235 | // Add new as needed. 236 | pinfo.RefIDs += queryStr 237 | pinfo.RefNum++ 238 | _, err = x.Id(pinfo.ID).AllCols().Update(pinfo) 239 | return pinfo.ID, err 240 | } 241 | 242 | // SavePkgInfo saves package information. 243 | func SavePkgInfo(pinfo *PkgInfo, updateRefs bool) (err error) { 244 | if len(pinfo.Synopsis) > 255 { 245 | pinfo.Synopsis = pinfo.Synopsis[:255] 246 | } 247 | 248 | pinfo.PkgVer = PackageVersion 249 | 250 | switch { 251 | case pinfo.IsGaeRepo: 252 | pinfo.Priority = 70 253 | case pinfo.IsGoSubrepo: 254 | pinfo.Priority = 80 255 | case pinfo.IsGoRepo: 256 | pinfo.Priority = 99 257 | } 258 | 259 | // When package is not created, there is no ID so check will certainly fail. 260 | var ignoreCheckRefs bool 261 | 262 | // Create or update package info itself. 263 | // Note(Unknwon): do this because we need ID field later. 264 | if pinfo.ID == 0 { 265 | ignoreCheckRefs = true 266 | pinfo.Views = 1 267 | 268 | // First time created, check PkgRef. 269 | ref := new(PkgRef) 270 | has, err := x.Where("import_path=?", pinfo.ImportPath).Get(ref) 271 | if err != nil { 272 | return fmt.Errorf("get PkgRef: %v", err) 273 | } else if has { 274 | pinfo.RefNum = ref.RefNum 275 | pinfo.RefIDs = ref.RefIDs 276 | if _, err = x.Id(ref.ID).Delete(ref); err != nil { 277 | return fmt.Errorf("delete PkgRef: %v", err) 278 | } 279 | } 280 | 281 | _, err = x.Insert(pinfo) 282 | } else { 283 | _, err = x.Id(pinfo.ID).AllCols().Update(pinfo) 284 | } 285 | if err != nil { 286 | return fmt.Errorf("update package info: %v", err) 287 | } 288 | 289 | // Update package import references. 290 | // Note(Unknwon): I just don't see the value of who imports STD 291 | // when you don't even import and uses what objects. 292 | if updateRefs && !pinfo.IsGoRepo { 293 | var buf bytes.Buffer 294 | paths := strings.Split(pinfo.ImportPaths, "|") 295 | for i := range paths { 296 | if base.IsGoRepoPath(paths[i]) { 297 | continue 298 | } 299 | 300 | refID, err := updateRef(pinfo.ID, paths[i]) 301 | if err != nil { 302 | return fmt.Errorf("updateRef: %v", err) 303 | } else if refID == 0 { 304 | continue 305 | } 306 | buf.WriteString("$") 307 | buf.WriteString(com.ToStr(refID)) 308 | buf.WriteString("|") 309 | } 310 | pinfo.ImportIDs = buf.String() 311 | 312 | if !ignoreCheckRefs { 313 | // Check packages who import this is still importing. 314 | checkRefs(pinfo) 315 | } 316 | _, err = x.Id(pinfo.ID).AllCols().Update(pinfo) 317 | return err 318 | } 319 | return nil 320 | } 321 | 322 | // GetPkgInfo returns package information by given import path. 323 | func GetPkgInfo(importPath string) (*PkgInfo, error) { 324 | if len(importPath) == 0 { 325 | return nil, ErrEmptyPackagePath 326 | } 327 | 328 | pinfo := new(PkgInfo) 329 | has, err := x.Where("import_path=?", importPath).Get(pinfo) 330 | if err != nil { 331 | return nil, err 332 | } else if !has { 333 | return nil, ErrPackageNotFound 334 | } else if pinfo.PkgVer < PackageVersion { 335 | pinfo.Etag = "" 336 | return pinfo, ErrPackageVersionTooOld 337 | } 338 | 339 | if !pinfo.HasJSFile() { 340 | pinfo.Etag = "" 341 | return pinfo, ErrPackageVersionTooOld 342 | } 343 | 344 | return pinfo, nil 345 | } 346 | 347 | // GetSubPkgs returns sub-projects by given sub-directories. 348 | func GetSubPkgs(importPath string, dirs []string) []*PkgInfo { 349 | pinfos := make([]*PkgInfo, 0, len(dirs)) 350 | for _, dir := range dirs { 351 | if len(dir) == 0 { 352 | continue 353 | } 354 | 355 | fullPath := importPath + "/" + dir 356 | if pinfo, err := GetPkgInfo(fullPath); err == nil { 357 | pinfo.Name = dir 358 | pinfos = append(pinfos, pinfo) 359 | } else { 360 | pinfos = append(pinfos, &PkgInfo{ 361 | Name: dir, 362 | ImportPath: fullPath, 363 | }) 364 | } 365 | } 366 | return pinfos 367 | } 368 | 369 | // GetPkgInfosByPaths returns a list of packages by given import paths. 370 | func GetPkgInfosByPaths(paths []string) []*PkgInfo { 371 | pinfos := make([]*PkgInfo, 0, len(paths)) 372 | for _, p := range paths { 373 | if len(p) == 0 { 374 | continue 375 | } 376 | 377 | if pinfo, err := GetPkgInfo(p); err == nil { 378 | pinfo.Name = path.Base(p) 379 | pinfos = append(pinfos, pinfo) 380 | } else { 381 | pinfos = append(pinfos, &PkgInfo{ 382 | Name: path.Base(p), 383 | ImportPath: p, 384 | }) 385 | } 386 | } 387 | return pinfos 388 | } 389 | 390 | // GetPkgInfoByID returns package information by given ID. 391 | func GetPkgInfoByID(id int64) (*PkgInfo, error) { 392 | pinfo := new(PkgInfo) 393 | has, err := x.Id(id).Get(pinfo) 394 | if err != nil { 395 | return nil, err 396 | } else if !has { 397 | return nil, ErrPackageNotFound 398 | } else if pinfo.PkgVer < PackageVersion { 399 | return pinfo, ErrPackageVersionTooOld 400 | } 401 | 402 | if !pinfo.HasJSFile() { 403 | return pinfo, ErrPackageVersionTooOld 404 | } 405 | 406 | return pinfo, nil 407 | } 408 | 409 | // GetPkgInfosByIDs returns a list of package info by given IDs. 410 | func GetPkgInfosByIDs(ids []int64) ([]*PkgInfo, error) { 411 | if len(ids) == 0 { 412 | return []*PkgInfo{}, nil 413 | } 414 | 415 | pkgInfos := make([]*PkgInfo, 0, len(ids)) 416 | return pkgInfos, x.Where("id > 0").In("id", base.Int64sToStrings(ids)).Find(&pkgInfos) 417 | } 418 | 419 | func getRepos(trueCondition string) ([]*PkgInfo, error) { 420 | pkgs := make([]*PkgInfo, 0, 100) 421 | return pkgs, x.Desc("views").Where(trueCondition+"=?", true).Find(&pkgs) 422 | } 423 | 424 | func GetGoRepos() ([]*PkgInfo, error) { 425 | return getRepos("is_go_repo") 426 | } 427 | 428 | func GetGoSubepos() ([]*PkgInfo, error) { 429 | return getRepos("is_go_subrepo") 430 | } 431 | 432 | func GetGAERepos() ([]*PkgInfo, error) { 433 | return getRepos("is_gae_repo") 434 | } 435 | 436 | // SearchPkgInfo searches package information by given keyword. 437 | func SearchPkgInfo(limit int, keyword string) ([]*PkgInfo, error) { 438 | if len(keyword) == 0 { 439 | return nil, nil 440 | } 441 | pkgs := make([]*PkgInfo, 0, limit) 442 | return pkgs, x.Limit(limit).Desc("priority").Desc("stars").Desc("views").Where("import_path like ?", "%"+keyword+"%").Find(&pkgs) 443 | } 444 | 445 | func DeletePackageByPath(importPath string) error { 446 | _, err := x.Delete(&PkgInfo{ImportPath: importPath}) 447 | return err 448 | } 449 | 450 | func NumMonthlyActivePackages() int64 { 451 | count, _ := x.Where("last_viewed >= ?", time.Now().Add(-30*24*time.Hour).Unix()).Count(new(PkgInfo)) 452 | return count 453 | } 454 | 455 | func NumWeeklyActivePackages() int64 { 456 | count, _ := x.Where("last_viewed >= ?", time.Now().Add(-7*24*time.Hour).Unix()).Count(new(PkgInfo)) 457 | return count 458 | } 459 | 460 | func NumDailyActivePackages() int64 { 461 | count, _ := x.Where("last_viewed >= ?", time.Now().Add(-24*time.Hour).Unix()).Count(new(PkgInfo)) 462 | return count 463 | } 464 | -------------------------------------------------------------------------------- /internal/httplib/httplib.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Usage: 16 | // 17 | // import "github.com/astaxie/beego/httplib" 18 | // 19 | // b := httplib.Post("http://beego.me/") 20 | // b.Param("username","astaxie") 21 | // b.Param("password","123456") 22 | // b.PostFile("uploadfile1", "httplib.pdf") 23 | // b.PostFile("uploadfile2", "httplib.txt") 24 | // str, err := b.String() 25 | // if err != nil { 26 | // t.Fatal(err) 27 | // } 28 | // fmt.Println(str) 29 | // 30 | // more docs http://beego.me/docs/module/httplib.md 31 | package httplib 32 | 33 | import ( 34 | "bytes" 35 | "crypto/tls" 36 | "encoding/json" 37 | "encoding/xml" 38 | "io" 39 | "io/ioutil" 40 | "log" 41 | "mime/multipart" 42 | "net" 43 | "net/http" 44 | "net/http/cookiejar" 45 | "net/http/httputil" 46 | "net/url" 47 | "os" 48 | "strings" 49 | "sync" 50 | "time" 51 | ) 52 | 53 | var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false} 54 | var defaultCookieJar http.CookieJar 55 | var settingMutex sync.Mutex 56 | 57 | // createDefaultCookie creates a global cookiejar to store cookies. 58 | func createDefaultCookie() { 59 | settingMutex.Lock() 60 | defer settingMutex.Unlock() 61 | defaultCookieJar, _ = cookiejar.New(nil) 62 | } 63 | 64 | // Overwrite default settings 65 | func SetDefaultSetting(setting BeegoHttpSettings) { 66 | settingMutex.Lock() 67 | defer settingMutex.Unlock() 68 | defaultSetting = setting 69 | if defaultSetting.ConnectTimeout == 0 { 70 | defaultSetting.ConnectTimeout = 60 * time.Second 71 | } 72 | if defaultSetting.ReadWriteTimeout == 0 { 73 | defaultSetting.ReadWriteTimeout = 60 * time.Second 74 | } 75 | } 76 | 77 | // return *BeegoHttpRequest with specific method 78 | func newBeegoRequest(url, method string) *BeegoHttpRequest { 79 | var resp http.Response 80 | req := http.Request{ 81 | Method: method, 82 | Header: make(http.Header), 83 | Proto: "HTTP/1.1", 84 | ProtoMajor: 1, 85 | ProtoMinor: 1, 86 | } 87 | return &BeegoHttpRequest{url, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil} 88 | } 89 | 90 | // Get returns *BeegoHttpRequest with GET method. 91 | func Get(url string) *BeegoHttpRequest { 92 | return newBeegoRequest(url, "GET") 93 | } 94 | 95 | // Post returns *BeegoHttpRequest with POST method. 96 | func Post(url string) *BeegoHttpRequest { 97 | return newBeegoRequest(url, "POST") 98 | } 99 | 100 | // Put returns *BeegoHttpRequest with PUT method. 101 | func Put(url string) *BeegoHttpRequest { 102 | return newBeegoRequest(url, "PUT") 103 | } 104 | 105 | // Delete returns *BeegoHttpRequest DELETE method. 106 | func Delete(url string) *BeegoHttpRequest { 107 | return newBeegoRequest(url, "DELETE") 108 | } 109 | 110 | // Head returns *BeegoHttpRequest with HEAD method. 111 | func Head(url string) *BeegoHttpRequest { 112 | return newBeegoRequest(url, "HEAD") 113 | } 114 | 115 | // BeegoHttpSettings 116 | type BeegoHttpSettings struct { 117 | ShowDebug bool 118 | UserAgent string 119 | ConnectTimeout time.Duration 120 | ReadWriteTimeout time.Duration 121 | TlsClientConfig *tls.Config 122 | Proxy func(*http.Request) (*url.URL, error) 123 | Transport http.RoundTripper 124 | EnableCookie bool 125 | } 126 | 127 | // BeegoHttpRequest provides more useful methods for requesting one url than http.Request. 128 | type BeegoHttpRequest struct { 129 | url string 130 | req *http.Request 131 | params map[string]string 132 | files map[string]string 133 | setting BeegoHttpSettings 134 | resp *http.Response 135 | body []byte 136 | } 137 | 138 | // Change request settings 139 | func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest { 140 | b.setting = setting 141 | return b 142 | } 143 | 144 | // SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password. 145 | func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest { 146 | b.req.SetBasicAuth(username, password) 147 | return b 148 | } 149 | 150 | // SetEnableCookie sets enable/disable cookiejar 151 | func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest { 152 | b.setting.EnableCookie = enable 153 | return b 154 | } 155 | 156 | // SetUserAgent sets User-Agent header field 157 | func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest { 158 | b.setting.UserAgent = useragent 159 | return b 160 | } 161 | 162 | // Debug sets show debug or not when executing request. 163 | func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest { 164 | b.setting.ShowDebug = isdebug 165 | return b 166 | } 167 | 168 | // SetTimeout sets connect time out and read-write time out for BeegoRequest. 169 | func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest { 170 | b.setting.ConnectTimeout = connectTimeout 171 | b.setting.ReadWriteTimeout = readWriteTimeout 172 | return b 173 | } 174 | 175 | // SetTLSClientConfig sets tls connection configurations if visiting https url. 176 | func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest { 177 | b.setting.TlsClientConfig = config 178 | return b 179 | } 180 | 181 | // Header add header item string in request. 182 | func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest { 183 | b.req.Header.Set(key, value) 184 | return b 185 | } 186 | 187 | // Set the protocol version for incoming requests. 188 | // Client requests always use HTTP/1.1. 189 | func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest { 190 | if len(vers) == 0 { 191 | vers = "HTTP/1.1" 192 | } 193 | 194 | major, minor, ok := http.ParseHTTPVersion(vers) 195 | if ok { 196 | b.req.Proto = vers 197 | b.req.ProtoMajor = major 198 | b.req.ProtoMinor = minor 199 | } 200 | 201 | return b 202 | } 203 | 204 | // SetCookie add cookie into request. 205 | func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest { 206 | b.req.Header.Add("Cookie", cookie.String()) 207 | return b 208 | } 209 | 210 | // Set transport to 211 | func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest { 212 | b.setting.Transport = transport 213 | return b 214 | } 215 | 216 | // Set http proxy 217 | // example: 218 | // 219 | // func(req *http.Request) (*url.URL, error) { 220 | // u, _ := url.ParseRequestURI("http://127.0.0.1:8118") 221 | // return u, nil 222 | // } 223 | func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest { 224 | b.setting.Proxy = proxy 225 | return b 226 | } 227 | 228 | // Param adds query param in to request. 229 | // params build query string as ?key1=value1&key2=value2... 230 | func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest { 231 | b.params[key] = value 232 | return b 233 | } 234 | 235 | func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest { 236 | b.files[formname] = filename 237 | return b 238 | } 239 | 240 | // Body adds request raw body. 241 | // it supports string and []byte. 242 | func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest { 243 | switch t := data.(type) { 244 | case string: 245 | bf := bytes.NewBufferString(t) 246 | b.req.Body = ioutil.NopCloser(bf) 247 | b.req.ContentLength = int64(len(t)) 248 | case []byte: 249 | bf := bytes.NewBuffer(t) 250 | b.req.Body = ioutil.NopCloser(bf) 251 | b.req.ContentLength = int64(len(t)) 252 | } 253 | return b 254 | } 255 | 256 | func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { 257 | if b.resp.StatusCode != 0 { 258 | return b.resp, nil 259 | } 260 | var paramBody string 261 | if len(b.params) > 0 { 262 | var buf bytes.Buffer 263 | for k, v := range b.params { 264 | buf.WriteString(url.QueryEscape(k)) 265 | buf.WriteByte('=') 266 | buf.WriteString(url.QueryEscape(v)) 267 | buf.WriteByte('&') 268 | } 269 | paramBody = buf.String() 270 | paramBody = paramBody[0 : len(paramBody)-1] 271 | } 272 | 273 | if b.req.Method == "GET" && len(paramBody) > 0 { 274 | if strings.Index(b.url, "?") != -1 { 275 | b.url += "&" + paramBody 276 | } else { 277 | b.url = b.url + "?" + paramBody 278 | } 279 | } else if b.req.Method == "POST" && b.req.Body == nil { 280 | if len(b.files) > 0 { 281 | pr, pw := io.Pipe() 282 | bodyWriter := multipart.NewWriter(pw) 283 | go func() { 284 | for formname, filename := range b.files { 285 | fileWriter, err := bodyWriter.CreateFormFile(formname, filename) 286 | if err != nil { 287 | log.Fatal(err) 288 | } 289 | fh, err := os.Open(filename) 290 | if err != nil { 291 | log.Fatal(err) 292 | } 293 | //iocopy 294 | _, err = io.Copy(fileWriter, fh) 295 | fh.Close() 296 | if err != nil { 297 | log.Fatal(err) 298 | } 299 | } 300 | for k, v := range b.params { 301 | bodyWriter.WriteField(k, v) 302 | } 303 | bodyWriter.Close() 304 | pw.Close() 305 | }() 306 | b.Header("Content-Type", bodyWriter.FormDataContentType()) 307 | b.req.Body = ioutil.NopCloser(pr) 308 | } else if len(paramBody) > 0 { 309 | b.Header("Content-Type", "application/x-www-form-urlencoded") 310 | b.Body(paramBody) 311 | } 312 | } 313 | 314 | url, err := url.Parse(b.url) 315 | if err != nil { 316 | return nil, err 317 | } 318 | 319 | b.req.URL = url 320 | 321 | trans := b.setting.Transport 322 | 323 | if trans == nil { 324 | // create default transport 325 | trans = &http.Transport{ 326 | TLSClientConfig: b.setting.TlsClientConfig, 327 | Proxy: b.setting.Proxy, 328 | Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout), 329 | } 330 | } else { 331 | // if b.transport is *http.Transport then set the settings. 332 | if t, ok := trans.(*http.Transport); ok { 333 | if t.TLSClientConfig == nil { 334 | t.TLSClientConfig = b.setting.TlsClientConfig 335 | } 336 | if t.Proxy == nil { 337 | t.Proxy = b.setting.Proxy 338 | } 339 | if t.Dial == nil { 340 | t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout) 341 | } 342 | } 343 | } 344 | 345 | var jar http.CookieJar 346 | if b.setting.EnableCookie { 347 | if defaultCookieJar == nil { 348 | createDefaultCookie() 349 | } 350 | jar = defaultCookieJar 351 | } else { 352 | jar = nil 353 | } 354 | 355 | client := &http.Client{ 356 | Transport: trans, 357 | Jar: jar, 358 | } 359 | 360 | if b.setting.UserAgent != "" && b.req.Header.Get("User-Agent") == "" { 361 | b.req.Header.Set("User-Agent", b.setting.UserAgent) 362 | } 363 | 364 | if b.setting.ShowDebug { 365 | dump, err := httputil.DumpRequest(b.req, true) 366 | if err != nil { 367 | println(err.Error()) 368 | } 369 | println(string(dump)) 370 | } 371 | 372 | resp, err := client.Do(b.req) 373 | if err != nil { 374 | return nil, err 375 | } 376 | b.resp = resp 377 | return resp, nil 378 | } 379 | 380 | // String returns the body string in response. 381 | // it calls Response inner. 382 | func (b *BeegoHttpRequest) String() (string, error) { 383 | data, err := b.Bytes() 384 | if err != nil { 385 | return "", err 386 | } 387 | 388 | return string(data), nil 389 | } 390 | 391 | // Bytes returns the body []byte in response. 392 | // it calls Response inner. 393 | func (b *BeegoHttpRequest) Bytes() ([]byte, error) { 394 | if b.body != nil { 395 | return b.body, nil 396 | } 397 | resp, err := b.getResponse() 398 | if err != nil { 399 | return nil, err 400 | } 401 | if resp.Body == nil { 402 | return nil, nil 403 | } 404 | defer resp.Body.Close() 405 | data, err := ioutil.ReadAll(resp.Body) 406 | if err != nil { 407 | return nil, err 408 | } 409 | b.body = data 410 | return data, nil 411 | } 412 | 413 | // ToFile saves the body data in response to one file. 414 | // it calls Response inner. 415 | func (b *BeegoHttpRequest) ToFile(filename string) error { 416 | f, err := os.Create(filename) 417 | if err != nil { 418 | return err 419 | } 420 | defer f.Close() 421 | 422 | resp, err := b.getResponse() 423 | if err != nil { 424 | return err 425 | } 426 | if resp.Body == nil { 427 | return nil 428 | } 429 | defer resp.Body.Close() 430 | _, err = io.Copy(f, resp.Body) 431 | return err 432 | } 433 | 434 | // ToJson returns the map that marshals from the body bytes as json in response . 435 | // it calls Response inner. 436 | func (b *BeegoHttpRequest) ToJson(v interface{}) error { 437 | data, err := b.Bytes() 438 | if err != nil { 439 | return err 440 | } 441 | err = json.Unmarshal(data, v) 442 | return err 443 | } 444 | 445 | // ToXml returns the map that marshals from the body bytes as xml in response . 446 | // it calls Response inner. 447 | func (b *BeegoHttpRequest) ToXml(v interface{}) error { 448 | data, err := b.Bytes() 449 | if err != nil { 450 | return err 451 | } 452 | err = xml.Unmarshal(data, v) 453 | return err 454 | } 455 | 456 | // Response executes request client gets response mannually. 457 | func (b *BeegoHttpRequest) Response() (*http.Response, error) { 458 | return b.getResponse() 459 | } 460 | 461 | // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. 462 | func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) { 463 | return func(netw, addr string) (net.Conn, error) { 464 | conn, err := net.DialTimeout(netw, addr, cTimeout) 465 | if err != nil { 466 | return nil, err 467 | } 468 | conn.SetDeadline(time.Now().Add(rwTimeout)) 469 | return conn, nil 470 | } 471 | } 472 | -------------------------------------------------------------------------------- /internal/doc/walker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Gary Burd 2 | // Copyright 2013 Unknown 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 5 | // not use this file except in compliance with the License. You may obtain 6 | // a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | // License for the specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package doc 17 | 18 | import ( 19 | "bytes" 20 | "errors" 21 | "fmt" 22 | "go/ast" 23 | "go/build" 24 | "go/doc" 25 | "go/parser" 26 | "go/printer" 27 | "go/token" 28 | "io" 29 | "io/ioutil" 30 | "os" 31 | "path" 32 | "regexp" 33 | "strings" 34 | "unicode" 35 | "unicode/utf8" 36 | 37 | "github.com/unknwon/com" 38 | ) 39 | 40 | // WalkDepth indicates how far the process goes. 41 | type WalkDepth uint 42 | 43 | const ( 44 | WD_Imports WalkDepth = iota 45 | WD_All 46 | ) 47 | 48 | // WalkType indicates which way to get data for walker. 49 | type WalkType uint 50 | 51 | const ( 52 | WT_Local WalkType = iota 53 | WT_Memory 54 | WT_Zip 55 | WT_TarGz 56 | WT_Http 57 | ) 58 | 59 | // WalkMode indicates which things to do. 60 | type WalkMode uint 61 | 62 | const ( 63 | WM_All WalkMode = 1 << iota 64 | WM_NoReadme 65 | WM_NoExample 66 | ) 67 | 68 | type WalkRes struct { 69 | WalkDepth 70 | WalkType 71 | WalkMode 72 | RootPath string // For WT_Local mode. 73 | Srcs []*Source // For WT_Memory mode. 74 | BuildAll bool 75 | } 76 | 77 | // ------------------------------ 78 | // WT_Local 79 | // ------------------------------ 80 | 81 | func (w *Walker) setLocalContext(ctxt *build.Context) { 82 | ctxt.IsAbsPath = path.IsAbs 83 | } 84 | 85 | // ------------------------------ 86 | // WT_Memory 87 | // ------------------------------ 88 | 89 | func (w *Walker) readDir(dir string) ([]os.FileInfo, error) { 90 | if dir != w.Pdoc.ImportPath { 91 | panic("unexpected") 92 | } 93 | fis := make([]os.FileInfo, 0, len(w.SrcFiles)) 94 | for _, src := range w.SrcFiles { 95 | fis = append(fis, src) 96 | } 97 | return fis, nil 98 | } 99 | 100 | func (w *Walker) openFile(path string) (io.ReadCloser, error) { 101 | if strings.HasPrefix(path, w.Pdoc.ImportPath+"/") { 102 | if src, ok := w.SrcFiles[path[len(w.Pdoc.ImportPath)+1:]]; ok { 103 | return ioutil.NopCloser(bytes.NewReader(src.Data())), nil 104 | } 105 | } 106 | return nil, os.ErrNotExist 107 | } 108 | 109 | func (w *Walker) setMemoryContext(ctxt *build.Context) { 110 | ctxt.JoinPath = path.Join 111 | ctxt.IsAbsPath = path.IsAbs 112 | ctxt.IsDir = func(path string) bool { return true } 113 | ctxt.HasSubdir = func(root, dir string) (rel string, ok bool) { panic("unexpected") } 114 | ctxt.ReadDir = func(dir string) (fi []os.FileInfo, err error) { return w.readDir(dir) } 115 | ctxt.OpenFile = func(path string) (r io.ReadCloser, err error) { return w.openFile(path) } 116 | } 117 | 118 | var badSynopsisPrefixes = []string{ 119 | "Autogenerated by Thrift Compiler", 120 | "Automatically generated ", 121 | "Auto-generated by ", 122 | "Copyright ", 123 | "COPYRIGHT ", 124 | `THE SOFTWARE IS PROVIDED "AS IS"`, 125 | "TODO: ", 126 | "vim:", 127 | } 128 | 129 | // Synopsis extracts the first sentence from s. All runs of whitespace are 130 | // replaced by a single space. 131 | func synopsis(s string) string { 132 | parts := strings.SplitN(s, "\n\n", 2) 133 | s = parts[0] 134 | 135 | var buf []byte 136 | const ( 137 | other = iota 138 | period 139 | space 140 | ) 141 | last := space 142 | Loop: 143 | for i := 0; i < len(s); i++ { 144 | b := s[i] 145 | switch b { 146 | case ' ', '\t', '\r', '\n': 147 | switch last { 148 | case period: 149 | break Loop 150 | case other: 151 | buf = append(buf, ' ') 152 | last = space 153 | } 154 | case '.': 155 | last = period 156 | buf = append(buf, b) 157 | default: 158 | last = other 159 | buf = append(buf, b) 160 | } 161 | } 162 | 163 | // Ensure that synopsis fits an App Engine datastore text property. 164 | const m = 297 165 | if len(buf) > m { 166 | buf = buf[:m] 167 | if i := bytes.LastIndex(buf, []byte{' '}); i >= 0 { 168 | buf = buf[:i] 169 | } 170 | buf = append(buf, " ..."...) 171 | } 172 | 173 | s = string(buf) 174 | 175 | r, n := utf8.DecodeRuneInString(s) 176 | if n < 0 || unicode.IsPunct(r) || unicode.IsSymbol(r) { 177 | // ignore Markdown headings, editor settings, Go build constraints, and * in poorly formatted block comments. 178 | s = "" 179 | } else { 180 | for _, prefix := range badSynopsisPrefixes { 181 | if strings.HasPrefix(s, prefix) { 182 | s = "" 183 | break 184 | } 185 | } 186 | } 187 | 188 | return strings.TrimRight(s, " \t\n\r") 189 | } 190 | 191 | // poorMansImporter returns a (dummy) package object named 192 | // by the last path component of the provided package path 193 | // (as is the convention for packages). This is sufficient 194 | // to resolve package identifiers without doing an actual 195 | // import. It never returns an error. 196 | // 197 | func poorMansImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) { 198 | pkg := imports[path] 199 | if pkg == nil { 200 | // Guess the package name without importing it. Start with the last 201 | // element of the path. 202 | name := path[strings.LastIndex(path, "/")+1:] 203 | 204 | // Trim commonly used prefixes and suffixes containing illegal name 205 | // runes. 206 | name = strings.TrimSuffix(name, ".go") 207 | name = strings.TrimSuffix(name, "-go") 208 | name = strings.TrimPrefix(name, "go.") 209 | name = strings.TrimPrefix(name, "go-") 210 | name = strings.TrimPrefix(name, "biogo.") 211 | 212 | pkg = ast.NewObj(ast.Pkg, name) 213 | pkg.Data = ast.NewScope(nil) // required by ast.NewPackage for dot-import 214 | imports[path] = pkg 215 | } 216 | return pkg, nil 217 | } 218 | 219 | type sliceWriter struct{ p *[]byte } 220 | 221 | func (w sliceWriter) Write(p []byte) (int, error) { 222 | *w.p = append(*w.p, p...) 223 | return len(p), nil 224 | } 225 | 226 | func (w *Walker) printNode(node interface{}) string { 227 | w.Buf = w.Buf[:0] 228 | err := (&printer.Config{Mode: printer.UseSpaces, Tabwidth: 4}).Fprint(sliceWriter{&w.Buf}, w.Fset, node) 229 | if err != nil { 230 | return err.Error() 231 | } 232 | return string(w.Buf) 233 | } 234 | 235 | var exampleOutputRx = regexp.MustCompile(`(?i)//[[:space:]]*output:`) 236 | 237 | func (w *Walker) getExamples() { 238 | var docs []*Example 239 | for _, e := range w.Examples { 240 | e.Name = strings.TrimPrefix(e.Name, "_") 241 | 242 | output := e.Output 243 | code := w.printNode(&printer.CommentedNode{ 244 | Node: e.Code, 245 | Comments: e.Comments, 246 | }) 247 | 248 | // additional formatting if this is a function body 249 | if i := len(code); i >= 2 && code[0] == '{' && code[i-1] == '}' { 250 | // remove surrounding braces 251 | code = code[1 : i-1] 252 | // unindent 253 | code = strings.Replace(code, "\n ", "\n", -1) 254 | // remove output comment 255 | if j := exampleOutputRx.FindStringIndex(code); j != nil { 256 | code = strings.TrimSpace(code[:j[0]]) 257 | } 258 | } else { 259 | // drop output, as the output comment will appear in the code 260 | output = "" 261 | } 262 | 263 | // play := "" 264 | // if e.Play != nil { 265 | // w.buf = w.buf[:0] 266 | // if err := format.Node(sliceWriter{&w.buf}, w.fset, e.Play); err != nil { 267 | // play = err.Error() 268 | // } else { 269 | // play = string(w.buf) 270 | // } 271 | // } 272 | 273 | docs = append(docs, &Example{ 274 | Name: e.Name, 275 | Doc: e.Doc, 276 | Code: code, 277 | Output: output, 278 | }) 279 | //Play: play 280 | } 281 | 282 | w.Pdoc.Examples = docs 283 | } 284 | 285 | func (w *Walker) printDecl(decl ast.Node) string { 286 | var d Code 287 | d, w.Buf = printDecl(decl, w.Fset, w.Buf) 288 | return d.Text 289 | } 290 | 291 | func (w *Walker) printPos(pos token.Pos) string { 292 | position := w.Fset.Position(pos) 293 | src := w.SrcFiles[position.Filename] 294 | if src == nil || src.BrowseUrl == "" { 295 | // src can be nil when line comments are used (//line :). 296 | return "" 297 | } 298 | return src.BrowseUrl + fmt.Sprintf(w.LineFmt, position.Line) 299 | } 300 | 301 | func (w *Walker) values(vdocs []*doc.Value) (vals []*Value) { 302 | for _, d := range vdocs { 303 | vals = append(vals, &Value{ 304 | Decl: w.printDecl(d.Decl), 305 | URL: w.printPos(d.Decl.Pos()), 306 | Doc: d.Doc, 307 | }) 308 | } 309 | 310 | return vals 311 | } 312 | 313 | // printCode returns function or method code from source files. 314 | func (w *Walker) printCode(decl ast.Node) string { 315 | pos := decl.Pos() 316 | posPos := w.Fset.Position(pos) 317 | src := w.SrcFiles[posPos.Filename] 318 | if src == nil || src.BrowseUrl == "" { 319 | // src can be nil when line comments are used (//line :). 320 | return "" 321 | } 322 | 323 | code, ok := w.SrcLines[posPos.Filename] 324 | // Check source file line arrays. 325 | if !ok { 326 | // Split source file to array and save into map when at the 1st time. 327 | w.SrcLines[posPos.Filename] = strings.Split(string(src.Data()), "\n") 328 | code = w.SrcLines[posPos.Filename] 329 | } 330 | 331 | // Get code. 332 | var buf bytes.Buffer 333 | l := len(code) 334 | CutCode: 335 | for i := posPos.Line; i < l; i++ { 336 | // Check end of code block. 337 | switch { 338 | case len(code[i]) > 0 && code[i][0] == '}': // Normal end. 339 | break CutCode 340 | case (i == posPos.Line) && len(code[i]) == 0 && (strings.Index(code[i-1], "{") == -1): // Package `builtin`. 341 | break CutCode 342 | case len(code[i-1]) > 4 && code[i-1][:4] == "func" && 343 | code[i-1][len(code[i-1])-1] == '}': // One line functions. 344 | line := code[i-1] 345 | buf.WriteString(" ") 346 | buf.WriteString(line[strings.Index(line, "{")+1 : len(line)-1]) 347 | buf.WriteByte('\n') 348 | break CutCode 349 | } 350 | 351 | buf.WriteString(code[i]) 352 | buf.WriteByte('\n') 353 | } 354 | return buf.String() 355 | } 356 | 357 | func (w *Walker) funcs(fdocs []*doc.Func) (funcs []*Func, ifuncs []*Func) { 358 | isBuiltIn := w.Pdoc.ImportPath == "builtin" 359 | for _, d := range fdocs { 360 | if unicode.IsUpper(rune(d.Name[0])) || isBuiltIn { 361 | // var exampleName string 362 | // switch { 363 | // case d.Recv == "": 364 | // exampleName = d.Name 365 | // case d.Recv[0] == '*': 366 | // exampleName = d.Recv[1:] + "_" + d.Name 367 | // default: 368 | // exampleName = d.Recv + "_" + d.Name 369 | // } 370 | funcs = append(funcs, &Func{ 371 | Decl: w.printDecl(d.Decl), 372 | URL: w.printPos(d.Decl.Pos()), 373 | Doc: d.Doc, 374 | Name: d.Name, 375 | Code: w.printCode(d.Decl), 376 | // Recv: d.Recv, 377 | // Examples: w.getExamples(exampleName), 378 | }) 379 | continue 380 | } 381 | 382 | ifuncs = append(ifuncs, &Func{ 383 | Decl: w.printDecl(d.Decl), 384 | URL: w.printPos(d.Decl.Pos()), 385 | Doc: d.Doc, 386 | Name: d.Name, 387 | Code: w.printCode(d.Decl), 388 | }) 389 | } 390 | 391 | return funcs, ifuncs 392 | } 393 | 394 | func (w *Walker) types(tdocs []*doc.Type) (tps []*Type, itps []*Type) { 395 | isBuiltIn := w.Pdoc.ImportPath == "builtin" 396 | for _, d := range tdocs { 397 | funcs, ifuncs := w.funcs(d.Funcs) 398 | meths, imeths := w.funcs(d.Methods) 399 | 400 | if unicode.IsUpper(rune(d.Name[0])) || isBuiltIn { 401 | tps = append(tps, &Type{ 402 | Doc: d.Doc, 403 | Name: d.Name, 404 | Decl: w.printDecl(d.Decl), 405 | URL: w.printPos(d.Decl.Pos()), 406 | Consts: w.values(d.Consts), 407 | Vars: w.values(d.Vars), 408 | Funcs: funcs, 409 | IFuncs: ifuncs, 410 | Methods: meths, 411 | IMethods: imeths, 412 | // Examples: w.getExamples(d.Name), 413 | }) 414 | continue 415 | } 416 | 417 | itps = append(itps, &Type{ 418 | Doc: d.Doc, 419 | Name: d.Name, 420 | Decl: w.printDecl(d.Decl), 421 | URL: w.printPos(d.Decl.Pos()), 422 | Consts: w.values(d.Consts), 423 | Vars: w.values(d.Vars), 424 | Funcs: funcs, 425 | IFuncs: ifuncs, 426 | Methods: meths, 427 | IMethods: imeths, 428 | }) 429 | } 430 | return tps, itps 431 | } 432 | 433 | func (w *Walker) isCgo() bool { 434 | for _, name := range w.Pdoc.Imports { 435 | if name == "C" || name == "os/user" { 436 | return true 437 | } 438 | } 439 | return false 440 | } 441 | 442 | var goEnvs = []struct{ GOOS, GOARCH string }{ 443 | {"linux", "amd64"}, 444 | {"darwin", "amd64"}, 445 | {"windows", "amd64"}, 446 | } 447 | 448 | // Build generates documentation from given source files through 'WalkType'. 449 | func (w *Walker) Build(wr *WalkRes) (*Package, error) { 450 | ctxt := build.Context{ 451 | CgoEnabled: true, 452 | ReleaseTags: build.Default.ReleaseTags, 453 | BuildTags: build.Default.BuildTags, 454 | Compiler: "gc", 455 | } 456 | 457 | if w.Pdoc.PkgDecl == nil { 458 | w.Pdoc.PkgDecl = &PkgDecl{} 459 | } 460 | 461 | // Check 'WalkType'. 462 | switch wr.WalkType { 463 | case WT_Local: 464 | // Check root path. 465 | if len(wr.RootPath) == 0 { 466 | return nil, errors.New("WT_Local: empty root path") 467 | } else if !com.IsDir(wr.RootPath) { 468 | return nil, errors.New("WT_Local: cannot find specific directory or it's a file") 469 | } 470 | 471 | w.setLocalContext(&ctxt) 472 | return nil, errors.New("Hasn't supported yet!") 473 | case WT_Memory: 474 | // Convert source files. 475 | w.SrcFiles = make(map[string]*Source) 476 | w.Pdoc.Readme = make(map[string][]byte) 477 | for _, src := range wr.Srcs { 478 | srcName := strings.ToLower(src.Name()) // For readme comparation. 479 | switch { 480 | case strings.HasSuffix(src.Name(), ".go"): 481 | w.SrcFiles[src.Name()] = src 482 | case len(w.Pdoc.Tag) > 0 || (wr.WalkMode&WM_NoReadme != 0): 483 | // This means we are not on the latest version of the code, 484 | // so we do not collect the README files. 485 | continue 486 | case strings.HasPrefix(srcName, "readme_zh") || strings.HasPrefix(srcName, "readme_cn"): 487 | w.Pdoc.Readme["zh"] = src.Data() 488 | case strings.HasPrefix(srcName, "readme"): 489 | w.Pdoc.Readme["en"] = src.Data() 490 | } 491 | } 492 | 493 | // Check source files. 494 | if w.SrcFiles == nil { 495 | return nil, errors.New("WT_Memory: no Go source file") 496 | } 497 | 498 | w.setMemoryContext(&ctxt) 499 | 500 | default: 501 | return nil, errors.New("Hasn't supported yet!") 502 | } 503 | 504 | var err error 505 | var bpkg *build.Package 506 | 507 | for _, env := range goEnvs { 508 | ctxt.GOOS = env.GOOS 509 | ctxt.GOARCH = env.GOARCH 510 | 511 | bpkg, err = ctxt.ImportDir(w.Pdoc.ImportPath, 0) 512 | // Continue if there are no Go source files; we still want the directory info. 513 | _, nogo := err.(*build.NoGoError) 514 | if err != nil { 515 | if nogo { 516 | err = nil 517 | } else { 518 | return nil, errors.New("Walker.Build -> ImportDir: " + err.Error()) 519 | } 520 | } 521 | } 522 | 523 | w.Pdoc.IsCmd = bpkg.IsCommand() 524 | w.Pdoc.Synopsis = synopsis(bpkg.Doc) 525 | 526 | w.Pdoc.Imports = bpkg.Imports 527 | w.Pdoc.IsCgo = w.isCgo() 528 | w.Pdoc.TestImports = bpkg.TestImports 529 | 530 | // Check depth. 531 | if wr.WalkDepth <= WD_Imports { 532 | return w.Pdoc, nil 533 | } 534 | 535 | w.Fset = token.NewFileSet() 536 | // Parse the Go files 537 | files := make(map[string]*ast.File) 538 | for _, name := range append(bpkg.GoFiles, bpkg.CgoFiles...) { 539 | file, err := parser.ParseFile(w.Fset, name, w.SrcFiles[name].Data(), parser.ParseComments) 540 | if err != nil { 541 | return nil, errors.New("Walker.Build -> parse Go files: " + err.Error()) 542 | continue 543 | } 544 | w.Pdoc.Files = append(w.Pdoc.Files, w.SrcFiles[name]) 545 | // w.Pdoc.SourceSize += int64(len(w.SrcFiles[name].Data())) 546 | files[name] = file 547 | } 548 | 549 | w.apkg, _ = ast.NewPackage(w.Fset, files, poorMansImporter, nil) 550 | 551 | // Find examples in the test files. 552 | for _, name := range append(bpkg.TestGoFiles, bpkg.XTestGoFiles...) { 553 | file, err := parser.ParseFile(w.Fset, name, w.SrcFiles[name].Data(), parser.ParseComments) 554 | if err != nil { 555 | return nil, errors.New("Walker.Build -> find examples: " + err.Error()) 556 | continue 557 | } 558 | w.Pdoc.TestFiles = append(w.Pdoc.TestFiles, w.SrcFiles[name]) 559 | //w.pdoc.TestSourceSize += len(w.srcs[name].data) 560 | 561 | if wr.WalkMode&WM_NoExample != 0 { 562 | continue 563 | } 564 | w.Examples = append(w.Examples, doc.Examples(file)...) 565 | } 566 | 567 | mode := doc.Mode(0) 568 | if w.Pdoc.ImportPath == "builtin" || wr.BuildAll { 569 | mode |= doc.AllDecls 570 | } 571 | pdoc := doc.New(w.apkg, w.Pdoc.ImportPath, mode) 572 | 573 | // Get doc. 574 | pdoc.Doc = strings.TrimRight(pdoc.Doc, " \t\n\r") 575 | var buf bytes.Buffer 576 | doc.ToHTML(&buf, pdoc.Doc, nil) 577 | w.Pdoc.Doc = buf.String() 578 | // Highlight first sentence. 579 | w.Pdoc.Doc = strings.Replace(w.Pdoc.Doc, "

", "

", 1) 580 | w.Pdoc.Doc = strings.Replace(w.Pdoc.Doc, "

", "

", 1) 581 | 582 | if wr.WalkMode&WM_NoExample == 0 { 583 | w.getExamples() 584 | } 585 | 586 | w.SrcLines = make(map[string][]string) 587 | w.Pdoc.Consts = w.values(pdoc.Consts) 588 | w.Pdoc.Funcs, w.Pdoc.Ifuncs = w.funcs(pdoc.Funcs) 589 | w.Pdoc.Types, w.Pdoc.Itypes = w.types(pdoc.Types) 590 | w.Pdoc.Vars = w.values(pdoc.Vars) 591 | w.Pdoc.ImportPaths = strings.Join(pdoc.Imports, "|") 592 | w.Pdoc.ImportNum = int64(len(pdoc.Imports)) 593 | //w.Pdoc.Notes = w.notes(pdoc.Notes) 594 | 595 | return w.Pdoc, nil 596 | } 597 | --------------------------------------------------------------------------------