├── contributions └── .gitkeep ├── web ├── templates │ ├── robots.txt │ ├── 404.html │ ├── 500.html │ ├── index.html │ ├── search.html │ ├── components │ │ └── uml_list.html │ └── base.html ├── index.yaml ├── static │ ├── img │ │ ├── logo.png │ │ ├── favicon.ico │ │ ├── top_image.png │ │ ├── github_octocat.png │ │ ├── icon_state.svg │ │ ├── icon_component.svg │ │ ├── icon_sequence.svg │ │ ├── icon_activity.svg │ │ ├── icon_usecase.svg │ │ └── icon_class.svg │ ├── js │ │ └── main.js │ └── css │ │ └── main.css ├── app.yaml ├── Makefile ├── .gcloudignore ├── main.go ├── handler.go └── uml.go ├── renderer ├── Dockerfile ├── app.yaml └── Makefile ├── scraping ├── package.json ├── scraping.js ├── package-lock.json └── results │ └── 20180125_01.txt ├── indexer ├── queue.yaml ├── app.go ├── app.yaml ├── middleware.go ├── indexer_test.go ├── Makefile ├── syntax_checker.go ├── renderer.go ├── handler.go └── indexer.go ├── docs ├── architecture.png └── architexture.puml ├── syntaxchecker ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── app.yaml ├── Dockerfile ├── Makefile ├── build.gradle ├── gradlew.bat ├── src │ └── main │ │ └── kotlin │ │ └── com │ │ └── yfuruyama │ │ └── syntaxchecker │ │ └── Main.kt └── gradlew ├── .gitignore ├── util └── gen_app_yaml.go └── README.md /contributions/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/templates/robots.txt: -------------------------------------------------------------------------------- 1 | Allow: / 2 | -------------------------------------------------------------------------------- /web/index.yaml: -------------------------------------------------------------------------------- 1 | indexes: 2 | # AUTOGENERATED 3 | -------------------------------------------------------------------------------- /renderer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM plantuml/plantuml-server:jetty 2 | -------------------------------------------------------------------------------- /scraping/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "chromy": "latest" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /indexer/queue.yaml: -------------------------------------------------------------------------------- 1 | queue: 2 | - name: index-create-queue 3 | target: indexer 4 | rate: 10/s 5 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfuruyama/real-world-plantuml/HEAD/docs/architecture.png -------------------------------------------------------------------------------- /web/static/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfuruyama/real-world-plantuml/HEAD/web/static/img/logo.png -------------------------------------------------------------------------------- /web/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfuruyama/real-world-plantuml/HEAD/web/static/img/favicon.ico -------------------------------------------------------------------------------- /web/static/img/top_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfuruyama/real-world-plantuml/HEAD/web/static/img/top_image.png -------------------------------------------------------------------------------- /web/templates/404.html: -------------------------------------------------------------------------------- 1 | {{define "content"}} 2 |
9 |
10 | ## For development
11 |
12 | ### web
13 |
14 | Run server
15 |
16 | ```
17 | make run GA_TRACKING_ID=${GA_TRACKING_ID}
18 | ```
19 |
20 | Register dummy UML: access to `/debug/dummy_uml` in your browser
21 |
22 | ### indexer
23 |
24 | Run server
25 |
26 | ```
27 | make run GITHUB_API_TOKEN=${GITHUB_API_TOKEN} GCS_BUCKET=${GCS_BUCKET}
28 | ```
29 |
30 | ### renderer
31 |
32 | Run server
33 |
34 | ```
35 | make run
36 | ```
37 |
38 | ### scraping
39 |
40 | Launch Chrome with remote debugging enabled
41 |
42 | ```
43 | /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222
44 | ```
45 |
46 | Run scraping script
47 |
48 | ```
49 | cd scraping
50 | npm install
51 | node scraping.js > results/YYYYMMDD_01.txt
52 | ```
53 |
--------------------------------------------------------------------------------
/docs/architexture.puml:
--------------------------------------------------------------------------------
1 | @startuml
2 | cloud {
3 | [GitHub] as github
4 | }
5 |
6 | package "Local Machine" {
7 | [nodejs] as scrape
8 | }
9 |
10 | package "GCP" {
11 | database "Cloud Datastore" as datastore {
12 | [entity]
13 | }
14 |
15 | database "Cloud Storage" as gcs {
16 | [bucket]
17 | }
18 |
19 | node "App Engine" {
20 | frame "Standard" {
21 | [indexer]
22 | [web]
23 | }
24 | frame "Flexible" {
25 | [syntax_checker]
26 | [renderer]
27 | }
28 | database "TaskQueue" as taskqueue {
29 | [index-create-queue] as queue
30 | }
31 | [Search API] as searchapi
32 | }
33 | }
34 |
35 | actor User as user
36 |
37 | scrape --> github : 1. scraping
38 | scrape -r-> bucket : 2. put GitHub urls
39 | bucket -d-> indexer : 3. notification
40 | indexer -l-> queue : 4. put task
41 | queue -r-> indexer : 5. execute task
42 | indexer --> syntax_checker : 6. syntax check
43 | indexer --> renderer : 7. rendering
44 | indexer -u-> entity : 8. put
45 | web -u-> entity : 9. get
46 | web -d-> searchapi : 10. search
47 | web -r-> user : 11. show web page
48 |
49 | @enduml
50 |
--------------------------------------------------------------------------------
/syntaxchecker/build.gradle:
--------------------------------------------------------------------------------
1 | buildscript {
2 | ext.kotlin_version = '1.1.0-dev-1159'
3 | ext.plantuml_version = '1.2017.20'
4 | ext.repo = 'https://repo.gradle.org/gradle/repo'
5 |
6 | repositories {
7 | maven { url = repo}
8 | }
9 | dependencies {
10 | classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
11 | }
12 | }
13 |
14 | apply plugin: 'kotlin'
15 | apply plugin: 'application'
16 |
17 | mainClassName = 'com.yfuruyama.syntaxchecker.MainKt'
18 |
19 | jar {
20 | baseName = 'com.yfuruyama.syntaxchecker'
21 | version = '0.1.0'
22 | }
23 |
24 | repositories {
25 | maven { url = repo}
26 | }
27 |
28 | dependencies {
29 | compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
30 | compile "org.glassfish.jersey.containers:jersey-container-jetty-http:2.23.1"
31 | compile "org.glassfish.jersey.media:jersey-media-json-jackson:2.23.1"
32 | compile "com.fasterxml.jackson.module:jackson-module-kotlin:2.5.5-2"
33 | compile "net.sourceforge.plantuml:plantuml:$plantuml_version"
34 |
35 | testCompile 'org.spockframework:spock-core:1.0-groovy-2.4'
36 | testCompile "org.jetbrains.spek:spek:1.0.25"
37 | }
38 |
39 | task wrapper(type: Wrapper) {
40 | gradleVersion = '2.14'
41 | }
--------------------------------------------------------------------------------
/indexer/indexer_test.go:
--------------------------------------------------------------------------------
1 | package indexer
2 |
3 | import (
4 | "testing"
5 |
6 | "google.golang.org/appengine/aetest"
7 | )
8 |
9 | func TestFindSources(t *testing.T) {
10 | ctx, done, err := aetest.NewContext()
11 | if err != nil {
12 | t.Fatal(err)
13 | }
14 | defer done()
15 |
16 | var tests = []struct {
17 | text string
18 | expected []string
19 | }{
20 | {
21 | `
22 | @startuml
23 | alice -> bob
24 | @enduml
25 |
26 | @startuml
27 | bob -> alice
28 | @enduml
29 | `, []string{"@startuml\nalice -> bob\n@enduml", "@startuml\nbob -> alice\n@enduml"}},
30 | {
31 | `
32 | @startuml
33 | alice -> bob
34 | @enduml
35 | @enduml
36 | @startuml
37 | `, []string{"@startuml\nalice -> bob\n@enduml"}},
38 | {
39 | `
40 | @enduml
41 | @startuml
42 | alice -> bob
43 | @enduml
44 | @startuml
45 | `, []string{"@startuml\nalice -> bob\n@enduml"}},
46 | }
47 |
48 | for _, test := range tests {
49 | got := findSources(ctx, test.text)
50 | if !isSameSources(got, test.expected) {
51 | t.Errorf("not expected sources: got=%#v, expected=%#v", got, test.expected)
52 | }
53 | }
54 | }
55 |
56 | func isSameSources(got []string, expected []string) bool {
57 | if len(got) != len(expected) {
58 | return false
59 | }
60 | for i := 0; i < len(got); i++ {
61 | if got[i] != expected[i] {
62 | return false
63 | }
64 | }
65 | return true
66 | }
67 |
--------------------------------------------------------------------------------
/indexer/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: test
2 |
3 | PORT=8083
4 | API_PORT=8084
5 | ADMIN_PORT=8085
6 | RENDERER_BASE_URL=http://localhost:8086
7 | SYNTAX_CHECKER_BASE_URL=http://localhost:8087
8 |
9 | # Must be set
10 | GITHUB_API_TOKEN=xxx
11 | GCS_BUCKET=xxx
12 |
13 | all:
14 | test
15 |
16 | test:
17 | go test -v ./...
18 |
19 | run:
20 | GITHUB_API_TOKEN=$(GITHUB_API_TOKEN) RENDERER_BASE_URL=$(RENDERER_BASE_URL) SYNTAX_CHECKER_BASE_URL=$(SYNTAX_CHECKER_BASE_URL) go run ../util/gen_app_yaml.go --in app.yaml --out app.dist.yaml
21 | dev_appserver.py --port=$(PORT) --api_port=$(API_PORT) --admin_port=$(ADMIN_PORT) --logs_path=/tmp/log_indexer.db --storage_path=/tmp/storage.db --search_indexes_path=/tmp/search.db --clear_search_indexes=false --default_gcs_bucket_name=$(GCS_BUCKET) app.dist.yaml
22 |
23 | notify:
24 | curl -X POST http://localhost:$(PORT)/_ah/push-handlers/gcs_notification --data '{"message": {"attributes":{"objectId":"$(OBJECT_ID)", "eventType":"OBJECT_FINALIZE"}, "messageId":"xxx"}, "subscription" :"xxx"}'
25 |
26 | deploy:
27 | GITHUB_API_TOKEN=$(GITHUB_API_TOKEN) RENDERER_BASE_URL=$(RENDERER_BASE_URL) SYNTAX_CHECKER_BASE_URL=$(SYNTAX_CHECKER_BASE_URL) go run ../util/gen_app_yaml.go --in app.yaml --out app.dist.yaml
28 | gcloud --project=$(PROJECT) app deploy app.dist.yaml --version=$(VERSION)
29 |
30 | deploy_queue:
31 | gcloud --project=$(PROJECT) app deploy queue.yaml
32 |
--------------------------------------------------------------------------------
/scraping/scraping.js:
--------------------------------------------------------------------------------
1 | const Chromy = require('chromy')
2 |
3 | let chromy = new Chromy({
4 | visible: true,
5 | launchBrowser: false,
6 | });
7 |
8 | async function scrape(searchWord) {
9 | var page = 1;
10 |
11 | while (true) {
12 | let url = 'https://github.com/search?type=Code&q=' + encodeURIComponent(searchWord) + '&p=' + page;
13 | console.error('scrape url:' + url);
14 | await chromy.goto(url);
15 |
16 | let repos = await chromy.evaluate(() => {
17 | return Array.prototype.map.call(document.getElementsByClassName('code-list-item'), function(e) { return e.getElementsByTagName('a')[2].href; });
18 | });
19 | repos.forEach(function(r) {
20 | console.log(r);
21 | });
22 |
23 | let hasNext = await chromy.evaluate(() => {
24 | if (document.getElementsByClassName('next_page').length == 0) {
25 | return false;
26 | }
27 | return document.getElementsByClassName('next_page disabled').length == 0 ? true : false;
28 | });
29 | if (!hasNext) {
30 | console.error('no more page');
31 | break;
32 | }
33 |
34 | page++;
35 |
36 | // sleep
37 | await new Promise(resolve => {
38 | setTimeout(() => {
39 | resolve();
40 | }, 10000);
41 | });
42 | }
43 | }
44 |
45 | const searchWords = [
46 | 'startuml enduml size:>100 license:mit language:Text',
47 | 'startuml enduml size:>100 license:mit language:Markdown',
48 | 'startuml enduml size:>100 license:mit extension:puml',
49 | 'startuml enduml size:>100 license:mit extension:uml',
50 | 'startuml enduml size:>100 license:mit extension:plantuml',
51 | ];
52 |
53 | (async function() {
54 | for (const word of searchWords) {
55 | await scrape(word);
56 | }
57 | })();
58 |
59 | chromy.close();
60 |
--------------------------------------------------------------------------------
/web/static/img/icon_state.svg:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/web/templates/components/uml_list.html:
--------------------------------------------------------------------------------
1 | {{define "uml_list"}}
2 |
3 |