├── .github ├── ISSUE_TEMPLATE │ └── issue.md ├── issue_template.md └── pull_request_template.md ├── .gitignore ├── .travis.yml ├── API.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LOGS.md ├── Makefile ├── README.md ├── SECURITY.md ├── TESTING.md ├── annotations.go ├── annotations_test.go ├── calc_metric.sh ├── cmd ├── annotations │ └── annotations.go ├── api │ └── api.go ├── calc_metric │ └── calc_metric.go ├── columns │ └── columns.go ├── devstats │ └── devstats.go ├── get_repos │ └── get_repos.go ├── gha2db │ └── gha2db.go ├── gha2db_sync │ └── gha2db_sync.go ├── ghapi2db │ └── ghapi2db.go ├── hide_data │ └── hide_data.go ├── import_affs │ └── import_affs.go ├── merge_dbs │ └── merge_dbs.go ├── replacer │ └── replacer.go ├── runq │ └── runq.go ├── splitcrons │ └── splitcrons.go ├── sqlitedb │ └── sqlitedb.go ├── structure │ └── structure.go ├── sync_issues │ └── sync_issues.go ├── tags │ └── tags.go ├── tsplit │ └── tsplit.go ├── vars │ └── vars.go ├── webhook │ ├── example_webhook_payload.json │ ├── example_webhook_payload_deploy.dat │ ├── example_webhook_payload_no_deploy.dat │ └── webhook.go └── website_data │ └── website_data.go ├── columns.sh ├── const.go ├── context.go ├── context_test.go ├── convert.go ├── convert_test.go ├── cron ├── backup_artificial.sh ├── cron_db_backup.sh └── sysctl_config.sh ├── dbtest.sh ├── devel ├── api_com_contrib_repo_grp.sh ├── api_com_stats_repo_grp.sh ├── api_companies.sh ├── api_companies_table.sh ├── api_countries.sh ├── api_cumulative_counts.sh ├── api_dev_act_cnt.sh ├── api_dev_act_cnt_comp.sh ├── api_dev_act_cnt_comp_repos.sh ├── api_dev_act_cnt_example.sh ├── api_dev_act_cnt_repos.sh ├── api_events.sh ├── api_health.sh ├── api_list_apis.sh ├── api_list_projects.sh ├── api_ranges.sh ├── api_repo_groups.sh ├── api_repos.sh ├── api_site_stats.sh ├── cronctl.sh ├── db.sh ├── mass_replace.sh ├── sync_lock.sh ├── sync_unlock.sh ├── test_webhook.sh ├── wait_for_command.sh └── webhook.sh ├── env.go ├── env_test.go ├── error.go ├── exec.go ├── find.sh ├── find_and_replace.sh ├── for_each_go_file.sh ├── gha.go ├── gha_test.go ├── ghapi.go ├── git ├── git_files.sh ├── git_loc.sh ├── git_reset_pull.sh ├── git_tags.sh └── last_tag.sh ├── go.mod ├── hash.go ├── io.go ├── json.go ├── log.go ├── map.go ├── map_test.go ├── mgetc.go ├── mgetc_test.go ├── pg_conn.go ├── pg_test.go ├── regexp_test.go ├── series_test.go ├── signal.go ├── splitcrons.sh ├── string.go ├── string_test.go ├── structure.go ├── tags.go ├── test ├── compare.go └── time.go ├── threads.go ├── threads_test.go ├── time.go ├── time_test.go ├── ts_points.go ├── unicode.go ├── unicode_test.go ├── vet_files.sh └── yaml.go /.github/ISSUE_TEMPLATE/issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue 3 | about: Standard issue 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please add `[feature request]` to the issue title when requesting new features, dashboards, projects etc. 11 | Please add `[bug]` to the issue title when reporting a bug. 12 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | jsons/*.json 2 | jsons/*.yaml 3 | *.swp 4 | *.db 5 | /go.sum 6 | /skip_dates.yaml 7 | /j.json 8 | /g.db 9 | /tmp 10 | /db2influx 11 | /structure 12 | /gha2db 13 | /calc_metric 14 | /gha2db_sync 15 | /runq 16 | /import_affs 17 | /annotations 18 | /tags 19 | /columns 20 | /webhook 21 | /devstats 22 | /get_repos 23 | /merge_dbs 24 | /replacer 25 | /vars 26 | /ghapi2db 27 | /hide_data 28 | /sqlitedb 29 | /website_data 30 | /sync_issues 31 | /idb_backup 32 | /idb_tags 33 | /idb_tst 34 | /idb_vars 35 | /merge_pdbs 36 | /pdb_vars 37 | /z2influx 38 | /api 39 | /devstats_api_server 40 | /tsplit 41 | /splitcrons 42 | /FROM 43 | /TO 44 | errors.txt 45 | run.log 46 | out 47 | .DS_Store 48 | temp.sql 49 | drop.sql 50 | /TODO 51 | /MAIL 52 | /wip.sql 53 | /log 54 | /debug 55 | /node_modules/ 56 | sqlite/*.sql 57 | /*.csv 58 | /devstats.tar 59 | /csv/*.csv 60 | /report_6months.txt 61 | /report_alltime.txt 62 | *.secret 63 | /error.yaml 64 | /in.json 65 | /new-values.yaml 66 | /values.yaml 67 | !/contributors_and_emails.csv 68 | !/contributing_actors*.csv 69 | !/k8s_contributors_and_emails.csv 70 | !/top_50_k8s_yearly_contributors.csv 71 | !/k8s_yearly_contributors_with_50.csv 72 | 73 | # IDE Config 74 | .idea 75 | *.g 76 | env.env 77 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | go: 4 | - 1.15 5 | before_install: 6 | - go get -u golang.org/x/lint/golint 7 | - go get -u golang.org/x/tools/cmd/goimports 8 | - go get -u github.com/jgautheron/usedexports 9 | - go get -u github.com/kisielk/errcheck 10 | - go get -u github.com/rs/cors 11 | - go get -u github.com/json-iterator/go 12 | - go get -u github.com/google/go-github/github 13 | # - go get -u github.com/jgautheron/goconst/cmd/goconst 14 | # - go get -u github.com/lib/pq 15 | # - go get -u golang.org/x/oauth2 16 | # - go get -u golang.org/x/text/transform 17 | # - go get -u golang.org/x/text/unicode/norm 18 | # - go get -u gopkg.in/yaml.v2 19 | # - go get -u github.com/mattn/go-sqlite3 20 | - sudo -u postgres createdb gha 21 | - sudo -u postgres psql gha -c "create user gha_admin with password 'pwd'" 22 | - sudo -u postgres psql gha -c 'grant all privileges on database "gha" to gha_admin' 23 | - sudo -u postgres psql gha -c "alter user gha_admin createdb" 24 | - sudo -u postgres psql gha -c "create user ro_user with password 'pwd'" 25 | - sudo -u postgres psql gha -c 'grant all privileges on database "gha" to ro_user' 26 | - sudo -u postgres psql gha -c "create user devstats_team with password 'pwd'" 27 | - sudo -u postgres psql gha -c 'grant all privileges on database "gha" to devstats_team' 28 | script: 29 | - cd /home/travis/gopath/src/github.com/cncf/devstatscode 30 | - make 31 | - make test 32 | - GHA2DB_PROJECT=kubernetes GHA2DB_LOCAL=1 PG_PORT=5433 PG_PASS=pwd ./dbtest.sh 33 | addons: 34 | postgresql: "13" 35 | apt: 36 | packages: 37 | - postgresql-13 38 | - postgresql-client-13 39 | env: 40 | global: 41 | - PGPORT=5433 42 | notifications: 43 | webhooks: 44 | - https://teststats.cncf.io:2982/hook 45 | - https://devstats.cncf.io:2982/hook 46 | - https://devstats.cd.foundation:2982/hook 47 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at lukaszgryglicki@o2.pl. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to devstats 2 | If You see any error, or if you have suggestion please create [issue and/or PR](https://github.com/cncf/devstats). 3 | 4 | # Coding standards 5 | - Please follow coding standards required for Go language. 6 | - This is checked during the `make` step which calls the following static analysis/lint tools: `fmt lint imports vet const usedexports`. 7 | - When adding new functionality please add test coverage please that will be executed by `make test`. 8 | - If adding new database functionality and/or new metrics, please add new test covergage that will be executed by: 9 | - `GHA2DB_PROJECT=kubernetes IDB_HOST='...' PG_PASS='...' IDB_PASS='...' ./dbtest.sh`. 10 | - New metrics test coverage should be added in `metrics_test.go`. 11 | 12 | # Working locally 13 | Please see [Development](https://github.com/cncf/devstats/blob/master/DEVELOPMENT.md). 14 | 15 | # Testing 16 | Please see [Tests](https://github.com/cncf/devstats/blob/master/TESTING.md). 17 | 18 | # Vulnerabilities 19 | Please use GitHub [issues](https://github.com/cncf/devstats/issues) to report any vulnerability found. 20 | 21 | # Adding new project 22 | To add new project follow [adding new project](https://github.com/cncf/devstats/blob/master/ADDING_NEW_PROJECT.md) instructions. 23 | -------------------------------------------------------------------------------- /LOGS.md: -------------------------------------------------------------------------------- 1 | # Project Metrics API logs 2 | 3 | To check logs for http://cncf.io/project-metrics: 4 | - `` k exec -itn devstats-prod devstats-postgres-0 -- psql devstats ``. 5 | - Then: `` select * from gha_logs where prog = 'api' and lower(msg) like '%cached values%' order by dt desc limit 10; ``. 6 | - Then: `` select dt, run_dt, prog, msg from gha_logs where proj = 'all' and lower(msg) like '%contributors_and_orgs_count%' order by dt desc limit 10; ``. 7 | - Metric executions: `` select dt, run_dt, msg from gha_logs where proj = 'all' and prog = 'gha2db_sync' and msg like '%Contributors and organizations%' order by dt desc limit 10; ``. 8 | - What exactly got executed: `` select dt, run_dt, prog, msg from gha_logs where proj = 'all' and dt >= '2025-05-22 20:22:38.966613' and dt <= '2025-05-22 20:23:21.965162' order by dt; ``. 9 | - Last execution details: `` select dt, run_dt, prog, msg from gha_logs where proj = 'all' and dt >= (select dt from gha_logs where proj = 'all' and prog = 'gha2db_sync' and msg like '%Contributors and organizations% ...' order by dt desc limit 1) and dt <= (select dt from gha_logs where proj = 'all' and prog = 'gha2db_sync' and msg like '%Contributors and organizations% ... %' order by dt desc limit 1) order by dt; ``. 10 | - Get most recent metrics values (possibly from cache): `` curl -s -H 'Content-Type: application/json' https://devstats.cncf.io/api/v1 -d'{"api":"CumulativeCounts","payload":{"project":"all","metric":"contributors"}}' | jq -r '.' ``. 11 | 12 | 13 | # Columns additions/deletions 14 | 15 | To check which columns were added/deleted recently: 16 | - `` k exec -itn devstats-prod devstats-postgres-0 -- psql devstats ``. 17 | - Additions: `` select dt, run_dt, proj, msg from gha_logs where prog = 'columns' and msg like '%Added column%' order by dt desc limit 100; ``. 18 | - Deletions: `` select dt, run_dt, proj, msg from gha_logs where prog = 'columns' and msg like '%Need to delete columns%' order by dt desc limit 100; ``. 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO_LIB_FILES=pg_conn.go error.go mgetc.go map.go threads.go gha.go json.go time.go context.go exec.go structure.go log.go hash.go unicode.go const.go string.go annotations.go env.go ghapi.go io.go tags.go yaml.go ts_points.go convert.go signal.go 2 | GO_BIN_FILES=cmd/structure/structure.go cmd/runq/runq.go cmd/gha2db/gha2db.go cmd/calc_metric/calc_metric.go cmd/gha2db_sync/gha2db_sync.go cmd/import_affs/import_affs.go cmd/annotations/annotations.go cmd/tags/tags.go cmd/webhook/webhook.go cmd/devstats/devstats.go cmd/get_repos/get_repos.go cmd/merge_dbs/merge_dbs.go cmd/replacer/replacer.go cmd/vars/vars.go cmd/ghapi2db/ghapi2db.go cmd/columns/columns.go cmd/hide_data/hide_data.go cmd/sqlitedb/sqlitedb.go cmd/website_data/website_data.go cmd/sync_issues/sync_issues.go cmd/api/api.go cmd/tsplit/tsplit.go cmd/splitcrons/splitcrons.go 3 | GO_TEST_FILES=context_test.go gha_test.go map_test.go mgetc_test.go threads_test.go time_test.go unicode_test.go string_test.go regexp_test.go annotations_test.go env_test.go convert_test.go 4 | GO_DBTEST_FILES=pg_test.go series_test.go 5 | GO_LIBTEST_FILES=test/compare.go test/time.go 6 | GO_BIN_CMDS=github.com/cncf/devstatscode/cmd/structure github.com/cncf/devstatscode/cmd/runq github.com/cncf/devstatscode/cmd/gha2db github.com/cncf/devstatscode/cmd/calc_metric github.com/cncf/devstatscode/cmd/gha2db_sync github.com/cncf/devstatscode/cmd/import_affs github.com/cncf/devstatscode/cmd/annotations github.com/cncf/devstatscode/cmd/tags github.com/cncf/devstatscode/cmd/webhook github.com/cncf/devstatscode/cmd/devstats github.com/cncf/devstatscode/cmd/get_repos github.com/cncf/devstatscode/cmd/merge_dbs github.com/cncf/devstatscode/cmd/replacer github.com/cncf/devstatscode/cmd/vars github.com/cncf/devstatscode/cmd/ghapi2db github.com/cncf/devstatscode/cmd/columns github.com/cncf/devstatscode/cmd/hide_data github.com/cncf/devstatscode/cmd/sqlitedb github.com/cncf/devstatscode/cmd/website_data github.com/cncf/devstatscode/cmd/sync_issues github.com/cncf/devstatscode/cmd/api github.com/cncf/devstatscode/cmd/tsplit github.com/cncf/devstatscode/cmd/splitcrons 7 | BUILD_TIME=`date -u '+%Y-%m-%d_%I:%M:%S%p'` 8 | COMMIT=`git rev-parse HEAD` 9 | HOSTNAME=`uname -a | sed "s/ /_/g"` 10 | GO_VERSION=`go version | sed "s/ /_/g"` 11 | #for race CGO_ENABLED=1 12 | #GO_ENV=CGO_ENABLED=1 13 | GO_ENV=CGO_ENABLED=0 14 | # -ldflags '-s -w': create release binary - without debug info 15 | GO_BUILD=go build -ldflags "-s -w -X github.com/cncf/devstatscode.BuildStamp=$(BUILD_TIME) -X github.com/cncf/devstatscode.GitHash=$(COMMIT) -X github.com/cncf/devstatscode.HostName=$(HOSTNAME) -X github.com/cncf/devstatscode.GoVersion=$(GO_VERSION)" 16 | #GO_BUILD=go build -ldflags "-s -w -X github.com/cncf/devstatscode.BuildStamp=$(BUILD_TIME) -X github.com/cncf/devstatscode.GitHash=$(COMMIT) -X github.com/cncf/devstatscode.HostName=$(HOSTNAME) -X github.com/cncf/devstatscode.GoVersion=$(GO_VERSION)" -race 17 | # -ldflags '-s': instal stripped binary 18 | #GO_INSTALL=go install 19 | #For static gcc linking 20 | GCC_STATIC= 21 | #GCC_STATIC=-ldflags '-extldflags "-static"' 22 | GO_INSTALL=go install -ldflags "-s -w -X github.com/cncf/devstatscode.BuildStamp=$(BUILD_TIME) -X github.com/cncf/devstatscode.GitHash=$(COMMIT) -X github.com/cncf/devstatscode.HostName=$(HOSTNAME) -X github.com/cncf/devstatscode.GoVersion=$(GO_VERSION)" 23 | GO_FMT=gofmt -s -w 24 | GO_LINT=golint -set_exit_status 25 | GO_VET=go vet 26 | GO_IMPORTS=goimports -w 27 | GO_USEDEXPORTS=usedexports -ignore 'sqlitedb.go|vendor' 28 | GO_ERRCHECK=errcheck -asserts -ignore '[FS]?[Pp]rint*' -ignoretests 29 | GO_TEST=go test 30 | BINARIES=structure gha2db calc_metric gha2db_sync import_affs annotations tags webhook devstats get_repos merge_dbs replacer vars ghapi2db columns hide_data website_data sync_issues runq api sqlitedb tsplit splitcrons 31 | CRON_SCRIPTS=cron/cron_db_backup.sh cron/sysctl_config.sh cron/backup_artificial.sh 32 | UTIL_SCRIPTS=devel/wait_for_command.sh devel/cronctl.sh devel/sync_lock.sh devel/sync_unlock.sh devel/db.sh 33 | GIT_SCRIPTS=git/git_reset_pull.sh git/git_files.sh git/git_tags.sh git/last_tag.sh git/git_loc.sh 34 | STRIP=strip 35 | 36 | all: check ${BINARIES} 37 | 38 | structure: cmd/structure/structure.go ${GO_LIB_FILES} 39 | ${GO_ENV} ${GO_BUILD} -o structure cmd/structure/structure.go 40 | 41 | runq: cmd/runq/runq.go ${GO_LIB_FILES} 42 | ${GO_ENV} ${GO_BUILD} -o runq cmd/runq/runq.go 43 | 44 | api: cmd/api/api.go ${GO_LIB_FILES} 45 | ${GO_ENV} ${GO_BUILD} -o api cmd/api/api.go 46 | 47 | # go build -o gha2db.g cmd/gha2db/gha2db.go 48 | gha2db: cmd/gha2db/gha2db.go ${GO_LIB_FILES} 49 | ${GO_ENV} ${GO_BUILD} -o gha2db cmd/gha2db/gha2db.go 50 | 51 | calc_metric: cmd/calc_metric/calc_metric.go ${GO_LIB_FILES} 52 | ${GO_ENV} ${GO_BUILD} -o calc_metric cmd/calc_metric/calc_metric.go 53 | 54 | import_affs: cmd/import_affs/import_affs.go ${GO_LIB_FILES} 55 | ${GO_ENV} ${GO_BUILD} -o import_affs cmd/import_affs/import_affs.go 56 | 57 | gha2db_sync: cmd/gha2db_sync/gha2db_sync.go ${GO_LIB_FILES} 58 | ${GO_ENV} ${GO_BUILD} -o gha2db_sync cmd/gha2db_sync/gha2db_sync.go 59 | 60 | devstats: cmd/devstats/devstats.go ${GO_LIB_FILES} 61 | ${GO_ENV} ${GO_BUILD} -o devstats cmd/devstats/devstats.go 62 | 63 | annotations: cmd/annotations/annotations.go ${GO_LIB_FILES} 64 | ${GO_ENV} ${GO_BUILD} -o annotations cmd/annotations/annotations.go 65 | 66 | tags: cmd/tags/tags.go ${GO_LIB_FILES} 67 | ${GO_ENV} ${GO_BUILD} -o tags cmd/tags/tags.go 68 | 69 | columns: cmd/columns/columns.go ${GO_LIB_FILES} 70 | ${GO_ENV} ${GO_BUILD} -o columns cmd/columns/columns.go 71 | 72 | webhook: cmd/webhook/webhook.go ${GO_LIB_FILES} 73 | ${GO_ENV} ${GO_BUILD} -o webhook cmd/webhook/webhook.go 74 | 75 | get_repos: cmd/get_repos/get_repos.go ${GO_LIB_FILES} 76 | ${GO_ENV} ${GO_BUILD} -o get_repos cmd/get_repos/get_repos.go 77 | 78 | merge_dbs: cmd/merge_dbs/merge_dbs.go ${GO_LIB_FILES} 79 | ${GO_ENV} ${GO_BUILD} -o merge_dbs cmd/merge_dbs/merge_dbs.go 80 | 81 | vars: cmd/vars/vars.go ${GO_LIB_FILES} 82 | ${GO_ENV} ${GO_BUILD} -o vars cmd/vars/vars.go 83 | 84 | ghapi2db: cmd/ghapi2db/ghapi2db.go ${GO_LIB_FILES} 85 | ${GO_ENV} ${GO_BUILD} -o ghapi2db cmd/ghapi2db/ghapi2db.go 86 | 87 | sync_issues: cmd/sync_issues/sync_issues.go ${GO_LIB_FILES} 88 | ${GO_ENV} ${GO_BUILD} -o sync_issues cmd/sync_issues/sync_issues.go 89 | 90 | replacer: cmd/replacer/replacer.go ${GO_LIB_FILES} 91 | ${GO_ENV} ${GO_BUILD} -o replacer cmd/replacer/replacer.go 92 | 93 | hide_data: cmd/hide_data/hide_data.go ${GO_LIB_FILES} 94 | ${GO_ENV} ${GO_BUILD} -o hide_data cmd/hide_data/hide_data.go 95 | 96 | website_data: cmd/website_data/website_data.go ${GO_LIB_FILES} 97 | ${GO_ENV} ${GO_BUILD} -o website_data cmd/website_data/website_data.go 98 | 99 | sqlitedb: cmd/sqlitedb/sqlitedb.go ${GO_LIB_FILES} 100 | ${GO_BUILD} ${GCC_STATIC} -o sqlitedb cmd/sqlitedb/sqlitedb.go 101 | 102 | tsplit: cmd/tsplit/tsplit.go ${GO_LIB_FILES} 103 | ${GO_ENV} ${GO_BUILD} -o tsplit cmd/tsplit/tsplit.go 104 | 105 | splitcrons: cmd/splitcrons/splitcrons.go ${GO_LIB_FILES} 106 | ${GO_ENV} ${GO_BUILD} -o splitcrons cmd/splitcrons/splitcrons.go 107 | 108 | fmt: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 109 | ./for_each_go_file.sh "${GO_FMT}" 110 | 111 | lint: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 112 | ./for_each_go_file.sh "${GO_LINT}" 113 | 114 | vet: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 115 | ./vet_files.sh "${GO_VET}" 116 | 117 | imports: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 118 | ./for_each_go_file.sh "${GO_IMPORTS}" 119 | 120 | usedexports: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 121 | ${GO_USEDEXPORTS} ./... 122 | 123 | errcheck: ${GO_BIN_FILES} ${GO_LIB_FILES} ${GO_TEST_FILES} ${GO_DBTEST_FILES} ${GO_LIBTEST_FILES} 124 | ${GO_ERRCHECK} $(go list ./... | grep -v /vendor/) 125 | 126 | test: 127 | ${GO_TEST} ${GO_TEST_FILES} 128 | 129 | dbtest: 130 | ${GO_TEST} ${GO_DBTEST_FILES} 131 | 132 | # check: fmt lint imports vet usedexports errcheck 133 | check: fmt lint imports vet usedexports 134 | 135 | install: ${BINARIES} 136 | cp -v ${UTIL_SCRIPTS} ${GOPATH}/bin 137 | [ ! -f /tmp/deploy.wip ] || exit 1 138 | ifneq (${NOWAIT},1) 139 | wait_for_command.sh 'devstats,devstats_others,devstats_kubernetes,devstats_allprj' 900 || exit 2 140 | endif 141 | ${GO_INSTALL} ${GO_BIN_CMDS} 142 | cp -v ${CRON_SCRIPTS} ${GOPATH}/bin 143 | cp -v ${GIT_SCRIPTS} ${GOPATH}/bin 144 | 145 | strip: ${BINARIES} 146 | ${STRIP} ${BINARIES} 147 | 148 | clean: 149 | rm -f ${BINARIES} 150 | 151 | .PHONY: test 152 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/cncf/devstatscode.svg?branch=master)](https://travis-ci.org/cncf/devstatscode) 2 | [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1357/badge)](https://bestpractices.coreinfrastructure.org/projects/1357) 3 | 4 | # DevStats code 5 | 6 | This is a code repository for [DevStats](https://github.com/cncf/devstats) used to display [CNCF projects dashboards](https://devstats.cncf.io), [CDF projects dashboards](https://devstats.cd.foundation), [GraphQL projects dashboards](https://devstats.graphql.org) and [example Kubernetes/helm deployment](https://cncf.devstats-demo.net). 7 | 8 | Authors: Łukasz Gryglicki , Justyna Gryglicka , Josh Berkus . 9 | 10 | # Building and installing 11 | 12 | - Follow [this guide](https://github.com/cncf/devstats-helm-example/blob/master/README.md) to see how to deploy on Kubernetes using Helm. 13 | - Follow [this guide](https://github.com/cncf/devstats-helm-graphql/blob/master/README.md) to see GraphQL deployment using Kubernetes & Helm. 14 | - Follow [this guide](https://github.com/cncf/devstats/blob/master/INSTALL_UBUNTU18.md#devstats-installation-on-ubuntu) for installing on bare metal. 15 | - Follow [this guide](https://github.com/cncf/devstats-example/blob/master/README.md) to deploy your own project on bare metal (this example deployes Homebrew statistics). 16 | - Fetch dependency libraries. 17 | - `make` then `make test` finally `make install`. 18 | 19 | # Adding new projects 20 | 21 | See `cncf/devstats-helm`:`ADDING_NEW_PROJECTS.md` for informations about how to add more projects on Kubernetes/Helm deployment. 22 | See `cncf/devstats`:`ADDING_NEW_PROJECT.md` for informations about how to add more projects on bare metal deployment. 23 | 24 | # API 25 | 26 | API documentation is available [here](https://github.com/cncf/devstatscode/blob/master/API.md). 27 | 28 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please send security vulnerability reports to lgryglicki@cncf.io 6 | -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 1. To execute tests just run `make test`, do not set any environment variables for them, one of tests is to check default environment. 3 | 2. To check all sources using multiple go tools (like fmt, lint, imports, vet, goconst, usedexports), run `make check`. 4 | 3. To check Travis CI payloads use `PG_PASS=pwd GET=1 ./devel/webhook.sh` and then `./devel/test_webhook.sh`. 5 | 4. To check annotations run: `PG_PASS=pwd PG_DB=dbtest GHA2DB_PROJECT=kubernetes GHA2DB_LOCAL=1 go test series_test.go -run TestProcessAnnotations`. 6 | 5. Continuous deployment instructions are [here](https://github.com/cncf/devstats/blob/master/CONTINUOUS_DEPLOYMENT.md). 7 | 6. To testDB/metrics: `PG_DB=dbtest PG_PASS=... make dbtest`. 8 | -------------------------------------------------------------------------------- /annotations_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | 8 | lib "github.com/cncf/devstatscode" 9 | testlib "github.com/cncf/devstatscode/test" 10 | ) 11 | 12 | func TestGetFakeAnnotations(t *testing.T) { 13 | // Example data 14 | ft := testlib.YMDHMS 15 | startDate := []time.Time{ft(2014), ft(2015), ft(2015), ft(2012)} 16 | joinDate := []time.Time{ft(2015), ft(2015), ft(2014), ft(2013)} 17 | 18 | // Test cases 19 | var testCases = []struct { 20 | startDate time.Time 21 | joinDate time.Time 22 | expectedAnnotations lib.Annotations 23 | }{ 24 | { 25 | startDate: startDate[0], 26 | joinDate: joinDate[0], 27 | expectedAnnotations: lib.Annotations{ 28 | Annotations: []lib.Annotation{ 29 | { 30 | Name: "Project start", 31 | Description: lib.ToYMDDate(startDate[0]) + " - project starts", 32 | Date: startDate[0], 33 | }, 34 | { 35 | Name: "First CNCF project join date", 36 | Description: lib.ToYMDDate(joinDate[0]), 37 | Date: joinDate[0], 38 | }, 39 | }, 40 | }, 41 | }, 42 | { 43 | startDate: startDate[1], 44 | joinDate: joinDate[1], 45 | expectedAnnotations: lib.Annotations{ 46 | Annotations: []lib.Annotation{}, 47 | }, 48 | }, 49 | { 50 | startDate: startDate[2], 51 | joinDate: joinDate[2], 52 | expectedAnnotations: lib.Annotations{ 53 | Annotations: []lib.Annotation{}, 54 | }, 55 | }, 56 | { 57 | startDate: startDate[3], 58 | joinDate: joinDate[3], 59 | expectedAnnotations: lib.Annotations{ 60 | Annotations: []lib.Annotation{}, 61 | }, 62 | }, 63 | { 64 | startDate: startDate[0], 65 | joinDate: joinDate[3], 66 | expectedAnnotations: lib.Annotations{ 67 | Annotations: []lib.Annotation{}, 68 | }, 69 | }, 70 | { 71 | startDate: startDate[3], 72 | joinDate: joinDate[0], 73 | expectedAnnotations: lib.Annotations{ 74 | Annotations: []lib.Annotation{}, 75 | }, 76 | }, 77 | } 78 | // Execute test cases 79 | for index, test := range testCases { 80 | expected := test.expectedAnnotations 81 | got := lib.GetFakeAnnotations(test.startDate, test.joinDate) 82 | if (len(expected.Annotations) > 0 || len(got.Annotations) > 0) && !reflect.DeepEqual(expected.Annotations, got.Annotations) { 83 | t.Errorf( 84 | "test number %d, expected:\n%+v\n%+v\n got, start date: %s, join date: %s", 85 | index+1, 86 | expected, 87 | got, 88 | lib.ToYMDDate(test.startDate), 89 | lib.ToYMDDate(test.joinDate), 90 | ) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /calc_metric.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # on k8s node-0: 'k exec -n devstats-prod devstats-postgres-0 -- pg_dump tuf > tuf.dump' 3 | # sftp root@node-0: 'mget tuf.dump' 4 | # here: 'PGPASSWORD=[redacted] createdb -Ugha_admin -hlocalhost -p5432 tuf' 5 | # here: 'PGPASSWORD=[redacted] psql -Ugha_admin -hlocalhost -p5432 tuf < tuf.dump' 6 | if [ -z "${PG_PASS}" ] 7 | then 8 | echo "$0: you must specify PG_PASS=..." 9 | exit 1 10 | fi 11 | make fmt && make calc_metric || exit 2 12 | cd ../devstats || exit 3 13 | # GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 GHA2DB_ST=1 GHA2DB_NCPUS=1 ../devstatscode/calc_metric events_hll ./events_hll.sql '2024-01-01 0' '2024-10-01 0' w 'skip_past,hll' 14 | # GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-01-01 0' q 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr' > ../devstatscode/out @>&1 15 | # GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 GHA2DB_ST=1 GHA2DB_NCPUS=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-10-01 0' q 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr' 16 | # GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-10-01 0' m 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr' 17 | # GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries_commiters.sql '2014-01-01 0' '2024-10-01 0' m 'hll,merge_series:prjcntr,drop:sprjcntr,skip_escape_series_name' 18 | GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries_commiters.sql '2014-01-01 0' '2024-10-01 0' m 'multivalue,hll,skip_escape_series_name,merge_series:prjcntr,drop:sprjcntr' 19 | -------------------------------------------------------------------------------- /cmd/annotations/annotations.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | yaml "gopkg.in/yaml.v2" 8 | ) 9 | 10 | // makeAnnotations: Insert TSDB annotations starting after `dt` 11 | func makeAnnotations() { 12 | // Environment context parse 13 | var ctx lib.Ctx 14 | ctx.Init() 15 | lib.SetupTimeoutSignal(&ctx) 16 | 17 | // Needs GHA2DB_PROJECT variable set 18 | if ctx.Project == "" { 19 | lib.Fatalf("you have to set project via GHA2DB_PROJECT environment variable") 20 | } 21 | 22 | // Local or cron mode? 23 | dataPrefix := ctx.DataDir 24 | if ctx.Local { 25 | dataPrefix = "./" 26 | } 27 | 28 | // Read defined projects 29 | data, err := lib.ReadFile(&ctx, dataPrefix+ctx.ProjectsYaml) 30 | lib.FatalOnError(err) 31 | var projects lib.AllProjects 32 | lib.FatalOnError(yaml.Unmarshal(data, &projects)) 33 | 34 | // Get current project's main repo and annotation regexp 35 | proj, ok := projects.Projects[ctx.Project] 36 | if !ok { 37 | lib.Fatalf("project '%s' not found in '%s'", ctx.Project, ctx.ProjectsYaml) 38 | } 39 | ctx.SharedDB = proj.SharedDB 40 | ctx.ProjectMainRepo = proj.MainRepo 41 | 42 | // Get annotations using GitHub API and add annotations and quick ranges to TSDB 43 | if proj.MainRepo != "" { 44 | annotations := lib.GetAnnotations(&ctx, proj.MainRepo, proj.AnnotationRegexp) 45 | lib.ProcessAnnotations(&ctx, &annotations, []*time.Time{proj.StartDate, proj.JoinDate, proj.IncubatingDate, proj.GraduatedDate, proj.ArchivedDate}) 46 | } else if proj.StartDate != nil { 47 | var annotations lib.Annotations 48 | if proj.JoinDate != nil { 49 | annotations = lib.GetFakeAnnotations(*proj.StartDate, *proj.JoinDate) 50 | } else { 51 | annotations.Annotations = append( 52 | annotations.Annotations, 53 | lib.Annotation{ 54 | Name: "Project start", 55 | Description: lib.ToYMDDate(*proj.StartDate) + " - project starts", 56 | Date: *proj.StartDate, 57 | }, 58 | ) 59 | } 60 | lib.ProcessAnnotations(&ctx, &annotations, []*time.Time{nil, nil, proj.IncubatingDate, proj.GraduatedDate, proj.ArchivedDate}) 61 | } 62 | } 63 | 64 | func main() { 65 | dtStart := time.Now() 66 | makeAnnotations() 67 | dtEnd := time.Now() 68 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 69 | } 70 | -------------------------------------------------------------------------------- /cmd/hide_data/hide_data.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/csv" 6 | "encoding/hex" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | lib "github.com/cncf/devstatscode" 15 | yaml "gopkg.in/yaml.v2" 16 | ) 17 | 18 | type replaceConfig struct { 19 | table string 20 | column string 21 | } 22 | 23 | func processHidden(ctx *lib.Ctx) { 24 | replaces := []replaceConfig{ 25 | { 26 | table: "gha_actors", 27 | column: "login", 28 | }, 29 | { 30 | table: "gha_actors", 31 | column: "name", 32 | }, 33 | { 34 | table: "gha_actors_emails", 35 | column: "email", 36 | }, 37 | { 38 | table: "gha_actors_names", 39 | column: "name", 40 | }, 41 | { 42 | table: "gha_actors_affiliations", 43 | column: "company_name", 44 | }, 45 | { 46 | table: "gha_actors_affiliations", 47 | column: "original_company_name", 48 | }, 49 | { 50 | table: "gha_companies", 51 | column: "name", 52 | }, 53 | { 54 | table: "gha_events", 55 | column: "dup_actor_login", 56 | }, 57 | { 58 | table: "gha_payloads", 59 | column: "dup_actor_login", 60 | }, 61 | { 62 | table: "gha_commits", 63 | column: "dup_actor_login", 64 | }, 65 | { 66 | table: "gha_commits", 67 | column: "dup_author_login", 68 | }, 69 | { 70 | table: "gha_commits", 71 | column: "dup_committer_login", 72 | }, 73 | { 74 | table: "gha_commits", 75 | column: "author_name", 76 | }, 77 | { 78 | table: "gha_commits", 79 | column: "author_email", 80 | }, 81 | { 82 | table: "gha_commits", 83 | column: "committer_name", 84 | }, 85 | { 86 | table: "gha_commits", 87 | column: "committer_email", 88 | }, 89 | { 90 | table: "gha_commits_roles", 91 | column: "actor_login", 92 | }, 93 | { 94 | table: "gha_commits_roles", 95 | column: "actor_name", 96 | }, 97 | { 98 | table: "gha_commits_roles", 99 | column: "actor_email", 100 | }, 101 | { 102 | table: "gha_pages", 103 | column: "dup_actor_login", 104 | }, 105 | { 106 | table: "gha_comments", 107 | column: "dup_actor_login", 108 | }, 109 | { 110 | table: "gha_comments", 111 | column: "dup_user_login", 112 | }, 113 | { 114 | table: "gha_reviews", 115 | column: "dup_actor_login", 116 | }, 117 | { 118 | table: "gha_reviews", 119 | column: "dup_user_login", 120 | }, 121 | { 122 | table: "gha_issues", 123 | column: "dup_actor_login", 124 | }, 125 | { 126 | table: "gha_issues", 127 | column: "dup_actor_login", 128 | }, 129 | { 130 | table: "gha_issues", 131 | column: "dup_user_login", 132 | }, 133 | { 134 | table: "gha_milestones", 135 | column: "dup_actor_login", 136 | }, 137 | { 138 | table: "gha_milestones", 139 | column: "dupn_creator_login", 140 | }, 141 | { 142 | table: "gha_issues_labels", 143 | column: "dup_actor_login", 144 | }, 145 | { 146 | table: "gha_forkees", 147 | column: "dup_actor_login", 148 | }, 149 | { 150 | table: "gha_forkees", 151 | column: "dup_owner_login", 152 | }, 153 | { 154 | table: "gha_releases", 155 | column: "dup_actor_login", 156 | }, 157 | { 158 | table: "gha_releases", 159 | column: "dup_author_login", 160 | }, 161 | { 162 | table: "gha_assets", 163 | column: "dup_actor_login", 164 | }, 165 | { 166 | table: "gha_assets", 167 | column: "dup_uploader_login", 168 | }, 169 | { 170 | table: "gha_pull_requests", 171 | column: "dup_actor_login", 172 | }, 173 | { 174 | table: "gha_pull_requests", 175 | column: "dup_user_login", 176 | }, 177 | { 178 | table: "gha_branches", 179 | column: "dupn_forkee_name", 180 | }, 181 | { 182 | table: "gha_branches", 183 | column: "dupn_user_login", 184 | }, 185 | { 186 | table: "gha_teams", 187 | column: "dup_actor_login", 188 | }, 189 | { 190 | table: "gha_texts", 191 | column: "actor_login", 192 | }, 193 | { 194 | table: "gha_issues_events_labels", 195 | column: "actor_login", 196 | }, 197 | } 198 | configFile := lib.HideCfgFile 199 | shaMap := lib.GetHidden(ctx, configFile) 200 | 201 | dataPrefix := ctx.DataDir 202 | if ctx.Local { 203 | dataPrefix = "./" 204 | } 205 | 206 | data, err := ioutil.ReadFile(dataPrefix + ctx.ProjectsYaml) 207 | lib.FatalOnError(err) 208 | 209 | var projects lib.AllProjects 210 | lib.FatalOnError(yaml.Unmarshal(data, &projects)) 211 | 212 | orders := []int{} 213 | projectsMap := make(map[int]string) 214 | for name, proj := range projects.Projects { 215 | if lib.IsProjectDisabled(ctx, name, proj.Disabled) { 216 | continue 217 | } 218 | orders = append(orders, proj.Order) 219 | projectsMap[proj.Order] = name 220 | } 221 | sort.Ints(orders) 222 | 223 | only := make(map[string]struct{}) 224 | onlyS := os.Getenv("ONLY") 225 | bOnly := false 226 | if onlyS != "" { 227 | onlyA := strings.Split(onlyS, " ") 228 | for _, item := range onlyA { 229 | if item == "" { 230 | continue 231 | } 232 | only[item] = struct{}{} 233 | } 234 | bOnly = true 235 | } 236 | 237 | tasks := [][3]string{} 238 | dbs := []string{} 239 | for _, order := range orders { 240 | name := projectsMap[order] 241 | if bOnly { 242 | _, ok := only[name] 243 | if !ok { 244 | continue 245 | } 246 | } 247 | proj := projects.Projects[name] 248 | for sha, anon := range shaMap { 249 | tasks = append(tasks, [3]string{proj.PDB, sha, anon}) 250 | } 251 | dbs = append(dbs, proj.PDB) 252 | } 253 | lib.Printf("Processing databases: %+v\n", dbs) 254 | thrN := lib.GetThreadsNum(ctx) 255 | ch := make(chan bool) 256 | nThreads := 0 257 | for _, task := range tasks { 258 | go func(ch chan bool, task [3]string) { 259 | con := lib.PgConnDB(ctx, task[0]) 260 | defer func() { lib.FatalOnError(con.Close()) }() 261 | for _, replace := range replaces { 262 | res := lib.ExecSQLWithErr( 263 | con, 264 | ctx, 265 | fmt.Sprintf( 266 | "update %s set %s = %s where encode(digest(%s, 'sha1'), 'hex') = %s", 267 | replace.table, 268 | replace.column, 269 | lib.NValue(1), 270 | replace.column, 271 | lib.NValue(2), 272 | ), 273 | lib.AnyArray{ 274 | task[2], 275 | task[1], 276 | }..., 277 | ) 278 | rows, err := res.RowsAffected() 279 | lib.FatalOnError(err) 280 | if rows > 0 { 281 | lib.Printf("DB: %s, table: %s, column: %s, sha: %s, updated %d rows\n", task[0], replace.table, replace.column, task[1], rows) 282 | } 283 | } 284 | ch <- true 285 | }(ch, task) 286 | nThreads++ 287 | if nThreads >= thrN { 288 | <-ch 289 | nThreads-- 290 | } 291 | } 292 | for nThreads > 0 { 293 | <-ch 294 | nThreads-- 295 | } 296 | } 297 | 298 | func hideData(ctx *lib.Ctx, args []string) { 299 | shaMap := lib.GetHidden(ctx, lib.HideCfgFile) 300 | added := false 301 | for _, argo := range args { 302 | arg := strings.TrimSpace(argo) 303 | hash := sha1.New() 304 | hash.Write([]byte(arg)) 305 | sha := hex.EncodeToString(hash.Sum(nil)) 306 | _, ok := shaMap[sha] 307 | if ok { 308 | lib.Printf("Skipping '%s', SHA1 '%s' - already added\n", arg, sha) 309 | continue 310 | } 311 | shaMap[sha] = "" 312 | added = true 313 | } 314 | if !added { 315 | return 316 | } 317 | var writer *csv.Writer 318 | oFile, err := os.Create(lib.HideCfgFile) 319 | lib.FatalOnError(err) 320 | defer func() { _ = oFile.Close() }() 321 | writer = csv.NewWriter(oFile) 322 | defer writer.Flush() 323 | err = writer.Write([]string{"sha1"}) 324 | lib.FatalOnError(err) 325 | for sha := range shaMap { 326 | err = writer.Write([]string{sha}) 327 | lib.FatalOnError(err) 328 | } 329 | } 330 | 331 | func main() { 332 | var ctx lib.Ctx 333 | dtStart := time.Now() 334 | ctx.Init() 335 | lib.SetupTimeoutSignal(&ctx) 336 | if len(os.Args) < 2 { 337 | processHidden(&ctx) 338 | } else { 339 | hideData(&ctx, os.Args[1:]) 340 | } 341 | dtEnd := time.Now() 342 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 343 | } 344 | -------------------------------------------------------------------------------- /cmd/merge_dbs/merge_dbs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "reflect" 7 | "time" 8 | 9 | lib "github.com/cncf/devstatscode" 10 | "github.com/lib/pq" 11 | ) 12 | 13 | func mergePDBs() { 14 | // Environment context parse 15 | var ctx lib.Ctx 16 | ctx.Init() 17 | lib.SetupTimeoutSignal(&ctx) 18 | 19 | if len(ctx.InputDBs) < 1 { 20 | lib.Fatalf("required at least 1 input database, got %d: %+v", len(ctx.InputDBs), ctx.InputDBs) 21 | return 22 | } 23 | if ctx.OutputDB == "" { 24 | lib.Fatalf("output database required") 25 | return 26 | } 27 | 28 | // Connect to input Postgres DBs 29 | ci := []*sql.DB{} 30 | iNames := []string{} 31 | for _, iName := range ctx.InputDBs { 32 | c := lib.PgConnDB(&ctx, iName) 33 | ci = append(ci, c) 34 | iNames = append(iNames, iName) 35 | } 36 | 37 | // Defer closing all input connections 38 | defer func() { 39 | for _, c := range ci { 40 | lib.FatalOnError(c.Close()) 41 | } 42 | }() 43 | 44 | // Connect to the output Postgres DB 45 | co := lib.PgConnDB(&ctx, ctx.OutputDB) 46 | // Defer close output connection 47 | defer func() { lib.FatalOnError(co.Close()) }() 48 | 49 | // process this tables 50 | // 1st pass uses 1st condition 51 | // 2nd pass uses 2nd condition 52 | // "-" means that this pass is skipped 53 | // Some tables are commented out because we're going to 54 | // run other tools on merged database to fill them 55 | tableData := [][]string{ 56 | {"gha_actors", "id > 0", "id <= 0"}, 57 | //{"gha_actors_affiliations", "", "-"}, 58 | //{"gha_actors_emails", "", "-"}, 59 | //{"gha_actors_names", "", "-"}, 60 | {"gha_assets", "", "-"}, 61 | {"gha_branches", "", "-"}, 62 | {"gha_comments", "", "-"}, 63 | {"gha_reviews", "", "-"}, 64 | {"gha_commits", "", "-"}, 65 | {"gha_commits_files", "", "-"}, 66 | {"gha_commits_roles", "", "-"}, 67 | //{"gha_companies", "", "-"}, 68 | //{"gha_computed", "", "-"}, 69 | {"gha_events", "id > 0", "id <= 0"}, 70 | //{"gha_events_commits_files", "", "-"}, 71 | {"gha_forkees", "", "-"}, 72 | {"gha_issues", "id > 0", "id <= 0"}, 73 | {"gha_issues_assignees", "", "-"}, 74 | {"gha_issues_events_labels", "", "-"}, 75 | {"gha_issues_labels", "", "-"}, 76 | {"gha_issues_pull_requests", "", "-"}, 77 | {"gha_labels", "id > 0", "id <= 0"}, 78 | //{"gha_logs", "", "-"}, 79 | {"gha_milestones", "", "-"}, 80 | {"gha_orgs", "", "-"}, 81 | {"gha_pages", "", "-"}, 82 | {"gha_payloads", "event_id > 0", "event_id <= 0"}, 83 | //{"gha_postprocess_scripts", "", "-"}, 84 | {"gha_pull_requests", "", "-"}, 85 | {"gha_pull_requests_assignees", "", "-"}, 86 | {"gha_pull_requests_requested_reviewers", "", "-"}, 87 | {"gha_releases", "", "-"}, 88 | {"gha_releases_assets", "", "-"}, 89 | {"gha_repos", "", "-"}, 90 | {"gha_repo_groups", "", "-"}, 91 | {"gha_repos_langs", "", "-"}, 92 | {"gha_skip_commits", "", "-"}, 93 | {"gha_teams", "", "-"}, 94 | {"gha_teams_repositories", "", "-"}, 95 | {"gha_texts", "", "-"}, 96 | // {"gha_parsed", "", "-"}, 97 | // {"gha_last_computed", "", "-"}, 98 | } 99 | 100 | for pass, passInfo := range []string{"1st pass", "2nd pass"} { 101 | for i, data := range tableData { 102 | table := data[0] 103 | cond := data[pass+1] 104 | if cond == "-" { 105 | continue 106 | } 107 | allRows := 0 108 | allErrs := 0 109 | allIns := 0 110 | for dbi, c := range ci { 111 | // First get row count 112 | rc := 0 113 | queryRoot := "from " + table 114 | if cond != "" { 115 | queryRoot += " where " + cond 116 | } 117 | row := c.QueryRow("select count(*) " + queryRoot) 118 | lib.FatalOnError(row.Scan(&rc)) 119 | 120 | // Now get all data 121 | lib.Printf( 122 | "%s: start table: #%d: %s, DB #%d: %s, rows: %d...\n", 123 | passInfo, i, table, dbi, iNames[dbi], rc, 124 | ) 125 | rows := lib.QuerySQLWithErr( 126 | c, 127 | &ctx, 128 | "select * "+queryRoot, 129 | ) 130 | //defer func() { lib.FatalOnError(rows.Close()) }() 131 | // Now unknown rows, with unknown types 132 | columns, err := rows.Columns() 133 | lib.FatalOnError(err) 134 | 135 | // Vals to hold any type as []interface{} 136 | nColumns := len(columns) 137 | vals := make([]interface{}, nColumns) 138 | cols := "(" 139 | for i, col := range columns { 140 | vals[i] = new(interface{}) 141 | cols += "\"" + col + "\", " 142 | } 143 | cols = cols[:len(cols)-2] + ")" 144 | 145 | // Get results into `results` array of maps 146 | rowCount := 0 147 | errCount := 0 148 | insCount := 0 149 | // For ProgressInfo() 150 | dtStart := time.Now() 151 | lastTime := dtStart 152 | for rows.Next() { 153 | lib.FatalOnError(rows.Scan(vals...)) 154 | _, err := lib.ExecSQL( 155 | co, 156 | &ctx, 157 | "insert into "+table+cols+" "+lib.NValues(nColumns), 158 | vals..., 159 | ) 160 | if err != nil { 161 | switch e := err.(type) { 162 | case *pq.Error: 163 | if e.Code.Name() != "unique_violation" { 164 | // Problem here usually means different columns order because it uses unordered inserts like 165 | // insert into table_name ($1, $2, $3) 166 | lib.Printf("Failing values:\n") 167 | for vi, vv := range vals { 168 | lib.Printf("%d: %+v\n", vi, reflect.ValueOf(vv).Elem()) 169 | } 170 | lib.FatalOnError(err) 171 | } 172 | default: 173 | lib.FatalOnError(err) 174 | } 175 | errCount++ 176 | } else { 177 | insCount++ 178 | } 179 | rowCount++ 180 | lib.ProgressInfo( 181 | rowCount, rc, dtStart, &lastTime, time.Duration(10)*time.Second, 182 | fmt.Sprintf("%s: table #%d %s, DB #%d %s", passInfo, i, table, dbi, iNames[dbi]), 183 | ) 184 | } 185 | lib.FatalOnError(rows.Err()) 186 | lib.FatalOnError(rows.Close()) 187 | perc := 0.0 188 | if rowCount > 0 { 189 | perc = float64(errCount) * 100.0 / (float64(rowCount)) 190 | } 191 | lib.Printf( 192 | "%s: done table: #%d: %s, DB #%d: %s, rows: %d, inserted: %d, collisions: %d (%.3f%%)\n", 193 | passInfo, i, table, dbi, iNames[dbi], rowCount, insCount, errCount, perc, 194 | ) 195 | allRows += rowCount 196 | allErrs += errCount 197 | allIns += insCount 198 | } 199 | perc := 0.0 200 | if allRows > 0 { 201 | perc = float64(allErrs) * 100.0 / (float64(allRows)) 202 | } 203 | lib.Printf( 204 | "%s: done table: #%d: %s, all rows: %d, inserted: %d, collisions: %d (%.3f%%)\n", 205 | passInfo, i, table, allRows, allIns, allErrs, perc, 206 | ) 207 | } 208 | } 209 | } 210 | 211 | func main() { 212 | dtStart := time.Now() 213 | mergePDBs() 214 | dtEnd := time.Now() 215 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 216 | fmt.Printf("Consider running './devel/remove_db_dups.sh' if you merged into existing database.\n") 217 | } 218 | -------------------------------------------------------------------------------- /cmd/replacer/replacer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | 11 | lib "github.com/cncf/devstatscode" 12 | ) 13 | 14 | // replacer - replace regexp or string with regesp or string 15 | // possible modes: 16 | // rr, rr0: regexp to regexp, trailing 0 means that we're allowing no hits 17 | // rs, rs0: regexp to string (so you cannot use $1, $2 ... matchings from FROM in the TO pattern) 18 | // ss, ss0: string to string, ususally both string are big and read from file, like MODE=ss FROM=`cat in` TO=`cat out` FILES=`find ...` ./devel/mass_replace.sh 19 | func replacer(from, to, fn, mode string) { 20 | if from == "-" { 21 | from = "" 22 | } 23 | if to == "-" { 24 | to = "" 25 | } 26 | bytes, err := ioutil.ReadFile(fn) 27 | if err != nil { 28 | fmt.Printf("Error: %v\n", err) 29 | os.Exit(1) 30 | } 31 | contents := string(bytes) 32 | nReplaces := -1 33 | snReplaces := os.Getenv("NREPLACES") 34 | if snReplaces != "" { 35 | nReplaces, err = strconv.Atoi(snReplaces) 36 | lib.FatalOnError(err) 37 | if nReplaces < 1 { 38 | lib.Fatalf("NREPLACES must be positive") 39 | } 40 | } 41 | replaceFrom := -1 42 | sReplaceFrom := os.Getenv("REPLACEFROM") 43 | if sReplaceFrom != "" { 44 | replaceFrom, err = strconv.Atoi(sReplaceFrom) 45 | lib.FatalOnError(err) 46 | if replaceFrom < 1 { 47 | lib.Fatalf("REPLACEFROM must be positive") 48 | } 49 | l := len(contents) 50 | if replaceFrom >= l { 51 | lib.Fatalf("REPLACEFROM must be less than filename length %d", l) 52 | } 53 | } 54 | var newContents string 55 | switch mode { 56 | case "rr", "rr0", "rs", "rs0": 57 | re := regexp.MustCompile(from) 58 | if mode[:2] == "rs" { 59 | newContents = re.ReplaceAllLiteralString(contents, to) 60 | } else { 61 | newContents = re.ReplaceAllString(contents, to) 62 | } 63 | if contents == newContents { 64 | fmt.Printf("Nothing replaced in: %s\n", fn) 65 | if mode == "rr" || mode == "rs" { 66 | os.Exit(1) 67 | } 68 | return 69 | } 70 | fmt.Printf("Hits: %s\n", fn) 71 | case "ss", "ss0": 72 | if replaceFrom < 0 { 73 | newContents = strings.Replace(contents, from, to, nReplaces) 74 | if contents == newContents { 75 | fmt.Printf("Nothing replaced in: %s\n", fn) 76 | if mode == "ss" { 77 | os.Exit(1) 78 | } 79 | return 80 | } 81 | } else { 82 | contents1 := contents[:replaceFrom] 83 | contents2 := contents[replaceFrom:] 84 | newContents = contents1 + strings.Replace(contents2, from, to, nReplaces) 85 | if contents == newContents { 86 | fmt.Printf("Nothing replaced in: %s\n", fn) 87 | if mode == "ss" { 88 | os.Exit(1) 89 | } 90 | return 91 | } 92 | } 93 | fmt.Printf("Hits: %s\n", fn) 94 | default: 95 | fmt.Printf("Unknown mode '%s'\n", mode) 96 | os.Exit(1) 97 | } 98 | info, err := os.Stat(fn) 99 | if err != nil { 100 | fmt.Printf("Error: %v\n", err) 101 | os.Exit(1) 102 | } 103 | err = ioutil.WriteFile(fn, []byte(newContents), info.Mode()) 104 | if err != nil { 105 | fmt.Printf("Error: %v\n", err) 106 | os.Exit(1) 107 | } 108 | } 109 | 110 | func main() { 111 | from := os.Getenv("FROM") 112 | if from == "" { 113 | fmt.Printf("You need to set 'FROM' env variable\n") 114 | os.Exit(1) 115 | } 116 | to := os.Getenv("TO") 117 | noTo := os.Getenv("NO_TO") 118 | if to == "" && noTo == "" { 119 | fmt.Printf("You need to set 'TO' env variable or specify NO_TO\n") 120 | os.Exit(1) 121 | } 122 | mode := os.Getenv("MODE") 123 | if mode == "" { 124 | fmt.Printf("You need to set 'MODE' env variable\n") 125 | os.Exit(1) 126 | } 127 | if len(os.Args) < 2 { 128 | fmt.Printf("You need to provide a file name\n") 129 | os.Exit(1) 130 | } 131 | fn := os.Args[1] 132 | // fmt.Printf("File: '%s': '%s' -> '%s' (mode: %s)\n", fn, from, to, mode) 133 | replacer(from, to, fn, mode) 134 | } 135 | -------------------------------------------------------------------------------- /cmd/runq/runq.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | lib "github.com/cncf/devstatscode" 12 | ) 13 | 14 | func runq(sqlFile string, params []string) (ctx lib.Ctx) { 15 | // Environment context parse 16 | ctx.Init() 17 | lib.SetupTimeoutSignal(&ctx) 18 | 19 | // SQL arguments number 20 | if len(params)%2 > 0 { 21 | lib.Printf("Must provide correct parameter value pairs: %+v\n", params) 22 | os.Exit(1) 23 | } 24 | 25 | // SQL arguments parse 26 | replaces := make(map[string]string) 27 | paramName := "" 28 | for index, param := range params { 29 | if index%2 == 0 { 30 | replaces[param] = "" 31 | paramName = param 32 | } else { 33 | // Support special "readfile:replacement.dat" mode 34 | if len(param) >= 10 && param[:9] == "readfile:" { 35 | fn := param[9:] 36 | if ctx.Debug > 0 { 37 | lib.Printf("Reading file: %s\n", fn) 38 | } 39 | bytes, err := lib.ReadFile(&ctx, fn) 40 | lib.FatalOnError(err) 41 | param = string(bytes) 42 | } 43 | replaces[paramName] = param 44 | paramName = "" 45 | } 46 | } 47 | 48 | // Local or cron mode? 49 | dataPrefix := ctx.DataDir 50 | if ctx.Local { 51 | dataPrefix = "./" 52 | } 53 | // Absolute mode only allowed in 'runq' tool. 54 | if ctx.Absolute { 55 | dataPrefix = "" 56 | } 57 | 58 | // Read and eventually transform SQL file. 59 | bytes, err := lib.ReadFile(&ctx, dataPrefix+sqlFile) 60 | lib.FatalOnError(err) 61 | sqlQuery := string(bytes) 62 | qrPeriod := "" 63 | qrFrom := "" 64 | qrTo := "" 65 | qr := false 66 | for from, to := range replaces { 67 | // Special replace 'qr' 'period,from,to' is used for {{period.alias.name}} replacements 68 | if from == "qr" { 69 | qrAry := strings.Split(to, ",") 70 | qr = true 71 | qrPeriod, qrFrom, qrTo = qrAry[0], qrAry[1], qrAry[2] 72 | continue 73 | } 74 | sqlQuery = strings.Replace(sqlQuery, from, to, -1) 75 | } 76 | if qr { 77 | sHours := "" 78 | sqlQuery, sHours = lib.PrepareQuickRangeQuery(sqlQuery, qrPeriod, qrFrom, qrTo) 79 | sqlQuery = strings.Replace(sqlQuery, "{{range}}", sHours, -1) 80 | } 81 | sqlQuery = strings.Replace(sqlQuery, "{{rnd}}", lib.RandString(), -1) 82 | if ctx.Explain { 83 | sqlQuery = strings.Replace(sqlQuery, "select\n", "explain select\n", -1) 84 | } 85 | if ctx.DryRun { 86 | if ctx.Debug >= 0 { 87 | lib.Printf("%s\n", sqlQuery) 88 | } else { 89 | fmt.Printf("%s\n", sqlQuery) 90 | } 91 | return 92 | } 93 | 94 | // Connect to Postgres DB 95 | c := lib.PgConn(&ctx) 96 | defer func() { lib.FatalOnError(c.Close()) }() 97 | 98 | // Execute SQL 99 | rows := lib.QuerySQLWithErr(c, &ctx, sqlQuery) 100 | defer func() { lib.FatalOnError(rows.Close()) }() 101 | 102 | // Now unknown rows, with unknown types 103 | columns, err := rows.Columns() 104 | lib.FatalOnError(err) 105 | // Make columns unique 106 | for i := range columns { 107 | columns[i] += strconv.Itoa(i) 108 | } 109 | 110 | // Vals to hold any type as []interface{} 111 | vals := make([]interface{}, len(columns)) 112 | for i := range columns { 113 | vals[i] = new([]byte) 114 | } 115 | 116 | // Get results into `results` array of maps 117 | var results []map[string]string 118 | rowCount := 0 119 | for rows.Next() { 120 | rowMap := make(map[string]string) 121 | lib.FatalOnError(rows.Scan(vals...)) 122 | for index, val := range vals { 123 | value := "" 124 | if val != nil { 125 | value = string(*val.(*[]byte)) 126 | } 127 | rowMap[columns[index]] = value 128 | } 129 | results = append(results, rowMap) 130 | rowCount++ 131 | } 132 | lib.FatalOnError(rows.Err()) 133 | 134 | if len(results) < 1 { 135 | lib.Printf("Metric returned no data\n") 136 | return 137 | } 138 | 139 | // Compute column Lengths 140 | columnLengths := make(map[string]int) 141 | indexLen := 1 142 | for index, column := range columns { 143 | if index == 10 { 144 | indexLen++ 145 | } 146 | maxLen := len(column) - indexLen 147 | for _, row := range results { 148 | valLen := len(row[column]) 149 | if valLen > maxLen { 150 | maxLen = valLen 151 | } 152 | } 153 | columnLengths[column] = maxLen 154 | } 155 | 156 | var writer *csv.Writer 157 | if ctx.CSVFile != "" { 158 | // Write output CSV 159 | oFile, err := os.Create(ctx.CSVFile) 160 | lib.FatalOnError(err) 161 | defer func() { _ = oFile.Close() }() 162 | writer = csv.NewWriter(oFile) 163 | defer writer.Flush() 164 | } 165 | 166 | // Upper frame of the header row 167 | output := "/" 168 | for _, column := range columns { 169 | strFormat := fmt.Sprintf("%%-%ds", columnLengths[column]) 170 | value := strings.Repeat("-", columnLengths[column]) 171 | output += fmt.Sprintf(strFormat, value) + "+" 172 | } 173 | output = output[:len(output)-1] + "\\\n" 174 | lib.Printf("%s", output) 175 | 176 | // Header row 177 | output = "|" 178 | indexLen = 1 179 | hdr := []string{} 180 | for index, column := range columns { 181 | if index == 10 { 182 | indexLen++ 183 | } 184 | strFormat := fmt.Sprintf("%%-%ds", columnLengths[column]) 185 | output += fmt.Sprintf(strFormat, column[:len(column)-indexLen]) + "|" 186 | hdr = append(hdr, column[:len(column)-indexLen]) 187 | } 188 | output += "\n" 189 | lib.Printf("%s", output) 190 | if writer != nil { 191 | err = writer.Write(hdr) 192 | } 193 | 194 | // Frame between header row and data rows 195 | output = "+" 196 | for _, column := range columns { 197 | strFormat := fmt.Sprintf("%%-%ds", columnLengths[column]) 198 | value := strings.Repeat("-", columnLengths[column]) 199 | output += fmt.Sprintf(strFormat, value) + "+" 200 | } 201 | output = output[:len(output)-1] + "+\n" 202 | lib.Printf("%s", output) 203 | 204 | // Data rows loop 205 | for _, row := range results { 206 | // data row 207 | output = "|" 208 | vals := []string{} 209 | for _, column := range columns { 210 | value := row[column] 211 | strFormat := fmt.Sprintf("%%-%ds", columnLengths[column]) 212 | output += fmt.Sprintf(strFormat, value) + "|" 213 | vals = append(vals, value) 214 | } 215 | if writer != nil { 216 | err = writer.Write(vals) 217 | } 218 | output = strings.Replace(output[:len(output)-1]+"|\n", "%", "%%", -1) 219 | lib.Printf("%s", output) 220 | } 221 | 222 | // Frame below data rows 223 | output = "\\" 224 | for _, column := range columns { 225 | strFormat := fmt.Sprintf("%%-%ds", columnLengths[column]) 226 | value := strings.Repeat("-", columnLengths[column]) 227 | output += fmt.Sprintf(strFormat, value) + "+" 228 | } 229 | output = output[:len(output)-1] + "/\n" 230 | lib.Printf("%s", output) 231 | 232 | lib.Printf("Rows: %v\n", rowCount) 233 | if writer != nil { 234 | lib.Printf("%s written\n", ctx.CSVFile) 235 | } 236 | return 237 | } 238 | 239 | func main() { 240 | dtStart := time.Now() 241 | if len(os.Args) < 2 { 242 | lib.Printf("Required SQL file name [param1 value1 [param2 value2 ...]]\n") 243 | lib.Printf("Special replace 'qr' 'period,from,to' is used for {{period.alias.name}} replacements\n") 244 | os.Exit(1) 245 | } 246 | ctx := runq(os.Args[1], os.Args[2:]) 247 | dtEnd := time.Now() 248 | if ctx.Debug >= 0 { 249 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /cmd/structure/structure.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | ) 8 | 9 | func main() { 10 | dtStart := time.Now() 11 | // Environment context parse 12 | var ctx lib.Ctx 13 | ctx.Init() 14 | lib.SetupTimeoutSignal(&ctx) 15 | 16 | // Create database if needed 17 | createdDatabase := lib.CreateDatabaseIfNeeded(&ctx) 18 | 19 | // If we are using existing database, then display warnings 20 | // And ask for continue 21 | if !createdDatabase { 22 | if ctx.Table { 23 | lib.Printf("This program will recreate DB structure (dropping all existing data)\n") 24 | } 25 | lib.Printf("Continue? (y/n) ") 26 | c := lib.Mgetc(&ctx) 27 | lib.Printf("\n") 28 | if c == "y" { 29 | lib.Structure(&ctx) 30 | } 31 | } else { 32 | lib.Structure(&ctx) 33 | } 34 | dtEnd := time.Now() 35 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 36 | } 37 | -------------------------------------------------------------------------------- /cmd/tags/tags.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | yaml "gopkg.in/yaml.v2" 8 | ) 9 | 10 | // Insert TSDB tags 11 | func calcTags() { 12 | // Environment context parse 13 | var ctx lib.Ctx 14 | ctx.Init() 15 | lib.SetupTimeoutSignal(&ctx) 16 | 17 | // Connect to Postgres DB 18 | con := lib.PgConn(&ctx) 19 | defer func() { lib.FatalOnError(con.Close()) }() 20 | 21 | // Local or cron mode? 22 | dataPrefix := ctx.DataDir 23 | if ctx.Local { 24 | dataPrefix = "./" 25 | } 26 | 27 | // Read tags to generate 28 | data, err := lib.ReadFile(&ctx, dataPrefix+ctx.TagsYaml) 29 | if err != nil { 30 | lib.FatalOnError(err) 31 | return 32 | } 33 | var allTags lib.Tags 34 | lib.FatalOnError(yaml.Unmarshal(data, &allTags)) 35 | 36 | // Per project directory for SQL files 37 | dir := lib.Metrics 38 | if ctx.Project != "" { 39 | dir += ctx.Project + "/" 40 | } 41 | 42 | thrN := lib.GetThreadsNum(&ctx) 43 | // Iterate tags 44 | ch := make(chan bool) 45 | nThreads := 0 46 | // Use integer index to pass to go rountine 47 | for i := range allTags.Tags { 48 | go func(ch chan bool, idx int) { 49 | // Refer to current tag using index passed to anonymous function 50 | tg := &allTags.Tags[idx] 51 | if ctx.Debug > 0 { 52 | lib.Printf("Start Tag '%s' --> '%s'\n", tg.Name, tg.SeriesName) 53 | } 54 | 55 | // Process tag 56 | lib.ProcessTag(con, &ctx, tg, [][]string{}) 57 | 58 | if ctx.Debug > 0 { 59 | lib.Printf("End Tag '%s' --> '%s'\n", tg.Name, tg.SeriesName) 60 | } 61 | // Synchronize go routine 62 | if ch != nil { 63 | ch <- true 64 | if ctx.Debug > 0 { 65 | lib.Printf("Synced tag '%s' --> '%s'\n", tg.Name, tg.SeriesName) 66 | } 67 | } 68 | }(ch, i) 69 | // go routine called with 'ch' channel to sync and tag index 70 | nThreads++ 71 | if nThreads >= thrN { 72 | if ctx.Debug > 0 { 73 | lib.Printf("threading: %d >= %d, waiting on the channel\n", nThreads, thrN) 74 | } 75 | <-ch 76 | nThreads-- 77 | if ctx.Debug > 0 { 78 | lib.Printf("threading: thread joined, num threads: %d\n", nThreads) 79 | } 80 | } 81 | } 82 | // Usually all work happens on '<-ch' 83 | lib.Printf("Final %d threads join\n", nThreads) 84 | for nThreads > 0 { 85 | <-ch 86 | nThreads-- 87 | if ctx.Debug > 0 { 88 | lib.Printf("threading: fianl thread joined, num threads: %d\n", nThreads) 89 | } 90 | } 91 | } 92 | 93 | func main() { 94 | dtStart := time.Now() 95 | calcTags() 96 | dtEnd := time.Now() 97 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 98 | } 99 | -------------------------------------------------------------------------------- /cmd/tsplit/tsplit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | // You should use files like 'graduated.secret' - they start from first graduated ' ' line 12 | // and end on last line before first inclubating line ' ' 13 | func tsplit(size int, kind, in string, dbg bool) (out string) { 14 | ary := strings.Split(in, "\n") 15 | lines := []string{} 16 | for _, item := range ary { 17 | if strings.TrimSpace(item) == "" { 18 | continue 19 | } 20 | lines = append(lines, item) 21 | } 22 | offset := "" 23 | for _, line := range lines { 24 | if strings.TrimSpace(line) == "" { 25 | off := strings.Index(line, "") 26 | if off > 0 { 27 | offset = line[:off] 28 | } 29 | break 30 | } 31 | } 32 | skips := []string{"", "", "colspan"} 33 | imageLines, linkLines := []string{}, []string{} 34 | for _, line := range lines { 35 | skipLine := false 36 | for _, skip := range skips { 37 | if strings.Contains(line, skip) { 38 | skipLine = true 39 | break 40 | } 41 | } 42 | if skipLine { 43 | continue 44 | } 45 | // fmt.Printf("considering line: %s\n", line) 46 | imgLine := strings.Contains(line, `class="cncf-proj"`) 47 | if imgLine { 48 | imageLines = append(imageLines, line) 49 | } else { 50 | linkLines = append(linkLines, line) 51 | } 52 | } 53 | linkReplacer := strings.NewReplacer(` class="cncf-bl"`, ``, ` class="cncf-br"`, ``, ` class="cncf-bl cncf-br"`, ``, ` class="cncf-br cncf-bl"`, ``) 54 | for i, line := range linkLines { 55 | linkLines[i] = linkReplacer.Replace(line) 56 | } 57 | imageReplacer := strings.NewReplacer(`class="cncf-bb cncf-bl"`, `class="cncf-bb"`, `class="cncf-bb cncf-br"`, `class="cncf-bb"`, `class="cncf-bb cncf-bl cncf-br"`, `class="cncf-bb"`, `class="cncf-bb cncf-br cncf-bl"`, `class="cncf-bb"`) 58 | for i, line := range imageLines { 59 | imageLines[i] = imageReplacer.Replace(line) 60 | } 61 | nItems := len(imageLines) 62 | nSections := nItems / size 63 | if nItems%size != 0 { 64 | nSections++ 65 | } 66 | outLines := []string{} 67 | for section := 0; section < nSections; section++ { 68 | from := section * size 69 | to := from + size 70 | if to > nItems { 71 | to = nItems 72 | } 73 | n := to - from 74 | if dbg { 75 | fmt.Fprintf(os.Stderr, "section %d: %d-%d (%d items)\n", section, from, to, n) 76 | } 77 | outLines = append(outLines, offset+"") 78 | outLines = append(outLines, offset+fmt.Sprintf(` %s`, n, kind)) 79 | outLines = append(outLines, offset+"") 80 | outLines = append(outLines, offset+"") 81 | lastTo := to - 1 82 | for i := from; i < to; i++ { 83 | if i == from && i == lastTo { 84 | outLines = append(outLines, strings.Replace(linkLines[i], "", ``, -1)) 85 | continue 86 | } 87 | if i == from { 88 | outLines = append(outLines, strings.Replace(linkLines[i], "", ``, -1)) 89 | continue 90 | } 91 | if i == lastTo { 92 | outLines = append(outLines, strings.Replace(linkLines[i], "", ``, -1)) 93 | continue 94 | } 95 | outLines = append(outLines, linkLines[i]) 96 | } 97 | outLines = append(outLines, offset+"") 98 | outLines = append(outLines, offset+"") 99 | for i := from; i < to; i++ { 100 | if i == from && i == lastTo { 101 | outLines = append(outLines, strings.Replace(imageLines[i], ``, ``, -1)) 102 | continue 103 | } 104 | if i == from { 105 | outLines = append(outLines, strings.Replace(imageLines[i], ``, ``, -1)) 106 | continue 107 | } 108 | if i == lastTo { 109 | outLines = append(outLines, strings.Replace(imageLines[i], ``, ``, -1)) 110 | continue 111 | } 112 | outLines = append(outLines, imageLines[i]) 113 | } 114 | outLines = append(outLines, offset+"") 115 | } 116 | if dbg { 117 | fmt.Fprintf(os.Stderr, "Links %d:\n%s\n", len(linkLines), strings.Join(linkLines, "\n")) 118 | fmt.Fprintf(os.Stderr, "Images %d:\n%s\n", len(imageLines), strings.Join(imageLines, "\n")) 119 | } 120 | out = strings.Join(outLines, "\n") 121 | return 122 | } 123 | 124 | func main() { 125 | kind := os.Getenv("KIND") 126 | if kind == "" { 127 | fmt.Printf("You need to specify kind via KIND=Graduated|Incubating|Sandbox\n") 128 | return 129 | } 130 | ssize := os.Getenv("SIZE") 131 | if ssize == "" { 132 | fmt.Printf("You need to specify size via SIZE=n (usually 9, 10, 11, 12)\n") 133 | return 134 | } 135 | size, err := strconv.Atoi(ssize) 136 | if err != nil { 137 | fmt.Printf("error: %+v\n", err) 138 | return 139 | } 140 | data, err := ioutil.ReadAll(os.Stdin) 141 | if err != nil { 142 | fmt.Printf("error: %+v\n", err) 143 | return 144 | } 145 | fmt.Printf("%s\n", tsplit(size, kind, string(data), os.Getenv("DEBUG") != "")) 146 | } 147 | -------------------------------------------------------------------------------- /cmd/vars/vars.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | lib "github.com/cncf/devstatscode" 13 | yaml "gopkg.in/yaml.v2" 14 | ) 15 | 16 | // vars contain list of Postgres variables to set 17 | type pvars struct { 18 | Vars []pvar `yaml:"vars"` 19 | } 20 | 21 | // pvar contain each Postgres data 22 | type pvar struct { 23 | Name string `yaml:"name"` 24 | Type string `yaml:"type"` 25 | Value string `yaml:"value"` 26 | Command []string `yaml:"command"` 27 | Replaces [][]string `yaml:"replaces"` 28 | Disabled bool `yaml:"disabled"` 29 | NoWrite bool `yaml:"no_write"` 30 | Queries [][]string `yaml:"queries"` 31 | Loops [][]int `yaml:"loops"` 32 | QueriesBefore bool `yaml:"queries_before"` 33 | QueriesAfter bool `yaml:"queries_after"` 34 | LoopsBefore bool `yaml:"loops_before"` 35 | LoopsAfter bool `yaml:"loops_after"` 36 | } 37 | 38 | func processLoops(str string, loops [][]int) string { 39 | for _, loop := range loops { 40 | loopN := loop[0] 41 | from := loop[1] 42 | to := loop[2] 43 | inc := loop[3] 44 | start := fmt.Sprintf("loop:%d:start", loopN) 45 | end := fmt.Sprintf("loop:%d:end", loopN) 46 | rep := fmt.Sprintf("loop:%d:i", loopN) 47 | for { 48 | iStart := strings.Index(str, start) 49 | if iStart < 0 { 50 | break 51 | } 52 | iEnd := strings.Index(str, end) 53 | if iEnd < 0 { 54 | break 55 | } 56 | lStart := len(start) 57 | lEnd := len(end) 58 | before := str[0:iStart] 59 | body := str[iStart+lStart : iEnd] 60 | after := str[iEnd+lEnd:] 61 | out := before 62 | for i := from; i < to; i += inc { 63 | lBody := strings.Replace(body, rep, strconv.Itoa(i), -1) 64 | out += lBody 65 | } 66 | out += after 67 | str = out 68 | } 69 | } 70 | return str 71 | } 72 | 73 | func processQueries(str string, queries map[string]map[string][][]string) string { 74 | for name, query := range queries { 75 | for mp, values := range query { 76 | pref := name + ":" + mp 77 | for r, columns := range values { 78 | for c, value := range columns { 79 | rep := fmt.Sprintf("%s:%d:%d", pref, r, c) 80 | str = strings.Replace(str, rep, value, -1) 81 | } 82 | } 83 | } 84 | } 85 | return str 86 | } 87 | 88 | func handleQuery(c *sql.DB, ctx *lib.Ctx, queries map[string]map[string][][]string, queryData []string) { 89 | // Name to store query results 90 | name := queryData[0] 91 | _, ok := queries[name] 92 | if ok { 93 | lib.Fatalf("query '%s' already defined", name) 94 | } 95 | 96 | // Execute SQL 97 | sql := queryData[1] 98 | rows := lib.QuerySQLWithErr(c, ctx, sql) 99 | defer func() { lib.FatalOnError(rows.Close()) }() 100 | 101 | // Columns metadata 102 | columns, err := rows.Columns() 103 | lib.FatalOnError(err) 104 | columnsMap := make(map[string]int) 105 | for i, col := range columns { 106 | columnsMap[col] = i 107 | } 108 | resultsMap := make(map[string]int) 109 | for _, mp := range queryData[2:] { 110 | i, ok := columnsMap[mp] 111 | if !ok { 112 | lib.Fatalf("column '%s' not found in query results: %+v", mp, columns) 113 | } 114 | resultsMap[mp] = i 115 | } 116 | 117 | // Vals to hold any type as []interface{} 118 | vals := make([]interface{}, len(columns)) 119 | for i := range columns { 120 | vals[i] = new([]byte) 121 | } 122 | 123 | queries[name] = make(map[string][][]string) 124 | // Values 125 | for rows.Next() { 126 | lib.FatalOnError(rows.Scan(vals...)) 127 | svals := []string{} 128 | for _, val := range vals { 129 | value := "" 130 | if val != nil { 131 | value = string(*val.(*[]byte)) 132 | } 133 | svals = append(svals, value) 134 | } 135 | for mp, i := range resultsMap { 136 | svalue := svals[i] 137 | key := mp + ":" + svalue 138 | _, ok := queries[name][key] 139 | if !ok { 140 | queries[name][key] = [][]string{svals} 141 | } else { 142 | queries[name][key] = append(queries[name][key], svals) 143 | } 144 | } 145 | } 146 | lib.FatalOnError(rows.Err()) 147 | } 148 | 149 | // Insert Postgres vars 150 | func pdbVars() { 151 | // Environment context parse 152 | var ctx lib.Ctx 153 | ctx.Init() 154 | lib.SetupTimeoutSignal(&ctx) 155 | 156 | // Connect to Postgres DB 157 | c := lib.PgConn(&ctx) 158 | defer func() { lib.FatalOnError(c.Close()) }() 159 | 160 | // Local or cron mode? 161 | dataPrefix := ctx.DataDir 162 | if ctx.Local { 163 | dataPrefix = "./" 164 | } 165 | 166 | // Read vars to generate 167 | data, err := lib.ReadFile(&ctx, dataPrefix+ctx.VarsYaml) 168 | if err != nil { 169 | lib.FatalOnError(err) 170 | return 171 | } 172 | var allVars pvars 173 | lib.FatalOnError(yaml.Unmarshal(data, &allVars)) 174 | 175 | // All key name - values are stored in map 176 | // So next keys can replace strings using previous key values 177 | replaces := make(map[string]string) 178 | // Also make environemnt variables available too 179 | for _, e := range os.Environ() { 180 | pair := strings.Split(e, "=") 181 | replaces["$"+pair[0]] = pair[1] 182 | } 183 | // Queries 184 | queries := make(map[string]map[string][][]string) 185 | // Iterate vars 186 | for _, va := range allVars.Vars { 187 | // If given variable name is in the exclude list, skip it 188 | _, skip := ctx.ExcludeVars[va.Name] 189 | if ctx.Debug > 0 { 190 | lib.Printf( 191 | "Variable Name '%s', Value '%s', Type '%s', Command %v, Replaces %v, Queries: %v, Loops: %v, Disabled: %v, Skip: %v, NoWrite: %v\n", 192 | va.Name, va.Value, va.Type, va.Command, va.Replaces, va.Queries, va.Loops, va.Disabled, skip, va.NoWrite, 193 | ) 194 | } 195 | if skip || va.Disabled { 196 | continue 197 | } 198 | if va.Type == "" || va.Name == "" || (va.Value == "" && len(va.Command) == 0) { 199 | lib.Printf("Incorrect variable configuration, skipping\n") 200 | continue 201 | } 202 | 203 | // Handle queries 204 | for _, queryData := range va.Queries { 205 | handleQuery(c, &ctx, queries, queryData) 206 | } 207 | 208 | if len(va.Command) > 0 { 209 | for i := range va.Command { 210 | va.Command[i] = strings.Replace(va.Command[i], "{{datadir}}", dataPrefix, -1) 211 | } 212 | for i := range va.Command { 213 | va.Command[i] = strings.Replace(va.Command[i], "{{project}}", ctx.Project, -1) 214 | } 215 | cmdBytes, err := exec.Command(va.Command[0], va.Command[1:]...).CombinedOutput() 216 | if err != nil { 217 | lib.Printf("Failed command: %s %v\n", va.Command[0], va.Command[1:]) 218 | lib.FatalOnError(err) 219 | return 220 | } 221 | outString := strings.TrimSpace(string(cmdBytes)) 222 | if outString != "" { 223 | // Process queries and loops (first pass) 224 | if va.LoopsBefore { 225 | outString = processLoops(outString, va.Loops) 226 | } 227 | if va.QueriesBefore { 228 | outString = processQueries(outString, queries) 229 | } 230 | 231 | // Handle replacements using variables defined so far 232 | for _, repl := range va.Replaces { 233 | if len(repl) != 2 { 234 | lib.Fatalf("Replacement definition should be array with 2 elements, got: %v", repl) 235 | } 236 | var ( 237 | ok bool 238 | replTo string 239 | ) 240 | // Handle direct string replacements 241 | if len(repl[1]) > 0 && repl[1][0:1] == ":" { 242 | ok = true 243 | replTo = repl[1][1:] 244 | } else { 245 | replTo, ok = replaces[repl[1]] 246 | if !ok { 247 | lib.Fatalf("Variable '%s' requests replacing '%s', but not such variable is defined, defined: %v", va.Name, repl[1], replaces) 248 | } 249 | } 250 | // If 'replace from' starts with ':' then do not use [[ and ]] when replacing. 251 | // That means you can replace non-template parts 252 | if len(repl[0]) > 1 && repl[0][0:1] == ":" { 253 | outString = strings.Replace(outString, repl[0][1:], replTo, -1) 254 | } else { 255 | outString = strings.Replace(outString, "[["+repl[0]+"]]", replTo, -1) 256 | // Make replacements results available as variables too 257 | if repl[0] != repl[1] { 258 | replaces[repl[0]] = replTo 259 | } 260 | } 261 | } 262 | // Process queries and loops (second pass after variables/replacements processing) 263 | if va.LoopsAfter { 264 | outString = processLoops(outString, va.Loops) 265 | } 266 | if va.QueriesAfter { 267 | outString = processQueries(outString, queries) 268 | } 269 | va.Value = outString 270 | if ctx.Debug > 0 { 271 | lib.Printf("Name '%s', New Value '%s', Type '%s'\n", va.Name, va.Value, va.Type) 272 | } 273 | } 274 | } 275 | replaces[va.Name] = va.Value 276 | 277 | write := !va.NoWrite 278 | // If only selected variables mode is on, the check if we want to include this variable 279 | if write && len(ctx.OnlyVars) > 0 { 280 | _, write = ctx.OnlyVars[va.Name] 281 | } 282 | 283 | if !ctx.SkipPDB && write { 284 | lib.ExecSQLWithErr( 285 | c, 286 | &ctx, 287 | "insert into gha_vars(name, value_"+va.Type+") "+lib.NValues(2)+ 288 | " on conflict(name) do update set "+ 289 | "value_"+va.Type+" = "+lib.NValue(3)+" where gha_vars.name = "+lib.NValue(4), 290 | va.Name, 291 | va.Value, 292 | va.Value, 293 | va.Name, 294 | ) 295 | } else if ctx.Debug > 0 { 296 | lib.Printf("Skipping postgres vars write\n") 297 | } 298 | } 299 | } 300 | 301 | func main() { 302 | dtStart := time.Now() 303 | pdbVars() 304 | dtEnd := time.Now() 305 | lib.Printf("Time: %v\n", dtEnd.Sub(dtStart)) 306 | } 307 | -------------------------------------------------------------------------------- /cmd/webhook/example_webhook_payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "author_email": "lukaszgryglicki@o2.pl", 3 | "author_name": "Lukasz Gryglicki", 4 | "base_commit": "ec70161a3d8ad9d97012c576a5a3bc32004e7f82", 5 | "branch": "master", 6 | "build_url": "https://travis-ci.org/cncfdevstats/builds/286422580", 7 | "commit": "ec70161a3d8ad9d97012c576a5a3bc32004e7f82", 8 | "commit_id": 8.380144e+07, 9 | "committed_at": "2017-10-11T09:08:24Z", 10 | "committer_email": "lukaszgryglicki@o2.pl", 11 | "committer_name": "Lukasz Gryglicki", 12 | "compare_url": "https://github.com/cncfdevstats/compare/4eaf0b0c1e6f...ec70161a3d8a", 13 | "config": { 14 | ".result": "configured", 15 | "addons": { 16 | "postgresql": "9.6" 17 | }, 18 | "before_install": [ 19 | "go get -u github.com/golang/lint/golint", 20 | "go get golang.org/x/tools/cmd/goimports", 21 | "go get github.com/jgautheron/goconst/cmd/goconst", 22 | "go get github.com/jgautheron/usedexports", 23 | "go get github.com/influxdata/influxdb/client/v2", 24 | "go get github.com/lib/pq", 25 | "go get golang.org/x/text/transform", 26 | "go get golang.org/x/text/unicode/norm", 27 | "go get gopkg.in/yaml.v2", 28 | "sudo -u postgres createdb gha", 29 | "sudo -u postgres psql gha -c \"create user gha_admin with password 'pwd';\"", 30 | "sudo -u postgres psql gha -c 'grant all privileges on database \"gha\" to gha_admin;'", 31 | "sudo -u postgres psql gha -c \"alter user gha_admin createdb;\"", 32 | "curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add -", 33 | "source /etc/lsb-release", 34 | "echo \"deb https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable\" | sudo tee /etc/apt/sources.list.d/influxdb.list", 35 | "sudo apt-get update", 36 | "sudo apt-get install influxdb", 37 | "sudo service influxdb start" 38 | ], 39 | "dist": "trusty", 40 | "go": [ 41 | 1.9 42 | ], 43 | "group": "stable", 44 | "language": "go", 45 | "notifications": { 46 | "webhooks": "http://cncftest.io:1982/hook" 47 | }, 48 | "script": [ 49 | "mv /home/travis/gopath/src/github.com/cncfdevstats /home/travis/gopath/src/devstats", 50 | "cd /home/travis/gopath/src/devstats", 51 | "make", 52 | "make test", 53 | "PG_PASS=pwd ./dbtest.sh" 54 | ], 55 | "services": [ 56 | "postgresql" 57 | ] 58 | }, 59 | "duration": 127, 60 | "finished_at": "2017-10-11T09:12:12Z", 61 | "head_commit": null, 62 | "id": 2.8642258e+08, 63 | "matrix": [ 64 | { 65 | "allow_failure": false, 66 | "author_email": "lukaszgryglicki@o2.pl", 67 | "author_name": "Lukasz Gryglicki", 68 | "branch": "master", 69 | "commit": "ec70161a3d8ad9d97012c576a5a3bc32004e7f82", 70 | "committed_at": "2017-10-11T09:08:24Z", 71 | "committer_email": "lukaszgryglicki@o2.pl", 72 | "committer_name": "Lukasz Gryglicki", 73 | "compare_url": "https://github.com/cncfdevstats/compare/4eaf0b0c1e6f...ec70161a3d8a", 74 | "config": { 75 | ".result": "configured", 76 | "addons": { 77 | "postgresql": "9.6" 78 | }, 79 | "before_install": [ 80 | "go get -u github.com/golang/lint/golint", 81 | "go get golang.org/x/tools/cmd/goimports", 82 | "go get github.com/jgautheron/goconst/cmd/goconst", 83 | "go get github.com/jgautheron/usedexports", 84 | "go get github.com/influxdata/influxdb/client/v2", 85 | "go get github.com/lib/pq", 86 | "go get golang.org/x/text/transform", 87 | "go get golang.org/x/text/unicode/norm", 88 | "go get gopkg.in/yaml.v2", 89 | "sudo -u postgres createdb gha", 90 | "sudo -u postgres psql gha -c \"create user gha_admin with password 'pwd';\"", 91 | "sudo -u postgres psql gha -c 'grant all privileges on database \"gha\" to gha_admin;'", 92 | "sudo -u postgres psql gha -c \"alter user gha_admin createdb;\"", 93 | "curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add -", 94 | "source /etc/lsb-release", 95 | "echo \"deb https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable\" | sudo tee /etc/apt/sources.list.d/influxdb.list", 96 | "sudo apt-get update", 97 | "sudo apt-get install influxdb", 98 | "sudo service influxdb start" 99 | ], 100 | "dist": "trusty", 101 | "go": 1.9, 102 | "group": "stable", 103 | "language": "go", 104 | "notifications": { 105 | "webhooks": "http://cncftest.io:1982/hook" 106 | }, 107 | "os": "linux", 108 | "script": [ 109 | "mv /home/travis/gopath/src/github.com/cncfdevstats /home/travis/gopath/src/devstats", 110 | "cd /home/travis/gopath/src/devstats", 111 | "make", 112 | "make test", 113 | "PG_PASS=pwd ./dbtest.sh" 114 | ], 115 | "services": [ 116 | "postgresql" 117 | ] 118 | }, 119 | "finished_at": null, 120 | "id": 2.86422581e+08, 121 | "message": "Add TravisCI webhook call, and receiver\n\nSigned-off-by: Lukasz Gryglicki \u003clukaszgryglicki@o2.pl\u003e", 122 | "number": "26.1", 123 | "parent_id": 2.8642258e+08, 124 | "repository_id": 1.4566882e+07, 125 | "result": 0, 126 | "started_at": null, 127 | "state": "passed", 128 | "status": 0 129 | } 130 | ], 131 | "message": "Add TravisCI webhook call, and receiver\n\nSigned-off-by: Lukasz Gryglicki \u003clukaszgryglicki@o2.pl\u003e", 132 | "number": "26", 133 | "pull_request": false, 134 | "pull_request_number": null, 135 | "pull_request_title": null, 136 | "repository": { 137 | "id": 1.4566882e+07, 138 | "name": "devstats", 139 | "owner_name": "cncf", 140 | "url": null 141 | }, 142 | "result": 0, 143 | "result_message": "Passed", 144 | "started_at": "2017-10-11T09:10:05Z", 145 | "state": "passed", 146 | "status": 0, 147 | "status_message": "Passed", 148 | "tag": null, 149 | "type": "push" 150 | } 151 | 152 | -------------------------------------------------------------------------------- /cmd/webhook/example_webhook_payload_deploy.dat: -------------------------------------------------------------------------------- 1 | payload=%7B%22id%22%3A286422580%2C%22number%22%3A%2226%22%2C%22config%22%3A%7B%22language%22%3A%22go%22%2C%22go%22%3A%5B1.9%5D%2C%22before_install%22%3A%5B%22go+get+-u+github.com%2Fgolang%2Flint%2Fgolint%22%2C%22go+get+golang.org%2Fx%2Ftools%2Fcmd%2Fgoimports%22%2C%22go+get+github.com%2Fjgautheron%2Fgoconst%2Fcmd%2Fgoconst%22%2C%22go+get+github.com%2Fjgautheron%2Fusedexports%22%2C%22go+get+github.com%2Finfluxdata%2Finfluxdb%2Fclient%2Fv2%22%2C%22go+get+github.com%2Flib%2Fpq%22%2C%22go+get+golang.org%2Fx%2Ftext%2Ftransform%22%2C%22go+get+golang.org%2Fx%2Ftext%2Funicode%2Fnorm%22%2C%22go+get+gopkg.in%2Fyaml.v2%22%2C%22sudo+-u+postgres+createdb+gha%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22create+user+gha_admin+with+password+%27pwd%27%3B%5C%22%22%2C%22sudo+-u+postgres+psql+gha+-c+%27grant+all+privileges+on+database+%5C%22gha%5C%22+to+gha_admin%3B%27%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22alter+user+gha_admin+createdb%3B%5C%22%22%2C%22curl+-sL+https%3A%2F%2Frepos.influxdata.com%2Finfluxdb.key+%7C+sudo+apt-key+add+-%22%2C%22source+%2Fetc%2Flsb-release%22%2C%22echo+%5C%22deb+https%3A%2F%2Frepos.influxdata.com%2F%24%7BDISTRIB_ID%2C%2C%7D+%24%7BDISTRIB_CODENAME%7D+stable%5C%22+%7C+sudo+tee+%2Fetc%2Fapt%2Fsources.list.d%2Finfluxdb.list%22%2C%22sudo+apt-get+update%22%2C%22sudo+apt-get+install+influxdb%22%2C%22sudo+service+influxdb+start%22%5D%2C%22script%22%3A%5B%22mv+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fgithub.com%2Fcncf%2Fdevstats+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22cd+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22make%22%2C%22make+test%22%2C%22PG_PASS%3Dpwd+.%2Fdbtest.sh%22%5D%2C%22services%22%3A%5B%22postgresql%22%5D%2C%22addons%22%3A%7B%22postgresql%22%3A%229.6%22%7D%2C%22notifications%22%3A%7B%22webhooks%22%3A%22http%3A%2F%2Fcncftest.io%3A1982%2Fhook%22%7D%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22trusty%22%7D%2C%22type%22%3A%22push%22%2C%22state%22%3A%22passed%22%2C%22status%22%3A0%2C%22result%22%3A0%2C%22status_message%22%3A%22Passed%22%2C%22result_message%22%3A%22Passed%22%2C%22started_at%22%3A%222017-10-11T09%3A10%3A05Z%22%2C%22finished_at%22%3A%222017-10-11T09%3A12%3A12Z%22%2C%22duration%22%3A127%2C%22build_url%22%3A%22https%3A%2F%2Ftravis-ci.org%2Fcncf%2Fdevstats%2Fbuilds%2F286422580%22%2C%22commit_id%22%3A83801440%2C%22commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22base_commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22head_commit%22%3Anull%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22Add+TravisCI+%5Bdeploy%5D+webhook+call%2C+and+receiver%5Cn%5CnSigned-off-by%3A+Lukasz+Gryglicki+%3Clukaszgryglicki%40o2.pl%3E%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcncf%2Fdevstats%2Fcompare%2F4eaf0b0c1e6f...ec70161a3d8a%22%2C%22committed_at%22%3A%222017-10-11T09%3A08%3A24Z%22%2C%22author_name%22%3A%22Lukasz+Gryglicki%22%2C%22author_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22committer_name%22%3A%22Lukasz+Gryglicki%22%2C%22committer_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22pull_request%22%3Afalse%2C%22pull_request_number%22%3Anull%2C%22pull_request_title%22%3Anull%2C%22tag%22%3Anull%2C%22repository%22%3A%7B%22id%22%3A14566882%2C%22name%22%3A%22devstats%22%2C%22owner_name%22%3A%22cncf%22%2C%22url%22%3Anull%7D%2C%22matrix%22%3A%5B%7B%22id%22%3A286422581%2C%22repository_id%22%3A14566882%2C%22parent_id%22%3A286422580%2C%22number%22%3A%2226.1%22%2C%22state%22%3A%22passed%22%2C%22config%22%3A%7B%22language%22%3A%22go%22%2C%22go%22%3A1.9%2C%22before_install%22%3A%5B%22go+get+-u+github.com%2Fgolang%2Flint%2Fgolint%22%2C%22go+get+golang.org%2Fx%2Ftools%2Fcmd%2Fgoimports%22%2C%22go+get+github.com%2Fjgautheron%2Fgoconst%2Fcmd%2Fgoconst%22%2C%22go+get+github.com%2Fjgautheron%2Fusedexports%22%2C%22go+get+github.com%2Finfluxdata%2Finfluxdb%2Fclient%2Fv2%22%2C%22go+get+github.com%2Flib%2Fpq%22%2C%22go+get+golang.org%2Fx%2Ftext%2Ftransform%22%2C%22go+get+golang.org%2Fx%2Ftext%2Funicode%2Fnorm%22%2C%22go+get+gopkg.in%2Fyaml.v2%22%2C%22sudo+-u+postgres+createdb+gha%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22create+user+gha_admin+with+password+%27pwd%27%3B%5C%22%22%2C%22sudo+-u+postgres+psql+gha+-c+%27grant+all+privileges+on+database+%5C%22gha%5C%22+to+gha_admin%3B%27%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22alter+user+gha_admin+createdb%3B%5C%22%22%2C%22curl+-sL+https%3A%2F%2Frepos.influxdata.com%2Finfluxdb.key+%7C+sudo+apt-key+add+-%22%2C%22source+%2Fetc%2Flsb-release%22%2C%22echo+%5C%22deb+https%3A%2F%2Frepos.influxdata.com%2F%24%7BDISTRIB_ID%2C%2C%7D+%24%7BDISTRIB_CODENAME%7D+stable%5C%22+%7C+sudo+tee+%2Fetc%2Fapt%2Fsources.list.d%2Finfluxdb.list%22%2C%22sudo+apt-get+update%22%2C%22sudo+apt-get+install+influxdb%22%2C%22sudo+service+influxdb+start%22%5D%2C%22script%22%3A%5B%22mv+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fgithub.com%2Fcncf%2Fdevstats+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22cd+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22make%22%2C%22make+test%22%2C%22PG_PASS%3Dpwd+.%2Fdbtest.sh%22%5D%2C%22services%22%3A%5B%22postgresql%22%5D%2C%22addons%22%3A%7B%22postgresql%22%3A%229.6%22%7D%2C%22notifications%22%3A%7B%22webhooks%22%3A%22http%3A%2F%2Fcncftest.io%3A1982%2Fhook%22%7D%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22trusty%22%2C%22os%22%3A%22linux%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22+Add+TravisCI+%5Bdeploy%5D+webhook+call%2C+and+receiver%5Cn%5CnSigned-off-by%3A+Lukasz+Gryglicki+%3Clukaszgryglicki%40o2.pl%3E%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcncf%2Fdevstats%2Fcompare%2F4eaf0b0c1e6f...ec70161a3d8a%22%2C%22started_at%22%3Anull%2C%22finished_at%22%3Anull%2C%22committed_at%22%3A%222017-10-11T09%3A08%3A24Z%22%2C%22author_name%22%3A%22Lukasz+Gryglicki%22%2C%22author_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22committer_name%22%3A%22Lukasz+Gryglicki%22%2C%22committer_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22allow_failure%22%3Afalse%7D%5D%7D 2 | -------------------------------------------------------------------------------- /cmd/webhook/example_webhook_payload_no_deploy.dat: -------------------------------------------------------------------------------- 1 | payload=%7B%22id%22%3A286422580%2C%22number%22%3A%2226%22%2C%22config%22%3A%7B%22language%22%3A%22go%22%2C%22go%22%3A%5B1.9%5D%2C%22before_install%22%3A%5B%22go+get+-u+github.com%2Fgolang%2Flint%2Fgolint%22%2C%22go+get+golang.org%2Fx%2Ftools%2Fcmd%2Fgoimports%22%2C%22go+get+github.com%2Fjgautheron%2Fgoconst%2Fcmd%2Fgoconst%22%2C%22go+get+github.com%2Fjgautheron%2Fusedexports%22%2C%22go+get+github.com%2Finfluxdata%2Finfluxdb%2Fclient%2Fv2%22%2C%22go+get+github.com%2Flib%2Fpq%22%2C%22go+get+golang.org%2Fx%2Ftext%2Ftransform%22%2C%22go+get+golang.org%2Fx%2Ftext%2Funicode%2Fnorm%22%2C%22go+get+gopkg.in%2Fyaml.v2%22%2C%22sudo+-u+postgres+createdb+gha%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22create+user+gha_admin+with+password+%27pwd%27%3B%5C%22%22%2C%22sudo+-u+postgres+psql+gha+-c+%27grant+all+privileges+on+database+%5C%22gha%5C%22+to+gha_admin%3B%27%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22alter+user+gha_admin+createdb%3B%5C%22%22%2C%22curl+-sL+https%3A%2F%2Frepos.influxdata.com%2Finfluxdb.key+%7C+sudo+apt-key+add+-%22%2C%22source+%2Fetc%2Flsb-release%22%2C%22echo+%5C%22deb+https%3A%2F%2Frepos.influxdata.com%2F%24%7BDISTRIB_ID%2C%2C%7D+%24%7BDISTRIB_CODENAME%7D+stable%5C%22+%7C+sudo+tee+%2Fetc%2Fapt%2Fsources.list.d%2Finfluxdb.list%22%2C%22sudo+apt-get+update%22%2C%22sudo+apt-get+install+influxdb%22%2C%22sudo+service+influxdb+start%22%5D%2C%22script%22%3A%5B%22mv+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fgithub.com%2Fcncf%2Fdevstats+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22cd+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22make%22%2C%22make+test%22%2C%22PG_PASS%3Dpwd+.%2Fdbtest.sh%22%5D%2C%22services%22%3A%5B%22postgresql%22%5D%2C%22addons%22%3A%7B%22postgresql%22%3A%229.6%22%7D%2C%22notifications%22%3A%7B%22webhooks%22%3A%22http%3A%2F%2Fcncftest.io%3A1982%2Fhook%22%7D%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22trusty%22%7D%2C%22type%22%3A%22push%22%2C%22state%22%3A%22passed%22%2C%22status%22%3A0%2C%22result%22%3A0%2C%22status_message%22%3A%22Passed%22%2C%22result_message%22%3A%22Passed%22%2C%22started_at%22%3A%222017-10-11T09%3A10%3A05Z%22%2C%22finished_at%22%3A%222017-10-11T09%3A12%3A12Z%22%2C%22duration%22%3A127%2C%22build_url%22%3A%22https%3A%2F%2Ftravis-ci.org%2Fcncf%2Fdevstats%2Fbuilds%2F286422580%22%2C%22commit_id%22%3A83801440%2C%22commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22base_commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22head_commit%22%3Anull%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22Add+TravisCI+webhook+call%2C+and+receiver%5Cn%5CnSigned-off-by%3A+Lukasz+Gryglicki+%3Clukaszgryglicki%40o2.pl%3E%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcncf%2Fdevstats%2Fcompare%2F4eaf0b0c1e6f...ec70161a3d8a%22%2C%22committed_at%22%3A%222017-10-11T09%3A08%3A24Z%22%2C%22author_name%22%3A%22Lukasz+Gryglicki%22%2C%22author_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22committer_name%22%3A%22Lukasz+Gryglicki%22%2C%22committer_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22pull_request%22%3Afalse%2C%22pull_request_number%22%3Anull%2C%22pull_request_title%22%3Anull%2C%22tag%22%3Anull%2C%22repository%22%3A%7B%22id%22%3A14566882%2C%22name%22%3A%22devstats%22%2C%22owner_name%22%3A%22cncf%22%2C%22url%22%3Anull%7D%2C%22matrix%22%3A%5B%7B%22id%22%3A286422581%2C%22repository_id%22%3A14566882%2C%22parent_id%22%3A286422580%2C%22number%22%3A%2226.1%22%2C%22state%22%3A%22passed%22%2C%22config%22%3A%7B%22language%22%3A%22go%22%2C%22go%22%3A1.9%2C%22before_install%22%3A%5B%22go+get+-u+github.com%2Fgolang%2Flint%2Fgolint%22%2C%22go+get+golang.org%2Fx%2Ftools%2Fcmd%2Fgoimports%22%2C%22go+get+github.com%2Fjgautheron%2Fgoconst%2Fcmd%2Fgoconst%22%2C%22go+get+github.com%2Fjgautheron%2Fusedexports%22%2C%22go+get+github.com%2Finfluxdata%2Finfluxdb%2Fclient%2Fv2%22%2C%22go+get+github.com%2Flib%2Fpq%22%2C%22go+get+golang.org%2Fx%2Ftext%2Ftransform%22%2C%22go+get+golang.org%2Fx%2Ftext%2Funicode%2Fnorm%22%2C%22go+get+gopkg.in%2Fyaml.v2%22%2C%22sudo+-u+postgres+createdb+gha%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22create+user+gha_admin+with+password+%27pwd%27%3B%5C%22%22%2C%22sudo+-u+postgres+psql+gha+-c+%27grant+all+privileges+on+database+%5C%22gha%5C%22+to+gha_admin%3B%27%22%2C%22sudo+-u+postgres+psql+gha+-c+%5C%22alter+user+gha_admin+createdb%3B%5C%22%22%2C%22curl+-sL+https%3A%2F%2Frepos.influxdata.com%2Finfluxdb.key+%7C+sudo+apt-key+add+-%22%2C%22source+%2Fetc%2Flsb-release%22%2C%22echo+%5C%22deb+https%3A%2F%2Frepos.influxdata.com%2F%24%7BDISTRIB_ID%2C%2C%7D+%24%7BDISTRIB_CODENAME%7D+stable%5C%22+%7C+sudo+tee+%2Fetc%2Fapt%2Fsources.list.d%2Finfluxdb.list%22%2C%22sudo+apt-get+update%22%2C%22sudo+apt-get+install+influxdb%22%2C%22sudo+service+influxdb+start%22%5D%2C%22script%22%3A%5B%22mv+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fgithub.com%2Fcncf%2Fdevstats+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22cd+%2Fhome%2Ftravis%2Fgopath%2Fsrc%2Fdevstats%22%2C%22make%22%2C%22make+test%22%2C%22PG_PASS%3Dpwd+.%2Fdbtest.sh%22%5D%2C%22services%22%3A%5B%22postgresql%22%5D%2C%22addons%22%3A%7B%22postgresql%22%3A%229.6%22%7D%2C%22notifications%22%3A%7B%22webhooks%22%3A%22http%3A%2F%2Fcncftest.io%3A1982%2Fhook%22%7D%2C%22.result%22%3A%22configured%22%2C%22group%22%3A%22stable%22%2C%22dist%22%3A%22trusty%22%2C%22os%22%3A%22linux%22%7D%2C%22status%22%3A0%2C%22result%22%3A0%2C%22commit%22%3A%22ec70161a3d8ad9d97012c576a5a3bc32004e7f82%22%2C%22branch%22%3A%22master%22%2C%22message%22%3A%22+Add+TravisCI+webhook+call%2C+and+receiver%5Cn%5CnSigned-off-by%3A+Lukasz+Gryglicki+%3Clukaszgryglicki%40o2.pl%3E%22%2C%22compare_url%22%3A%22https%3A%2F%2Fgithub.com%2Fcncf%2Fdevstats%2Fcompare%2F4eaf0b0c1e6f...ec70161a3d8a%22%2C%22started_at%22%3Anull%2C%22finished_at%22%3Anull%2C%22committed_at%22%3A%222017-10-11T09%3A08%3A24Z%22%2C%22author_name%22%3A%22Lukasz+Gryglicki%22%2C%22author_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22committer_name%22%3A%22Lukasz+Gryglicki%22%2C%22committer_email%22%3A%22lukaszgryglicki%40o2.pl%22%2C%22allow_failure%22%3Afalse%7D%5D%7D 2 | -------------------------------------------------------------------------------- /columns.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "${PG_PASS}" ] 3 | then 4 | echo "$0: you must specify PG_PASS=..." 5 | exit 1 6 | fi 7 | make fmt && make columns || exit 2 8 | cd ../devstats || exit 3 9 | PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 GHA2DB_DEBUG=2 GHA2DB_COLUMNS_YAML=devel/test_columns.yaml ../devstatscode/columns 10 | -------------------------------------------------------------------------------- /const.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | // Today - common constant string 4 | const Today string = "today" 5 | 6 | // DefaultDataDir - common constant string 7 | const DefaultDataDir string = "/etc/gha2db/" 8 | 9 | // Retry - common constant string 10 | const Retry string = "retry" 11 | 12 | // Password - common constant string 13 | const Password string = "password" 14 | 15 | // GHAAdmin - common constant string 16 | const GHAAdmin string = "gha_admin" 17 | 18 | // Quarter - common constant string 19 | const Quarter string = "quarter" 20 | 21 | // Now - common constant string 22 | const Now string = "now" 23 | 24 | // GHA - common constant string 25 | const GHA string = "gha" 26 | 27 | // Localhost - common constant string 28 | const Localhost string = "localhost" 29 | 30 | // Devstats - common constant string 31 | const Devstats string = "devstats" 32 | 33 | // DevstatsCode - common constant string 34 | const DevstatsCode string = "devstatscode" 35 | 36 | // TimeoutError - common constant string 37 | const TimeoutError string = "{\"error\":\"timeout\"}\n" 38 | 39 | // EngineIsClosedError - common constant string 40 | const EngineIsClosedError string = "engine is closed" 41 | 42 | // LocalGitScripts - common constant string 43 | const LocalGitScripts string = "./git/" 44 | 45 | // Metrics - common constant string 46 | const Metrics string = "metrics/" 47 | 48 | // Unset - common constant string 49 | const Unset string = "{{unset}}" 50 | 51 | // TimeCol - common constant string 52 | const TimeCol string = "time" 53 | 54 | // SeriesCol - common constant string 55 | const SeriesCol string = "series" 56 | 57 | // PeriodCol - common constant string 58 | const PeriodCol string = "period" 59 | 60 | // Null - common constant string 61 | const Null string = "null" 62 | 63 | // HideCfgFile - common constant string 64 | const HideCfgFile string = "hide/hide.csv" 65 | 66 | // All - common constant string 67 | const All string = "all" 68 | 69 | // ALL - common constant string 70 | const ALL string = "All" 71 | 72 | // Kubernetes - common constant string 73 | const Kubernetes string = "kubernetes" 74 | 75 | // Abuse - common constant string 76 | const Abuse string = "abuse" 77 | 78 | // NotFound - common constant string 79 | const NotFound string = "not_found" 80 | 81 | // IssueIsDeleted - common constant string 82 | const IssueIsDeleted string = "issue_is_deleted" 83 | 84 | // MovedPermanently - common constant string 85 | const MovedPermanently string = "moved_permanently" 86 | 87 | // Merged - common constant string 88 | const Merged string = "merged" 89 | 90 | // InvalidCatalogName - common constant string 91 | const InvalidCatalogName string = "invalid_catalog_name" 92 | 93 | // Nil - common constant string 94 | const Nil string = "(nil)" 95 | 96 | // Reconnect - common constant string 97 | const Reconnect string = "reconnect" 98 | 99 | // OK - common constant string 100 | const OK string = "ok" 101 | 102 | // RepoNamesQuery - common constant string 103 | const RepoNamesQuery string = "select distinct name from gha_repos where name like '%_/_%' and name not like '%/%/%'" 104 | 105 | // DevActCnt - common constant string 106 | const DevActCnt string = "DevActCnt" 107 | 108 | // DevActCntComp - common constant string 109 | const DevActCntComp string = "DevActCntComp" 110 | 111 | // ComContribRepoGrp - common constant string 112 | const ComContribRepoGrp string = "ComContribRepoGrp" 113 | 114 | // CompaniesTable - common constant string 115 | const CompaniesTable string = "CompaniesTable" 116 | 117 | // ComStatsRepoGrp - common constant string 118 | const ComStatsRepoGrp string = "ComStatsRepoGrp" 119 | 120 | // Health - common constant string 121 | const Health string = "Health" 122 | 123 | // Events - common constant string 124 | const Events string = "Events" 125 | 126 | // ListAPIs - common constant string 127 | const ListAPIs string = "ListAPIs" 128 | 129 | // CumulativeCounts - common constant string 130 | const CumulativeCounts string = "CumulativeCounts" 131 | 132 | // ListProjects - common constant string 133 | const ListProjects string = "ListProjects" 134 | 135 | // RepoGroups - common constant string 136 | const RepoGroups string = "RepoGroups" 137 | 138 | // Ranges - common constant string 139 | const Ranges string = "Ranges" 140 | 141 | // Repos - common constant string 142 | const Repos string = "Repos" 143 | 144 | // Countries - common constant string 145 | const Countries string = "Countries" 146 | 147 | // Companies - common constant string 148 | const Companies string = "Companies" 149 | 150 | // SiteStats - common constant string 151 | const SiteStats string = "SiteStats" 152 | 153 | // Day - common constant string 154 | const Day string = "day" 155 | 156 | // Week - common constant string 157 | const Week string = "week" 158 | 159 | // Hour - common constant string 160 | const Hour string = "hour" 161 | 162 | // Month - common constant string 163 | const Month string = "month" 164 | 165 | // Year - common constant string 166 | const Year string = "year" 167 | -------------------------------------------------------------------------------- /convert.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import "math" 4 | 5 | // GetFloatFromInterface if an interface is of numeric type, return its value as float64 6 | func GetFloatFromInterface(i interface{}) (float64, bool) { 7 | switch i := i.(type) { 8 | case float64: 9 | return i, true 10 | case float32: 11 | return float64(i), true 12 | case int64: 13 | return float64(i), true 14 | case int32: 15 | return float64(i), true 16 | case int16: 17 | return float64(i), true 18 | case int8: 19 | return float64(i), true 20 | case int: 21 | return float64(i), true 22 | case uint64: 23 | return float64(i), true 24 | case uint32: 25 | return float64(i), true 26 | case uint16: 27 | return float64(i), true 28 | case uint8: 29 | return float64(i), true 30 | case uint: 31 | return float64(i), true 32 | default: 33 | return math.NaN(), false 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /convert_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | 7 | lib "github.com/cncf/devstatscode" 8 | ) 9 | 10 | func TestGetFloatFromInterface(t *testing.T) { 11 | // Test cases 12 | var testCases = []struct { 13 | input interface{} 14 | expectedFloat float64 15 | expectedOK bool 16 | }{ 17 | {input: float64(0.0), expectedFloat: float64(0.0), expectedOK: true}, 18 | {input: float64(1.0), expectedFloat: float64(1.0), expectedOK: true}, 19 | {input: float64(-1.5), expectedFloat: float64(-1.5), expectedOK: true}, 20 | {input: float32(2.0), expectedFloat: float64(2.0), expectedOK: true}, 21 | {input: int64(3.0), expectedFloat: float64(3.0), expectedOK: true}, 22 | {input: int64(-33.0), expectedFloat: float64(-33.0), expectedOK: true}, 23 | {input: int32(4.0), expectedFloat: float64(4.0), expectedOK: true}, 24 | {input: int16(5.0), expectedFloat: float64(5.0), expectedOK: true}, 25 | {input: int8(6.0), expectedFloat: float64(6.0), expectedOK: true}, 26 | {input: int(7.0), expectedFloat: float64(7.0), expectedOK: true}, 27 | {input: uint64(8.0), expectedFloat: float64(8.0), expectedOK: true}, 28 | {input: uint32(9.0), expectedFloat: float64(9.0), expectedOK: true}, 29 | {input: uint16(10.0), expectedFloat: float64(10.0), expectedOK: true}, 30 | {input: uint8(11.0), expectedFloat: float64(11.0), expectedOK: true}, 31 | {input: uint(12.0), expectedFloat: float64(12.0), expectedOK: true}, 32 | {input: string("123"), expectedFloat: float64(math.NaN()), expectedOK: false}, 33 | {input: string("xyz"), expectedFloat: float64(math.NaN()), expectedOK: false}, 34 | } 35 | // Execute test cases 36 | for index, test := range testCases { 37 | expectedFloat := test.expectedFloat 38 | expectedOK := test.expectedOK 39 | gotFloat, gotOK := lib.GetFloatFromInterface(test.input) 40 | if gotOK != expectedOK { 41 | t.Errorf( 42 | "test number %d, expected %v, got %v", 43 | index+1, expectedOK, gotOK, 44 | ) 45 | } 46 | if math.IsNaN(gotFloat) && math.IsNaN(expectedFloat) { 47 | continue 48 | } 49 | if gotFloat != expectedFloat { 50 | t.Errorf( 51 | "test number %d, expected %v, got %v", 52 | index+1, expectedFloat, gotFloat, 53 | ) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /cron/backup_artificial.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -z "${NOBACKUP}" ] 3 | then 4 | exit 0 5 | fi 6 | if [ -z "$1" ] 7 | then 8 | echo "$0: you need to provide database name as an argument" 9 | exit 1 10 | fi 11 | db=$1 12 | if [ "$db" = "devstats" ] 13 | then 14 | exit 0 15 | fi 16 | cd /tmp || exit 1 17 | function finish { 18 | cd /tmp 19 | rm $db.* 20 | } 21 | trap finish EXIT 22 | db.sh psql $db -tAc "copy (select * from gha_events where id > 281474976710656) TO '/tmp/$db.events.tsv'" || exit 2 23 | db.sh psql $db -tAc "copy (select * from gha_payloads where event_id > 281474976710656) TO '/tmp/$db.payloads.tsv'" || exit 3 24 | db.sh psql $db -tAc "copy (select * from gha_issues where event_id > 281474976710656) TO '/tmp/$db.issues.tsv'" || exit 4 25 | db.sh psql $db -tAc "copy (select * from gha_pull_requests where event_id > 281474976710656) TO '/tmp/$db.prs.tsv'" || exit 5 26 | db.sh psql $db -tAc "copy (select * from gha_milestones where event_id > 281474976710656) TO '/tmp/$db.milestones.tsv'" || exit 6 27 | db.sh psql $db -tAc "copy (select * from gha_issues_labels where event_id > 281474976710656) TO '/tmp/$db.labels.tsv'" || exit 7 28 | db.sh psql $db -tAc "copy (select * from gha_issues_assignees where event_id > 281474976710656) TO '/tmp/$db.issue_assignees.tsv'" || exit 8 29 | db.sh psql $db -tAc "copy (select * from gha_pull_requests_assignees where event_id > 281474976710656) TO '/tmp/$db.pr_assignees.tsv'" || exit 9 30 | db.sh psql $db -tAc "copy (select * from gha_pull_requests_requested_reviewers where event_id > 281474976710656) TO '/tmp/$db.pr_reviewers.tsv'" || exit 10 31 | db.sh psql $db -tAc "copy (select * from gha_issues_events_labels where event_id > 281474976710656) TO '/tmp/$db.issues_events_labels.tsv'" || exit 11 32 | db.sh psql $db -tAc "copy (select * from gha_texts where event_id > 281474976710656) TO '/tmp/$db.texts.tsv'" || exit 12 33 | rm -f $db.tar* || exit 13 34 | tar cf $db.tar $db.*.tsv || exit 14 35 | xz $db.tar || exit 15 36 | mv $db.tar.xz /var/www/html/ || exit 16 37 | -------------------------------------------------------------------------------- /cron/cron_db_backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -z "${NOBACKUP}" ] 3 | then 4 | exit 0 5 | fi 6 | echo "Backup start:" && date 7 | db.sh pg_dump -Fc $1 -f /tmp/$1.dump || exit 1 8 | mv /tmp/$1.dump /var/www/html/ || exit 2 9 | backup_artificial.sh "$1" || exit 3 10 | date && echo "Backup OK" 11 | 12 | -------------------------------------------------------------------------------- /cron/sysctl_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /sbin/sysctl net.ipv4.tcp_tw_reuse=1 3 | /sbin/sysctl vm.max_map_count=262144 4 | -------------------------------------------------------------------------------- /dbtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PG_DB=dbtest make dbtest 3 | -------------------------------------------------------------------------------- /devel/api_com_contrib_repo_grp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "$0: please specify timestamp from as a 2nd arg" 10 | exit 2 11 | fi 12 | if [ -z "$3" ] 13 | then 14 | echo "$0: please specify timestamp to as a 3rd arg" 15 | exit 3 16 | fi 17 | if [ -z "$4" ] 18 | then 19 | echo "$0: please specify repository group as a 4th arg" 20 | exit 4 21 | fi 22 | if [ -z "$5" ] 23 | then 24 | echo "$0: please specify period as a 5th arg" 25 | exit 5 26 | fi 27 | if [ -z "$API_URL" ] 28 | then 29 | API_URL="http://127.0.0.1:8080/api/v1" 30 | fi 31 | project="${1}" 32 | from="${2}" 33 | to="${3}" 34 | repo_group="${4}" 35 | period="${5}" 36 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"ComContribRepoGrp\",\"payload\":{\"project\":\"${project}\",\"from\":\"${from}\",\"to\":\"${to}\",\"repository_group\":\"${repo_group}\",\"period\":\"${period}\"}}" 2>/dev/null | jq 37 | -------------------------------------------------------------------------------- /devel/api_com_stats_repo_grp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$API_URL" ] 3 | then 4 | API_URL="http://127.0.0.1:8080/api/v1" 5 | fi 6 | if [ -z "$1" ] 7 | then 8 | echo "$0: please specify project name as a 1st arg" 9 | exit 1 10 | fi 11 | if [ -z "$2" ] 12 | then 13 | echo "$0: please specify timestamp from as a 2nd arg" 14 | exit 2 15 | fi 16 | if [ -z "$3" ] 17 | then 18 | echo "$0: please specify timestamp to as a 3rd arg" 19 | exit 3 20 | fi 21 | project="${1}" 22 | from="${2}" 23 | to="${3}" 24 | period="${4}" 25 | metric="${5}" 26 | repository_group="${6}" 27 | companies="${7}" 28 | if [ -z "$period" ] 29 | then 30 | period='7 Days MA' 31 | fi 32 | if [ -z "$metric" ] 33 | then 34 | metric='Contributions' 35 | fi 36 | if [ -z "$repository_group" ] 37 | then 38 | repository_group='All' 39 | fi 40 | if [ -z "$companies" ] 41 | then 42 | companies='["All"]' 43 | fi 44 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"ComStatsRepoGrp\",\"payload\":{\"project\":\"${project}\",\"from\":\"${from}\",\"to\":\"${to}\",\"period\":\"${period}\",\"metric\":\"${metric}\",\"repository_group\":\"${repository_group}\",\"companies\":${companies}}}" 2>/dev/null | jq 45 | -------------------------------------------------------------------------------- /devel/api_companies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Companies\",\"payload\":{\"project\":\"${project}\"}}" 2>/dev/null | jq 13 | -------------------------------------------------------------------------------- /devel/api_companies_table.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 2 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | range="${2}" 13 | metric="${3}" 14 | if [ -z "$range" ] 15 | then 16 | range='Last decade' 17 | fi 18 | if [ -z "$metric" ] 19 | then 20 | metric='Contributions' 21 | fi 22 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"CompaniesTable\",\"payload\":{\"project\":\"${project}\",\"range\":\"${range}\",\"metric\":\"${metric}\"}}" 2>/dev/null | jq 23 | -------------------------------------------------------------------------------- /devel/api_countries.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | raw='' 13 | if [ ! -z "$2" ] 14 | then 15 | raw="${2}" 16 | fi 17 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Countries\",\"payload\":{\"project\":\"${project}\",\"raw\":\"${raw}\"}}" 2>/dev/null | jq 18 | -------------------------------------------------------------------------------- /devel/api_cumulative_counts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "$0: please specify metric as a 2nd arg" 10 | exit 2 11 | fi 12 | if [ -z "$API_URL" ] 13 | then 14 | export API_URL="http://127.0.0.1:8080/api/v1" 15 | fi 16 | if [ -z "$ORIGIN" ] 17 | then 18 | export ORIGIN='https://teststats.cncf.io' 19 | fi 20 | project="${1}" 21 | metric="${2}" 22 | if [ -z "$DEBUG" ] 23 | then 24 | curl -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"CumulativeCounts\",\"payload\":{\"project\":\"${project}\",\"metric\":\"${metric}\"}}" | jq 25 | else 26 | echo curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"CumulativeCounts\",\"payload\":{\"project\":\"${project}\",\"metric\":\"${metric}\"}}" 27 | curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"CumulativeCounts\",\"payload\":{\"project\":\"${project}\",\"metric\":\"${metric}\"}}" 28 | fi 29 | -------------------------------------------------------------------------------- /devel/api_dev_act_cnt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 2 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | range="${2}" 13 | metric="${3}" 14 | repository_group="${4}" 15 | country="${5}" 16 | github_id="${6}" 17 | if [ -z "$range" ] 18 | then 19 | range='Last decade' 20 | fi 21 | if [ -z "$metric" ] 22 | then 23 | metric='Contributions' 24 | fi 25 | if [ -z "$repository_group" ] 26 | then 27 | repository_group='All' 28 | fi 29 | if [ -z "$country" ] 30 | then 31 | country='All' 32 | fi 33 | if [ -z "$github_id" ] 34 | then 35 | github_id='' 36 | fi 37 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"DevActCnt\",\"payload\":{\"project\":\"${project}\",\"range\":\"${range}\",\"metric\":\"${metric}\",\"repository_group\":\"${repository_group}\",\"country\":\"${country}\",\"github_id\":\"${github_id}\",\"bg\":\"${BG}\"}}" 2>/dev/null | jq -rS . 38 | -------------------------------------------------------------------------------- /devel/api_dev_act_cnt_comp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 2 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | range="${2}" 13 | metric="${3}" 14 | repository_group="${4}" 15 | country="${5}" 16 | companies="${6}" 17 | github_id="${7}" 18 | if [ -z "$range" ] 19 | then 20 | range='Last decade' 21 | fi 22 | if [ -z "$metric" ] 23 | then 24 | metric='Contributions' 25 | fi 26 | if [ -z "$repository_group" ] 27 | then 28 | repository_group='All' 29 | fi 30 | if [ -z "$country" ] 31 | then 32 | country='All' 33 | fi 34 | if [ -z "$companies" ] 35 | then 36 | companies='["All"]' 37 | fi 38 | if [ -z "$github_id" ] 39 | then 40 | github_id='' 41 | fi 42 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"DevActCntComp\",\"payload\":{\"project\":\"${project}\",\"range\":\"${range}\",\"metric\":\"${metric}\",\"repository_group\":\"${repository_group}\",\"country\":\"${country}\",\"companies\":${companies},\"github_id\":\"${github_id}\",\"bg\":\"${BG}\"}}" 2>/dev/null | jq -rS . 43 | -------------------------------------------------------------------------------- /devel/api_dev_act_cnt_comp_repos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 2 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | range="${2}" 13 | metric="${3}" 14 | repository="${4}" 15 | country="${5}" 16 | companies="${6}" 17 | github_id="${7}" 18 | if [ -z "$range" ] 19 | then 20 | range='Last decade' 21 | fi 22 | if [ -z "$metric" ] 23 | then 24 | metric='Contributions' 25 | fi 26 | if [ -z "$repository" ] 27 | then 28 | echo "$0: you must specify repository" 29 | exit 3 30 | fi 31 | if [ -z "$country" ] 32 | then 33 | country='All' 34 | fi 35 | if [ -z "$companies" ] 36 | then 37 | companies='["All"]' 38 | fi 39 | if [ -z "$github_id" ] 40 | then 41 | github_id='' 42 | fi 43 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"DevActCntComp\",\"payload\":{\"project\":\"${project}\",\"range\":\"${range}\",\"metric\":\"${metric}\",\"repository\":\"${repository}\",\"country\":\"${country}\",\"companies\":${companies},\"github_id\":\"${github_id}\",\"bg\":\"${BG}\"}}" 2>/dev/null | jq -rS . 44 | -------------------------------------------------------------------------------- /devel/api_dev_act_cnt_example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 3 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' '' 4 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 5 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' '' 6 | 7 | ./devel/api_dev_act_cnt.sh kubernete 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 8 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - noww' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 9 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Eventsa' 'SIG Apps' 'United States' 'janetkuo' 10 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Appsa' 'United States' 'janetkuo' 11 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United Statesa' 'janetkuo' 12 | ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'not_exist_for_sure' 13 | 14 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernete 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 15 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - noww' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 16 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Eventsa' 'SIG Apps' 'United States' 'janetkuo' 17 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Appsa' 'United States' 'janetkuo' 18 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United Statesa' 'janetkuo' 19 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'not_exist_for_sure' 20 | 21 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernete 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 22 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - noww' 'GitHub Events' 'SIG Apps' 'United States' 'janetkuo' 23 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Eventsa' 'SIG Apps' 'United States' 'janetkuo' 24 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Appsa' 'United States' 'janetkuo' 25 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United Statesa' 'janetkuo' 26 | API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt.sh kubernetes 'v1.17.0 - now' 'GitHub Events' 'SIG Apps' 'United States' 'not_exist_for_sure' 27 | 28 | ./devel/api_dev_act_cnt_comp.sh kubernetes 'v1.17.0 - now' 'Reviews' 'SIG Apps' 'All' '["Google", "Red Hat"]' 'janetkuo' 29 | ./devel/api_dev_act_cnt_comp.sh all 'Last month' 'PRs' 'Prometheus' 'All' '["Google", "Red Hat"]' 30 | 31 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt_comp.sh kubernetes 'v1.17.0 - now' 'Reviews' 'SIG Apps' 'All' '["Google", "Red Hat"]' 'janetkuo' 32 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt_comp.sh all 'Last month' 'PRs' 'Prometheus' 'All' '["Google", "Red Hat"]' 33 | API_URL='https://devstats.cncf.io/api/v1' ./devel/api_dev_act_cnt_comp.sh all 34 | 35 | BG=1 API_URL="https://teststats.cncf.io/api/v1" ./devel/api_dev_act_cnt.sh kubernetes 'range:2021-08-10,2021-08-15' 'Contributions' 'SIG Apps' 'United States' '' 36 | BG=1 API_URL="https://teststats.cncf.io/api/v1" ./devel/api_dev_act_cnt_repos.sh kubernetes 'range:2021-08-10,2021-08-15' 'Commits' 'kubernetes/test-infra' 'United States' '' 37 | BG=1 API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt_comp.sh kubernetes 'range:2021-08-10,2021-08-15' 'Reviews' 'SIG Apps' 'All' '["Google", "Red Hat"]' '' 38 | BG=1 API_URL='https://teststats.cncf.io/api/v1' ./devel/api_dev_act_cnt_comp_repos.sh kubernetes 'range:2021-08-10,2021-08-15' 'Reviews' 'kubernetes/test-infra' 'All' '["Google", "Red Hat"]' '' 39 | -------------------------------------------------------------------------------- /devel/api_dev_act_cnt_repos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 2 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | range="${2}" 13 | metric="${3}" 14 | repository="${4}" 15 | country="${5}" 16 | github_id="${6}" 17 | if [ -z "$range" ] 18 | then 19 | range='Last decade' 20 | fi 21 | if [ -z "$metric" ] 22 | then 23 | metric='Contributions' 24 | fi 25 | if [ -z "$repository" ] 26 | then 27 | echo "$0: you must specify repository" 28 | exit 3 29 | fi 30 | if [ -z "$country" ] 31 | then 32 | country='All' 33 | fi 34 | if [ -z "$github_id" ] 35 | then 36 | github_id='' 37 | fi 38 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"DevActCnt\",\"payload\":{\"project\":\"${project}\",\"range\":\"${range}\",\"metric\":\"${metric}\",\"repository\":\"${repository}\",\"country\":\"${country}\",\"github_id\":\"${github_id}\",\"bg\":\"${BG}\"}}" 2>/dev/null | jq -rS . 39 | -------------------------------------------------------------------------------- /devel/api_events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "$0: please specify timestamp from as a 2nd arg" 10 | exit 2 11 | fi 12 | if [ -z "$3" ] 13 | then 14 | echo "$0: please specify timestamp to as a 3rd arg" 15 | exit 3 16 | fi 17 | if [ -z "$API_URL" ] 18 | then 19 | export API_URL="http://127.0.0.1:8080/api/v1" 20 | fi 21 | if [ -z "$ORIGIN" ] 22 | then 23 | export ORIGIN='https://teststats.cncf.io' 24 | fi 25 | project="${1}" 26 | from="${2}" 27 | to="${3}" 28 | if [ -z "$DEBUG" ] 29 | then 30 | curl -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Events\",\"payload\":{\"project\":\"${project}\",\"from\":\"${from}\",\"to\":\"${to}\"}}" | jq 31 | else 32 | echo curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Events\",\"payload\":{\"project\":\"${project}\",\"from\":\"${from}\",\"to\":\"${to}\"}}" 33 | curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Events\",\"payload\":{\"project\":\"${project}\",\"from\":\"${from}\",\"to\":\"${to}\"}}" 34 | fi 35 | -------------------------------------------------------------------------------- /devel/api_health.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | echo 'Note that last API call should succeed and all other should fail due to expected errors' 13 | curl http://127.0.0.1:8080/api/v1 -d"xyz" 2>/dev/null | jq 14 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"health\",\"payload\":{\"project\":\"${project}\"}}" 2>/dev/null | jq 15 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Health\",\"payloada\":{\"project\":\"${project}\"}}" 2>/dev/null | jq 16 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Health\",\"payload\":{\"projecta\":\"${project}\"}}" 2>/dev/null | jq 17 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Health\",\"payload\":{\"project\":{\"obj\":\"val\"}}}" 2>/dev/null | jq 18 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Health\",\"payload\":{\"project\":\"${project}xx\"}}" 2>/dev/null | jq 19 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Health\",\"payload\":{\"project\":\"${project}\"}}" 2>/dev/null | jq 20 | -------------------------------------------------------------------------------- /devel/api_list_apis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$API_URL" ] 3 | then 4 | API_URL="http://127.0.0.1:8080/api/v1" 5 | fi 6 | project="${1}" 7 | if [ ! -z "$RAW" ] 8 | then 9 | curl -H "Content-Type: application/json" "${API_URL}" -d'{"api":"ListAPIs"}' 10 | else 11 | curl -H "Content-Type: application/json" "${API_URL}" -d'{"api":"ListAPIs"}' 2>/dev/null | jq 12 | fi 13 | -------------------------------------------------------------------------------- /devel/api_list_projects.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$API_URL" ] 3 | then 4 | API_URL="http://127.0.0.1:8080/api/v1" 5 | fi 6 | project="${1}" 7 | if [ ! -z "$RAW" ] 8 | then 9 | curl -H "Content-Type: application/json" "${API_URL}" -d'{"api":"ListProjects"}' 10 | else 11 | curl -H "Content-Type: application/json" "${API_URL}" -d'{"api":"ListProjects"}' 2>/dev/null | jq 12 | fi 13 | -------------------------------------------------------------------------------- /devel/api_ranges.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | raw='' 13 | if [ ! -z "$2" ] 14 | then 15 | raw="${2}" 16 | fi 17 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Ranges\",\"payload\":{\"project\":\"${project}\",\"raw\":\"${raw}\"}}" 2>/dev/null | jq 18 | -------------------------------------------------------------------------------- /devel/api_repo_groups.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | raw='' 13 | if [ ! -z "$2" ] 14 | then 15 | raw="${2}" 16 | fi 17 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"RepoGroups\",\"payload\":{\"project\":\"${project}\",\"raw\":\"${raw}\"}}" 2>/dev/null | jq 18 | -------------------------------------------------------------------------------- /devel/api_repos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | project="${1}" 12 | groups='["All"]' 13 | if [ ! -z "$2" ] 14 | then 15 | # Example correct value: '["SIG Apps", "Other", "Not Specified"]' 16 | groups="${2}" 17 | fi 18 | curl -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"Repos\",\"payload\":{\"project\":\"${project}\",\"repository_group\":${groups}}}" 2>/dev/null | jq 19 | -------------------------------------------------------------------------------- /devel/api_site_stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "$0: please specify project name as a 1st arg" 5 | exit 1 6 | fi 7 | if [ -z "$API_URL" ] 8 | then 9 | export API_URL="http://127.0.0.1:8080/api/v1" 10 | fi 11 | if [ -z "$ORIGIN" ] 12 | then 13 | export ORIGIN='https://teststats.cncf.io' 14 | fi 15 | project="${1}" 16 | from="${2}" 17 | to="${3}" 18 | if [ -z "$DEBUG" ] 19 | then 20 | curl -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"SiteStats\",\"payload\":{\"project\":\"${project}\"}}" | jq 21 | else 22 | echo curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"SiteStats\",\"payload\":{\"project\":\"${project}\"}}" 23 | curl -i -s -H "Origin: ${ORIGIN}" -H "Content-Type: application/json" "${API_URL}" -d"{\"api\":\"SiteStats\",\"payload\":{\"project\":\"${project}\"}}" 24 | fi 25 | -------------------------------------------------------------------------------- /devel/cronctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOLOCK=1 skip 3 | if [ ! -z "$NOLOCK" ] 4 | then 5 | exit 0 6 | fi 7 | set -o pipefail 8 | if ([ -z "$1" ] || [ -z "$2" ]) 9 | then 10 | echo "Usage $0 command on|off" 11 | exit 1 12 | fi 13 | crontab -l > /tmp/crontab.tmp 14 | if [ "$2" = "off" ] 15 | then 16 | MODE=rr0 FROM="(?m)^([^#].*\s+$1\s+.*)$" TO='#$1' replacer /tmp/crontab.tmp > /dev/null || exit 1 17 | elif [ "$2" = "on" ] 18 | then 19 | MODE=rr0 FROM="(?m)^#(.*\s+$1\s+.*)$" TO='$1' replacer /tmp/crontab.tmp > /dev/null || exit 2 20 | else 21 | echo "Usage $0 command on|off" 22 | rm -f /tmp/crontab.tmp 23 | exit 1 24 | fi 25 | crontab /tmp/crontab.tmp || exit 3 26 | rm -f /tmp/crontab.tmp 27 | -------------------------------------------------------------------------------- /devel/db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # DBDEBUG=1 - verbose operations 3 | if [ -z "$PG_PASS" ] 4 | then 5 | echo "$0: you need to set PG_PASS=... $*" 6 | exit 1 7 | fi 8 | if [ -z "$1" ] 9 | then 10 | echo "$0 you need to specify at least one argument" 11 | exit 2 12 | fi 13 | if [ -z "$PG_HOST" ] 14 | then 15 | PG_HOST=127.0.0.1 16 | fi 17 | if [ -z "$PG_PORT" ] 18 | then 19 | PG_PORT=5432 20 | fi 21 | if [ -z "$PG_USER" ] 22 | then 23 | if [ -z "$PG_ADMIN_USER" ] 24 | then 25 | PG_USER=postgres 26 | else 27 | PG_USER="${PG_ADMIN_USER}" 28 | fi 29 | fi 30 | cmd=${1} 31 | shift 32 | if [ ! -z "$DBDEBUG" ] 33 | then 34 | echo "PGPASSWORD=... '${cmd}' -U '${PG_USER}' -h '${PG_HOST}' -p '${PG_PORT}' '${@}'" >&2 35 | fi 36 | PGPASSWORD="${PG_PASS}" "${cmd}" -U "${PG_USER}" -h "${PG_HOST}" -p "${PG_PORT}" "${@}" 37 | -------------------------------------------------------------------------------- /devel/mass_replace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Examples: 3 | # MODE=rr FROM=`cat input` TO=`cat output` FILES=`find abc/ -type f -not -iname 'something.txt'` ./devel/mass_replace.sh 4 | # MODE=ss FROM=`cat input` TO=`cat output` FILES=`ls grafana/dashboards/{all,cncf,cni,containerd,coredns,envoy,fluentd,grpc,jaeger,linkerd,kubernetes,notary,opencontainers,opentracing,prometheus,rkt,rook,tuf,vitess}/*` ./devel/mass_replace.sh 5 | # MODE=ss0 FROM=CNCF TO='[[full_name]]' FILES=`find grafana/dashboards/cncf/ -type f -not -iname dashboards.json` ./devel/mass_replace.sh 6 | # MODE=ss FROM=' "title": "' TO=' "title": "[[full_name]] ' FILES=`find grafana/dashboards/cncf/ -name "top_commenters.json" -or -name "project_statistics.json" -or -name "companies_summary.json" -or -name "prs_authors_companies_histogram.json" -or -name "developers_summary.json" -or -name "prs_authors_histogram.json"` ./devel/mass_replace.sh 7 | # MODE=rs0 FROM='(?m)^.*"uid": "\w+",\n' TO='-' replacer input.json 8 | # MODE=rr0 FROM='(?m)(^.*)"uid": "(\w+)",' TO='$1"uid": "placeholder",' replacer input.json 9 | # MODE=rr FROM='(?m);;;(.*)$' TO=';;;$1 # {{repo_groups}}' FILES=`find metrics/ -iname "gaps.yaml"` ./devel/mass_replace.sh 10 | if [ -z "${FROM}" ] 11 | then 12 | echo "You need to set FROM, example FROM=abc TO=xyz FILES='f1 f2' $0" 13 | exit 1 14 | fi 15 | if [ -z "${TO}" ] 16 | then 17 | echo "You need to set TO, example FROM=abc TO=xyz FILES='f1 f2' $0" 18 | exit 2 19 | fi 20 | if [ -z "${FILES}" ] 21 | then 22 | echo "You need to set FILES, example FROM=abc TO=xyz FILES='f1 f2' $0" 23 | exit 3 24 | fi 25 | for f in ${FILES} 26 | do 27 | replacer $f || exit 4 28 | done 29 | echo 'OK' 30 | -------------------------------------------------------------------------------- /devel/sync_lock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOLOCK=1 skip 3 | if [ ! -z "$NOLOCK" ] 4 | then 5 | exit 0 6 | fi 7 | if [ -z "$SKIPLOCK" ] 8 | then 9 | if [ -f "/tmp/deploy.wip" ] 10 | then 11 | echo "another deploy process is running, exiting" 12 | exit 1 13 | fi 14 | wait_for_command.sh 'devstats,devstats_others,devstats_kubernetes,devstats_allprj' 900 || exit 2 15 | cronctl.sh devstats off || exit 3 16 | if [ -z "$FROM_WEBHOOK" ] 17 | then 18 | wait_for_command.sh webhook 900 || exit 4 19 | cronctl.sh webhook off || exit 5 20 | killall webhook 21 | fi 22 | echo 'All sync and deploy jobs stopped and disabled' 23 | fi 24 | -------------------------------------------------------------------------------- /devel/sync_unlock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOLOCK=1 skip 3 | if [ ! -z "$NOLOCK" ] 4 | then 5 | exit 0 6 | fi 7 | if [ -z "$SKIPLOCK" ] 8 | then 9 | cronctl.sh devstats on || exit 1 10 | if [ -z "$FROM_WEBHOOK" ] 11 | then 12 | cronctl.sh webhook on || exit 3 13 | fi 14 | echo 'All sync and deploy jobs enabled' 15 | fi 16 | -------------------------------------------------------------------------------- /devel/test_webhook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Talk to local test webhook" 3 | curl -H "Content-Type: application/json" -d "@cmd/webhook/example_webhook_payload_deploy.dat" -X POST http://127.0.0.1:1986/test 4 | #curl -H "Content-Type: application/json" -d "@cmd/webhook/example_webhook_payload_no_deploy.dat" -X POST http://127.0.0.1:1986/test 5 | echo "" 6 | echo "Done" 7 | -------------------------------------------------------------------------------- /devel/wait_for_command.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # NOLOCK=1 skip 3 | if [ ! -z "$NOLOCK" ] 4 | then 5 | exit 0 6 | fi 7 | set -o pipefail 8 | 9 | if ([ -z "$1" ] || [ -z "$2" ]) 10 | then 11 | echo "Usage $0 command max_seconds" 12 | echo "Usage $0 'command1,command2,..,commandN' max_seconds" 13 | exit 1 14 | fi 15 | 16 | commands=${1//,/ } 17 | trials=0 18 | maxTrials=$2 19 | 20 | while true 21 | do 22 | info="" 23 | running=0 24 | all=0 25 | for cmd in $commands 26 | do 27 | pid="/tmp/$cmd.pid" 28 | if [ -e $pid ] 29 | then 30 | if [ -z "$info" ] 31 | then 32 | info="${cmd} is running: ${pid} exists" 33 | else 34 | info="${info}, ${cmd} is running: ${pid} exists" 35 | fi 36 | running=$((running+1)) 37 | fi 38 | all=$((all+1)) 39 | done 40 | if [ "$running" -ge "1" ] 41 | then 42 | info="${info}, ${running}/${all} running" 43 | if ( [ "$trials" -eq "0" ] || [ ! "$info" = "$lastinfo" ] ) 44 | then 45 | echo "${info}, waiting" 46 | lastinfo=$info 47 | fi 48 | sleep 1 49 | trials=$((trials+1)) 50 | if [ "$trials" -ge "$maxTrials" ] 51 | then 52 | echo "${info}, waited $maxTrials seconds, exiting" 53 | exit 1 54 | fi 55 | else 56 | break 57 | fi 58 | done 59 | 60 | s=0 61 | for cmd in $commands 62 | do 63 | s=$((s+1)) 64 | done 65 | 66 | if [ "$trials" -gt "0" ] 67 | then 68 | if [ "$s" -ge "2" ] 69 | then 70 | echo "$1 were running waited $trials seconds" 71 | else 72 | echo "$1 was running waited $trials seconds" 73 | fi 74 | else 75 | if [ "$s" -ge "2" ] 76 | then 77 | echo "$1 were not running" 78 | else 79 | echo "$1 was not running" 80 | fi 81 | fi 82 | -------------------------------------------------------------------------------- /devel/webhook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Add GHA2DB_SKIP_FULL_DEPLOY=1 to skip handling [deploy] 3 | #GHA2DB_CMDDEBUG=2 GHA2DB_DEBUG=2 GHA2DB_SKIP_VERIFY_PAYLOAD=1 GHA2DB_PROJECT_ROOT=`pwd` GHA2DB_DEPLOY_BRANCHES="master,disaster" GHA2DB_DEPLOY_STATUSES="Passed,Fixed" GHA2DB_DEPLOY_RESULTS="0" GHA2DB_DEPLOY_TYPES="push" GHA2DB_WHROOT="/test" GHA2DB_WHPORT=1986 GHA2DB_WHHOST="0.0.0.0" GET=1 webhook 4 | # No debug output 5 | GHA2DB_SKIP_VERIFY_PAYLOAD=1 GHA2DB_PROJECT_ROOT=`pwd` GHA2DB_DEPLOY_BRANCHES="master,disaster" GHA2DB_DEPLOY_STATUSES="Passed,Fixed" GHA2DB_DEPLOY_RESULTS="0" GHA2DB_DEPLOY_TYPES="push" GHA2DB_WHROOT="/test" GHA2DB_WHPORT=1986 GHA2DB_WHHOST="0.0.0.0" GET=1 webhook 6 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "sort" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | var gEnvMap map[string]string = make(map[string]string) 13 | 14 | // EnvSyncer - support auto updating env variables via "env.env" file 15 | func EnvSyncer() { 16 | for { 17 | time.Sleep(30 * time.Second) 18 | UpdateEnv(true) 19 | } 20 | } 21 | 22 | // UpdateEnv - update (eventually) env using env.env file 23 | func UpdateEnv(useLog bool) { 24 | ef, err := os.Open("env.env") 25 | if err != nil { 26 | return 27 | } 28 | defer ef.Close() 29 | sc := bufio.NewScanner(ef) 30 | for sc.Scan() { 31 | line := sc.Text() 32 | ary := strings.Split(line, "=") 33 | if len(ary) < 2 { 34 | continue 35 | } 36 | k := strings.TrimSpace(ary[0]) 37 | if k == "" { 38 | continue 39 | } 40 | // v := strings.TrimSpace(ary[1]) 41 | v := strings.TrimSpace(strings.Join(ary[1:], "=")) 42 | os.Setenv(k, v) 43 | cv, ok := gEnvMap[k] 44 | if !ok || cv != v { 45 | if useLog { 46 | if IsLogInitialized() { 47 | Printf("new environment overwrite: '%s' --> '%s'\n", k, v) 48 | } 49 | } else { 50 | fmt.Printf("new environment overwrite: '%s' --> '%s'\n", k, v) 51 | } 52 | } 53 | gEnvMap[k] = v 54 | } 55 | } 56 | 57 | // EnvReplace - replace all environment variables starting with "prefix" 58 | // with contents of variables with "suffix" added - if defined 59 | // If prefix is "DB_" and suffix is "_SRC" then: 60 | // So if there is "DB_HOST_SRC" variable defined - it will replace "DB_HOST" and so on 61 | func EnvReplace(prefix, suffix string) map[string]string { 62 | if suffix == "" { 63 | return map[string]string{} 64 | } 65 | oldEnv := make(map[string]string) 66 | var environ []string 67 | for _, e := range os.Environ() { 68 | environ = append(environ, e) 69 | } 70 | sort.Strings(environ) 71 | pLen := len(prefix) 72 | for _, e := range environ { 73 | l := pLen 74 | eLen := len(e) 75 | if l > eLen { 76 | l = eLen 77 | } 78 | if pLen == 0 || e[0:l] == prefix { 79 | pair := strings.Split(e, "=") 80 | eSuff := os.Getenv(pair[0] + suffix) 81 | if eSuff != "" { 82 | oldEnv[pair[0]] = pair[1] 83 | FatalOnError(os.Setenv(pair[0], eSuff)) 84 | } 85 | } 86 | } 87 | sLen := len(suffix) 88 | for _, e := range environ { 89 | pair := strings.Split(e, "=") 90 | eLen := len(pair[0]) 91 | lS := eLen - sLen 92 | if lS <= 0 { 93 | continue 94 | } 95 | lP := pLen 96 | if lP > eLen { 97 | lP = eLen 98 | } 99 | if (pLen == 0 || e[0:lP] == prefix) && pair[0][lS:] == suffix { 100 | eName := pair[0][:lS] 101 | _, ok := oldEnv[eName] 102 | if !ok { 103 | oldEnv[eName] = Unset 104 | FatalOnError(os.Setenv(eName, pair[1])) 105 | } 106 | } 107 | } 108 | return oldEnv 109 | } 110 | 111 | // EnvRestore - restores all environment variables given in the map 112 | func EnvRestore(env map[string]string) { 113 | for envName, envValue := range env { 114 | if envValue == Unset { 115 | FatalOnError(os.Unsetenv(envName)) 116 | } else { 117 | FatalOnError(os.Setenv(envName, envValue)) 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /env_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | lib "github.com/cncf/devstatscode" 9 | testlib "github.com/cncf/devstatscode/test" 10 | ) 11 | 12 | func TestEnv(t *testing.T) { 13 | 14 | // Test cases 15 | var testCases = []struct { 16 | name string 17 | environment map[string]string 18 | prefix string 19 | suffix string 20 | newEnvs []string 21 | expectedSave map[string]string 22 | expectedEnv map[string]string 23 | }{ 24 | { 25 | "No op", 26 | map[string]string{}, 27 | "", 28 | "", 29 | []string{}, 30 | map[string]string{}, 31 | map[string]string{}, 32 | }, 33 | { 34 | "No replaces", 35 | map[string]string{"a": "A", "c": "C", "b": "B"}, 36 | "", 37 | "", 38 | []string{}, 39 | map[string]string{}, 40 | map[string]string{"a": "A", "c": "C", "b": "B"}, 41 | }, 42 | { 43 | "No suffix", 44 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B"}, 45 | "pref_", 46 | "", 47 | []string{}, 48 | map[string]string{}, 49 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B"}, 50 | }, 51 | { 52 | "No prefix and no suffix hit", 53 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B"}, 54 | "", 55 | "_suff", 56 | []string{}, 57 | map[string]string{}, 58 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B"}, 59 | }, 60 | { 61 | "No prefix with suffix hit", 62 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B", "pref_a_suff": "D"}, 63 | "", 64 | "_suff", 65 | []string{}, 66 | map[string]string{"pref_a": "A"}, 67 | map[string]string{"pref_a": "D", "pref_c": "C", "pref_b": "B", "pref_a_suff": "D"}, 68 | }, 69 | { 70 | "Prefix and suffix", 71 | map[string]string{"pref_a": "A", "pref_c": "C", "pref_b": "B", "pref_a_suff": "D", "a": "A", "a_suff": "D"}, 72 | "pref_", 73 | "_suff", 74 | []string{}, 75 | map[string]string{"pref_a": "A"}, 76 | map[string]string{"pref_a": "D", "pref_c": "C", "pref_b": "B", "pref_a_suff": "D", "a": "A", "a_suff": "D"}, 77 | }, 78 | { 79 | "Replace all starting with 'a' with suffix 2", 80 | map[string]string{"a1": "1", "a2": "2", "b1": "3", "b2": "4", "a12": "5", "a22": "6", "b12": "7", "b22": "8"}, 81 | "a", 82 | "2", 83 | []string{}, 84 | map[string]string{"a": "{{unset}}", "a1": "1", "a2": "2"}, 85 | map[string]string{"a1": "5", "a2": "6", "b1": "3", "b2": "4", "a12": "5", "a22": "6", "b12": "7", "b22": "8"}, 86 | }, 87 | { 88 | "Need to save empty variables too", 89 | map[string]string{"a": "", "b": "B", "c": "", "a2": "1", "b2": "2", "c2": "3"}, 90 | "", 91 | "2", 92 | []string{}, 93 | map[string]string{"a": "", "b": "B", "c": ""}, 94 | map[string]string{"a": "1", "b": "2", "c": "3", "a2": "1", "b2": "2", "c2": "3"}, 95 | }, 96 | { 97 | "Replace nonexisting var", 98 | map[string]string{"pref_a_suff": "new_value"}, 99 | "pref_", 100 | "_suff", 101 | []string{"pref_a"}, 102 | map[string]string{"pref_a": "{{unset}}"}, 103 | map[string]string{"pref_a": "new_value", "pref_a_suff": "new_value"}, 104 | }, 105 | { 106 | "Crazy", 107 | map[string]string{"aa": "2", "a": "1", "aaaa": "4", "aaa": "3"}, 108 | "a", 109 | "a", 110 | []string{}, 111 | map[string]string{"a": "1", "aa": "2", "aaa": "3"}, 112 | map[string]string{"a": "2", "aa": "3", "aaa": "4", "aaaa": "4"}, 113 | }, 114 | } 115 | 116 | // Execute test cases 117 | for index, test := range testCases { 118 | // Remember initial environment 119 | currEnv := make(map[string]string) 120 | for key := range test.environment { 121 | currEnv[key] = os.Getenv(key) 122 | } 123 | 124 | // Set new environment 125 | for key, value := range test.environment { 126 | err := os.Setenv(key, value) 127 | if err != nil { 128 | t.Error(err.Error()) 129 | } 130 | } 131 | 132 | // Call EnvReplace 133 | saved := lib.EnvReplace(test.prefix, test.suffix) 134 | 135 | // Get replaced environment 136 | replacedEnv := make(map[string]string) 137 | for key := range test.environment { 138 | replacedEnv[key] = os.Getenv(key) 139 | } 140 | for _, key := range test.newEnvs { 141 | replacedEnv[key] = os.Getenv(key) 142 | } 143 | 144 | // Call EnvRestore 145 | lib.EnvRestore(saved) 146 | 147 | // Get restored environment 148 | restoredEnv := make(map[string]string) 149 | for key := range test.environment { 150 | restoredEnv[key] = os.Getenv(key) 151 | } 152 | 153 | // Remove the test environment 154 | for key := range test.environment { 155 | //err := os.Setenv(key, currEnv[key]) 156 | err := os.Unsetenv(key) 157 | if err != nil { 158 | t.Error(err.Error()) 159 | } 160 | } 161 | 162 | // Maps are not directly compareable (due to unknown key order) - need to transorm them 163 | testlib.MakeComparableMapStr(&test.environment) 164 | testlib.MakeComparableMapStr(&test.expectedSave) 165 | testlib.MakeComparableMapStr(&test.expectedEnv) 166 | testlib.MakeComparableMapStr(&saved) 167 | testlib.MakeComparableMapStr(&replacedEnv) 168 | testlib.MakeComparableMapStr(&restoredEnv) 169 | 170 | // Check if we got expected values 171 | got := fmt.Sprintf("%+v", replacedEnv) 172 | expected := fmt.Sprintf("%+v", test.expectedEnv) 173 | if got != expected { 174 | t.Errorf( 175 | "Test case number %d \"%s\"\nExpected replaced env:\n%+v\nGot:\n%+v\n", 176 | index+1, test.name, expected, got, 177 | ) 178 | } 179 | got = fmt.Sprintf("%+v", saved) 180 | expected = fmt.Sprintf("%+v", test.expectedSave) 181 | if got != expected { 182 | t.Errorf( 183 | "Test case number %d \"%s\"\nExpected saved env:\n%+v\nGot:\n%+v\n", 184 | index+1, test.name, expected, got, 185 | ) 186 | } 187 | got = fmt.Sprintf("%+v", restoredEnv) 188 | expected = fmt.Sprintf("%+v", test.environment) 189 | if got != expected { 190 | t.Errorf( 191 | "Test case number %d \"%s\"\nExpected restored env:\n%+v\nGot:\n%+v\n", 192 | index+1, test.name, expected, got, 193 | ) 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime/debug" 7 | "strings" 8 | "time" 9 | 10 | "github.com/lib/pq" 11 | ) 12 | 13 | // FatalOnError displays error message (if error present) and exits program 14 | func FatalOnError(err error) string { 15 | if err != nil { 16 | tm := time.Now() 17 | switch e := err.(type) { 18 | case *pq.Error: 19 | errName := e.Code.Name() 20 | if errName == "too_many_connections" { 21 | fmt.Fprintf(os.Stderr, "PqError: code=%s, name=%s, detail=%s\n", e.Code, errName, e.Detail) 22 | fmt.Fprintf(os.Stderr, "Warning: too many postgres connections: %+v: '%s'\n", tm, err.Error()) 23 | return Retry 24 | } else if errName == "cannot_connect_now" { 25 | fmt.Fprintf(os.Stderr, "PqError: code=%s, name=%s, detail=%s\n", e.Code, errName, e.Detail) 26 | fmt.Fprintf(os.Stderr, "Warning: DB shutting down: %+v: '%s', sleeping 15 minutes to settle\n", tm, err.Error()) 27 | time.Sleep(time.Duration(900) * time.Second) 28 | tm = time.Now() 29 | fmt.Fprintf(os.Stderr, "Warning: DB shutting down: %+v: '%s', waited 15 minutes, retrying\n", tm, err.Error()) 30 | return Reconnect 31 | } 32 | Printf("PqError: code=%s, name=%s, detail=%s\n", e.Code, errName, e.Detail) 33 | fmt.Fprintf(os.Stderr, "PqError: code=%s, name=%s, detail=%s\n", e.Code, errName, e.Detail) 34 | if os.Getenv("DURABLE_PQ") != "" && os.Getenv("DURABLE_PQ") != "0" && os.Getenv("DURABLE_PQ") != "false" { 35 | switch errName { 36 | case "program_limit_exceeded", "undefined_column", "invalid_catalog_name", "character_not_in_repertoire": 37 | Printf("%s error is not retryable, even with DURABLE_PQ\n", errName) 38 | default: 39 | fmt.Fprintf(os.Stderr, "retrying with DURABLE_PQ\n") 40 | return Reconnect 41 | } 42 | } 43 | default: 44 | fmt.Fprintf(os.Stderr, "ErrorType: %T, error: %+v\n", e, e) 45 | fmt.Fprintf(os.Stderr, "ErrorType: %T, error: %+v\n", e, e) 46 | } 47 | if strings.Contains(err.Error(), "driver: bad connection") { 48 | fmt.Fprintf(os.Stderr, "Warning: bad driver, retrying\n") 49 | return Reconnect 50 | } 51 | if strings.Contains(err.Error(), "cannot assign requested address") { 52 | fmt.Fprintf(os.Stderr, "Warning: cannot assign requested address, retrying in 15 minutes\n") 53 | time.Sleep(time.Duration(900) * time.Second) 54 | fmt.Fprintf(os.Stderr, "Warning: cannot assign requested address - waited 15 minutes, retrying\n") 55 | return Reconnect 56 | } 57 | /* 58 | if strings.Contains(err.Error(), "database is closed") { 59 | Printf("Warning: database is closed, retrying\n") 60 | return Reconnect 61 | } 62 | */ 63 | fmt.Fprintf(os.Stderr, "Error(time=%+v):\nError: '%s'\nStacktrace:\n%s\n", tm, err.Error(), string(debug.Stack())) 64 | if os.Getenv("NO_FATAL_DELAY") == "" { 65 | time.Sleep(time.Duration(60) * time.Second) 66 | } 67 | panic(fmt.Sprintf("stacktrace: %+v", err)) 68 | } 69 | return OK 70 | } 71 | 72 | // Fatalf - it will call FatalOnError using fmt.Errorf with args provided 73 | func Fatalf(f string, a ...interface{}) { 74 | FatalOnError(fmt.Errorf(f, a...)) 75 | } 76 | 77 | // FatalNoLog displays error message (if error present) and exits program, should be used for very early init state 78 | func FatalNoLog(err error) string { 79 | if err != nil { 80 | tm := time.Now() 81 | fmt.Fprintf(os.Stderr, "Error(time=%+v):\nError: '%s'\nStacktrace:\n", tm, err.Error()) 82 | time.Sleep(time.Duration(60) * time.Second) 83 | panic(fmt.Sprintf("stacktrace: %+v", err)) 84 | } 85 | return OK 86 | } 87 | -------------------------------------------------------------------------------- /exec.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // logCommand - output command and arguments 14 | func logCommand(ctx *Ctx, cmdAndArgs []string, env map[string]string) { 15 | if !ctx.ExecQuiet { 16 | Printf("Command, arguments, environment:\n%+v\n%+v\n", cmdAndArgs, env) 17 | fmt.Fprintf(os.Stdout, "Command and arguments:\n%+v\n%+v\n", cmdAndArgs, env) 18 | } 19 | } 20 | 21 | // ExecCommand - execute command given by array of strings with eventual environment map 22 | func ExecCommand(ctx *Ctx, cmdAndArgs []string, env map[string]string) (string, error) { 23 | // Execution time 24 | dtStart := time.Now() 25 | 26 | // STDOUT pipe size 27 | pipeSize := 0x100 28 | 29 | // Command & arguments 30 | command := cmdAndArgs[0] 31 | arguments := cmdAndArgs[1:] 32 | if ctx.CmdDebug > 0 { 33 | var args []string 34 | for _, arg := range cmdAndArgs { 35 | argLen := len(arg) 36 | if argLen > 0x200 { 37 | arg = arg[0:0x100] + "..." + arg[argLen-0x100:argLen] 38 | } 39 | if strings.Contains(arg, " ") { 40 | args = append(args, "'"+arg+"'") 41 | } else { 42 | args = append(args, arg) 43 | } 44 | } 45 | Printf("%s\n", strings.Join(args, " ")) 46 | } 47 | cmd := exec.Command(command, arguments...) 48 | 49 | // Environment setup (if any) 50 | if len(env) > 0 { 51 | newEnv := os.Environ() 52 | for key, value := range env { 53 | newEnv = append(newEnv, key+"="+value) 54 | } 55 | cmd.Env = newEnv 56 | if ctx.CmdDebug > 0 { 57 | Printf("Environment Override: %+v\n", env) 58 | if ctx.CmdDebug > 2 { 59 | Printf("Full Environment: %+v\n", newEnv) 60 | } 61 | } 62 | } 63 | 64 | // Capture STDOUT (non buffered - all at once when command finishes), only used on error and when no buffered/piped version used 65 | // Which means it is used on error when CmdDebug <= 1 66 | // In CmdDebug > 1 mode, we're displaying STDOUT during execution, and storing results to 'outputStr' 67 | // Capture STDERR (non buffered - all at once when command finishes) 68 | var ( 69 | stdOut bytes.Buffer 70 | stdErr bytes.Buffer 71 | outputStr string 72 | ) 73 | cmd.Stderr = &stdErr 74 | if ctx.CmdDebug <= 1 { 75 | cmd.Stdout = &stdOut 76 | } 77 | 78 | // Pipe command's STDOUT during execution (if CmdDebug > 1) 79 | // Or just starts command when no STDOUT debug 80 | if ctx.CmdDebug > 1 { 81 | stdOutPipe, e := cmd.StdoutPipe() 82 | if e != nil { 83 | logCommand(ctx, cmdAndArgs, env) 84 | if ctx.ExecFatal { 85 | FatalOnError(e) 86 | } else { 87 | return "", e 88 | } 89 | } 90 | e = cmd.Start() 91 | if e != nil { 92 | logCommand(ctx, cmdAndArgs, env) 93 | if ctx.ExecFatal { 94 | FatalOnError(e) 95 | } else { 96 | return "", e 97 | } 98 | } 99 | buffer := make([]byte, pipeSize, pipeSize) 100 | nBytes, e := stdOutPipe.Read(buffer) 101 | for e == nil && nBytes > 0 { 102 | Printf("%s", buffer[:nBytes]) 103 | outputStr += string(buffer[:nBytes]) 104 | nBytes, e = stdOutPipe.Read(buffer) 105 | } 106 | if e != io.EOF { 107 | logCommand(ctx, cmdAndArgs, env) 108 | if ctx.ExecFatal { 109 | FatalOnError(e) 110 | } else { 111 | return "", e 112 | } 113 | } 114 | } else { 115 | e := cmd.Start() 116 | if e != nil { 117 | logCommand(ctx, cmdAndArgs, env) 118 | if ctx.ExecFatal { 119 | FatalOnError(e) 120 | } else { 121 | return "", e 122 | } 123 | } 124 | } 125 | // Wait for command to finish 126 | err := cmd.Wait() 127 | 128 | // If error - then output STDOUT, STDERR and error info 129 | if err != nil { 130 | if ctx.CmdDebug <= 1 { 131 | outStr := stdOut.String() 132 | if len(outStr) > 0 && !ctx.ExecQuiet { 133 | Printf("%v\n", outStr) 134 | } 135 | } 136 | errStr := stdErr.String() 137 | if len(errStr) > 0 && !ctx.ExecQuiet { 138 | Printf("STDERR:\n%v\n", errStr) 139 | } 140 | if err != nil { 141 | logCommand(ctx, cmdAndArgs, env) 142 | if ctx.ExecFatal { 143 | FatalOnError(err) 144 | } else { 145 | return stdOut.String(), err 146 | } 147 | } 148 | } 149 | 150 | // If CmdDebug > 1 display STDERR contents as well (if any) 151 | if ctx.CmdDebug > 1 { 152 | errStr := stdErr.String() 153 | if len(errStr) > 0 { 154 | Printf("Errors:\n%v\n", errStr) 155 | } 156 | } 157 | if ctx.CmdDebug > 0 { 158 | info := strings.Join(cmdAndArgs, " ") 159 | lenInfo := len(info) 160 | if lenInfo > 0x280 { 161 | info = info[0:0x140] + "..." + info[lenInfo-0x140:lenInfo] 162 | } 163 | dtEnd := time.Now() 164 | Printf("%s ... %+v\n", info, dtEnd.Sub(dtStart)) 165 | } 166 | outStr := "" 167 | if ctx.ExecOutput { 168 | if ctx.CmdDebug <= 1 { 169 | outStr = stdOut.String() 170 | } else { 171 | outStr = outputStr 172 | } 173 | } 174 | return outStr, nil 175 | } 176 | -------------------------------------------------------------------------------- /find.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "You need to provide path as first arument" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "You need to provide file name pattern as a second argument" 10 | exit 1 11 | fi 12 | if [ -z "$3" ] 13 | then 14 | echo "You need to provide regexp pattern to search for as a third argument" 15 | exit 1 16 | fi 17 | find "$1" -type f -iname "$2" -not -name "out" -not -path '*.git/*' -exec grep -EHIn "$3" "{}" \; | tee -a out 18 | -------------------------------------------------------------------------------- /find_and_replace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$FROM" ] 3 | then 4 | export FROM="`cat ./FROM`" 5 | fi 6 | if [ -z "$TO" ] 7 | then 8 | export TO="`cat ./TO`" 9 | fi 10 | if [ -z "$MODE" ] 11 | then 12 | export MODE=ss0 13 | fi 14 | if [ -z "$FILES" ] 15 | then 16 | if [ -z "$1" ] 17 | then 18 | echo "You need to provide path as first arument" 19 | exit 1 20 | fi 21 | if [ -z "$2" ] 22 | then 23 | echo "You need to provide file name pattern as a second argument" 24 | exit 2 25 | fi 26 | if [ -z "$3" ] 27 | then 28 | echo "You need to provide regexp pattern to search for as a third argument" 29 | exit 3 30 | fi 31 | find "$1" -type f -iname "$2" -not -name "out" -not -path '*.git/*' -exec grep -El "$3" "{}" \; > out 32 | export FILES=`cat out` 33 | fi 34 | if [ -z "$DRY" ] 35 | then 36 | ./devel/mass_replace.sh 37 | else 38 | echo "from: '$FROM'" 39 | echo "to: '$TO'" 40 | echo "mode: '$MODE'" 41 | echo "files: '$FILES'" 42 | fi 43 | -------------------------------------------------------------------------------- /for_each_go_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for f in `find . -type f -iname "*.go" -not -path "./vendor/*"` 3 | do 4 | $1 "$f" || exit 1 5 | done 6 | exit 0 7 | -------------------------------------------------------------------------------- /git/git_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "Arguments required: path sha, none given" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "Arguments required: path sha, only path given" 10 | exit 2 11 | fi 12 | 13 | cd "$1" || exit 3 14 | git show -s --format=%ct "$2" || exit 4 15 | #files=`git diff-tree --no-commit-id --name-only -M8 -m -r "$2"` || exit 5 16 | #files=`git diff-tree --no-commit-id --name-only -r "$2"` || exit 5 17 | files=`git diff-tree --no-commit-id --name-only -M7 -r "$2"` || exit 5 18 | for file in $files 19 | do 20 | file_and_size=`git ls-tree -r -l "$2" "$file" | awk '{print $5 "♂♀" $4}'` 21 | if [ -z "$file_and_size" ] 22 | then 23 | echo "$file♂♀-1" 24 | else 25 | echo "$file_and_size" 26 | fi 27 | done 28 | -------------------------------------------------------------------------------- /git/git_loc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "Arguments required: path sha, none given" 5 | exit 1 6 | fi 7 | if [ -z "$2" ] 8 | then 9 | echo "Arguments required: path sha, only path given" 10 | exit 2 11 | fi 12 | 13 | cd "$1" || exit 3 14 | output=`git show "$2" --shortstat --oneline` 15 | if [ ! "$?" = "0" ] 16 | then 17 | exit 4 18 | fi 19 | output=`echo "$output" | tail -1` 20 | echo $output 21 | -------------------------------------------------------------------------------- /git/git_reset_pull.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "Argument required: path to call git-reset and the git-pull" 5 | exit 1 6 | fi 7 | 8 | cd "$1" || exit 2 9 | git fetch origin || exit 3 10 | git reset --hard origin/master || exit 4 11 | git pull || exit 5 12 | -------------------------------------------------------------------------------- /git/git_tags.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "Argument required: repo path" 5 | exit 1 6 | fi 7 | 8 | cd "$1" || exit 3 9 | git tag -l --format="%(refname:short)♂♀%(creatordate:unix)♂♀%(subject)" 10 | -------------------------------------------------------------------------------- /git/last_tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ -z "$1" ] 3 | then 4 | echo "Argument required: repo path" 5 | exit 1 6 | fi 7 | 8 | cd "$1" || exit 2 9 | git describe --abbrev=0 --tags || echo "-" 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cncf/devstatscode 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/google/go-github/v38 v38.1.0 7 | github.com/json-iterator/go v1.1.12 8 | github.com/lib/pq v1.10.9 9 | github.com/mattn/go-sqlite3 v1.14.22 10 | github.com/rs/cors v1.11.0 11 | golang.org/x/oauth2 v0.20.0 12 | golang.org/x/text v0.22.0 13 | gopkg.in/yaml.v2 v2.4.0 14 | ) 15 | 16 | require ( 17 | github.com/google/go-querystring v1.0.0 // indirect 18 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 19 | github.com/modern-go/reflect2 v1.0.2 // indirect 20 | golang.org/x/crypto v0.35.0 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /hash.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "fmt" 5 | "hash/fnv" 6 | "strconv" 7 | ) 8 | 9 | // HashStrings - returns unique Hash for strings array 10 | // This value is supposed to be used as ID (negative) to mark it was artificially generated 11 | func HashStrings(strs []string) int { 12 | h := fnv.New64a() 13 | s := "" 14 | for _, str := range strs { 15 | s += str 16 | } 17 | _, _ = h.Write([]byte(s)) 18 | res := int(h.Sum64()) 19 | if res > 0 { 20 | res *= -1 21 | } 22 | if res == -0x8000000000000000 { 23 | return HashStrings(append(strs, "a")) 24 | } 25 | return res 26 | } 27 | 28 | // HashObject takes map[string]interface{} and keys from []string and returns hash string 29 | // from given keys from map 30 | func HashObject(iv map[string]interface{}, keys []string) string { 31 | h := fnv.New64a() 32 | s := "" 33 | for _, key := range keys { 34 | v, ok := iv[key] 35 | if !ok { 36 | Fatalf("HashObject: %+v missing %s key", iv, key) 37 | } 38 | s += fmt.Sprintf("%v", v) 39 | } 40 | _, _ = h.Write([]byte(s)) 41 | return strconv.FormatUint(h.Sum64(), 36) 42 | } 43 | 44 | // HashArray takes []interface{} and returns hash string 45 | // from given keys from map 46 | func HashArray(ia []interface{}) string { 47 | h := fnv.New64a() 48 | s := "" 49 | for _, iv := range ia { 50 | s += fmt.Sprintf("%v", iv) 51 | } 52 | _, _ = h.Write([]byte(s)) 53 | return strconv.FormatUint(h.Sum64(), 36) 54 | } 55 | -------------------------------------------------------------------------------- /io.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | ) 7 | 8 | // ReadFile tries to read any filename, but have a fallback 9 | // it attempts to replace current project name with shared: /proj/ -> /shared/ 10 | // This is to allow reading files that can be shared between projects 11 | func ReadFile(ctx *Ctx, path string) ([]byte, error) { 12 | data, err := ioutil.ReadFile(path) 13 | if err == nil || ctx.Project == "" { 14 | if ctx.Debug > 0 { 15 | Printf("lib.ReadFile('%s'): ok\n", path) 16 | } 17 | return data, err 18 | } 19 | path = strings.Replace(path, "/"+ctx.Project+"/", "/shared/", -1) 20 | data, err = ioutil.ReadFile(path) 21 | if err == nil && ctx.Debug > 0 { 22 | Printf("lib.ReadFile('%s'): ok", path) 23 | } 24 | if err != nil { 25 | Printf("lib.ReadFile('%s'): error: %+v", path, err) 26 | } 27 | return data, err 28 | } 29 | -------------------------------------------------------------------------------- /json.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | jsoniter "github.com/json-iterator/go" 7 | ) 8 | 9 | // PrettyPrintJSON - pretty formats raw JSON bytes 10 | func PrettyPrintJSON(jsonBytes []byte) []byte { 11 | var jsonObj interface{} 12 | FatalOnError(jsoniter.Unmarshal(jsonBytes, &jsonObj)) 13 | pretty, err := jsoniter.MarshalIndent(jsonObj, "", " ") 14 | FatalOnError(err) 15 | return pretty 16 | } 17 | 18 | // ObjectToJSON - serialize given object as JSON 19 | func ObjectToJSON(obj interface{}, fn string) { 20 | jsonBytes, err := jsoniter.Marshal(obj) 21 | FatalOnError(err) 22 | pretty := PrettyPrintJSON(jsonBytes) 23 | FatalOnError(ioutil.WriteFile(fn, pretty, 0644)) 24 | } 25 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // Holds data needed to make DB calls 13 | type logContext struct { 14 | ctx Ctx 15 | con *sql.DB 16 | prog string 17 | proj string 18 | runDt time.Time 19 | } 20 | 21 | // This is the *only* global variable used in entire toolset. 22 | // I want to save passing context and DB to all Printf(...) calls. 23 | // This variable is initialized *only* once, and must be guared by the mutex 24 | // to avoid initializing it from multiple go routines 25 | var ( 26 | logCtx *logContext 27 | logCtxMutex sync.RWMutex 28 | logOnce sync.Once 29 | logInitialized = false 30 | logInitMtx sync.Mutex 31 | BuildStamp = "None" 32 | GitHash = "None" 33 | HostName = "None" 34 | GoVersion = "None" 35 | ) 36 | 37 | // Returns new context when not yet created 38 | func newLogContext() *logContext { 39 | var ctx Ctx 40 | ctx.Init() 41 | ctx.PgDB = Devstats 42 | con := PgConn(&ctx) 43 | progSplit := strings.Split(os.Args[0], "/") 44 | prog := progSplit[len(progSplit)-1] 45 | now := time.Now() 46 | if ctx.Debug >= 0 { 47 | info := fmt.Sprintf("Compiled %s, commit: %s on %s using %s", BuildStamp, GitHash, HostName, GoVersion) 48 | fmt.Printf("%s\n", info) 49 | _, _ = ExecSQL( 50 | con, 51 | &ctx, 52 | "insert into gha_logs(prog, proj, run_dt, msg) "+NValues(4), 53 | prog, 54 | ctx.Project, 55 | now, 56 | info, 57 | ) 58 | } 59 | defer func() { 60 | logInitMtx.Lock() 61 | logInitialized = true 62 | logInitMtx.Unlock() 63 | }() 64 | return &logContext{ 65 | ctx: ctx, 66 | con: con, 67 | prog: prog, 68 | proj: ctx.Project, 69 | runDt: now, 70 | } 71 | } 72 | 73 | // logToDB writes message to database 74 | func logToDB(format string, args ...interface{}) (err error) { 75 | logCtxMutex.RLock() 76 | defer func() { logCtxMutex.RUnlock() }() 77 | if logCtx.ctx.LogToDB == false { 78 | return 79 | } 80 | msg := strings.Trim(fmt.Sprintf(format, args...), " \t\n\r") 81 | _, err = ExecSQL( 82 | logCtx.con, 83 | &logCtx.ctx, 84 | "insert into gha_logs(prog, proj, run_dt, msg) "+NValues(4), 85 | logCtx.prog, 86 | logCtx.proj, 87 | logCtx.runDt, 88 | msg, 89 | ) 90 | return 91 | } 92 | 93 | // Printf is a wrapper around Printf(...) that supports logging. 94 | func Printf(format string, args ...interface{}) (n int, err error) { 95 | // Initialize context once 96 | logOnce.Do(func() { logCtx = newLogContext() }) 97 | // Avoid query out on adding to logs itself 98 | // it would print any text with its particular logs DB insert which 99 | // would result in stdout mess 100 | logCtxMutex.Lock() 101 | qOut := logCtx.ctx.QOut 102 | logCtx.ctx.QOut = false 103 | logCtxMutex.Unlock() 104 | defer func() { 105 | logCtxMutex.Lock() 106 | logCtx.ctx.QOut = qOut 107 | logCtxMutex.Unlock() 108 | }() 109 | 110 | // Actual logging to stdout & DB 111 | if logCtx.ctx.LogTime { 112 | n, err = fmt.Printf("%s %s/%s: "+format, append([]interface{}{ToYMDHMSDate(time.Now()), logCtx.proj, logCtx.prog}, args...)...) 113 | } else { 114 | n, err = fmt.Printf(format, args...) 115 | } 116 | err = logToDB(format, args...) 117 | return 118 | } 119 | 120 | // IsLogInitialized - check if log is initialized 121 | func IsLogInitialized() bool { 122 | logInitMtx.Lock() 123 | defer logInitMtx.Unlock() 124 | return logInitialized 125 | } 126 | 127 | // ClearDBLogs clears logs older by defined period (in context.go) 128 | // It clears logs on `devstats` database 129 | func ClearDBLogs() { 130 | // Environment context parse 131 | var ctx Ctx 132 | ctx.Init() 133 | 134 | // Point to logs database 135 | ctx.PgDB = Devstats 136 | 137 | // Connect to DB 138 | c := PgConn(&ctx) 139 | defer func() { _ = c.Close() }() 140 | 141 | // Clear logs older that defined period 142 | if !ctx.SkipPDB { 143 | fmt.Printf("Clearing old DB logs.\n") 144 | ExecSQLWithErr(c, &ctx, "delete from gha_logs where dt < now() - '"+ctx.ClearDBPeriod+"'::interval") 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /map.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | ) 7 | 8 | // SkipEmpty - skip one element arrays contining only empty string 9 | // This is what strings.Split() returns for empty input 10 | // We expect empty array or empty map returned in such cases 11 | func SkipEmpty(arr []string) []string { 12 | if len(arr) != 1 || len(arr) == 1 && arr[0] != "" { 13 | return arr 14 | } 15 | return []string{} 16 | } 17 | 18 | // StringsMapToArray this is a function that calls given function for all array items and returns array of items processed by this func 19 | // Example call: lib.StringsMapToArray(func(x string) string { return strings.TrimSpace(x) }, []string{" a", " b ", "c "}) 20 | func StringsMapToArray(f func(string) string, strArr []string) []string { 21 | strArr = SkipEmpty(strArr) 22 | outArr := make([]string, len(strArr)) 23 | for index, str := range strArr { 24 | outArr[index] = f(str) 25 | } 26 | return outArr 27 | } 28 | 29 | // StringsMapToSet this is a function that calls given function for all array items and returns set of items processed by this func 30 | // Example call: lib.StringsMapToSet(func(x string) string { return strings.TrimSpace(x) }, []string{" a", " b ", "c "}) 31 | func StringsMapToSet(f func(string) string, strArr []string) map[string]struct{} { 32 | strArr = SkipEmpty(strArr) 33 | outSet := make(map[string]struct{}) 34 | for _, str := range strArr { 35 | outSet[f(str)] = struct{}{} 36 | } 37 | return outSet 38 | } 39 | 40 | // StringsSetKeys - returns all keys from string map 41 | func StringsSetKeys(set map[string]struct{}) []string { 42 | outArr := make([]string, len(set)) 43 | index := 0 44 | for key := range set { 45 | outArr[index] = key 46 | index++ 47 | } 48 | sort.Strings(outArr) 49 | return outArr 50 | } 51 | 52 | // MapFromString - returns maps from string formatted as map[k1:v1 k2:v2 k3:v3 ...] 53 | func MapFromString(str string) map[string]string { 54 | if len(str) < 6 { 55 | return nil 56 | } 57 | l := len(str) 58 | if str[:4] != "map[" || str[l-1:l] != "]" { 59 | return nil 60 | } 61 | str = str[4 : l-1] 62 | ary := strings.Split(str, " ") 63 | var res map[string]string 64 | for _, data := range ary { 65 | d := strings.Split(data, ":") 66 | if len(d) == 2 { 67 | if res == nil { 68 | res = make(map[string]string) 69 | } 70 | res[d[0]] = d[1] 71 | } 72 | } 73 | return res 74 | } 75 | -------------------------------------------------------------------------------- /map_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | lib "github.com/cncf/devstatscode" 9 | testlib "github.com/cncf/devstatscode/test" 10 | ) 11 | 12 | func TestMapFromString(t *testing.T) { 13 | // Test cases 14 | var testCases = []struct { 15 | value string 16 | expected map[string]string 17 | }{ 18 | {value: "", expected: nil}, 19 | {value: "map[]", expected: nil}, 20 | {value: "map[:]", expected: map[string]string{"": ""}}, 21 | {value: "map[a:]", expected: map[string]string{"a": ""}}, 22 | {value: "map[:b]", expected: map[string]string{"": "b"}}, 23 | {value: "map[a:b]", expected: map[string]string{"a": "b"}}, 24 | {value: "map[a:b c:d", expected: nil}, 25 | {value: "map a:b c:d]", expected: nil}, 26 | {value: "map[a:b c:d]", expected: map[string]string{"a": "b", "c": "d"}}, 27 | } 28 | // Execute test cases 29 | for index, test := range testCases { 30 | got := lib.MapFromString(test.value) 31 | expected := test.expected 32 | testlib.MakeComparableMapStr(&expected) 33 | oriGot := got 34 | testlib.MakeComparableMapStr(&got) 35 | gotS := fmt.Sprintf("%+v", got) 36 | expectedS := fmt.Sprintf("%+v", expected) 37 | if gotS != expectedS { 38 | t.Errorf( 39 | "test number %d, expected %v got %v, test: %v", 40 | index+1, test.expected, oriGot, test, 41 | ) 42 | } 43 | } 44 | } 45 | 46 | func TestSkipEmpty(t *testing.T) { 47 | // Test cases 48 | var testCases = []struct { 49 | values []string 50 | expected []string 51 | }{ 52 | {values: []string{}, expected: []string{}}, 53 | {values: []string{""}, expected: []string{}}, 54 | {values: []string{" "}, expected: []string{" "}}, 55 | {values: []string{"a"}, expected: []string{"a"}}, 56 | {values: []string{"", ""}, expected: []string{"", ""}}, 57 | {values: []string{"", "a"}, expected: []string{"", "a"}}, 58 | {values: []string{"a", "b"}, expected: []string{"a", "b"}}, 59 | } 60 | // Execute test cases 61 | for index, test := range testCases { 62 | got := lib.SkipEmpty(test.values) 63 | if !testlib.CompareStringSlices(got, test.expected) { 64 | t.Errorf( 65 | "test number %d, expected %v length %d, got %v length %d", 66 | index+1, test.expected, len(test.expected), got, len(got), 67 | ) 68 | } 69 | } 70 | } 71 | 72 | func TestStringsMapToArray(t *testing.T) { 73 | // Test cases 74 | toLower := func(in string) string { 75 | return strings.ToLower(in) 76 | } 77 | var testCases = []struct { 78 | values []string 79 | function func(string) string 80 | expected []string 81 | }{ 82 | { 83 | values: []string{}, 84 | function: toLower, 85 | expected: []string{}, 86 | }, 87 | { 88 | values: []string{"A"}, 89 | function: toLower, 90 | expected: []string{"a"}, 91 | }, 92 | { 93 | values: []string{"A", "b", "Cd"}, 94 | function: toLower, 95 | expected: []string{"a", "b", "cd"}, 96 | }, 97 | } 98 | // Execute test cases 99 | for index, test := range testCases { 100 | got := lib.StringsMapToArray(test.function, test.values) 101 | if !testlib.CompareStringSlices(got, test.expected) { 102 | t.Errorf( 103 | "test number %d, expected %v, got %v", 104 | index+1, test.expected, got, 105 | ) 106 | } 107 | } 108 | } 109 | 110 | func TestStringsMapToSet(t *testing.T) { 111 | // Test cases 112 | stripFunc := func(x string) string { 113 | return strings.TrimSpace(x) 114 | } 115 | var testCases = []struct { 116 | values []string 117 | function func(string) string 118 | expected map[string]struct{} 119 | }{ 120 | { 121 | values: []string{}, 122 | function: stripFunc, 123 | expected: map[string]struct{}{}, 124 | }, 125 | { 126 | values: []string{" a\n\t"}, 127 | function: stripFunc, 128 | expected: map[string]struct{}{"a": {}}, 129 | }, 130 | { 131 | values: []string{"a ", " b", "\tc\t", "d e"}, 132 | function: stripFunc, 133 | expected: map[string]struct{}{ 134 | "a": {}, 135 | "b": {}, 136 | "c": {}, 137 | "d e": {}, 138 | }, 139 | }, 140 | } 141 | // Execute test cases 142 | for index, test := range testCases { 143 | got := lib.StringsMapToSet(test.function, test.values) 144 | if !testlib.CompareSets(got, test.expected) { 145 | t.Errorf( 146 | "test number %d, expected %v, got %v", 147 | index+1, test.expected, got, 148 | ) 149 | } 150 | } 151 | } 152 | 153 | func TestStringsSetKeys(t *testing.T) { 154 | // Test cases 155 | var testCases = []struct { 156 | set map[string]struct{} 157 | expected []string 158 | }{ 159 | { 160 | set: map[string]struct{}{}, 161 | expected: []string{}, 162 | }, 163 | { 164 | set: map[string]struct{}{"xyz": {}}, 165 | expected: []string{"xyz"}, 166 | }, 167 | { 168 | set: map[string]struct{}{"b": {}, "a": {}, "c": {}}, 169 | expected: []string{"a", "b", "c"}, 170 | }, 171 | } 172 | // Execute test cases 173 | for index, test := range testCases { 174 | got := lib.StringsSetKeys(test.set) 175 | if !testlib.CompareStringSlices(got, test.expected) { 176 | t.Errorf( 177 | "test number %d, expected %v, got %v", 178 | index+1, test.expected, got, 179 | ) 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /mgetc.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // Mgetc waits for single key press and return character pressed 8 | func Mgetc(ctx *Ctx) string { 9 | if ctx.Mgetc != "" { 10 | return ctx.Mgetc 11 | } 12 | b := make([]byte, 1) 13 | _, err := os.Stdin.Read(b) 14 | FatalOnError(err) 15 | return string(b) 16 | } 17 | -------------------------------------------------------------------------------- /mgetc_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "testing" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | ) 8 | 9 | func TestMgetc(t *testing.T) { 10 | // Environment context parse 11 | var ctx lib.Ctx 12 | ctx.Init() 13 | ctx.TestMode = true 14 | 15 | // Set context's Mgetc manually (don't need to repeat tests from context_test.go) 16 | ctx.Mgetc = "y" 17 | 18 | expected := "y" 19 | got := lib.Mgetc(&ctx) 20 | if got != expected { 21 | t.Errorf("expected %v, got %v", expected, got) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pg_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "database/sql" 5 | "testing" 6 | "time" 7 | 8 | lib "github.com/cncf/devstatscode" 9 | testlib "github.com/cncf/devstatscode/test" 10 | ) 11 | 12 | func TestCleanUTF8(t *testing.T) { 13 | // Test cases 14 | var testCases = []struct { 15 | value string 16 | expected string 17 | }{ 18 | {value: "value", expected: "value"}, 19 | {value: "val\x00ue", expected: "value"}, 20 | {value: "val\u0000ue", expected: "value"}, 21 | {value: "v\x00a\U00000000l\u0000ue", expected: "value"}, 22 | {value: "平仮名, ひらがな", expected: "平仮名, ひらがな"}, 23 | {value: "\u0000平仮名\x00ひらがな\U00000000", expected: "平仮名ひらがな"}, 24 | } 25 | // Execute test cases 26 | for index, test := range testCases { 27 | got := lib.CleanUTF8(test.value) 28 | if got != test.expected { 29 | t.Errorf("test number %d, expected %v, got %v", index+1, test.expected, got) 30 | } 31 | } 32 | } 33 | 34 | func TestTruncToBytes(t *testing.T) { 35 | // Test cases 36 | var testCases = []struct { 37 | value string 38 | n int 39 | expectedStr string 40 | expectedLen int 41 | }{ 42 | { 43 | value: "value", 44 | n: 3, 45 | expectedStr: "val", 46 | expectedLen: 3, 47 | }, 48 | { 49 | value: "平仮名, ひらがな", 50 | n: 6, 51 | expectedStr: "平仮", 52 | expectedLen: 6, 53 | }, 54 | { 55 | value: "平仮名, ひらがな", 56 | n: 8, 57 | expectedStr: "平仮", 58 | expectedLen: 6, 59 | }, 60 | { 61 | value: "平仮名, ひらがな", 62 | n: 9, 63 | expectedStr: "平仮名", 64 | expectedLen: 9, 65 | }, 66 | { 67 | value: "\u0000平仮名, \x00ひら\U00000000がな", 68 | n: 9, 69 | expectedStr: "平仮名", 70 | expectedLen: 9, 71 | }, 72 | } 73 | // Execute test cases 74 | for index, test := range testCases { 75 | gotStr := lib.TruncToBytes(test.value, test.n) 76 | if gotStr != test.expectedStr { 77 | t.Errorf("test number %d, expected string %v, got %v", index+1, test.expectedStr, gotStr) 78 | } 79 | gotLen := len(gotStr) 80 | if gotLen != test.expectedLen { 81 | t.Errorf("test number %d, expected length %v, got %v", index+1, test.expectedLen, gotLen) 82 | } 83 | } 84 | } 85 | 86 | func TestTruncStringOrNil(t *testing.T) { 87 | // Test cases 88 | sValues := []string{"value", "平仮名, ひらがな", "\u0000平仮名, \x00ひら\U00000000がな"} 89 | var testCases = []struct { 90 | value *string 91 | n int 92 | expected interface{} 93 | }{ 94 | { 95 | value: nil, 96 | n: 10, 97 | expected: nil, 98 | }, 99 | { 100 | value: &sValues[0], 101 | n: 3, 102 | expected: "val", 103 | }, 104 | { 105 | value: &sValues[1], 106 | n: 6, 107 | expected: "平仮", 108 | }, 109 | { 110 | value: &sValues[2], 111 | n: 9, 112 | expected: "平仮名", 113 | }, 114 | } 115 | // Execute test cases 116 | for index, test := range testCases { 117 | got := lib.TruncStringOrNil(test.value, test.n) 118 | if got != test.expected { 119 | t.Errorf("test number %d, expected %v, got %v", index+1, test.expected, got) 120 | } 121 | } 122 | } 123 | 124 | func TestBoolOrNil(t *testing.T) { 125 | result := lib.BoolOrNil(nil) 126 | if result != nil { 127 | t.Errorf("test nil case: expected , got %v", result) 128 | } 129 | val := true 130 | result = lib.BoolOrNil(&val) 131 | if result != val { 132 | t.Errorf("expected true, got %v", result) 133 | } 134 | } 135 | 136 | func TestNegatedBoolOrNil(t *testing.T) { 137 | result := lib.NegatedBoolOrNil(nil) 138 | if result != nil { 139 | t.Errorf("test nil case: expected , got %v", result) 140 | } 141 | val := true 142 | result = lib.NegatedBoolOrNil(&val) 143 | expected := !val 144 | if result != expected { 145 | t.Errorf("expected %v, got %v", expected, result) 146 | } 147 | } 148 | 149 | func TestTimeOrNil(t *testing.T) { 150 | result := lib.TimeOrNil(nil) 151 | if result != nil { 152 | t.Errorf("test nil case: expected , got %v", result) 153 | } 154 | val := time.Now() 155 | result = lib.TimeOrNil(&val) 156 | if result != val { 157 | t.Errorf("expected %v, got %v", val, result) 158 | } 159 | } 160 | 161 | func TestIntOrNil(t *testing.T) { 162 | result := lib.IntOrNil(nil) 163 | if result != nil { 164 | t.Errorf("test nil case: expected , got %v", result) 165 | } 166 | val := 2 167 | result = lib.IntOrNil(&val) 168 | if result != val { 169 | t.Errorf("expected %v, got %v", val, result) 170 | } 171 | } 172 | 173 | func TestFirstIntOrNil(t *testing.T) { 174 | nn1 := 1 175 | nn2 := 2 176 | var testCases = []struct { 177 | array []*int 178 | expected interface{} 179 | }{ 180 | {array: []*int{}, expected: nil}, 181 | {array: []*int{nil}, expected: nil}, 182 | {array: []*int{&nn1}, expected: nn1}, 183 | {array: []*int{nil, nil}, expected: nil}, 184 | {array: []*int{nil, &nn1}, expected: nn1}, 185 | {array: []*int{&nn1, nil}, expected: nn1}, 186 | {array: []*int{&nn1, &nn2}, expected: nn1}, 187 | {array: []*int{&nn2, &nn1}, expected: nn2}, 188 | {array: []*int{nil, &nn2, &nn1}, expected: nn2}, 189 | } 190 | // Execute test cases 191 | for index, test := range testCases { 192 | got := lib.FirstIntOrNil(test.array) 193 | if got != test.expected { 194 | t.Errorf("test number %d, expected %v, got %v", index+1, test.expected, got) 195 | } 196 | } 197 | } 198 | 199 | func TestStringOrNil(t *testing.T) { 200 | result := lib.StringOrNil(nil) 201 | if result != nil { 202 | t.Errorf("test nil case: expected , got %v", result) 203 | } 204 | val := "hello\x00 world" 205 | expected := "hello world" 206 | result = lib.StringOrNil(&val) 207 | if result != expected { 208 | t.Errorf("expected %v, got %v", val, result) 209 | } 210 | } 211 | 212 | func TestPostgres(t *testing.T) { 213 | // Environment context parse 214 | var ctx lib.Ctx 215 | ctx.Init() 216 | ctx.TestMode = true 217 | 218 | // Do not allow to run tests in "gha" database 219 | if ctx.PgDB != "dbtest" { 220 | t.Errorf("tests can only be run on \"dbtest\" database") 221 | return 222 | } 223 | 224 | // Drop database if exists 225 | lib.DropDatabaseIfExists(&ctx) 226 | 227 | // Create database if needed 228 | createdDatabase := lib.CreateDatabaseIfNeeded(&ctx) 229 | if !createdDatabase { 230 | t.Errorf("failed to create database \"%s\"", ctx.PgDB) 231 | } 232 | 233 | // Drop database after tests 234 | defer func() { 235 | // Drop database after tests 236 | lib.DropDatabaseIfExists(&ctx) 237 | }() 238 | 239 | // Connect to Postgres DB 240 | c := lib.PgConn(&ctx) 241 | defer func() { lib.FatalOnError(c.Close()) }() 242 | 243 | // Create example table 244 | lib.ExecSQLWithErr( 245 | c, 246 | &ctx, 247 | lib.CreateTable( 248 | "test(an_int int, a_string text, a_dt {{ts}}, primary key(an_int))", 249 | ), 250 | ) 251 | 252 | // Insert single row 253 | lib.ExecSQLWithErr( 254 | c, 255 | &ctx, 256 | "insert into test(an_int, a_string, a_dt) "+lib.NValues(3), 257 | lib.AnyArray{1, "string", time.Now()}..., 258 | ) 259 | 260 | // Get inserted int 261 | i := 0 262 | lib.FatalOnError(lib.QueryRowSQL(c, &ctx, "select an_int from test").Scan(&i)) 263 | if i != 1 { 264 | t.Errorf("expected to insert 1, got %v", i) 265 | } 266 | 267 | // Insert another row 268 | lib.ExecSQLWithErr( 269 | c, 270 | &ctx, 271 | "insert into test(an_int, a_string, a_dt) "+lib.NValues(3), 272 | lib.AnyArray{11, "another string", time.Now()}..., 273 | ) 274 | 275 | // Get all ints from database 276 | gotArr := getInts(c, &ctx) 277 | 278 | expectedArr := []int{1, 11} 279 | if !testlib.CompareIntSlices(gotArr, expectedArr) { 280 | t.Errorf("expected %v after two inserts, got %v", expectedArr, gotArr) 281 | } 282 | 283 | // Start transaction 284 | tx, err := c.Begin() 285 | if err != nil { 286 | t.Error(err.Error()) 287 | } 288 | 289 | // Insert another row 290 | lib.ExecSQLTxWithErr( 291 | tx, 292 | &ctx, 293 | "insert into test(an_int, a_string, a_dt) "+lib.NValues(3), 294 | lib.AnyArray{21, "this will be rolled back", time.Now()}..., 295 | ) 296 | 297 | // Rollback transaction 298 | lib.FatalOnError(tx.Rollback()) 299 | 300 | // Get all ints from database 301 | gotArr = getInts(c, &ctx) 302 | 303 | if !testlib.CompareIntSlices(gotArr, expectedArr) { 304 | t.Errorf("expected %v after rollback, got %v", expectedArr, gotArr) 305 | } 306 | 307 | // Start transaction 308 | tx, err = c.Begin() 309 | if err != nil { 310 | t.Error(err.Error()) 311 | } 312 | 313 | // Insert another row 314 | lib.ExecSQLTxWithErr( 315 | tx, 316 | &ctx, 317 | "insert into test(an_int, a_string, a_dt) "+lib.NValues(3), 318 | lib.AnyArray{31, "this will be committed", time.Now()}..., 319 | ) 320 | 321 | // Commit transaction 322 | lib.FatalOnError(tx.Commit()) 323 | 324 | // Get all ints from database 325 | gotArr = getInts(c, &ctx) 326 | 327 | expectedArr = []int{1, 11, 31} 328 | if !testlib.CompareIntSlices(gotArr, expectedArr) { 329 | t.Errorf("expected %v after commit, got %v", expectedArr, gotArr) 330 | } 331 | 332 | // Insert ignore row (that violetes primary key constraint) 333 | lib.ExecSQLWithErr( 334 | c, 335 | &ctx, 336 | lib.InsertIgnore("into test(an_int, a_string, a_dt) "+lib.NValues(3)), 337 | lib.AnyArray{1, "conflicting key", time.Now()}..., 338 | ) 339 | 340 | // Get all ints from database 341 | gotArr = getInts(c, &ctx) 342 | 343 | if !testlib.CompareIntSlices(gotArr, expectedArr) { 344 | t.Errorf("expected %v after insert ignore, got %v", expectedArr, gotArr) 345 | } 346 | } 347 | 348 | // getInts - gets all ints from database, sorted 349 | func getInts(c *sql.DB, ctx *lib.Ctx) []int { 350 | // Get inserted values 351 | rows := lib.QuerySQLWithErr(c, ctx, "select an_int from test order by an_int asc") 352 | defer func() { lib.FatalOnError(rows.Close()) }() 353 | 354 | var ( 355 | i int 356 | arr []int 357 | ) 358 | for rows.Next() { 359 | lib.FatalOnError(rows.Scan(&i)) 360 | arr = append(arr, i) 361 | } 362 | lib.FatalOnError(rows.Err()) 363 | return arr 364 | } 365 | -------------------------------------------------------------------------------- /signal.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "path/filepath" 7 | "strings" 8 | "syscall" 9 | "time" 10 | ) 11 | 12 | // FinishAfterTimeout - finish program 'prog' after given timeout 'seconds', exit with 'status' code 13 | func FinishAfterTimeout(prog string, seconds, status int) { 14 | time.Sleep(time.Duration(seconds) * time.Second) 15 | Printf("Program '%s' reached timeout after %d seconds, sending signal to exit %d\n", prog, seconds, status) 16 | err := syscall.Kill(syscall.Getpid(), syscall.SIGALRM) 17 | if err != nil { 18 | Printf("Error: %+v sending '%s' timeout signal after %d seconds, exiting %d status\n", err, prog, seconds, status) 19 | os.Exit(status) 20 | return 21 | } 22 | Printf("Program '%s': sent timeout signal after %d seconds, requesting %d exit status\n", prog, seconds, status) 23 | } 24 | 25 | // SetupTimeoutSignal - if GHA2DB_MAX_RUN_DURATION contains configuration for 'prog' 26 | // Then it is given as "...,prog:duration:exit_status:,..." - it means that the 'prog' 27 | // can only run 'duration' seconds, and after that time it receives timeout, logs it 28 | // and exists with 'exit_status' 29 | func SetupTimeoutSignal(ctx *Ctx) { 30 | prog := filepath.Base(os.Args[0]) 31 | ary := strings.Split(prog, ".") 32 | lAry := len(ary) 33 | if lAry > 1 { 34 | prog = strings.Join(ary[:lAry-1], ".") 35 | } 36 | if ctx.MaxRunDuration == nil { 37 | return 38 | } 39 | data, ok := ctx.MaxRunDuration[prog] 40 | if !ok { 41 | return 42 | } 43 | seconds, status := data[0], data[1] 44 | if data[0] <= 0 { 45 | return 46 | } 47 | go FinishAfterTimeout(prog, seconds, status) 48 | sigs := make(chan os.Signal, 1) 49 | signal.Notify(sigs, syscall.SIGALRM) 50 | go func() { 51 | for { 52 | sig := <-sigs 53 | Printf("Program '%s': timeout %v after %d seconds, will exit with %d code\n", prog, sig, seconds, status) 54 | os.Exit(status) 55 | } 56 | }() 57 | Printf("Program '%s': timeout handler installed: exit %d after %d seconds\n", prog, status, seconds) 58 | } 59 | -------------------------------------------------------------------------------- /splitcrons.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # value a|b means a in weekly mode, b in monthly mode 3 | # MONTHLY=1 (use 4-weeks - 28 days schedule instead of weekly one). 4 | # KUBERNETES_HOURS=24|36 (reserve this amount of time for Kubernetes project sync [3,30|48]). 5 | # ALL_HOURS=20|36 (reserve this amount of time for All CNCF project sync [3,30|48]). 6 | # GHA_OFFSET=4 (start at HH:04, to ensure GHA archives are already saved [2,10]). 7 | # SYNC_HOURS=2 (ensure syncing projects every 2 hours, only 1, 2 and 3 values are supported) 8 | # OFFSET_HOURS=-4 (we assume half of weekend is Sun 3 AM, and assume USA tz -7 (3-7=-4), [-84,84]) 9 | # ALWAYS_PATCH=1 (skip checking for difference and always call kubectl patch) 10 | # NEVER_PATCH=1 (do not execute kubectl patch - preview/dry mode) 11 | # ONLY_ENV=1 (only patch CJs env variables) 12 | # SKIP_AFFS_ENV=1 (skip patching env for affiliations cron jobs) 13 | # SKIP_SYNC_ENV=1 (skip patching env for affiliations cron jobs) 14 | # PATCH_ENV='AffSkipTemp,MaxHist,SkipAffsLock,AffsLockDB,NoDurable,DurablePQ,MaxRunDuration,SkipGHAPI,SkipGetRepos,NCPUs' 15 | # ONLY_SUSPEND=1 (only process suspend data) 16 | # SUSPEND_ALL=1 (suspend all cronjobs) 17 | # NO_SUSPEND_H=1 (do not process (un)suspend for hourly sync crons 18 | # NO_SUSPEND_A=1 (do not process (un)suspend for affiliations crons 19 | # DEBUG=1 - more verbose output 20 | # Examples: 21 | # ./splitcrons ../devstats-helm/devstats-helm/values.yaml new-values.yaml 22 | # MONTHLY=1 DEBUG=1 ./splitcrons devstats-helm/values.yaml new-values.yaml 23 | # ALWAYS_PATCH=1 MONTHLY=1 DEBUG=1 ./splitcronsa devstats-helm/values.yaml new-values.yaml 24 | # PATCH_ENV=NCPUs ALWAYS_PATCH=1 MONTHLY=1 DEBUG=1 ./splitcronsa devstats-helm/values.yaml new-values.yaml 25 | # PATCH_ENV=NCPUs ALWAYS_PATCH=1 MONTHLY=1 DEBUG=1 ./splitcrons devstats-helm/values.yaml new-values.yaml 26 | # MONTHLY=1 ONLY_SUSPEND=1 ./splitcrons devstats-helm/values.yaml new-values.yaml 27 | ./splitcrons devstats-helm/values.yaml new-values.yaml && echo "Now update devstats-helm/values.yaml with new-values.yaml" 28 | -------------------------------------------------------------------------------- /string.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/csv" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "math/rand" 11 | "os" 12 | "reflect" 13 | "regexp" 14 | "strconv" 15 | "strings" 16 | "sync" 17 | ) 18 | 19 | // PrepareQuickRangeQuery Perpares query using either ready `period` string or using `from` and `to` strings 20 | // Values to replace are specially encoded {{period:alias.column}} 21 | // Can either replace with: (alias.column >= now() - 'period'::interval) 22 | // Or (alias.column >= 'from' and alias.column < 'to') 23 | func PrepareQuickRangeQuery(sql, period, from, to string) (string, string) { 24 | start := 0 25 | startPatt := "{{period:" 26 | startPattLen := len(startPatt) 27 | endPatt := "}}" 28 | endPattLen := len(endPatt) 29 | lenSQL := len(sql) 30 | res := "" 31 | sHours := "0" 32 | periodMode := false 33 | if period != "" { 34 | periodMode = true 35 | sHours = IntervalHours(period) 36 | } else { 37 | if from != "" && to != "" { 38 | tFrom := TimeParseAny(from) 39 | tTo := TimeParseAny(to) 40 | from = ToYMDHMSDate(tFrom) 41 | to = ToYMDHMSDate(tTo) 42 | sHours = RangeHours(tFrom, tTo) 43 | } 44 | } 45 | for { 46 | idx1 := strings.Index(sql[start:], startPatt) 47 | if idx1 == -1 { 48 | break 49 | } 50 | idx2 := strings.Index(sql[start+idx1:], endPatt) 51 | col := sql[start+idx1+startPattLen : start+idx1+idx2] 52 | res += sql[start : start+idx1] 53 | if periodMode { 54 | res += " (" + col + " >= now() - '" + period + "'::interval) " 55 | } else { 56 | if from == "" || to == "" { 57 | return "You need to provide either non-empty `period` or non empty `from` and `to`", sHours 58 | } 59 | res += " (" + col + " >= '" + from + "' and " + col + " < '" + to + "') " 60 | } 61 | start += idx1 + idx2 + endPattLen 62 | } 63 | res += sql[start:lenSQL] 64 | if periodMode { 65 | res = strings.Replace(res, "{{from}}", "(now() -'"+period+"'::interval)", -1) 66 | res = strings.Replace(res, "{{to}}", "(now())", -1) 67 | } else { 68 | res = strings.Replace(res, "{{from}}", "'"+from+"'", -1) 69 | res = strings.Replace(res, "{{to}}", "'"+to+"'", -1) 70 | } 71 | return res, sHours 72 | } 73 | 74 | // SafeUTF8String - make sure string is UTF-8 valid 75 | func SafeUTF8String(input string) string { 76 | return string(bytes.ToValidUTF8([]byte(strings.Replace(input, "\x00", "", -1)), []byte(""))) 77 | } 78 | 79 | // Slugify replace all whitespace with "-", remove all non-word letters downcase 80 | func Slugify(arg string) string { 81 | re := regexp.MustCompile(`[^\w-]+`) 82 | arg = re.ReplaceAllLiteralString(arg, "-") 83 | return strings.ToLower(arg) 84 | } 85 | 86 | // GetHidden - return list of shas to replace 87 | func GetHidden(ctx *Ctx, configFile string) (shaMap map[string]string) { 88 | shaMap = make(map[string]string) 89 | f, err := os.Open(configFile) 90 | if err != nil { 91 | f, err = os.Open(ctx.DataDir + "/" + configFile) 92 | } 93 | if err == nil { 94 | defer func() { _ = f.Close() }() 95 | reader := csv.NewReader(f) 96 | for { 97 | row, err := reader.Read() 98 | if err == io.EOF { 99 | break 100 | } else if err != nil { 101 | FatalOnError(err) 102 | } 103 | sha := row[0] 104 | if sha == "sha1" { 105 | continue 106 | } 107 | shaMap[sha] = "anon-" + sha 108 | } 109 | } 110 | return 111 | } 112 | 113 | // MaybeHideFunc - use closure as a data storage 114 | func MaybeHideFunc(shas map[string]string) (f func(string) string) { 115 | cache := make(map[string]string) 116 | f = func(arg string) string { 117 | var sha string 118 | sha, ok := cache[arg] 119 | if !ok { 120 | hash := sha1.New() 121 | _, err := hash.Write([]byte(arg)) 122 | FatalOnError(err) 123 | sha = hex.EncodeToString(hash.Sum(nil)) 124 | cache[arg] = sha 125 | } 126 | anon, ok := shas[sha] 127 | if ok { 128 | return anon 129 | } 130 | return arg 131 | } 132 | return f 133 | } 134 | 135 | // MaybeHideFuncTS - use closure as a data storage - thread safe 136 | func MaybeHideFuncTS(shas map[string]string) (f func(string) string) { 137 | cache := make(map[string]string) 138 | mtx := &sync.RWMutex{} 139 | smtx := &sync.Mutex{} 140 | f = func(arg string) string { 141 | var sha string 142 | mtx.RLock() 143 | sha, ok := cache[arg] 144 | mtx.RUnlock() 145 | if !ok { 146 | hash := sha1.New() 147 | _, err := hash.Write([]byte(arg)) 148 | FatalOnError(err) 149 | sha = hex.EncodeToString(hash.Sum(nil)) 150 | mtx.Lock() 151 | cache[arg] = sha 152 | mtx.Unlock() 153 | } 154 | smtx.Lock() 155 | anon, ok := shas[sha] 156 | smtx.Unlock() 157 | if ok { 158 | return anon 159 | } 160 | return arg 161 | } 162 | return f 163 | } 164 | 165 | // RandString - return random string 166 | func RandString() string { 167 | return fmt.Sprintf("%x", rand.Uint64()) 168 | } 169 | 170 | // FormatRawBytes - format []uint8 string 171 | func FormatRawBytes(rawB []uint8) string { 172 | raw := fmt.Sprintf("%v", reflect.ValueOf(rawB)) 173 | op := strings.Index(raw, "[") + 1 174 | cl := strings.Index(raw, "]") 175 | ary := strings.Split(raw[op:cl], " ") 176 | formatted := "" 177 | for _, s := range ary { 178 | b, _ := strconv.ParseInt(s, 10, 32) 179 | formatted += fmt.Sprintf("%02x", b) 180 | } 181 | return fmt.Sprintf("%T(%d)", rawB, len(rawB)) + ":" + formatted + ":" + fmt.Sprintf("%+v", rawB) 182 | } 183 | 184 | // FormatRawInterface - format raw string that is probably value or pointer to either []uint8 or sql.RawBytes 185 | func FormatRawInterface(rawI interface{}) string { 186 | raw := fmt.Sprintf("%v", reflect.ValueOf(rawI)) 187 | op := strings.Index(raw, "[") + 1 188 | cl := strings.Index(raw, "]") 189 | ary := strings.Split(raw[op:cl], " ") 190 | formatted := "" 191 | for _, s := range ary { 192 | b, _ := strconv.ParseInt(s, 10, 32) 193 | formatted += fmt.Sprintf("%02x", b) 194 | } 195 | return fmt.Sprintf("%T", rawI) + ":" + formatted + ":" + fmt.Sprintf("%+v", rawI) 196 | } 197 | -------------------------------------------------------------------------------- /string_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "testing" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | ) 8 | 9 | func TestSafeUTF8String(t *testing.T) { 10 | // Test cases 11 | var testCases = []struct { 12 | before string 13 | after string 14 | }{ 15 | { 16 | before: "A b\x00C", 17 | after: "A bC", 18 | }, 19 | { 20 | before: string([]byte{0x41, 0x42, 0xff, 0xfe, 0x43}), 21 | after: string([]byte{0x41, 0x42, 0x43}), 22 | }, 23 | { 24 | before: string([]byte{0x00, 0x41, 0x42, 0xff, 0xff, 0x43, 0x00}), 25 | after: string([]byte{0x41, 0x42, 0x43}), 26 | }, 27 | } 28 | // Execute test cases 29 | for index, test := range testCases { 30 | after := lib.SafeUTF8String(test.before) 31 | if after != test.after { 32 | t.Errorf( 33 | "test number %d, expected '%v', got '%v'", 34 | index+1, test.after, after, 35 | ) 36 | } 37 | } 38 | } 39 | 40 | func TestSlugify(t *testing.T) { 41 | // Test cases 42 | var testCases = []struct { 43 | before string 44 | after string 45 | }{ 46 | { 47 | before: "A b C", 48 | after: "a-b-c", 49 | }, 50 | { 51 | before: "Hello, world\t bye", 52 | after: "hello-world-bye", 53 | }, 54 | { 55 | before: "Activity Repo Groups", 56 | after: "activity-repo-groups", 57 | }, 58 | { 59 | before: "Open issues/PRs", 60 | after: "open-issues-prs", 61 | }, 62 | } 63 | // Execute test cases 64 | for index, test := range testCases { 65 | after := lib.Slugify(test.before) 66 | if after != test.after { 67 | t.Errorf( 68 | "test number %d, expected '%v', got '%v'", 69 | index+1, test.after, after, 70 | ) 71 | } 72 | } 73 | } 74 | 75 | func TestMaybeHideFunc(t *testing.T) { 76 | // Test cases 77 | var testCases = []struct { 78 | shas map[string]string 79 | args []string 80 | results []string 81 | }{ 82 | { 83 | shas: map[string]string{ 84 | "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8": "anon-86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", 85 | "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98": "anon-e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98", 86 | "84a516841ba77a5b4648de2cd0dfcb30ea46dbb4": "anon-84a516841ba77a5b4648de2cd0dfcb30ea46dbb4", 87 | }, 88 | args: []string{ 89 | "a", 90 | "a", 91 | "b", 92 | "d", 93 | "c", 94 | "e", 95 | "a", 96 | "x", 97 | }, 98 | results: []string{ 99 | "anon-86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", 100 | "anon-86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", 101 | "anon-e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98", 102 | "d", 103 | "anon-84a516841ba77a5b4648de2cd0dfcb30ea46dbb4", 104 | "e", 105 | "anon-86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", 106 | "x", 107 | }, 108 | }, 109 | { 110 | shas: map[string]string{}, 111 | args: []string{ 112 | "a", 113 | "b", 114 | "c", 115 | }, 116 | results: []string{ 117 | "a", 118 | "b", 119 | "c", 120 | }, 121 | }, 122 | } 123 | // Execute test cases 124 | for index, test := range testCases { 125 | f := lib.MaybeHideFunc(test.shas) 126 | for i, arg := range test.args { 127 | res := f(arg) 128 | if res != test.results[i] { 129 | t.Errorf( 130 | "test number %d:%d, expected '%v', got '%v'", 131 | index+1, i+1, test.results[i], res, 132 | ) 133 | } 134 | } 135 | } 136 | } 137 | 138 | func TestPrepareQuickRangeQuery(t *testing.T) { 139 | // Test cases 140 | var testCases = []struct { 141 | sql string 142 | period string 143 | from string 144 | to string 145 | expected string 146 | hours string 147 | }{ 148 | { 149 | sql: "simplest period {{period:a}} case", 150 | period: "", 151 | from: "", 152 | to: "", 153 | expected: "You need to provide either non-empty `period` or non empty `from` and `to`", 154 | hours: "0", 155 | }, 156 | { 157 | sql: "simplest no-period case", 158 | period: "", 159 | from: "", 160 | to: "", 161 | expected: "simplest no-period case", 162 | hours: "0", 163 | }, 164 | { 165 | sql: "simplest no-period case", 166 | period: "1 month", 167 | from: "", 168 | to: "", 169 | expected: "simplest no-period case", 170 | hours: "730.500000", 171 | }, 172 | { 173 | sql: "simplest no-period case", 174 | period: "0 month", 175 | from: "", 176 | to: "", 177 | expected: "simplest no-period case", 178 | hours: "0.000000", 179 | }, 180 | { 181 | sql: "simplest no-period case", 182 | period: "-3 days", 183 | from: "", 184 | to: "", 185 | expected: "simplest no-period case", 186 | hours: "0.000000", 187 | }, 188 | { 189 | sql: "simplest no-period case", 190 | period: "", 191 | from: "2010-01-01 12:00:00", 192 | to: "2010-01-01 12:00:00", 193 | expected: "simplest no-period case", 194 | hours: "0", 195 | }, 196 | { 197 | sql: "simplest no-period case", 198 | period: "", 199 | from: "2010-01-01 12:00:00", 200 | to: "2010-01-01 13:00:00", 201 | expected: "simplest no-period case", 202 | hours: "1.000000", 203 | }, 204 | { 205 | sql: "simplest period {{period:a}} case", 206 | period: "1 day", 207 | from: "", 208 | to: "", 209 | expected: "simplest period (a >= now() - '1 day'::interval) case", 210 | hours: "24.000000", 211 | }, 212 | { 213 | sql: "simplest period {{period:a}} case", 214 | period: "", 215 | from: "2010-01-01 12:00:00", 216 | to: "2015-02-02 13:00:00", 217 | expected: "simplest period (a >= '2010-01-01 12:00:00' and a < '2015-02-02 13:00:00') case", 218 | hours: "44593.000000", 219 | }, 220 | { 221 | sql: "simplest period {{period:a}} case", 222 | period: "1 week", 223 | from: "2010-01-01 12:00:00", 224 | to: "2015-02-02 13:00:00", 225 | expected: "simplest period (a >= now() - '1 week'::interval) case", 226 | hours: "168.000000", 227 | }, 228 | { 229 | sql: "{{period:a.b.c}}{{period:c.d.e}}", 230 | period: "1 day", 231 | from: "", 232 | to: "", 233 | expected: " (a.b.c >= now() - '1 day'::interval) (c.d.e >= now() - '1 day'::interval) ", 234 | hours: "24.000000", 235 | }, 236 | { 237 | sql: "{{period:a.b.c}}{{period:c.d.e}}", 238 | period: "10 days", 239 | from: "", 240 | to: "", 241 | expected: " (a.b.c >= now() - '10 days'::interval) (c.d.e >= now() - '10 days'::interval) ", 242 | hours: "240.000000", 243 | }, 244 | { 245 | sql: "{{period:a.b.c}}{{period:c.d.e}}", 246 | period: "", 247 | from: "2015", 248 | to: "2016", 249 | expected: " (a.b.c >= '2015-01-01 00:00:00' and a.b.c < '2016-01-01 00:00:00') (c.d.e >= '2015-01-01 00:00:00' and c.d.e < '2016-01-01 00:00:00') ", 250 | hours: "8760.000000", 251 | }, 252 | { 253 | sql: "and ({{period:a.b.c}} and x is null) or {{period:c.d.e}}", 254 | period: "3 months", 255 | from: "", 256 | to: "", 257 | expected: "and ( (a.b.c >= now() - '3 months'::interval) and x is null) or (c.d.e >= now() - '3 months'::interval) ", 258 | hours: "2191.500000", 259 | }, 260 | { 261 | sql: "and ({{period:a.b.c}} and x is null) or {{period:c.d.e}}", 262 | period: "", 263 | from: "1982-07-16", 264 | to: "2017-12", 265 | expected: "and ( (a.b.c >= '1982-07-16 00:00:00' and a.b.c < '2017-12-01 00:00:00') and x is null) or (c.d.e >= '1982-07-16 00:00:00' and c.d.e < '2017-12-01 00:00:00') ", 266 | hours: "310128.000000", 267 | }, 268 | { 269 | sql: "and ({{period:a.b.c}} and x is null) or {{period:c.d.e}} and {{from}} - {{to}}", 270 | period: "", 271 | from: "1982-07-16", 272 | to: "2017-12", 273 | expected: "and ( (a.b.c >= '1982-07-16 00:00:00' and a.b.c < '2017-12-01 00:00:00') and x is null) or (c.d.e >= '1982-07-16 00:00:00' and c.d.e < '2017-12-01 00:00:00') and '1982-07-16 00:00:00' - '2017-12-01 00:00:00'", 274 | hours: "310128.000000", 275 | }, 276 | { 277 | sql: "and ({{period:a.b.c}} and x is null) or {{period:c.d.e}} and {{from}} or {{to}}", 278 | period: "3 months", 279 | from: "", 280 | to: "", 281 | expected: "and ( (a.b.c >= now() - '3 months'::interval) and x is null) or (c.d.e >= now() - '3 months'::interval) and (now() -'3 months'::interval) or (now())", 282 | hours: "2191.500000", 283 | }, 284 | } 285 | // Execute test cases 286 | for index, test := range testCases { 287 | expected := test.expected 288 | expectedHours := test.hours 289 | got, gotHours := lib.PrepareQuickRangeQuery(test.sql, test.period, test.from, test.to) 290 | if got != expected || gotHours != expectedHours { 291 | t.Errorf( 292 | "test number %d, expected '%v'/'%v', got '%v'/'%v'", 293 | index+1, expected, expectedHours, got, gotHours, 294 | ) 295 | } 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /tags.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | // Tags contain list of TSDB tags 12 | type Tags struct { 13 | Tags []Tag `yaml:"tags"` 14 | } 15 | 16 | // Tag contain each TSDB tag data 17 | type Tag struct { 18 | Name string `yaml:"name"` 19 | SQLFile string `yaml:"sql"` 20 | SeriesName string `yaml:"series_name"` 21 | NameTag string `yaml:"name_tag"` 22 | ValueTag string `yaml:"value_tag"` 23 | OtherTags map[string][2]string `yaml:"other_tags"` 24 | Limit int `yaml:"limit"` 25 | Disabled bool `yaml:"disabled"` 26 | } 27 | 28 | // ProcessTag - insert given Tag into Postgres TSDB 29 | func ProcessTag(con *sql.DB, ctx *Ctx, tg *Tag, replaces [][]string) { 30 | // Batch TS points 31 | var pts TSPoints 32 | 33 | // Skip disabled tags 34 | if tg.Disabled && !ctx.TestMode { 35 | return 36 | } 37 | 38 | // Local or cron mode 39 | dataPrefix := ctx.DataDir 40 | if ctx.Local { 41 | dataPrefix = "./" 42 | } 43 | 44 | // Per project directory for SQL files 45 | dir := Metrics 46 | if ctx.Project != "" { 47 | dir += ctx.Project + "/" 48 | } 49 | 50 | // Read SQL file 51 | bytes, err := ReadFile(ctx, dataPrefix+dir+tg.SQLFile+".sql") 52 | FatalOnError(err) 53 | sqlQuery := string(bytes) 54 | 55 | // Handle excluding bots 56 | bytes, err = ReadFile(ctx, dataPrefix+"util_sql/exclude_bots.sql") 57 | FatalOnError(err) 58 | excludeBots := string(bytes) 59 | 60 | // Transform SQL 61 | limit := tg.Limit 62 | if limit <= 0 { 63 | limit = 255 64 | } 65 | sqlQuery = strings.Replace(sqlQuery, "{{lim}}", strconv.Itoa(limit), -1) 66 | sqlQuery = strings.Replace(sqlQuery, "{{exclude_bots}}", excludeBots, -1) 67 | 68 | // Replaces 69 | for _, replace := range replaces { 70 | if len(replace) != 2 { 71 | FatalOnError(fmt.Errorf("replace(s) should have length 2, invalid: %+v", replace)) 72 | } 73 | sqlQuery = strings.Replace(sqlQuery, replace[0], replace[1], -1) 74 | } 75 | 76 | // Execute SQL 77 | rows := QuerySQLWithErr(con, ctx, sqlQuery) 78 | defer func() { FatalOnError(rows.Close()) }() 79 | 80 | // Drop current tags 81 | if !ctx.SkipTSDB { 82 | table := "t" + tg.SeriesName 83 | if TableExists(con, ctx, table) { 84 | ExecSQLWithErr(con, ctx, "truncate "+table) 85 | } 86 | } 87 | tm := TimeParseAny("2012-07-01") 88 | 89 | // Columns 90 | columns, err := rows.Columns() 91 | FatalOnError(err) 92 | colIdx := make(map[string]int) 93 | for i, column := range columns { 94 | colIdx[column] = i 95 | } 96 | 97 | // Iterate tag values 98 | tags := make(map[string]string) 99 | iVals := make([]interface{}, len(columns)) 100 | for i := range columns { 101 | iVals[i] = new([]byte) 102 | } 103 | got := false 104 | for rows.Next() { 105 | got = true 106 | FatalOnError(rows.Scan(iVals...)) 107 | sVals := []string{} 108 | for _, iVal := range iVals { 109 | sVal := "" 110 | if iVal != nil { 111 | sVal = string(*iVal.(*[]byte)) 112 | } 113 | sVals = append(sVals, sVal) 114 | } 115 | strVal := sVals[0] 116 | if tg.NameTag != "" { 117 | tags[tg.NameTag] = strVal 118 | } 119 | if tg.ValueTag != "" { 120 | tags[tg.ValueTag] = NormalizeName(strVal) 121 | } 122 | if tg.OtherTags != nil { 123 | for tName, tData := range tg.OtherTags { 124 | tValue := tData[0] 125 | cIdx, ok := colIdx[tValue] 126 | if !ok { 127 | Fatalf("other tag: name: %s: column %s not found", tName, tValue) 128 | } 129 | tags[tName] = sVals[cIdx] 130 | tNorm := strings.ToLower(tData[1]) 131 | if tNorm == "1" || tNorm == "t" || tNorm == "y" { 132 | tags[tName+"_norm"] = NormalizeName(sVals[cIdx]) 133 | } 134 | } 135 | } 136 | if ctx.Debug > 0 { 137 | Printf("'%s': %+v\n", tg.SeriesName, tags) 138 | } 139 | // Add batch point 140 | pt := NewTSPoint(ctx, tg.SeriesName, "", tags, nil, tm, false) 141 | AddTSPoint(ctx, &pts, pt) 142 | tm = tm.Add(time.Hour) 143 | } 144 | FatalOnError(rows.Err()) 145 | if !got { 146 | Printf("Warning: Tag '%+v' have no values\n", tg) 147 | } 148 | 149 | // Write the batch 150 | if !ctx.SkipTSDB { 151 | WriteTSPoints(ctx, con, &pts, "", []uint8{}, nil) 152 | } else if ctx.Debug > 0 { 153 | Printf("Skipping tags series write\n") 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /test/compare.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "regexp" 7 | "sort" 8 | ) 9 | 10 | var dotZero = regexp.MustCompile(`(\d+)(\.000+)`) 11 | 12 | // CompareIntSlices - compares two int slices 13 | func CompareIntSlices(s1 []int, s2 []int) bool { 14 | if len(s1) != len(s2) { 15 | return false 16 | } 17 | for index, value := range s1 { 18 | if value != s2[index] { 19 | return false 20 | } 21 | } 22 | return true 23 | } 24 | 25 | // CompareStringSlices - compares two string slices 26 | func CompareStringSlices(s1 []string, s2 []string) bool { 27 | if len(s1) != len(s2) { 28 | return false 29 | } 30 | for index, value := range s1 { 31 | if value != s2[index] { 32 | return false 33 | } 34 | } 35 | return true 36 | } 37 | 38 | // CompareSlices - compares two any type slices 39 | func CompareSlices(s1 []interface{}, s2 []interface{}) bool { 40 | if len(s1) != len(s2) { 41 | fmt.Printf("CompareSlices: len: %d != %d\n", len(s1), len(s2)) 42 | return false 43 | } 44 | for index, value := range s1 { 45 | // fmt.Printf("types: (%+v,%T) <=> (%+v,%T)\n", value, value, s2[index], s2[index]) 46 | if value != s2[index] { 47 | v1s := fmt.Sprintf("%+v", value) 48 | v2s := fmt.Sprintf("%+v", s2[index]) 49 | v1 := dotZero.ReplaceAllString(v1s, `$1`) 50 | v2 := dotZero.ReplaceAllString(v2s, `$1`) 51 | if v1 != v2 { 52 | fmt.Printf("CompareSlices: value:\n'%+v' not equal to:\n'%+v'\nwithout dots: '%+v' != '%+v'\n", value, s2[index], v1, v2) 53 | return false 54 | } 55 | // fmt.Printf("CompareSlices: OK after conv: value:\n'%+v' equal to:\n'%+v'\nwithout dots: '%+v' != '%+v'\n", value, s2[index], v1, v2) 56 | } 57 | } 58 | return true 59 | } 60 | 61 | // CompareStringSlices2D - compares two slices of string slices 62 | func CompareStringSlices2D(s1 [][]string, s2 [][]string) bool { 63 | if len(s1) != len(s2) { 64 | return false 65 | } 66 | for index, value := range s1 { 67 | if !CompareStringSlices(value, s2[index]) { 68 | return false 69 | } 70 | } 71 | return true 72 | } 73 | 74 | // CompareSlices2D - compares two slices of any type slices 75 | func CompareSlices2D(s1 [][]interface{}, s2 [][]interface{}) bool { 76 | if len(s1) != len(s2) { 77 | fmt.Printf("CompareSlices2D: len: %d != %d\n", len(s1), len(s2)) 78 | return false 79 | } 80 | for index, value := range s1 { 81 | if !CompareSlices(value, s2[index]) { 82 | fmt.Printf("CompareSlices2D: CompareSlices:\n'%+v' not equal to:\n'%+v'\n", value, s2[index]) 83 | return false 84 | } 85 | } 86 | return true 87 | } 88 | 89 | // CompareSets - comparses two string sets 90 | func CompareSets(s1 map[string]struct{}, s2 map[string]struct{}) bool { 91 | // Different if different length 92 | if len(s1) != len(s2) { 93 | return false 94 | } 95 | 96 | // Get maps keys 97 | k1 := make([]string, len(s1)) 98 | index := 0 99 | for key := range s1 { 100 | k1[index] = key 101 | index++ 102 | } 103 | k2 := make([]string, len(s2)) 104 | index = 0 105 | for key := range s2 { 106 | k2[index] = key 107 | index++ 108 | } 109 | 110 | // Map keys aren't sorted 111 | sort.Strings(k1) 112 | sort.Strings(k2) 113 | 114 | // Compare 115 | for index, key := range k1 { 116 | if key != k2[index] { 117 | return false 118 | } 119 | } 120 | return true 121 | } 122 | 123 | // MakeComparableMap - transforms input map { k1: v1, k2: v2, ..., kn: vn } 124 | // into map with single key being its string representation, works on map[string]bool type 125 | // Example: { "b": true, "a": false, "c": true } --> { "a:false,b:true,c:true,": true } 126 | // We cannot compare such maps directly because order of keys is not guaranteed 127 | func MakeComparableMap(m *map[string]bool) { 128 | // Get maps keys 129 | keyAry := make([]string, len(*m)) 130 | index := 0 131 | for key := range *m { 132 | keyAry[index] = key 133 | index++ 134 | } 135 | // Map keys aren't sorted 136 | sort.Strings(keyAry) 137 | 138 | // Create string with k:v sorted 139 | outStr := "" 140 | for _, key := range keyAry { 141 | outStr += fmt.Sprintf("%s:%v,", key, (*m)[key]) 142 | } 143 | // Replace original map 144 | newMap := make(map[string]bool) 145 | newMap[outStr] = true 146 | *m = newMap 147 | } 148 | 149 | // MakeComparableMapStr - transforms input map { k1: v1, k2: v2, ..., kn: vn } 150 | // into map with single key being its string representation, works on map[string]string type 151 | // Example: { "b": "x", "a": "y", "c": "z" } --> { "a:y,b:x,c:z,": true } 152 | // We cannot compare such maps directly because order of keys is not guaranteed 153 | func MakeComparableMapStr(m *map[string]string) { 154 | // Get maps keys 155 | keyAry := make([]string, len(*m)) 156 | index := 0 157 | for key := range *m { 158 | keyAry[index] = key 159 | index++ 160 | } 161 | // Map keys aren't sorted 162 | sort.Strings(keyAry) 163 | 164 | // Create string with k:v sorted 165 | outStr := "" 166 | for _, key := range keyAry { 167 | outStr += fmt.Sprintf("%s:%s,", key, (*m)[key]) 168 | } 169 | // Replace original map 170 | newMap := make(map[string]string) 171 | newMap[outStr] = "" 172 | *m = newMap 173 | } 174 | 175 | // MakeComparableMap2 - transforms input map { k1: { true: struct{}{}, false: struct{}{}, ...}, k2: { ... } ... } 176 | // into map with single key being its string representation, works on map[string]map[bool]struct{} type 177 | // Example: { "w": { true: struct{}{}, false: struct{}{}}, "y10": { false: struct{}{}} } --> { "w:t,w:f,y10:f,": { false: struct{}{} } } 178 | // We cannot compare such maps directly because order of keys is not guaranteed 179 | func MakeComparableMap2(m *map[string]map[bool]struct{}) { 180 | // Get maps keys 181 | keyAry := []string{} 182 | for key, val := range *m { 183 | for key2 := range val { 184 | kk := fmt.Sprintf("%v", key2)[0:1] 185 | keyAry = append(keyAry, fmt.Sprintf("%s:%s", key, kk)) 186 | } 187 | } 188 | // Map keys aren't sorted 189 | sort.Strings(keyAry) 190 | 191 | // Create string with k:v sorted 192 | outStr := "" 193 | for _, key := range keyAry { 194 | outStr += fmt.Sprintf("%s,", key) 195 | } 196 | // Replace original map 197 | newMap := make(map[string]map[bool]struct{}) 198 | newMap[outStr] = make(map[bool]struct{}) 199 | newMap[outStr][false] = struct{}{} 200 | *m = newMap 201 | } 202 | 203 | // MakeComparableMap2Int - transforms input map { k1: [v11, v12], k2: [v21, v22], ..., kn: [vn1, vn2] } 204 | // into map with single key being its string representation, works on map[string][2]int type 205 | // Example: { "b": [1, 2], "a": [3, 4], "c": [5, 6] } --> { "a:[3,4],b:[1,2],c:[5,6],":[0,0] } 206 | // We cannot compare such maps directly because order of keys is not guaranteed 207 | func MakeComparableMap2Int(m *map[string][2]int) { 208 | // Get maps keys 209 | keyAry := make([]string, len(*m)) 210 | index := 0 211 | for key := range *m { 212 | keyAry[index] = key 213 | index++ 214 | } 215 | // Map keys aren't sorted 216 | sort.Strings(keyAry) 217 | 218 | // Create string with k:v sorted 219 | outStr := "" 220 | for _, key := range keyAry { 221 | outStr += fmt.Sprintf("%s:%v,", key, (*m)[key]) 222 | } 223 | // Replace original map 224 | newMap := make(map[string][2]int) 225 | newMap[outStr] = [2]int{0, 0} 226 | fmt.Fprintf(os.Stderr, "%+v\n", newMap) 227 | *m = newMap 228 | } 229 | -------------------------------------------------------------------------------- /test/time.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | ) 8 | 9 | // YMDHMS - return time defined by args 10 | func YMDHMS(in ...int) time.Time { 11 | m := 1 12 | d := 1 13 | h := 0 14 | mi := 0 15 | s := 0 16 | l := len(in) 17 | if l >= 2 { 18 | m = in[1] 19 | } 20 | if l >= 3 { 21 | d = in[2] 22 | } 23 | if l >= 4 { 24 | h = in[3] 25 | } 26 | if l >= 5 { 27 | mi = in[4] 28 | } 29 | if l >= 6 { 30 | s = in[5] 31 | } 32 | t := time.Date( 33 | in[0], 34 | time.Month(m), 35 | d, 36 | h, 37 | mi, 38 | s, 39 | 0, 40 | time.UTC, 41 | ) 42 | if t.Year() != in[0] || t.Month() != time.Month(m) || t.Day() != d || t.Hour() != h || t.Minute() != mi || t.Second() != s { 43 | fmt.Printf("Expected to set date from %v, got %v\n", in, t) 44 | os.Exit(1) 45 | } 46 | return t 47 | } 48 | -------------------------------------------------------------------------------- /threads.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "runtime" 5 | ) 6 | 7 | // GetThreadsNum returns the number of available CPUs 8 | // If environment variable GHA2DB_ST is set it retuns 1 9 | // It can be used to debug single threaded verion 10 | // It runs on 95% CPU power by default 11 | func GetThreadsNum(ctx *Ctx) int { 12 | // Get CPUs setting from env 13 | ctx.SetCPUs() 14 | // Use environment variable to have singlethreaded version 15 | if ctx.NCPUs > 0 { 16 | n := runtime.NumCPU() 17 | if ctx.NCPUs > n { 18 | ctx.NCPUs = n 19 | } 20 | runtime.GOMAXPROCS(ctx.NCPUs) 21 | return ctx.NCPUs 22 | } 23 | if ctx.ST { 24 | return 1 25 | } 26 | thrN := runtime.NumCPU() 27 | // thrN = (thrN * 19) / 20 28 | runtime.GOMAXPROCS(thrN) 29 | //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 2 * thrN 30 | return thrN 31 | } 32 | -------------------------------------------------------------------------------- /threads_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "testing" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | ) 8 | 9 | func TestGetThreadsNum(t *testing.T) { 10 | // Environment context parse 11 | var ctx lib.Ctx 12 | ctx.Init() 13 | ctx.TestMode = true 14 | 15 | // Get actual number of threads available 16 | nThreads := lib.GetThreadsNum(&ctx) 17 | 18 | // Set context's ST/NCPUs manually (don't need to repeat tests from context_test.go) 19 | var testCases = []struct { 20 | ST bool 21 | NCPUs int 22 | expected int 23 | }{ 24 | {ST: false, NCPUs: 0, expected: nThreads}, 25 | {ST: false, NCPUs: 1, expected: 1}, 26 | {ST: false, NCPUs: -1, expected: nThreads}, 27 | {ST: false, NCPUs: 2, expected: 2}, 28 | {ST: true, NCPUs: 0, expected: 1}, 29 | {ST: true, NCPUs: 1, expected: 1}, 30 | {ST: true, NCPUs: -1, expected: 1}, 31 | {ST: true, NCPUs: 2, expected: 2}, 32 | {ST: true, NCPUs: nThreads + 1, expected: nThreads}, 33 | } 34 | // Execute test cases 35 | for index, test := range testCases { 36 | ctx.ST = test.ST 37 | ctx.NCPUs = test.NCPUs 38 | expected := test.expected 39 | got := lib.GetThreadsNum(&ctx) 40 | if got != expected { 41 | t.Errorf( 42 | "test number %d, expected to return %d threads, got %d (default is %d on this machine)", 43 | index+1, expected, got, nThreads, 44 | ) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /ts_points.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // TSPoint keeps single time series point 9 | type TSPoint struct { 10 | t time.Time 11 | added time.Time 12 | period string 13 | name string 14 | tags map[string]string 15 | fields map[string]interface{} 16 | } 17 | 18 | // TSPoints keeps batch of TSPoint values to write 19 | type TSPoints []TSPoint 20 | 21 | // Str - string pretty print 22 | func (p *TSPoint) Str() string { 23 | return fmt.Sprintf( 24 | "%s %s %s period: %s tags: %+v fields: %+v", 25 | ToYMDHDate(p.t), 26 | ToYMDHDate(p.added), 27 | p.name, 28 | p.period, 29 | p.tags, 30 | p.fields, 31 | ) 32 | } 33 | 34 | // Str - string pretty print 35 | func (ps *TSPoints) Str() string { 36 | s := "" 37 | for i, p := range *ps { 38 | s += fmt.Sprintf("#%d %s\n", i+1, p.Str()) 39 | } 40 | return s 41 | } 42 | 43 | // NewTSPoint returns new point as specified by args 44 | func NewTSPoint(ctx *Ctx, name, period string, tags map[string]string, fields map[string]interface{}, t time.Time, exact bool) TSPoint { 45 | var ( 46 | otags map[string]string 47 | ofields map[string]interface{} 48 | ) 49 | if tags != nil { 50 | otags = make(map[string]string) 51 | for k, v := range tags { 52 | otags[k] = v 53 | } 54 | } 55 | if fields != nil { 56 | ofields = make(map[string]interface{}) 57 | for k, v := range fields { 58 | ofields[k] = v 59 | } 60 | } 61 | var pt time.Time 62 | if exact { 63 | pt = t 64 | } else { 65 | pt = HourStart(t) 66 | } 67 | p := TSPoint{ 68 | t: pt, 69 | added: time.Now(), 70 | name: name, 71 | period: period, 72 | tags: otags, 73 | fields: ofields, 74 | } 75 | if ctx.Debug > 0 { 76 | Printf("NewTSPoint: %s\n", p.Str()) 77 | } 78 | return p 79 | } 80 | 81 | // AddTSPoint add single point to the batch 82 | func AddTSPoint(ctx *Ctx, pts *TSPoints, pt TSPoint) { 83 | if ctx.Debug > 0 { 84 | Printf("AddTSPoint: %s\n", pt.Str()) 85 | } 86 | *pts = append(*pts, pt) 87 | if ctx.Debug > 0 { 88 | Printf("AddTSPoint: point added, now %d points\n", len(*pts)) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /unicode.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "strings" 5 | 6 | "golang.org/x/text/transform" 7 | "golang.org/x/text/unicode/norm" 8 | ) 9 | 10 | // StripUnicode strip non-unicode and control characters from a string 11 | // From: https://rosettacode.org/wiki/Strip_control_codes_and_extended_characters_from_a_string#Go 12 | func StripUnicode(str string) string { 13 | isOk := func(r rune) bool { 14 | return r < 32 || r >= 127 15 | } 16 | t := transform.Chain(norm.NFKD, transform.RemoveFunc(isOk)) 17 | str, _, _ = transform.String(t, str) 18 | return str 19 | } 20 | 21 | // NormalizeName - clean DB string from ", ', -, /, ., " ",), (, ], [ trim leading and trailing space, lowercase 22 | // Normalize Unicode characters 23 | func NormalizeName(str string) string { 24 | r := strings.NewReplacer( 25 | "-", "", "/", "", ".", "", " ", "", ",", "", ";", "", ":", "", "`", "", "(", "", ")", "", "[", "", "]", "", "<", "", ">", "", "_", "", "\"", "", "'", "", 26 | ) 27 | return r.Replace(strings.ToLower(strings.TrimSpace(StripUnicode(str)))) 28 | } 29 | -------------------------------------------------------------------------------- /unicode_test.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "testing" 5 | 6 | lib "github.com/cncf/devstatscode" 7 | ) 8 | 9 | func TestStripUnicode(t *testing.T) { 10 | // Test cases 11 | var testCases = []struct { 12 | str, expected string 13 | }{ 14 | {str: "hello", expected: "hello"}, 15 | {str: "control:\t\n\r", expected: "control:"}, 16 | {str: "gżegżółką", expected: "gzegzoka"}, 17 | {str: "net_ease_网易有态", expected: "net_ease_"}, 18 | } 19 | // Execute test cases 20 | for index, test := range testCases { 21 | expected := test.expected 22 | got := lib.StripUnicode(test.str) 23 | if got != expected { 24 | t.Errorf( 25 | "test number %d, expected %v, got %v", 26 | index+1, expected, got, 27 | ) 28 | } 29 | } 30 | } 31 | 32 | // NormalizeName - clean DB string from ', ", -, /, ., " ", trim leading and trailing space, lowercase 33 | // Normalize Unicode characters 34 | func TestNormalizeName(t *testing.T) { 35 | // Test cases 36 | var testCases = []struct { 37 | str, expected string 38 | }{ 39 | {str: "hello", expected: "hello"}, 40 | {str: "control:\t\n\r", expected: "control"}, 41 | {str: "gżegżółką", expected: "gzegzoka"}, 42 | {str: "net_ease_网易有态", expected: "netease"}, 43 | {str: " see;hello-world/k8s.io, said: HE`MAN ", expected: "seehelloworldk8siosaidheman"}, 44 | {str: "Contributions (issues, PRs, git pushes)", expected: "contributionsissuesprsgitpushes"}, 45 | {str: "Exclude (ro[bot]nik)", expected: "excluderobotnik"}, 46 | {str: "comment\"sallcoted'ivoire", expected: "commentsallcotedivoire"}, 47 | {str: "Piraeus-Datastore", expected: "piraeusdatastore"}, 48 | } 49 | // Execute test cases 50 | for index, test := range testCases { 51 | expected := test.expected 52 | got := lib.NormalizeName(test.str) 53 | if got != expected { 54 | t.Errorf( 55 | "test number %d, expected %v, got %v", 56 | index+1, expected, got, 57 | ) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /vet_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | $1 *.go || exit 1 3 | for dir in `find ./cmd/ -mindepth 1 -type d` 4 | do 5 | $1 $dir/*.go || exit 1 6 | done 7 | exit 0 8 | -------------------------------------------------------------------------------- /yaml.go: -------------------------------------------------------------------------------- 1 | package devstatscode 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | yaml "gopkg.in/yaml.v2" 7 | ) 8 | 9 | // ObjectToYAML - serialize given object as YAML 10 | func ObjectToYAML(obj interface{}, fn string) { 11 | yamlBytes, err := yaml.Marshal(obj) 12 | FatalOnError(err) 13 | FatalOnError(ioutil.WriteFile(fn, yamlBytes, 0644)) 14 | } 15 | --------------------------------------------------------------------------------