├── .gitignore ├── LICENSE.txt ├── README.md ├── auto_sort_projects.py ├── display_scan_status_of_repository.py ├── follow_org.py ├── follow_repos_by_search_term.py ├── follow_repositories.py ├── follow_top_repos_by_star_count.py ├── lgtm.py ├── move_org_projects_under_project_list_then_unfollow.py ├── move_repositories_under_project_list.py ├── rebuild_all_following_projects.py ├── requirements.txt ├── topJavaMavenProjects.csv ├── unfollow_org.py └── utils ├── __init__.py ├── github_api.py └── github_dates.py /.gitignore: -------------------------------------------------------------------------------- 1 | config.yml 2 | 3 | # Created by https://www.toptal.com/developers/gitignore/api/java,pycharm+all,intellij+all,python,macos,windows,linux 4 | # Edit at https://www.toptal.com/developers/gitignore?templates=java,pycharm+all,intellij+all,python,macos,windows,linux 5 | 6 | ### Intellij+all ### 7 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 8 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 9 | 10 | # User-specific stuff 11 | .idea/**/workspace.xml 12 | .idea/**/tasks.xml 13 | .idea/**/usage.statistics.xml 14 | .idea/**/dictionaries 15 | .idea/**/shelf 16 | 17 | # Generated files 18 | .idea/**/contentModel.xml 19 | 20 | # Sensitive or high-churn files 21 | .idea/**/dataSources/ 22 | .idea/**/dataSources.ids 23 | .idea/**/dataSources.local.xml 24 | .idea/**/sqlDataSources.xml 25 | .idea/**/dynamic.xml 26 | .idea/**/uiDesigner.xml 27 | .idea/**/dbnavigator.xml 28 | 29 | # Gradle 30 | .idea/**/gradle.xml 31 | .idea/**/libraries 32 | 33 | # Gradle and Maven with auto-import 34 | # When using Gradle or Maven with auto-import, you should exclude module files, 35 | # since they will be recreated, and may cause churn. Uncomment if using 36 | # auto-import. 37 | # .idea/artifacts 38 | # .idea/compiler.xml 39 | # .idea/jarRepositories.xml 40 | # .idea/modules.xml 41 | # .idea/*.iml 42 | # .idea/modules 43 | # *.iml 44 | # *.ipr 45 | 46 | # CMake 47 | cmake-build-*/ 48 | 49 | # Mongo Explorer plugin 50 | .idea/**/mongoSettings.xml 51 | 52 | # File-based project format 53 | *.iws 54 | 55 | # IntelliJ 56 | out/ 57 | 58 | # mpeltonen/sbt-idea plugin 59 | .idea_modules/ 60 | 61 | # JIRA plugin 62 | atlassian-ide-plugin.xml 63 | 64 | # Cursive Clojure plugin 65 | .idea/replstate.xml 66 | 67 | # Crashlytics plugin (for Android Studio and IntelliJ) 68 | com_crashlytics_export_strings.xml 69 | crashlytics.properties 70 | crashlytics-build.properties 71 | fabric.properties 72 | 73 | # Editor-based Rest Client 74 | .idea/httpRequests 75 | 76 | # Android studio 3.1+ serialized cache file 77 | .idea/caches/build_file_checksums.ser 78 | 79 | ### Intellij+all Patch ### 80 | # Ignores the whole .idea folder and all .iml files 81 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 82 | 83 | .idea/ 84 | 85 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 86 | 87 | *.iml 88 | modules.xml 89 | .idea/misc.xml 90 | *.ipr 91 | 92 | # Sonarlint plugin 93 | .idea/sonarlint 94 | 95 | ### Java ### 96 | # Compiled class file 97 | *.class 98 | 99 | # Log file 100 | *.log 101 | 102 | # BlueJ files 103 | *.ctxt 104 | 105 | # Mobile Tools for Java (J2ME) 106 | .mtj.tmp/ 107 | 108 | # Package Files # 109 | *.jar 110 | *.war 111 | *.nar 112 | *.ear 113 | *.zip 114 | *.tar.gz 115 | *.rar 116 | 117 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 118 | hs_err_pid* 119 | 120 | ### Linux ### 121 | *~ 122 | 123 | # temporary files which can be created if a process still has a handle open of a deleted file 124 | .fuse_hidden* 125 | 126 | # KDE directory preferences 127 | .directory 128 | 129 | # Linux trash folder which might appear on any partition or disk 130 | .Trash-* 131 | 132 | # .nfs files are created when an open file is removed but is still being accessed 133 | .nfs* 134 | 135 | ### macOS ### 136 | # General 137 | .DS_Store 138 | .AppleDouble 139 | .LSOverride 140 | 141 | # Icon must end with two \r 142 | Icon 143 | 144 | # Thumbnails 145 | ._* 146 | 147 | # Files that might appear in the root of a volume 148 | .DocumentRevisions-V100 149 | .fseventsd 150 | .Spotlight-V100 151 | .TemporaryItems 152 | .Trashes 153 | .VolumeIcon.icns 154 | .com.apple.timemachine.donotpresent 155 | 156 | # Directories potentially created on remote AFP share 157 | .AppleDB 158 | .AppleDesktop 159 | Network Trash Folder 160 | Temporary Items 161 | .apdisk 162 | 163 | ### PyCharm+all ### 164 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 165 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 166 | 167 | # User-specific stuff 168 | 169 | # Generated files 170 | 171 | # Sensitive or high-churn files 172 | 173 | # Gradle 174 | 175 | # Gradle and Maven with auto-import 176 | # When using Gradle or Maven with auto-import, you should exclude module files, 177 | # since they will be recreated, and may cause churn. Uncomment if using 178 | # auto-import. 179 | # .idea/artifacts 180 | # .idea/compiler.xml 181 | # .idea/jarRepositories.xml 182 | # .idea/modules.xml 183 | # .idea/*.iml 184 | # .idea/modules 185 | # *.iml 186 | # *.ipr 187 | 188 | # CMake 189 | 190 | # Mongo Explorer plugin 191 | 192 | # File-based project format 193 | 194 | # IntelliJ 195 | 196 | # mpeltonen/sbt-idea plugin 197 | 198 | # JIRA plugin 199 | 200 | # Cursive Clojure plugin 201 | 202 | # Crashlytics plugin (for Android Studio and IntelliJ) 203 | 204 | # Editor-based Rest Client 205 | 206 | # Android studio 3.1+ serialized cache file 207 | 208 | ### PyCharm+all Patch ### 209 | # Ignores the whole .idea folder and all .iml files 210 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 211 | 212 | 213 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 214 | 215 | 216 | # Sonarlint plugin 217 | 218 | ### Python ### 219 | # Byte-compiled / optimized / DLL files 220 | __pycache__/ 221 | *.py[cod] 222 | *$py.class 223 | 224 | # C extensions 225 | *.so 226 | 227 | # Distribution / packaging 228 | .Python 229 | build/ 230 | develop-eggs/ 231 | dist/ 232 | downloads/ 233 | eggs/ 234 | .eggs/ 235 | lib/ 236 | lib64/ 237 | parts/ 238 | sdist/ 239 | var/ 240 | wheels/ 241 | pip-wheel-metadata/ 242 | share/python-wheels/ 243 | *.egg-info/ 244 | .installed.cfg 245 | *.egg 246 | MANIFEST 247 | 248 | # PyInstaller 249 | # Usually these files are written by a python script from a template 250 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 251 | *.manifest 252 | *.spec 253 | 254 | # Installer logs 255 | pip-log.txt 256 | pip-delete-this-directory.txt 257 | 258 | # Unit test / coverage reports 259 | htmlcov/ 260 | .tox/ 261 | .nox/ 262 | .coverage 263 | .coverage.* 264 | .cache 265 | nosetests.xml 266 | coverage.xml 267 | *.cover 268 | *.py,cover 269 | .hypothesis/ 270 | .pytest_cache/ 271 | pytestdebug.log 272 | 273 | # Translations 274 | *.mo 275 | *.pot 276 | 277 | # Django stuff: 278 | local_settings.py 279 | db.sqlite3 280 | db.sqlite3-journal 281 | 282 | # Flask stuff: 283 | instance/ 284 | .webassets-cache 285 | 286 | # Scrapy stuff: 287 | .scrapy 288 | 289 | # Sphinx documentation 290 | docs/_build/ 291 | doc/_build/ 292 | 293 | # PyBuilder 294 | target/ 295 | 296 | # Jupyter Notebook 297 | .ipynb_checkpoints 298 | 299 | # IPython 300 | profile_default/ 301 | ipython_config.py 302 | 303 | # pyenv 304 | .python-version 305 | 306 | # pipenv 307 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 308 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 309 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 310 | # install all needed dependencies. 311 | #Pipfile.lock 312 | 313 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 314 | __pypackages__/ 315 | 316 | # Celery stuff 317 | celerybeat-schedule 318 | celerybeat.pid 319 | 320 | # SageMath parsed files 321 | *.sage.py 322 | 323 | # Environments 324 | .env 325 | .venv 326 | env/ 327 | venv/ 328 | ENV/ 329 | env.bak/ 330 | venv.bak/ 331 | 332 | # Spyder project settings 333 | .spyderproject 334 | .spyproject 335 | 336 | # Rope project settings 337 | .ropeproject 338 | 339 | # mkdocs documentation 340 | /site 341 | 342 | # mypy 343 | .mypy_cache/ 344 | .dmypy.json 345 | dmypy.json 346 | 347 | # Pyre type checker 348 | .pyre/ 349 | 350 | # pytype static type analyzer 351 | .pytype/ 352 | 353 | ### Windows ### 354 | # Windows thumbnail cache files 355 | Thumbs.db 356 | Thumbs.db:encryptable 357 | ehthumbs.db 358 | ehthumbs_vista.db 359 | 360 | # Dump file 361 | *.stackdump 362 | 363 | # Folder config file 364 | [Dd]esktop.ini 365 | 366 | # Recycle Bin used on file shares 367 | $RECYCLE.BIN/ 368 | 369 | # Windows Installer files 370 | *.cab 371 | *.msi 372 | *.msix 373 | *.msm 374 | *.msp 375 | 376 | # Windows shortcuts 377 | *.lnk 378 | 379 | # End of https://www.toptal.com/developers/gitignore/api/java,pycharm+all,intellij+all,python,macos,windows,linux 380 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Jonathan Leitschuh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LGTM Hack Scripts 2 | 3 | Collection of python helper API's for interacting with [LGTM.com](https://lgtm.com) in ways the official API doesn't support. 4 | 5 | ## Why 6 | 7 | As a GitHub Security Lab bounty hunter the best way to run your CodeQL query against large numbers of projects is still 8 | through the [LGTM.com](https://lgtm.com) site. However, there is still no good way to import/subscribe to large numbers 9 | of projects on the LGTM site, nor through the official API. 10 | 11 | However, the [LGTM.com](https://lgtm.com) enables subscriptions to projects. The query console 12 | does allow you to run queries against large number of projects all at once. These are 'internal' APIs that are only 13 | surfaced through UI. By utilizing these 'internal' APIs we're able to leverage bulk import and subscription to projects. 14 | 15 | This can significantly up your LGTM BB research game. 16 | 17 | ## How 18 | 19 | Unfortunately, because this solution isn't officially supported, there are a few pieces of API information you'll 20 | need to extract manually while interacting with the [LGTM.com](https://lgtm.com) site. 21 | 22 | ### Using this Project 23 | 24 | In order to extract these 'keys' for uses by these scripts, we recommend that your browser's developer tools 25 | and inspect the various requests normally made under the 'Network' tab. This information should be put inside of a 26 | file named `config.yml` inside the repositories root directory. This `config.yml` file is already part of the 27 | `.gitignore` so adding it to this repository will not risk you accidentally committing it. 28 | 29 | The format of this `config.yml` is the following: 30 | 31 | ```yaml 32 | lgtm: 33 | nonce: # From the request header `LGTM-Nonce` 34 | long_session: # From the cookie named `lgtm_long_session` 35 | short_session: # From the cookie named `lgtm_short_session` 36 | api_version: # From a property named `api_version` inside of any JSON POST request made 37 | github: 38 | api_key: # The Github API token. You can create one here: https://github.com/settings/tokens/new. The token should have no permissions. 39 | ``` 40 | 41 | ## Commands 42 | 43 | ```bash 44 | # Finds all repositories for a specified Github organization and adds them to your LGTM's account's project list. 45 | # 46 | # For example, if you want to add repositories that use Java and Kotlin: 47 | # python3 follow_org.py netflix Java,Kotlin 48 | # 49 | # If you want to find all CodeQL-supported repositories regardless of the language used, 50 | # don't provide a second argument in the command: 51 | # python3 follow_org.py netflix 52 | # 53 | python3 follow_org.py 54 | 55 | 56 | # Finds all repositories for a specified Github organization and unfollows them from your LGTM account's project list. 57 | python3 unfollow_org.py 58 | 59 | # Finds all repositories for a specified Github Organization and adds them to your specified LGTM account's project list. 60 | python3 move_org_projects_under_project_list_then_unfollow.py 61 | 62 | # Finds repositories given a search term. Under the hood, the script searches Github for repositories that match the provided search term. 63 | python3 follow_repos_by_search_term.py 64 | 65 | # Finds top repositories that have a minimum 500 stars and use the provided programming language. 66 | python3 follow_top_repos_by_star_count.py 67 | 68 | # Re-runs failed project builds in an attempt to get the build to succeed. 69 | python3 rebuild_all_following_projects.py 70 | ``` 71 | 72 | ## Legal 73 | 74 | The author of this script assumes no liability for your use of this project, including, 75 | but not limited legal repercussions or being banned from [LGTM.com](https://lgtm.com). 76 | Please consult the [LGTM.com](https://lgtm.com) terms of service for more information. 77 | -------------------------------------------------------------------------------- /auto_sort_projects.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List 2 | 3 | from lgtm import LGTMSite, LGTMDataFilters, SimpleProject 4 | 5 | project_list_name_to_gh_org = { 6 | 'Google_Projects': [ 7 | 'google', 8 | 'GoogleCloudPlatform', 9 | 'GoogleContainerTools', 10 | 'googleapis', 11 | 'googlecolab', 12 | 'googlemaps', 13 | 'google-cloudsearch', 14 | 'flutter' 15 | ], 16 | 'Apache_Projects': ['apache'], 17 | 'AirBnB_Projects': ['airbnb'], 18 | 'Amazon_Projects': [ 19 | 'aws', 20 | 'awslabs', 21 | ], 22 | 'LinkedIn_Projects': ['linkedin'], 23 | 'Netflix_Projects': [ 24 | 'Netflix', 25 | 'nebula-plugins' 26 | ], 27 | 'OpenTracing_Projects': ['opentracing', 'opentracing-contrib'], 28 | 'Spring_Projects': ['spring-projects'], 29 | 'Jenkins_Projects': ['jenkinsci'], 30 | 'Cloudfoundry_Projects': ['cloudfoundry'], 31 | 'Square_Projects': ['square'], 32 | 'Gradle_Projects': ['gradle'], 33 | 'PortSwigger_Projects': ['PortSwigger'], 34 | 'Wildlfy_Projects': [ 35 | 'wildfly', 36 | 'jbosstools', 37 | 'keycloak', 38 | ], 39 | 'Graphite_Projects': ['graphite-project'], 40 | 'Openmrs_Projects': ['openmrs'], 41 | 'JetBrains_Projects': ['JetBrains'], 42 | 'Eclipse_Projects': [ 43 | 'eclipse', 44 | 'eclipse-ee4j', 45 | 'eclipse-theia', 46 | 'eclipse-vertx', 47 | ], 48 | 'Microsoft_Projects': ['microsoft', 'Azure'], 49 | 'PayPal_Projects': ['paypal'], 50 | 'Spinnaker_Projects': ['spinnaker'], 51 | 'Elastic_Projects': ['elastic'], 52 | 'Facebook_Projects': ['facebook'], 53 | 'DataDog_Projects': ['DataDog'], 54 | 'Micronaut_Projects': ['micronaut-projects'], 55 | 'Apple_Projects': ['apple'], 56 | 'Gradle_Plugin_Projects': ['nebula-plugins'], 57 | 'Open_Rewrite_Projects': ['openrewrite'], 58 | 'Oracle_Projects': ['oracle'], 59 | 'Sonatype_Projects': ['sonatype'], 60 | 'WPI_Projects': [ 61 | 'wpilibsuite', 62 | 'WPIRoboticsProjects', 63 | ], 64 | 'US_Government_Projects': [ 65 | 'CDCgov', 66 | 'NationalSecurityAgency', 67 | ], 68 | 'Mozilla_Projects': [ 69 | 'mozilla', 70 | 'mozilla-mobile', 71 | ], 72 | 'wso2_Projects': ['wso2'], 73 | } 74 | 75 | project_list_to_repo = { 76 | 'jOOq-Users': [ 77 | 'self-xdsd/self-storage', 78 | 'folio-org/mod-source-record-storage', 79 | 'ICIJ/datashare', 80 | 'hartwigmedical/hmftools', 81 | 'openforis/collect', 82 | 'jklingsporn/vertx-jooq', 83 | 'trib3/leakycauldron', 84 | 'ZupIT/charlescd', 85 | 'waikato-datamining/adams-applications' 86 | ], 87 | 'Gradle_Plugin_Projects': [ 88 | '47degrees/hood', 89 | 'autonomousapps/dependency-analysis-android-gradle-plugin', 90 | 'applandinc/appmap-gradle-plugin', 91 | 'anatawa12/auto-visitor', 92 | 'aim42/htmlSanityCheck', 93 | 'android-async-http/android-async-http', 94 | 'diffplug/spotless', 95 | 'diffplug/blowdryer', 96 | 'diffplug/spotless-changelog', 97 | 'spring-cloud/spring-cloud-contract', 98 | 'JetBrains/gradle-intellij-plugin', 99 | 'JetBrains/gradle-jps-compiler-plugin', 100 | 'JetBrains/gradle-grammar-kit-plugin', 101 | 'JetBrains/gradle-changelog-plugin', 102 | 'JetBrains/gradle-idea-ext-plugin', 103 | 'JetBrains/gradle-node-envs', 104 | 'JetBrains/gradle-python-envs', 105 | 'JetBrains/gradle-ruby-envs', 106 | 'JetBrains/gradle-intellij-plugin', 107 | 'openrewrite/rewrite-gradle-plugin', 108 | ], 109 | 'Top_Java_Projects': [ 110 | 'apereo/cas', 111 | 'spring-projects/spring-boot', 112 | 'iluwatar/java-design-patterns', 113 | 'square/retrofit', 114 | 'square/okhttp', 115 | 'zxing/zxing', 116 | 'libgdx/libgdx', 117 | 'google/guava', 118 | 'alibaba/dubbo', 119 | 'jfeinstein10/SlidingMenu', 120 | 'netty/netty', 121 | 'JakeWharton/ActionBarSherlock', 122 | 'chrisbanes/Android-PullToRefresh', 123 | 'alibaba/fastjson', 124 | 'deeplearning4j/deeplearning4j', 125 | 'JakeWharton/ViewPagerIndicator', 126 | 'alibaba/druid', 127 | 'liaohuqiu/android-Ultra-Pull-To-Refresh', 128 | 'mybatis/mybatis-3', 129 | 'springside/springside4', 130 | 'apache/storm', 131 | 'xetorthio/jedis', 132 | 'apache/hadoop', 133 | 'dropwizard/dropwizard', 134 | 'swagger-api/swagger-codegen', 135 | 'code4craft/webmagic', 136 | 'junit-team/junit', 137 | 'Trinea/android-common', 138 | 'clojure/clojure', 139 | 'nhaarman/ListViewAnimations', 140 | 'perwendel/spark', 141 | 'spring-projects/spring-mvc-showcase', 142 | 'square/dagger', 143 | 'swagger-api/swagger-core', 144 | 'jhy/jsoup', 145 | 'mcxiaoke/android-volley', 146 | 'Activiti/Activiti', 147 | 'spring-projects/spring-petclinic', 148 | 'openhab/openhab', 149 | 'JakeWharton/NineOldAndroids', 150 | 'wildfly/wildfly', 151 | 'Bukkit/Bukkit', 152 | 'jersey/jersey', 153 | 'NLPchina/ansj_seg', 154 | 'spring-projects/spring-security-oauth', 155 | 'eclipse/vert.x', 156 | 'apache/flink', 157 | 'neo4j/neo4j', 158 | 'google/guice', 159 | 'MyCATApache/Mycat-Server', 160 | 'apache/camel', 161 | 'druid-io/druid', 162 | 'naver/pinpoint', 163 | 'AsyncHttpClient/async-http-client', 164 | 'thinkaurelius/titan', 165 | 'stanfordnlp/CoreNLP', 166 | 'dropwizard/metrics', 167 | 'bauerca/drag-sort-listview', 168 | 'EnterpriseQualityCoding/FizzBuzzEnterpriseEdition', 169 | 'brettwooldridge/HikariCP', 170 | 'pardom/ActiveAndroid', 171 | 'google/auto', 172 | 'square/otto', 173 | 'openmrs/openmrs-core', 174 | 'alibaba/jstorm', 175 | 'b3log/solo', 176 | 'hankcs/HanLP', 177 | 'knightliao/disconf', 178 | 'facebook/presto', 179 | 'aws/aws-sdk-java', 180 | 'cucumber/cucumber-jvm', 181 | 'Atmosphere/atmosphere', 182 | 'yusuke/twitter4j', 183 | 'yasserg/crawler4j', 184 | 'alibaba/canal', 185 | 'gephi/gephi', 186 | 'NanoHttpd/nanohttpd', 187 | 'google/closure-compiler', 188 | 'JakeWharton/DiskLruCache', 189 | 'apache/hive', 190 | 'square/okio', 191 | 'scribejava/scribejava', 192 | 'checkstyle/checkstyle', 193 | 'roboguice/roboguice', 194 | 'hazelcast/hazelcast', 195 | 'antlr/antlr4', 196 | 'databricks/learning-spark', 197 | 'Alluxio/alluxio', 198 | 'jfinal/jfinal', 199 | 'apache/hbase', 200 | 'javaee-samples/javaee7-samples', 201 | 'joelittlejohn/jsonschema2pojo', 202 | 'dangdangdotcom/elastic-job', 203 | 'pxb1988/dex2jar', 204 | 'alibaba/DataX', 205 | 'shuzheng/zheng', 206 | 'Graylog2/graylog2-server', 207 | 'brianfrankcooper/YCSB', 208 | 'essentials/Essentials', 209 | 'kbastani/spring-cloud-microservice-example', 210 | 'square/javapoet', 211 | 'signalapp/Signal-Android', 212 | ], 213 | } 214 | 215 | # Flip the list 216 | gh_org_to_project_list_name: Dict[str, str] = {} 217 | for list_name in project_list_name_to_gh_org: 218 | for gh_org in project_list_name_to_gh_org[list_name]: 219 | gh_org_to_project_list_name[gh_org] = list_name 220 | 221 | gh_repo_to_project_list_name: Dict[str, str] = {} 222 | for list_name in project_list_to_repo: 223 | for gh_repo in project_list_to_repo[list_name]: 224 | gh_repo_to_project_list_name[gh_repo] = list_name 225 | 226 | site = LGTMSite.create_from_file() 227 | my_projects = site.get_my_projects() 228 | org_to_projects: Dict[str, List[SimpleProject]] = LGTMDataFilters.org_to_ids(my_projects) 229 | for org in org_to_projects: 230 | project_lists_to_use: List[Dict] = [] 231 | 232 | if org in gh_org_to_project_list_name: 233 | project_list_name = gh_org_to_project_list_name[org] 234 | project_list_id = site.get_or_create_project_list(project_list_name) 235 | project_lists_to_use.append({'id': project_list_id, 'name': project_list_name}) 236 | 237 | for project in org_to_projects[org]: 238 | if project.is_protoproject: 239 | print('Unable to add project to project list since it is a protoproject. %s' % project) 240 | continue 241 | 242 | # Create a list of project lists to add this project to 243 | project_lists_to_use_local = project_lists_to_use.copy() 244 | if project.display_name in gh_repo_to_project_list_name: 245 | project_list_name = gh_repo_to_project_list_name[project.display_name] 246 | project_list_id = site.get_or_create_project_list(project_list_name) 247 | project_lists_to_use_local.append({'id': project_list_id, 'name': project_list_name}) 248 | 249 | project_list_str = [project_list["name"] for project_list in project_lists_to_use_local] 250 | print(f'Adding repository {project.display_name} to project lists: {project_list_str}') 251 | # Add the project to the project lists 252 | for project_list in project_lists_to_use_local: 253 | site.load_into_project_list(project_list['id'], [project.key]) 254 | 255 | if project_lists_to_use_local: 256 | site.unfollow_repository(project) 257 | -------------------------------------------------------------------------------- /display_scan_status_of_repository.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from lgtm import LGTMSite 4 | 5 | 6 | def main(): 7 | parser = argparse.ArgumentParser( 8 | description='display the build status of the specified repositories' 9 | ) 10 | parser.add_argument('-i', '--infile', dest='in_file', required=True, type=argparse.FileType('r')) 11 | args = parser.parse_args() 12 | unloaded_count = 0 13 | total_count = 0 14 | for line in args.in_file: 15 | line_clean: str = line.strip() 16 | total_count += 1 17 | gh_project_path = line_clean.lstrip('https://github.com/') 18 | print('Checking status for %s' % gh_project_path) 19 | result = LGTMSite.retrieve_project(gh_project_path) 20 | print(result) 21 | if 'code' in result: 22 | unloaded_count += 1 23 | print('%d/%d projects loaded' % (total_count - unloaded_count, total_count)) 24 | 25 | 26 | if __name__ == "__main__": 27 | main() 28 | -------------------------------------------------------------------------------- /follow_org.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from lgtm import LGTMSite 3 | 4 | import utils.github_api 5 | import sys 6 | 7 | 8 | def get_languages() -> List[str]: 9 | default_languages: List[str] = [ 10 | 'Kotlin', 11 | 'Groovy', 12 | 'Java', 13 | 'C#', 14 | 'C', 15 | 'C++', 16 | 'Python', 17 | 'Javascript', 18 | 'TypeScript', 19 | 'Go' 20 | ] 21 | 22 | # If the user does not want to follow repos using a specific language(s), we 23 | # follow all repos that are CodeQL supported instead. 24 | if len(sys.argv) >= 3: 25 | languages = sys.argv[2].split(',') 26 | formatted_languages = [language.capitalize() for language in languages] 27 | return formatted_languages 28 | else: 29 | return default_languages 30 | 31 | 32 | def load_repository_list(org: str) -> List[str]: 33 | languages = get_languages() 34 | github = utils.github_api.create() 35 | repos = github.get_organization(org).get_repos(type='public') 36 | 37 | repos_to_load: List[str] = [] 38 | for repo in repos: 39 | if repo.archived or repo.fork: 40 | continue 41 | if repo.language in languages: 42 | print("Adding: " + repo.full_name) 43 | repos_to_load.append(repo.full_name) 44 | 45 | return repos_to_load 46 | 47 | 48 | org_to_follow = sys.argv[1] 49 | print('Following Org: %s' % org_to_follow) 50 | 51 | repository_list = load_repository_list(org_to_follow) 52 | 53 | site = LGTMSite.create_from_file() 54 | 55 | for repo_name in repository_list: 56 | repo_url: str = 'https://github.com/' + repo_name 57 | print(repo_url) 58 | site.follow_repository(repo_url) 59 | -------------------------------------------------------------------------------- /follow_repos_by_search_term.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from lgtm import LGTMSite 3 | import utils.github_dates 4 | import utils.github_api 5 | 6 | import sys 7 | import time 8 | 9 | def save_project_to_lgtm(site: 'LGTMSite', repo_name: str): 10 | print("About to save: " + repo_name) 11 | # Another throttle. Considering we are sending a request to Github 12 | # owned properties twice in a small time-frame, I would prefer for 13 | # this to be here. 14 | time.sleep(1) 15 | 16 | repo_url: str = 'https://github.com/' + repo_name 17 | site.follow_repository(repo_url) 18 | print("Saved the project: " + repo_name) 19 | 20 | def find_and_save_projects_to_lgtm(language: str, search_term: str): 21 | github = utils.github_api.create() 22 | site = LGTMSite.create_from_file() 23 | 24 | for date_range in utils.github_dates.generate_dates(): 25 | repos = github.search_repositories(query=f'language:{language} fork:false created:{date_range} {search_term}') 26 | 27 | for repo in repos: 28 | # Github has rate limiting in place hence why we add a sleep here. More info can be found here: 29 | # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting 30 | time.sleep(1) 31 | 32 | if repo.archived or repo.fork: 33 | continue 34 | 35 | save_project_to_lgtm(site, repo.full_name) 36 | 37 | if len(sys.argv) < 3: 38 | print("Please make sure you provided a language and search term") 39 | exit 40 | 41 | language = sys.argv[1].capitalize() 42 | search_term = sys.argv[2] 43 | 44 | print(f'Following repos for the {language} language that contain the \'{search_term}\' search term.') 45 | find_and_save_projects_to_lgtm(language, search_term) 46 | -------------------------------------------------------------------------------- /follow_repositories.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from lgtm import LGTMSite, LGTMRequestException 4 | 5 | 6 | def main(): 7 | parser = argparse.ArgumentParser( 8 | description='follow repositories from a list of newline delimited repositories' 9 | ) 10 | parser.add_argument('-i', '--infile', dest='in_file', required=True, type=argparse.FileType('r')) 11 | args = parser.parse_args() 12 | site = LGTMSite.create_from_file() 13 | for line in args.in_file: 14 | line_clean = line.strip() 15 | print('Following: %s' % line_clean) 16 | if not line_clean: 17 | continue 18 | try: 19 | if not line_clean.startswith('https://github.com'): 20 | line_clean = 'https://github.com/%s' % line_clean 21 | site.follow_repository(line_clean) 22 | except LGTMRequestException: 23 | print('Failed to follow: %s' % line_clean) 24 | 25 | 26 | if __name__ == "__main__": 27 | main() 28 | -------------------------------------------------------------------------------- /follow_top_repos_by_star_count.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from lgtm import LGTMSite 3 | 4 | import utils.github_dates 5 | import utils.github_api 6 | import sys 7 | import time 8 | 9 | 10 | def save_project_to_lgtm(site: 'LGTMSite', repo_name: str): 11 | print("Adding: " + repo_name) 12 | # Another throttle. Considering we are sending a request to Github 13 | # owned properties twice in a small time-frame, I would prefer for 14 | # this to be here. 15 | time.sleep(1) 16 | 17 | repo_url: str = 'https://github.com/' + repo_name 18 | site.follow_repository(repo_url) 19 | print("Saved the project: " + repo_name) 20 | 21 | 22 | def find_and_save_projects_to_lgtm(language: str): 23 | github = utils.github_api.create() 24 | site = LGTMSite.create_from_file() 25 | 26 | for date_range in utils.github_dates.generate_dates(): 27 | repos = github.search_repositories( 28 | query=f'stars:>500 created:{date_range} fork:false sort:stars language:{language}') 29 | 30 | for repo in repos: 31 | # Github has rate limiting in place hence why we add a sleep here. More info can be found here: 32 | # https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting 33 | time.sleep(1) 34 | 35 | if repo.archived or repo.fork: 36 | continue 37 | 38 | save_project_to_lgtm(site, repo.full_name) 39 | 40 | 41 | if len(sys.argv) < 2: 42 | print("Please provide a language you want to search") 43 | exit 44 | 45 | language = sys.argv[1].capitalize() 46 | 47 | print('Following the top repos for %s' % language) 48 | find_and_save_projects_to_lgtm(language) 49 | -------------------------------------------------------------------------------- /lgtm.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional, List, Dict, Callable 3 | 4 | import requests 5 | import yaml 6 | import time 7 | 8 | from requests.exceptions import SSLError 9 | 10 | 11 | class LGTMRequestException(Exception): 12 | pass 13 | 14 | 15 | @dataclass 16 | class LGTMSite: 17 | nonce: str 18 | long_session: str 19 | short_session: str 20 | api_version: str 21 | 22 | def _cookies(self): 23 | return { 24 | 'lgtm_long_session': self.long_session, 25 | 'lgtm_short_session': self.short_session 26 | } 27 | 28 | def _headers(self): 29 | return { 30 | 'LGTM-Nonce': self.nonce 31 | } 32 | 33 | def _make_lgtm_get(self, url: str) -> dict: 34 | r = LGTMSite._resilient_request(lambda: requests.get( 35 | url, 36 | cookies=self._cookies(), 37 | headers=self._headers() 38 | )) 39 | return r.json() 40 | 41 | def get_my_projects(self) -> List[dict]: 42 | url = 'https://lgtm.com/internal_api/v0.2/getMyProjects?apiVersion=' + self.api_version 43 | data = self._make_lgtm_get(url) 44 | if data['status'] == 'success': 45 | return data['data'] 46 | else: 47 | raise LGTMRequestException('LGTM GET request failed with response: %s' % str(data)) 48 | 49 | def get_my_projects_under_org(self, org: str) -> List['SimpleProject']: 50 | projects_sorted = LGTMDataFilters.org_to_ids(self.get_my_projects()) 51 | return LGTMDataFilters.extract_project_under_org(org, projects_sorted) 52 | 53 | def _make_lgtm_post(self, url: str, data: dict, retry_count: int = 0) -> dict: 54 | api_data = { 55 | 'apiVersion': self.api_version 56 | } 57 | full_data = {**api_data, **data} 58 | print(data) 59 | r = LGTMSite._resilient_request(lambda: requests.post( 60 | url, 61 | full_data, 62 | cookies=self._cookies(), 63 | headers=self._headers() 64 | )) 65 | try: 66 | data_returned = r.json() 67 | except ValueError as e: 68 | response_text = r.text 69 | raise LGTMRequestException(f'Failed to parse JSON. Response was: {response_text}') from e 70 | 71 | print(data_returned) 72 | if data_returned['status'] == 'success': 73 | if 'data' in data_returned: 74 | return data_returned['data'] 75 | else: 76 | return {} 77 | else: 78 | raise LGTMRequestException('LGTM POST request failed with response: %s' % str(data_returned)) 79 | 80 | def load_into_project_list(self, into_project: int, lgtm_project_ids: List[str]): 81 | url = "https://lgtm.com/internal_api/v0.2/updateProjectSelection" 82 | # Because LGTM uses some wacky format for it's application/x-www-form-urlencoded data 83 | list_serialized = ', '.join([('"' + str(elem) + '"') for elem in lgtm_project_ids]) 84 | data = { 85 | 'projectSelectionId': into_project, 86 | 'addedProjects': '[' + list_serialized + ']', 87 | 'removedProjects': '[]', 88 | } 89 | self._make_lgtm_post(url, data) 90 | 91 | def force_rebuild_all_proto_projects(self): 92 | org_to_projects = LGTMDataFilters.org_to_ids(self.get_my_projects()) 93 | for org in org_to_projects: 94 | for project in org_to_projects[org]: 95 | if not project.is_protoproject: 96 | continue 97 | 98 | time.sleep(1) 99 | self.force_rebuild_project(project) 100 | 101 | def force_rebuild_project(self, simple_project: 'SimpleProject'): 102 | url = 'https://lgtm.com/internal_api/v0.2/rebuildProtoproject' 103 | data = { 104 | **simple_project.make_post_data(), 105 | 'config': '' 106 | } 107 | try: 108 | self._make_lgtm_post(url, data) 109 | except LGTMRequestException: 110 | print('Failed rebuilding project. This may be because it is already being built. `%s`' % simple_project) 111 | 112 | def follow_repository(self, repository_url: str): 113 | url = "https://lgtm.com/internal_api/v0.2/followProject" 114 | data = { 115 | 'url': repository_url, 116 | 'apiVersion': self.api_version 117 | } 118 | self._make_lgtm_post(url, data) 119 | 120 | def unfollow_repository_by_id(self, project_id: str): 121 | url = "https://lgtm.com/internal_api/v0.2/unfollowProject" 122 | data = { 123 | 'project_key': project_id, 124 | } 125 | self._make_lgtm_post(url, data) 126 | 127 | def unfollow_repository(self, simple_project: 'SimpleProject'): 128 | print(f'Unfollowing: {simple_project.display_name}') 129 | url = "https://lgtm.com/internal_api/v0.2/unfollowProject" if not simple_project.is_protoproject \ 130 | else "https://lgtm.com/internal_api/v0.2/unfollowProtoproject" 131 | data = simple_project.make_post_data() 132 | self._make_lgtm_post(url, data) 133 | 134 | def unfollow_repository_by_org(self, org: str, include_protoproject: bool = False): 135 | projects_under_org = self.get_my_projects_under_org(org) 136 | for project in projects_under_org: 137 | if not include_protoproject and project.is_protoproject: 138 | print("Not unfollowing project since it is a protoproject. %s" % project) 139 | continue 140 | print('Unfollowing project %s' % project.display_name) 141 | self.unfollow_repository(project) 142 | 143 | def get_project_lists(self): 144 | url = 'https://lgtm.com/internal_api/v0.2/getUsedProjectSelections' 145 | return self._make_lgtm_post(url, {}) 146 | 147 | def get_project_list_by_name(self, list_name: str) -> Optional[int]: 148 | project_lists = self.get_project_lists() 149 | for project_list in project_lists: 150 | if project_list['name'] == list_name: 151 | return int(project_list['key']) 152 | return None 153 | 154 | def get_or_create_project_list(self, list_name: str) -> int: 155 | project_list_id = self.get_project_list_by_name(list_name) 156 | if project_list_id is not None: 157 | print('Found Project List with name: %s' % list_name) 158 | else: 159 | print('Creating Project List with name: %s' % list_name) 160 | project_list_id = self.create_project_list(list_name) 161 | return project_list_id 162 | 163 | def create_project_list(self, name: str) -> int: 164 | """ 165 | :param name: Name of the project list to create. 166 | :return: The key id for this project. 167 | """ 168 | url = 'https://lgtm.com/internal_api/v0.2/createProjectSelection' 169 | data = { 170 | 'name': name 171 | } 172 | response = self._make_lgtm_post(url, data) 173 | return int(response['key']) 174 | 175 | def add_org_to_project_list_by_list_key(self, org: str, project_list_key: int): 176 | projects_under_org = self.get_my_projects_under_org(org) 177 | ids = [] 178 | for project in projects_under_org: 179 | print('Adding `%s` project to project list' % project.display_name) 180 | ids.append(project.key) 181 | self.load_into_project_list(project_list_key, ids) 182 | 183 | def add_org_to_project_list_by_list_name(self, org: str, project_name: str): 184 | pass 185 | 186 | @staticmethod 187 | def _resilient_request(request_method: Callable[[], requests.Response], retry_count: int = 0): 188 | try: 189 | return request_method() 190 | except SSLError as e: 191 | if retry_count < 4: 192 | return LGTMSite._resilient_request(request_method, retry_count + 1) 193 | raise LGTMRequestException(f'SSL Error') from e 194 | 195 | @staticmethod 196 | def retrieve_project(gh_project_path: str): 197 | url = "https://lgtm.com/api/v1.0/projects/g/" + gh_project_path 198 | r = LGTMSite._resilient_request(lambda: requests.get(url)) 199 | return r.json() 200 | 201 | @staticmethod 202 | def retrieve_project_id(gh_project_path: str) -> Optional[int]: 203 | data_returned = LGTMSite.retrieve_project(gh_project_path) 204 | if 'id' in data_returned: 205 | return int(data_returned["id"]) 206 | else: 207 | return None 208 | 209 | @staticmethod 210 | def create_from_file() -> 'LGTMSite': 211 | with open("config.yml") as config_file: 212 | config = yaml.safe_load(config_file) 213 | lgtm: dict = config['lgtm'] 214 | return LGTMSite( 215 | nonce=lgtm['nonce'], 216 | long_session=lgtm['long_session'], 217 | short_session=lgtm['short_session'], 218 | api_version=lgtm['api_version'], 219 | ) 220 | 221 | 222 | @dataclass 223 | class SimpleProject: 224 | display_name: str # The repository name, e.g. `gradle/gradle` or `github/codeql` 225 | key: str # The LGTM key for this project. Unique identifier for LGTM.com 226 | is_protoproject: bool 227 | 228 | def make_post_data(self): 229 | data_dict_key = 'protoproject_key' if self.is_protoproject else 'project_key' 230 | return { 231 | data_dict_key: self.key 232 | } 233 | 234 | 235 | class LGTMDataFilters: 236 | 237 | @staticmethod 238 | def org_to_ids(projects: List[Dict]) -> Dict[str, List[SimpleProject]]: 239 | """ 240 | Converts the output from :func:`~lgtm.LGTMSite.get_my_projects` into a dic of GH org 241 | to list of projects including their GH id and LGTM id. 242 | """ 243 | org_to_ids = {} 244 | for project in projects: 245 | org: str 246 | display_name: str 247 | key: str 248 | is_protoproject: bool 249 | if 'protoproject' in project: 250 | the_project = project['protoproject'] 251 | if 'https://github.com/' not in the_project['cloneUrl']: 252 | # Not really concerned with BitBucket right now 253 | continue 254 | display_name = the_project['displayName'] 255 | org = display_name.split('/')[0] 256 | key = the_project['key'] 257 | is_protoproject = True 258 | elif 'realProject' in project: 259 | 260 | the_project = project['realProject'][0] 261 | if the_project['repoProvider'] != 'github_apps': 262 | # Not really concerned with BitBucket right now 263 | continue 264 | org = str(the_project['slug']).split('/')[1] 265 | display_name = the_project['displayName'] 266 | key = the_project['key'] 267 | is_protoproject = False 268 | else: 269 | raise KeyError('\'realProject\' nor \'protoproject\' in %s' % str(project)) 270 | 271 | ids_list: List[SimpleProject] 272 | if org in org_to_ids: 273 | ids_list = org_to_ids[org] 274 | else: 275 | ids_list = [] 276 | org_to_ids[org] = ids_list 277 | ids_list.append(SimpleProject( 278 | display_name=display_name, 279 | key=key, 280 | is_protoproject=is_protoproject 281 | )) 282 | 283 | return org_to_ids 284 | 285 | @staticmethod 286 | def extract_project_under_org(org: str, projects_sorted: Dict[str, List[SimpleProject]]) -> List[SimpleProject]: 287 | if org not in projects_sorted: 288 | print('org %s not found in projects list' % org) 289 | return [] 290 | return projects_sorted[org] 291 | -------------------------------------------------------------------------------- /move_org_projects_under_project_list_then_unfollow.py: -------------------------------------------------------------------------------- 1 | from lgtm import LGTMSite 2 | import sys 3 | 4 | project_list_name = sys.argv[1] 5 | github_org = sys.argv[2] 6 | print( 7 | 'Moving code under GH org `%s` to project list `%s`, then unfollowing org' % 8 | (github_org, project_list_name) 9 | ) 10 | 11 | site = LGTMSite.create_from_file() 12 | 13 | project_list_id = site.get_or_create_project_list(project_list_name) 14 | 15 | site.add_org_to_project_list_by_list_key(github_org, project_list_id) 16 | site.unfollow_repository_by_org(github_org) 17 | -------------------------------------------------------------------------------- /move_repositories_under_project_list.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from lgtm import LGTMSite 4 | 5 | 6 | def main(): 7 | parser = argparse.ArgumentParser( 8 | description='moves the specified list of projects under a project list' 9 | ) 10 | parser.add_argument('-l', '--list-name', dest='list_name', required=True) 11 | parser.add_argument('-i', '--infile', dest='in_file', required=True, type=argparse.FileType('r')) 12 | args = parser.parse_args() 13 | 14 | site = LGTMSite.create_from_file() 15 | 16 | project_list_id = site.get_or_create_project_list(args.list_name) 17 | 18 | ids = [] 19 | for line in args.in_file: 20 | line_clean: str = line.strip() 21 | gh_project_path = line_clean.lstrip('https://github.com/') 22 | the_id = LGTMSite.retrieve_project_id(gh_project_path) 23 | if the_id is not None: 24 | print('Loaded: %s' % gh_project_path) 25 | ids.append(str(the_id)) 26 | 27 | site.load_into_project_list(project_list_id, ids) 28 | 29 | 30 | if __name__ == "__main__": 31 | main() 32 | -------------------------------------------------------------------------------- /rebuild_all_following_projects.py: -------------------------------------------------------------------------------- 1 | from lgtm import LGTMSite 2 | 3 | site = LGTMSite.create_from_file() 4 | site.force_rebuild_all_proto_projects() 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2021.10.8 2 | cffi==1.15.0 3 | charset-normalizer==2.0.10 4 | Deprecated==1.2.13 5 | idna==3.3 6 | pycparser==2.21 7 | PyGithub==1.55 8 | PyJWT==2.3.0 9 | PyNaCl==1.5.0 10 | PyYAML==6.0 11 | requests==2.27.1 12 | urllib3==1.26.8 13 | wrapt==1.13.3 14 | -------------------------------------------------------------------------------- /topJavaMavenProjects.csv: -------------------------------------------------------------------------------- 1 | repository_url,forks,watchers,normForks+normWatchers 2 | https://github.com/spring-projects/spring-boot,8873,12605,523.55661589948068 3 | https://github.com/iluwatar/java-design-patterns,4731,19250,421.65921283865379 4 | https://github.com/square/retrofit,3299,20846,378.45158799039257 5 | https://github.com/square/okhttp,3538,19629,374.84806606746878 6 | https://github.com/zxing/zxing,4547,12526,337.2786157157621 7 | https://github.com/libgdx/libgdx,5078,9165,321.79670607167384 8 | https://github.com/google/guava,2994,16927,320.79701410067929 9 | https://github.com/alibaba/dubbo,4721,8520,299.16055838149504 10 | https://github.com/jfeinstein10/SlidingMenu,4805,7354,289.49512548068668 11 | https://github.com/netty/netty,4033,10088,287.5165372053915 12 | https://github.com/JakeWharton/ActionBarSherlock,4360,8061,278.46903525937762 13 | https://github.com/chrisbanes/Android-PullToRefresh,3898,8937,268.6370881376576 14 | https://github.com/alibaba/fastjson,2425,8578,201.43140744250849 15 | https://github.com/deeplearning4j/deeplearning4j,2264,7548,182.81435706900319 16 | https://github.com/JakeWharton/ViewPagerIndicator,2887,4572,175.65482614118363 17 | https://github.com/alibaba/druid,2370,6379,174.05754622964196 18 | https://github.com/liaohuqiu/android-Ultra-Pull-To-Refresh,1723,7960,164.31836614082468 19 | https://github.com/mybatis/mybatis-3,2513,4705,161.14112628210648 20 | https://github.com/springside/springside4,2507,3927,152.03307446226293 21 | https://github.com/apache/storm,2354,4305,149.7769923944943 22 | https://github.com/xetorthio/jedis,1822,5108,136.11489451356266 23 | https://github.com/apache/hadoop,2179,3482,132.91495017539168 24 | https://github.com/dropwizard/dropwizard,1958,4041,129.80405981287805 25 | https://github.com/swagger-api/swagger-codegen,1937,3911,127.42521205395445 26 | https://github.com/code4craft/webmagic,1686,4455,122.85810337155172 27 | https://github.com/junit-team/junit,1407,4918,116.16963030485292 28 | https://github.com/Trinea/android-common,1685,3776,115.09058942701145 29 | https://github.com/clojure/clojure,1069,5664,110.17242578801702 30 | https://github.com/nhaarman/ListViewAnimations,1306,4346,105.33416997849427 31 | https://github.com/perwendel/spark,967,5562,104.64108064110486 32 | https://github.com/spring-projects/spring-mvc-showcase,1860,2073,103.21552532131817 33 | https://github.com/square/dagger,763,6154,102.63410556285361 34 | https://github.com/swagger-api/swagger-core,1214,4136,99.002683426453331 35 | https://github.com/jhy/jsoup,1125,4214,96.07619359240843 36 | https://github.com/mcxiaoke/android-volley,1045,4419,94.98019376928967 37 | https://github.com/Activiti/Activiti,1574,2217,92.597973821042274 38 | https://github.com/spring-projects/spring-petclinic,1824,1122,90.853763970454168 39 | https://github.com/openhab/openhab,1227,3315,90.219635559937871 40 | https://github.com/JakeWharton/NineOldAndroids,1218,3306,89.731575694033864 41 | https://github.com/wildfly/wildfly,1576,1948,89.623392846131111 42 | https://github.com/Bukkit/Bukkit,1491,2235,89.246006471853207 43 | https://github.com/jersey/jersey,1593,1832,89.032205659136707 44 | https://github.com/NLPchina/ansj_seg,1250,3076,88.486249678677325 45 | https://github.com/spring-projects/spring-security-oauth,1642,1573,88.185450275573288 46 | https://github.com/eclipse/vert.x,1038,3507,84.304835030643702 47 | https://github.com/apache/flink,1353,2287,83.923961925343676 48 | https://github.com/neo4j/neo4j,961,3749,83.758323735685792 49 | https://github.com/google/guice,554,5220,83.05228907585338 50 | https://github.com/MyCATApache/Mycat-Server,1255,2561,82.841600189204414 51 | https://github.com/apache/camel,1570,1310,82.108054757669748 52 | https://github.com/druid-io/druid,908,3744,81.43026669358207 53 | https://github.com/naver/pinpoint,891,3802,81.361615334718124 54 | https://github.com/AsyncHttpClient/async-http-client,877,3853,81.34188533129489 55 | https://github.com/thinkaurelius/titan,762,4212,80.498038455915378 56 | https://github.com/stanfordnlp/CoreNLP,952,3217,77.320340430403988 57 | https://github.com/dropwizard/metrics,1043,2836,76.885447383344911 58 | https://github.com/bauerca/drag-sort-listview,1263,2002,76.82494056902118 59 | https://github.com/EnterpriseQualityCoding/FizzBuzzEnterpriseEdition,356,5371,76.285379827767827 60 | https://github.com/brettwooldridge/HikariCP,570,4337,73.692453598878487 61 | https://github.com/pardom/ActiveAndroid,748,3646,73.458991507757744 62 | https://github.com/google/auto,432,4834,73.4329634128338 63 | https://github.com/square/otto,660,3932,72.941671564817284 64 | https://github.com/openmrs/openmrs-core,1569,508,72.941228034843704 65 | https://github.com/alibaba/jstorm,904,2847,71.054112896736854 66 | https://github.com/b3log/solo,978,2420,69.36740971892992 67 | https://github.com/hankcs/HanLP,872,2783,68.954740076190774 68 | https://github.com/knightliao/disconf,934,2478,68.141744981977638 69 | https://github.com/facebook/presto,936,2429,67.669999870667084 70 | https://github.com/aws/aws-sdk-java,1163,1429,66.020956063346404 71 | https://github.com/cucumber/cucumber-jvm,1132,1494,65.432007528339113 72 | https://github.com/Atmosphere/atmosphere,676,3121,64.40094600683895 73 | https://github.com/yusuke/twitter4j,903,2244,64.151214977804074 74 | https://github.com/yasserg/crawler4j,942,2076,63.911200046273038 75 | https://github.com/alibaba/canal,864,2271,62.787139651300194 76 | https://github.com/gephi/gephi,977,1822,62.521394433260781 77 | https://github.com/NanoHttpd/nanohttpd,828,2370,62.370731285802677 78 | https://github.com/google/closure-compiler,535,3368,61.168767115601966 79 | https://github.com/JakeWharton/DiskLruCache,670,2845,61.003910566665844 80 | https://github.com/apache/hive,1129,1110,60.934864251680985 81 | https://github.com/square/okio,441,3669,60.564980988432822 82 | https://github.com/scribejava/scribejava,1080,1252,60.450566016875001 83 | https://github.com/checkstyle/checkstyle,860,2071,60.340424931407775 84 | https://github.com/roboguice/roboguice,740,2443,59.430211165830741 85 | https://github.com/hazelcast/hazelcast,788,2205,58.779510494638039 86 | https://github.com/antlr/antlr4,670,2633,58.592086916287087 87 | https://github.com/databricks/learning-spark,883,1755,57.731046497887007 88 | https://github.com/Alluxio/alluxio,1029,1123,56.797524364505982 89 | https://github.com/jfinal/jfinal,865,1703,56.368124859886166 90 | https://github.com/apache/hbase,926,1465,56.274504704069329 91 | https://github.com/javaee-samples/javaee7-samples,939,1402,56.114864040323205 92 | https://github.com/joelittlejohn/jsonschema2pojo,594,2689,55.972394011257997 93 | https://github.com/dangdangdotcom/elastic-job,746,2090,55.671411341436681 94 | https://github.com/pxb1988/dex2jar,448,3195,55.472473786395042 95 | https://github.com/alibaba/DataX,725,2064,54.475722354397007 96 | https://github.com/shuzheng/zheng,862,1498,53.907379854066704 97 | https://github.com/Graylog2/graylog2-server,463,3000,53.89683629916177 98 | https://github.com/brianfrankcooper/YCSB,868,1443,53.538784972186193 99 | https://github.com/essentials/Essentials,1077,636,53.314068556783511 100 | https://github.com/kbastani/spring-cloud-microservice-example,808,1574,52.45796912349752 101 | https://github.com/square/javapoet,388,3091,51.718174174314832 102 | -------------------------------------------------------------------------------- /unfollow_org.py: -------------------------------------------------------------------------------- 1 | from lgtm import LGTMSite 2 | import sys 3 | 4 | org_to_delete = sys.argv[1] 5 | print('Unfollowing Org: %s' % org_to_delete) 6 | 7 | site = LGTMSite.create_from_file() 8 | site.unfollow_repository_by_org(org_to_delete) 9 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JLLeitschuh/lgtm_hack_scripts/bcee06595bb3549c68057c7ef6631d9b04d3ca57/utils/__init__.py -------------------------------------------------------------------------------- /utils/github_api.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from github import Github 4 | import yaml 5 | 6 | 7 | def create() -> Github: 8 | hub_config_file = f'{os.path.expanduser("~")}/.config/hub' 9 | if os.path.exists(hub_config_file): 10 | with open(f'{os.path.expanduser("~")}/.config/hub') as hub_file: 11 | hub_config = yaml.safe_load(hub_file) 12 | return Github(login_or_token=hub_config['github.com'][0]['oauth_token']) 13 | else: 14 | with open("config.yml") as config_file: 15 | config = yaml.safe_load(config_file) 16 | github: dict = config['github'] 17 | return Github(github['api_key']) 18 | -------------------------------------------------------------------------------- /utils/github_dates.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from datetime import datetime 3 | 4 | 5 | def current_year() -> int: 6 | now = datetime.now() 7 | return now.year 8 | 9 | 10 | def generate_dates() -> List[str]: 11 | date_ranges: List[str] = [] 12 | 13 | # Github started in 2008 14 | year_range = list(range(2008, current_year() + 1)) 15 | 16 | for i, year in enumerate(year_range): 17 | date_ranges.append(f'{year}-01-01..{year}-12-31') 18 | 19 | return date_ranges 20 | --------------------------------------------------------------------------------