├── .env ├── LICENSE ├── README.md ├── crawled_repos ├── solana_devs.csv ├── solana_integrated_repos_2022.csv └── solana_integrated_repos_feb_2023.csv ├── local └── docker-compose.yml ├── nodemon.json ├── package.json ├── server.js └── src ├── common ├── cache.js ├── constant.js ├── githubAPI.js └── index.js ├── crons ├── common.cron.js ├── github.cron.js └── solanaResponseCache.cron.js ├── db ├── config │ └── config.js └── models │ ├── activities.js │ ├── developers.js │ ├── index.js │ ├── repotypes.js │ └── solanagithubrepos.js └── middleware ├── githubError.js ├── index.js ├── postgresError.js └── unknownError.js /.env: -------------------------------------------------------------------------------- 1 | NODE_ENV=development 2 | GITHUB_ACCESS_TOKEN= 3 | 4 | DB_HOST=localhost 5 | DB_PORT=5432 6 | DB_USERNAME=postgres 7 | DB_PASSWORD=solana 8 | DB_NAME=solana_analytics 9 | DB_DIALECT= 10 | 11 | PROD_DB_HOST= 12 | PROD_DB_PORT= 13 | PROD_DB_USERNAME= 14 | PROD_DB_PASSWORD= 15 | PROD_DB_NAME= 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # solana-developer-data 2 | 3 | Goal: Collecting Solana Developer Data from Github 4 | 5 | This repository is an example crawler that searches for Solana related activity on Github and collects them into a postgres db. 6 | 7 | The crawler uses the following to find Solana repositories: 8 | 9 | | Library | Query | Description | 10 | |-----------------|-------------------------------------------|------------------------------| 11 | | @solana/web3.js | solana/web3.js filename:package.json | Solana JS/TS SDK | 12 | | @solana/web3.js | solana/web3.js filename:package-lock.json | Solana JS/TS SDK | 13 | | @solana/web3.js | solana/web3.js filename:yarn.lock | Solana JS/TS SDK | 14 | | serum/anchor | serum/anchor filename:package.json | Anchor JS/TS SDK | 15 | | solana-program | solana-program filename:Cargo.toml | Solana Rust Program SDK | 16 | | anchor-lang | anchor-lang filename:Cargo.toml | Anchor Framework Program SDK | 17 | | Solnet | Solnet.Rpc filename:*.csproj | Solana C# SDK | 18 | | solana-go | gagliardetto/solana filename:mod.go | Solana Go SDK | 19 | | solana | "from solana rpc import" language:python | Solana Python SDK | 20 | 21 | ## How to Run Locally 22 | 23 | 1. Go into the `/local` folder and run `docker compose up` to start a docker container with a postgres db 24 | ```bash 25 | cd local 26 | docker compose up 27 | ``` 28 | 2. Get a [Github access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and place in the `.env` file under `GITHUB_ACCESS_TOKEN` 29 | 2. Run `npm install` 30 | 3. Run `npm run start` 31 | -------------------------------------------------------------------------------- /local/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | postgres: 4 | image: postgres:latest 5 | restart: always 6 | environment: 7 | - POSTGRES_USER=postgres 8 | - POSTGRES_PASSWORD=solana 9 | - POSTGRES_DB=solana_analytics 10 | logging: 11 | options: 12 | max-size: 10m 13 | max-file: "3" 14 | ports: 15 | - '5432:5432' -------------------------------------------------------------------------------- /nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "watch": [ 3 | "src" 4 | ], 5 | "ext": ".js", 6 | "ignore": [] 7 | } 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "solana_analytics", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node ./server.js", 8 | "db:create": "sequelize-cli db:create", 9 | "db:migrate": "sequelize-cli db:migrate", 10 | "db:g:migration": "sequelize-cli migration:generate --name", 11 | "db:g:seed": "sequelize-cli seed:generate --name", 12 | "db:seeds": "sequelize-cli db:seed:all", 13 | "test": "echo \"Error: no test specified\" && exit 1" 14 | }, 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "@solana/web3.js": "^1.36.0", 19 | "apollo-server-express": "^3.1.2", 20 | "app-module-path": "^2.2.0", 21 | "body-parser": "^1.19.0", 22 | "cors": "^2.8.5", 23 | "dotenv": "^10.0.0", 24 | "express": "^4.17.1", 25 | "graphql": "^15.5.1", 26 | "lodash": "^4.17.21", 27 | "morgan": "^1.10.0", 28 | "mysql2": "^2.3.3", 29 | "node-cache": "^5.1.2", 30 | "node-cron": "^3.0.0", 31 | "node-fetch": "^2.6.7", 32 | "octokit": "^1.3.0", 33 | "pg": "^8.7.1", 34 | "pg-hstore": "^2.3.4", 35 | "sequelize": "^6.6.5" 36 | }, 37 | "devDependencies": { 38 | "nodemon": "^2.0.12", 39 | "prettier": "^2.5.1", 40 | "sequelize-cli": "^6.2.0" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | const express = require("express"); 2 | const bodyParser = require("body-parser"); 3 | const cors = require("cors"); 4 | require("dotenv").config(); 5 | require("app-module-path").addPath(__dirname); 6 | 7 | // middleware 8 | const middleware = require("./src/middleware"); 9 | // db sync 10 | const db = require("./src/db/models"); 11 | db.sequelize.sync(); 12 | // cron 13 | const githubCronJob = require("./src/crons/github.cron").findSolanaReposCron; 14 | const fetchSolanaDevelopers = 15 | require("./src/crons/github.cron").fetchSolanaDevelopers; 16 | githubCronJob.start(); 17 | fetchSolanaDevelopers.start(); 18 | const enableFetchSolanaReposCronJob = 19 | require("./src/crons/github.cron").enableFetchSolanaRepos; 20 | enableFetchSolanaReposCronJob.start(); 21 | 22 | const solanaResponseCacheCronJob = 23 | require("./src/crons/solanaResponseCache.cron").solanaResponseCache; 24 | solanaResponseCacheCronJob.start(); 25 | 26 | const port = 8080; 27 | const app = express(); 28 | app.use(bodyParser.json()); 29 | app.use(require("morgan")("dev")); 30 | app.use(cors()); 31 | 32 | app.use(middleware.githubError); 33 | app.use(middleware.postgresError); 34 | app.use(middleware.unknownError); 35 | 36 | const server = app.listen(port, function () { 37 | const host = server.address().address; 38 | const port = server.address().port; 39 | 40 | console.log("App listening at //%s%s", host, port); 41 | }); 42 | -------------------------------------------------------------------------------- /src/common/cache.js: -------------------------------------------------------------------------------- 1 | const NodeCache = require("node-cache"); 2 | 3 | const responseCache = new NodeCache(); 4 | 5 | exports.responseCache = responseCache; 6 | 7 | exports.getResponseCacheValue = (name) => { 8 | const result = responseCache.get(name); 9 | if (result) { 10 | return result; 11 | } else { 12 | return []; 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /src/common/constant.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | DB: "postgres", 3 | SOLANA_WEB3_KEY_WORD: "solana/web3.js", 4 | NATIVE_KEY_WORD: "solana-program", 5 | ANCHOR_KEY_WORD: "anchor-lang", 6 | ANCHOR_WEB3_KEY_WORD: "serum/anchor", 7 | METAPLEX_KEY_WORD: "metaplex/js", 8 | GO_KEY_WORD: "gagliardetto/solana", 9 | DOT_NET_KEY_WORD: "Solnet.Rpc", 10 | WEB3_JS_TYPE: "web3.js", 11 | ANCHOR_JS_TYPE: "anchor.js", 12 | NFT_TYPE: "nft", 13 | GO_TYPE: "go", 14 | NATIVE_TYPE: "native", 15 | ANCHOR_TYPE: "anchor", 16 | DOT_NET_TYPE: "dotnet", 17 | }; -------------------------------------------------------------------------------- /src/common/githubAPI.js: -------------------------------------------------------------------------------- 1 | const { Octokit } = require("octokit"); 2 | const githubAPI = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); 3 | 4 | exports.getRepoContributorsActivity = ({ owner, repo }) => { 5 | return githubAPI.request("GET /repos/{owner}/{repo}/stats/contributors", { 6 | owner, 7 | repo, 8 | }); 9 | }; 10 | 11 | exports.getRepoActivity = ({ owner, repo }) => { 12 | return githubAPI.request("GET /repos/{owner}/{repo}/stats/commit_activity", { 13 | owner, 14 | repo, 15 | }); 16 | }; 17 | 18 | exports.getRepoInfo = ({ owner, repo }) => { 19 | return githubAPI.request("GET /repos/{owner}/{repo}", { 20 | owner, 21 | repo, 22 | }); 23 | }; 24 | 25 | exports.searchGithubRepos = ({ searchKey, per_page = 30, page = 1 }) => { 26 | return githubAPI.request("GET /search/code", { 27 | q: searchKey, 28 | per_page, 29 | page, 30 | }); 31 | }; 32 | 33 | exports.getRepoContributors = ({ owner, repo, per_page = 30, page = 1 }) => { 34 | return githubAPI.request("GET /repos/{owner}/{repo}/contributors", { 35 | owner, 36 | repo, 37 | per_page, 38 | page, 39 | }); 40 | }; 41 | 42 | exports.getGitUserInfo = ({ username }) => { 43 | return githubAPI.request("GET /users/{username}", { 44 | username, 45 | }); 46 | }; 47 | 48 | exports.getGetRepoCommits = ({ owner, repo, ...props }) => { 49 | return githubAPI.request("GET /repos/{owner}/{repo}/commits", { 50 | owner, 51 | repo, 52 | ...props, 53 | }); 54 | }; 55 | -------------------------------------------------------------------------------- /src/common/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | constant: require("./constant"), 3 | githubAPI: require("./githubAPI"), 4 | }; 5 | -------------------------------------------------------------------------------- /src/crons/common.cron.js: -------------------------------------------------------------------------------- 1 | const nodeCron = require("node-cron"); 2 | const isEmpty = require("lodash/isEmpty"); 3 | const { Op } = require("sequelize"); 4 | 5 | const common = require("../common"); 6 | const db = require("../db/models"); 7 | const SolanaGithubRepos = db.SolanaGithubRepos; 8 | const Developers = db.Developers; 9 | const Activities = db.Activities; 10 | 11 | let page = 1; 12 | const pageSize = 25; 13 | 14 | const minFilesize = 300; 15 | const maxFilesize = 5000; 16 | const filesizeInterval = 50; 17 | let filesizeIndex = 300; 18 | 19 | let searchIndex = 0; 20 | 21 | const UpdateSearchQuery = (filename, keyword) => { 22 | let sizeQuery = ""; 23 | if (filesizeIndex === minFilesize) { 24 | sizeQuery = ` size:<=${minFilesize}`; 25 | } else if (filesizeIndex > maxFilesize) { 26 | sizeQuery = ` size:>=${maxFilesize}`; 27 | } else { 28 | sizeQuery = ` size:${filesizeIndex - filesizeInterval}..${filesizeIndex}`; 29 | } 30 | 31 | return keyword + sizeQuery + ` filename:${filename}`; 32 | }; 33 | 34 | const UpdateGithubRepos = async ( 35 | page, 36 | pageSize, 37 | ecosystem, 38 | fileName, 39 | keyword, 40 | typeMap 41 | ) => { 42 | const searchKey = UpdateSearchQuery(fileName, keyword); 43 | const { data } = await common.githubAPI.searchGithubRepos({ 44 | searchKey, 45 | per_page: pageSize, 46 | page, 47 | }); 48 | let newData = []; 49 | await Promise.all( 50 | data && 51 | data.items.map(async (repo) => { 52 | const { repository } = repo; 53 | if (repository && repository.id && repository.name) { 54 | try { 55 | const { data } = await common.githubAPI.getRepoInfo({ 56 | owner: repository.owner.login, 57 | repo: repository.name, 58 | }); 59 | newData.push({ 60 | repoId: repository.id, 61 | name: repository.name, 62 | url: repository.html_url, 63 | owner: repository.owner.login, 64 | started: parseInt(new Date(data.created_at).getTime() / 1000), 65 | ecosystem: ecosystem, 66 | }); 67 | } catch (err) { 68 | console.log(err); 69 | } 70 | } 71 | }) 72 | ); 73 | const bulkData = newData.filter((item) => item !== undefined); 74 | if (bulkData && bulkData.length) { 75 | await SolanaGithubRepos.bulkCreate(bulkData, { 76 | fields: ["repoId", "name", "url", "owner", "started", "ecosystem"], 77 | ignoreDuplicates: true, 78 | returning: true, 79 | }); 80 | if (typeMap) { 81 | let repoTypeData = []; 82 | bulkData.map((repo) => { 83 | repoTypeData.push({ 84 | repoId: repo.repoId, 85 | type: typeMap[keyword], 86 | }); 87 | }); 88 | await RepoTypes.bulkCreate(repoTypeData, { 89 | fields: ["repoId", "type"], 90 | ignoreDuplicates: true, 91 | }); 92 | } 93 | } 94 | return data; 95 | }; 96 | 97 | exports.findRepos = async ( 98 | ecosystem, 99 | searchFileNamesOrder, 100 | searchKeyWordOrder, 101 | typeMap 102 | ) => { 103 | try { 104 | const data = await UpdateGithubRepos( 105 | page, 106 | pageSize, 107 | ecosystem, 108 | searchFileNamesOrder[searchIndex], 109 | searchKeyWordOrder[searchIndex], 110 | typeMap 111 | ); 112 | if (!data || data.items.length === 0) { 113 | page = 1; 114 | if (filesizeIndex > maxFilesize) { 115 | filesizeIndex = minFilesize; 116 | if (searchIndex === searchFileNamesOrder.length - 1) { 117 | console.log( 118 | `SolanaGithubRepos table was updated successfully with ${ecosystem}!` 119 | ); 120 | return; 121 | } else { 122 | searchIndex += 1; 123 | } 124 | } else { 125 | filesizeIndex += filesizeInterval; 126 | } 127 | } else { 128 | page += 1; 129 | } 130 | } catch (err) { 131 | console.log(err); 132 | if ( 133 | err.response && 134 | err.response.data.message === 135 | "Only the first 1000 search results are available" 136 | ) { 137 | page = 1; 138 | if (filesizeIndex > maxFilesize) { 139 | filesizeIndex = minFilesize; 140 | if (searchIndex === searchFileNamesOrder.length - 1) { 141 | console.log( 142 | `SolanaGithubRepos table was updated but you missed some ${ecosystem} repos because of 1000 limit` 143 | ); 144 | return; 145 | } else { 146 | searchIndex += 1; 147 | } 148 | } else { 149 | filesizeIndex += filesizeInterval; 150 | } 151 | } else { 152 | page = 1; 153 | if (filesizeIndex > maxFilesize) { 154 | filesizeIndex = minFilesize; 155 | if (searchIndex === searchFileNamesOrder.length - 1) { 156 | console.log( 157 | `SolanaGithubRepos table was updated but you missed some ${ecosystem} repos.` 158 | ); 159 | return; 160 | } else { 161 | searchIndex += 1; 162 | } 163 | } else { 164 | filesizeIndex += filesizeInterval; 165 | } 166 | } 167 | } 168 | }; 169 | 170 | // Developers Cron Parameters 171 | let repoPage = 0; 172 | const repoPageSize = 14; 173 | 174 | exports.fetchDevelopers = async (ecosystem) => { 175 | if (repoPage === 0) { 176 | // Comment to keep old Activities Data. 177 | // const oneYearAgo = new Date(); 178 | // oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); 179 | // const referenceTime = parseInt(oneYearAgo.getTime() / 1000); 180 | // try { 181 | // const response = await Activities.destroy({ 182 | // where: { 183 | // date: { 184 | // [Op.lt]: referenceTime, 185 | // }, 186 | // }, 187 | // truncate: false, 188 | // }); 189 | // console.log( 190 | // `${response} items of Activities table were deleted successfully!` 191 | // ); 192 | // } catch (err) { 193 | // console.log(err); 194 | // } 195 | repoPage++; 196 | return; 197 | } 198 | const currentTime = parseInt(new Date().getTime() / 1000); 199 | const week = 3600 * 24 * 7; 200 | const referenceTime = currentTime - week * 15; 201 | const offset = (repoPage - 1) * repoPageSize; 202 | const limit = repoPageSize; 203 | const repos = await SolanaGithubRepos.findAll({ 204 | where: { ecosystem: ecosystem }, 205 | offset, 206 | limit, 207 | }); 208 | if (repos && repos.length) { 209 | repoPage += 1; 210 | repos.forEach(async (element) => { 211 | const { repoId, owner, name } = element; 212 | let elementData = null; 213 | let isRepoValid = true; 214 | try { 215 | const { data } = await common.githubAPI.getRepoContributorsActivity({ 216 | owner, 217 | repo: name, 218 | }); 219 | elementData = data; 220 | } catch { 221 | // Comment because issue happens by relationships between SolanaGithubRepos and Activities modals 222 | // const response = await SolanaGithubRepos.destroy({ 223 | // where: { 224 | // repoId, 225 | // }, 226 | // truncate: false, 227 | // }); 228 | isRepoValid = false; 229 | } 230 | 231 | if (isRepoValid) { 232 | if (elementData && !isEmpty(elementData)) { 233 | elementData.forEach(async (contribution) => { 234 | const username = contribution.author.login; 235 | const activities = contribution.weeks.filter( 236 | (item) => item.c !== 0 && item.w > referenceTime 237 | ); 238 | let developer = await Developers.findOne({ 239 | where: { 240 | username, 241 | }, 242 | }); 243 | if (!developer) { 244 | try { 245 | const developerInfo = await common.githubAPI.getGitUserInfo({ 246 | username, 247 | }); 248 | developer = await Developers.create({ 249 | username, 250 | name: developerInfo.data.name, 251 | gitUrl: developerInfo.data.html_url, 252 | avatar: developerInfo.data.avatar_url, 253 | location: developerInfo.data.location, 254 | twitter: developerInfo.data.twitter_username, 255 | }); 256 | } catch (err) { 257 | console.log(err); 258 | } 259 | } 260 | try { 261 | if (!developer) { 262 | developer = await Developers.findOne({ 263 | where: { 264 | username, 265 | }, 266 | }); 267 | } 268 | const bulkData = activities.map((activity) => ({ 269 | commits: activity.c, 270 | additions: activity.a, 271 | delettioins: activity.d, 272 | date: activity.w, 273 | developerId: developer.id, 274 | repositoryId: element.id, 275 | })); 276 | if (bulkData && bulkData.length) { 277 | await Activities.bulkCreate(bulkData, { 278 | fields: [ 279 | "commits", 280 | "additions", 281 | "delettioins", 282 | "date", 283 | "developerId", 284 | "repositoryId", 285 | ], 286 | ignoreDuplicates: true, 287 | }); 288 | } 289 | } catch (err) { 290 | console.log(err); 291 | } 292 | }); 293 | } else { 294 | let commitsData = null; 295 | try { 296 | commitsData = await common.githubAPI.getGetRepoCommits({ 297 | owner, 298 | repo: name, 299 | per_page: 1, 300 | }); 301 | } catch { 302 | commitsData = null; 303 | } 304 | 305 | if (commitsData.data && commitsData.data.length) { 306 | const commit = commitsData.data[0].commit; 307 | const date = parseInt( 308 | new Date(commit.author.date).getTime() / 1000 309 | ); 310 | let developer = await Developers.findOne({ 311 | where: { 312 | username: owner, 313 | }, 314 | }); 315 | if (!developer) { 316 | try { 317 | const developerInfo = await common.githubAPI.getGitUserInfo({ 318 | username: owner, 319 | }); 320 | developer = await Developers.create({ 321 | username: owner, 322 | name: developerInfo.data.name, 323 | gitUrl: developerInfo.data.html_url, 324 | avatar: developerInfo.data.avatar_url, 325 | location: developerInfo.data.location, 326 | twitter: developerInfo.data.twitter_username, 327 | }); 328 | } catch (err) { 329 | console.log(err); 330 | } 331 | } 332 | try { 333 | if (!developer) { 334 | developer = await Developers.findOne({ 335 | where: { 336 | username: owner, 337 | }, 338 | }); 339 | } 340 | await Activities.create({ 341 | commits: 1, 342 | additions: 0, 343 | delettioins: 0, 344 | date, 345 | developerId: developer.id, 346 | repositoryId: element.id, 347 | }); 348 | } catch (err) { 349 | console.log(err); 350 | } 351 | } 352 | } 353 | } 354 | }); 355 | } else { 356 | isDevelopersCron = false; 357 | repoPage = 0; 358 | console.log("Developers table was updated successfully!"); 359 | return { 360 | isDevelopersCron: false, 361 | }; 362 | } 363 | }; 364 | -------------------------------------------------------------------------------- /src/crons/github.cron.js: -------------------------------------------------------------------------------- 1 | const nodeCron = require("node-cron"); 2 | const isEmpty = require("lodash/isEmpty"); 3 | const { Op } = require("sequelize"); 4 | 5 | const common = require("../common"); 6 | const db = require("../db/models"); 7 | const SolanaGithubRepos = db.SolanaGithubRepos; 8 | const RepoTypes = db.RepoTypes; 9 | const Developers = db.Developers; 10 | const Activities = db.Activities; 11 | 12 | let isEnabledCron = true; 13 | let isDevelopersCron = false; 14 | let page = 1; 15 | const pageSize = 25; 16 | 17 | const minFilesize = 300; 18 | const maxFilesize = 5000; 19 | const filesizeInterval = 50; 20 | let filesizeIndex = 300; 21 | 22 | let searchIndex = 0; 23 | 24 | let SEARCHFILENAMESORDER = [ 25 | "package.json", 26 | "package-lock.json", 27 | "yarn.lock", 28 | "mod.go", 29 | "Cargo.toml", 30 | "Cargo.toml", 31 | "package.json", 32 | "package-lock.json", 33 | "yarn.lock", 34 | "package.json", 35 | "package-lock.json", 36 | "yarn.lock", 37 | "*.csproj", 38 | ]; 39 | let SEARCHKEYWORDORDER = [ 40 | common.constant.ANCHOR_WEB3_KEY_WORD, 41 | common.constant.ANCHOR_WEB3_KEY_WORD, 42 | common.constant.ANCHOR_WEB3_KEY_WORD, 43 | common.constant.GO_KEY_WORD, 44 | common.constant.NATIVE_KEY_WORD, 45 | common.constant.ANCHOR_KEY_WORD, 46 | common.constant.METAPLEX_KEY_WORD, 47 | common.constant.METAPLEX_KEY_WORD, 48 | common.constant.METAPLEX_KEY_WORD, 49 | common.constant.SOLANA_WEB3_KEY_WORD, 50 | common.constant.SOLANA_WEB3_KEY_WORD, 51 | common.constant.SOLANA_WEB3_KEY_WORD, 52 | common.constant.DOT_NET_KEY_WORD, 53 | ]; 54 | const TYPEMAP = { 55 | [common.constant.SOLANA_WEB3_KEY_WORD]: common.constant.WEB3_JS_TYPE, 56 | [common.constant.ANCHOR_WEB3_KEY_WORD]: common.constant.ANCHOR_JS_TYPE, 57 | [common.constant.METAPLEX_KEY_WORD]: common.constant.NFT_TYPE, 58 | [common.constant.GO_KEY_WORD]: common.constant.GO_TYPE, 59 | [common.constant.NATIVE_KEY_WORD]: common.constant.NATIVE_TYPE, 60 | [common.constant.ANCHOR_KEY_WORD]: common.constant.ANCHOR_TYPE, 61 | [common.constant.DOT_NET_KEY_WORD]: common.constant.DOT_NET_TYPE, 62 | }; 63 | 64 | const UpdateSearchQuery = () => { 65 | let sizeQuery = ""; 66 | if (filesizeIndex === minFilesize) { 67 | sizeQuery = ` size:<=${minFilesize}`; 68 | } else if (filesizeIndex > maxFilesize) { 69 | sizeQuery = ` size:>=${maxFilesize}`; 70 | } else { 71 | sizeQuery = ` size:${filesizeIndex - filesizeInterval}..${filesizeIndex}`; 72 | } 73 | 74 | let filename = SEARCHFILENAMESORDER[searchIndex]; 75 | let keyword = SEARCHKEYWORDORDER[searchIndex]; 76 | 77 | return keyword + sizeQuery + ` filename:${filename}`; 78 | }; 79 | 80 | const UpdateSolanaGithubRepos = async (page, pageSize) => { 81 | const searchKey = UpdateSearchQuery(); 82 | const { data } = await common.githubAPI.searchGithubRepos({ 83 | searchKey, 84 | per_page: pageSize, 85 | page, 86 | }); 87 | let newData = []; 88 | await Promise.all( 89 | data && 90 | data.items.map(async (repo) => { 91 | const { repository } = repo; 92 | if (repository && repository.id && repository.name) { 93 | try { 94 | const { data } = await common.githubAPI.getRepoInfo({ 95 | owner: repository.owner.login, 96 | repo: repository.name, 97 | }); 98 | newData.push({ 99 | repoId: repository.id, 100 | name: repository.name, 101 | url: repository.html_url, 102 | owner: repository.owner.login, 103 | started: parseInt(new Date(data.created_at).getTime() / 1000), 104 | ecosystem: "solana", 105 | }); 106 | } catch (err) { 107 | console.log(err); 108 | } 109 | } 110 | }) 111 | ); 112 | const bulkData = newData.filter((item) => item !== undefined); 113 | if (bulkData && bulkData.length) { 114 | await SolanaGithubRepos.bulkCreate(bulkData, { 115 | fields: ["repoId", "name", "url", "owner", "started", "ecosystem"], 116 | ignoreDuplicates: true, 117 | returning: true, 118 | }); 119 | let repoTypeData = []; 120 | bulkData.map((repo) => { 121 | repoTypeData.push({ 122 | repoId: repo.repoId, 123 | type: TYPEMAP[SEARCHKEYWORDORDER[searchIndex]], 124 | }); 125 | }); 126 | await RepoTypes.bulkCreate(repoTypeData, { 127 | fields: ["repoId", "type"], 128 | ignoreDuplicates: true, 129 | }); 130 | } 131 | return data; 132 | }; 133 | 134 | exports.findSolanaReposCron = nodeCron.schedule("*/2 * * * *", async () => { 135 | if (!isEnabledCron) return; 136 | 137 | try { 138 | const data = await UpdateSolanaGithubRepos(page, pageSize); 139 | if (!data || data.items.length === 0) { 140 | page = 1; 141 | if (filesizeIndex > maxFilesize) { 142 | filesizeIndex = minFilesize; 143 | if (searchIndex === SEARCHFILENAMESORDER.length - 1) { 144 | isEnabledCron = false; 145 | console.log("SolanaGithubRepos table was updated successfully!"); 146 | searchIndex = 0; 147 | isDevelopersCron = true; 148 | } else { 149 | searchIndex += 1; 150 | } 151 | } else { 152 | filesizeIndex += filesizeInterval; 153 | } 154 | } else { 155 | page += 1; 156 | } 157 | } catch (err) { 158 | console.log(err); 159 | if ( 160 | err.response && 161 | err.response.data.message === 162 | "Only the first 1000 search results are available" 163 | ) { 164 | page = 1; 165 | if (filesizeIndex > maxFilesize) { 166 | filesizeIndex = minFilesize; 167 | if (searchIndex === SEARCHFILENAMESORDER.length - 1) { 168 | isEnabledCron = false; 169 | searchIndex = 0; 170 | console.log( 171 | "SolanaGithubRepos table was updated but you missed some repos because of 1000 limit" 172 | ); 173 | isDevelopersCron = true; 174 | } else { 175 | searchIndex += 1; 176 | } 177 | } else { 178 | filesizeIndex += filesizeInterval; 179 | } 180 | } else { 181 | page = 1; 182 | if (filesizeIndex > maxFilesize) { 183 | filesizeIndex = minFilesize; 184 | if (searchIndex === SEARCHFILENAMESORDER.length - 1) { 185 | isEnabledCron = false; 186 | searchIndex = 0; 187 | console.log( 188 | "SolanaGithubRepos table was updated but you missed some repos" 189 | ); 190 | isDevelopersCron = true; 191 | } else { 192 | searchIndex += 1; 193 | } 194 | } else { 195 | filesizeIndex += filesizeInterval; 196 | } 197 | } 198 | } 199 | }); 200 | 201 | exports.enableFetchSolanaRepos = nodeCron.schedule("0 0 */5 * *", () => { 202 | isEnabledCron = true; 203 | isDevelopersCron = false; 204 | console.log(`Start updating Solana Data at ${new Date()}`); 205 | }); 206 | 207 | // Developers Cron Parameters 208 | let solanaRepoPage = 0; 209 | const solanaRepoPageSize = 14; 210 | 211 | exports.fetchSolanaDevelopers = nodeCron.schedule("* * * * *", async () => { 212 | if (!isDevelopersCron) return; 213 | if (solanaRepoPage === 0) { 214 | solanaRepoPage++; 215 | return; 216 | } 217 | const currentTime = parseInt(new Date().getTime() / 1000); 218 | const week = 3600 * 24 * 7; 219 | const referenceTime = currentTime - week * 15; 220 | const offset = (solanaRepoPage - 1) * solanaRepoPageSize; 221 | const limit = solanaRepoPageSize; 222 | const repos = await SolanaGithubRepos.findAll({ 223 | where: { ecosystem: "solana" }, 224 | offset, 225 | limit, 226 | }); 227 | if (repos && repos.length) { 228 | solanaRepoPage += 1; 229 | repos.forEach(async (element) => { 230 | const { repoId, owner, name } = element; 231 | let elementData = null; 232 | let isRepoValid = true; 233 | try { 234 | const { data } = await common.githubAPI.getRepoContributorsActivity({ 235 | owner, 236 | repo: name, 237 | }); 238 | elementData = data; 239 | } catch { 240 | isRepoValid = false; 241 | } 242 | 243 | if (isRepoValid) { 244 | if (elementData && !isEmpty(elementData)) { 245 | elementData.forEach(async (contribution) => { 246 | const username = contribution.author.login; 247 | const activities = contribution.weeks.filter( 248 | (item) => item.c !== 0 && item.w > referenceTime 249 | ); 250 | let developer = await Developers.findOne({ 251 | where: { 252 | username, 253 | }, 254 | }); 255 | if (!developer) { 256 | try { 257 | const developerInfo = await common.githubAPI.getGitUserInfo({ 258 | username, 259 | }); 260 | developer = await Developers.create({ 261 | username, 262 | name: developerInfo.data.name, 263 | gitUrl: developerInfo.data.html_url, 264 | avatar: developerInfo.data.avatar_url, 265 | location: developerInfo.data.location, 266 | twitter: developerInfo.data.twitter_username, 267 | }); 268 | } catch (err) { 269 | console.log(err); 270 | } 271 | } 272 | try { 273 | if (!developer) { 274 | developer = await Developers.findOne({ 275 | where: { 276 | username, 277 | }, 278 | }); 279 | } 280 | const bulkData = activities.map((activity) => ({ 281 | commits: activity.c, 282 | additions: activity.a, 283 | delettioins: activity.d, 284 | date: activity.w, 285 | developerId: developer.id, 286 | repositoryId: element.id, 287 | })); 288 | if (bulkData && bulkData.length) { 289 | await Activities.bulkCreate(bulkData, { 290 | fields: [ 291 | "commits", 292 | "additions", 293 | "delettioins", 294 | "date", 295 | "developerId", 296 | "repositoryId", 297 | ], 298 | ignoreDuplicates: true, 299 | }); 300 | } 301 | } catch (err) { 302 | console.log(err); 303 | } 304 | }); 305 | } else { 306 | let commitsData = null; 307 | try { 308 | commitsData = await common.githubAPI.getGetRepoCommits({ 309 | owner, 310 | repo: name, 311 | per_page: 1, 312 | }); 313 | } catch { 314 | commitsData = null; 315 | } 316 | 317 | if (commitsData && commitsData.data && commitsData.data.length) { 318 | const commit = commitsData.data[0].commit; 319 | const date = parseInt( 320 | new Date(commit.author.date).getTime() / 1000 321 | ); 322 | let developer = await Developers.findOne({ 323 | where: { 324 | username: owner, 325 | }, 326 | }); 327 | if (!developer) { 328 | try { 329 | const developerInfo = await common.githubAPI.getGitUserInfo({ 330 | username: owner, 331 | }); 332 | developer = await Developers.create({ 333 | username: owner, 334 | name: developerInfo.data.name, 335 | gitUrl: developerInfo.data.html_url, 336 | avatar: developerInfo.data.avatar_url, 337 | location: developerInfo.data.location, 338 | twitter: developerInfo.data.twitter_username, 339 | }); 340 | } catch (err) { 341 | console.log(err); 342 | } 343 | } 344 | try { 345 | if (!developer) { 346 | developer = await Developers.findOne({ 347 | where: { 348 | username: owner, 349 | }, 350 | }); 351 | } 352 | await Activities.create({ 353 | commits: 1, 354 | additions: 0, 355 | delettioins: 0, 356 | date, 357 | developerId: developer.id, 358 | repositoryId: element.id, 359 | }); 360 | } catch (err) { 361 | console.log(err); 362 | } 363 | } 364 | } 365 | } 366 | }); 367 | } else { 368 | isDevelopersCron = false; 369 | solanaRepoPage = 0; 370 | console.log("Developers table was updated successfully!"); 371 | return; 372 | } 373 | }); 374 | -------------------------------------------------------------------------------- /src/crons/solanaResponseCache.cron.js: -------------------------------------------------------------------------------- 1 | const { Op, QueryTypes } = require("sequelize"); 2 | const nodeCron = require("node-cron"); 3 | const responseCache = require("../common/cache").responseCache; 4 | const db = require("../db/models"); 5 | const common = require("../common"); 6 | const SolanaGithubRepos = db.SolanaGithubRepos; 7 | const Activities = db.Activities; 8 | const SolanaTVLs = db.SolanaTVLs; 9 | const SolanaUSDCs = db.SolanaUSDCs; 10 | const StackOverflow = db.StackoverflowQuestions; 11 | const SolanaTransactions = db.SolanaTransactions; 12 | const SolanaNodes = db.SolanaNodes; 13 | const ProgramsDeployed = db.ProgramsDeployed; 14 | const RepoTypes = db.RepoTypes; 15 | 16 | exports.solanaResponseCache = nodeCron.schedule("0 0 */6 * * *", async () => { 17 | const currentTime = parseInt(new Date().getTime() / 1000); 18 | const week = 3600 * 24 * 7; 19 | // SolanaContributorsActivity 20 | const responseSolanaContributorsActivity = await Promise.all(Array.from( 21 | Array(52).keys() 22 | ).map(async (index) => { 23 | const referenceTime = currentTime - week * (index + 1); 24 | try { 25 | const data = await Activities.findAndCountAll({ 26 | include: [ 27 | { 28 | model: SolanaGithubRepos, 29 | attributes: ["ecosystem"], 30 | where: { 31 | ecosystem: "solana", 32 | }, 33 | }, 34 | ], 35 | where: { 36 | date: { 37 | [Op.and]: { 38 | [Op.lt]: referenceTime, 39 | [Op.gt]: referenceTime - 4 * week, 40 | }, 41 | }, 42 | }, 43 | distinct: true, 44 | col: "developerId", 45 | limit: 1, 46 | }); 47 | return { 48 | xData: referenceTime + week, 49 | yData: data.count, 50 | }; 51 | } catch (err) { 52 | console.log(err); 53 | } 54 | })); 55 | responseCache.set( 56 | "SolanaContributorsActivity", 57 | responseSolanaContributorsActivity 58 | ); 59 | 60 | // SolanaContributorsStatistics 61 | const responseSolanaContributorsStatistics = await Promise.all(Array.from( 62 | Array(53).keys() 63 | ).map(async (index) => { 64 | const referenceTime = currentTime - week * index; 65 | try { 66 | const data = await Activities.findAndCountAll({ 67 | include: [ 68 | { 69 | model: SolanaGithubRepos, 70 | attributes: ["ecosystem"], 71 | where: { 72 | ecosystem: "solana", 73 | }, 74 | }, 75 | ], 76 | where: { 77 | date: { 78 | [Op.and]: { 79 | [Op.lt]: referenceTime, 80 | }, 81 | }, 82 | }, 83 | distinct: true, 84 | col: "developerId", 85 | limit: 1, 86 | }); 87 | return { 88 | xData: referenceTime, 89 | yData: data.count, 90 | }; 91 | } catch (err) { 92 | console.log(err); 93 | } 94 | })); 95 | responseCache.set( 96 | "SolanaContributorsStatistics", 97 | responseSolanaContributorsStatistics 98 | ); 99 | 100 | // SolanaReposActivity 101 | const responseSolanaReposActivity = await Promise.all(Array.from(Array(52).keys()).map( 102 | async (index) => { 103 | const referenceTime = currentTime - week * (index + 1); 104 | try { 105 | const data = await Activities.findAndCountAll({ 106 | include: [ 107 | { 108 | model: SolanaGithubRepos, 109 | attributes: ["ecosystem"], 110 | where: { 111 | ecosystem: "solana", 112 | }, 113 | }, 114 | ], 115 | where: { 116 | date: { 117 | [Op.and]: { 118 | [Op.lt]: referenceTime, 119 | [Op.gt]: referenceTime - 4 * week, 120 | }, 121 | }, 122 | }, 123 | distinct: true, 124 | col: "repositoryId", 125 | limit: 1, 126 | }); 127 | return { 128 | xData: referenceTime + week, 129 | yData: data.count, 130 | }; 131 | } catch (err) { 132 | console.log(err); 133 | } 134 | } 135 | )); 136 | responseCache.set("SolanaReposActivity", responseSolanaReposActivity); 137 | 138 | // SolanaRecentRepoIds 139 | try { 140 | const data = await SolanaGithubRepos.findAll({ 141 | attributes: ["repoId", "ecosystem"], 142 | include: [ 143 | { 144 | model: Activities, 145 | attributes: ["date"], 146 | where: { 147 | date: { 148 | [Op.and]: { 149 | [Op.gt]: currentTime - 4 * week, 150 | }, 151 | }, 152 | }, 153 | }, 154 | ], 155 | where: { 156 | ecosystem: "solana", 157 | }, 158 | }); 159 | if (data) { 160 | const responseSolanaRecentRepoIds = data.map((item) => ({ 161 | repoId: item.repoId, 162 | })); 163 | responseCache.set("SolanaRecentRepoIds", responseSolanaRecentRepoIds); 164 | } 165 | } catch (err) { 166 | console.log(err); 167 | } 168 | 169 | // SolanaReposStatistics 170 | const responseSolanaReposStatistics = await Promise.all(Array.from(Array(53).keys()).map( 171 | async (index) => { 172 | const referenceTime = currentTime - week * index; 173 | try { 174 | const data = await SolanaGithubRepos.findAndCountAll({ 175 | where: { 176 | started: { 177 | [Op.lt]: referenceTime, 178 | }, 179 | ecosystem: "solana", 180 | }, 181 | limit: 1, 182 | }); 183 | return { 184 | xData: referenceTime, 185 | yData: data.count, 186 | }; 187 | } catch (err) { 188 | console.log(err); 189 | } 190 | } 191 | )); 192 | responseCache.set("SolanaReposStatistics", responseSolanaReposStatistics); 193 | 194 | const oneYearAgo = new Date(); 195 | oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); 196 | const oneYearAgoReferenceTime = oneYearAgo.getTime() / 1000; 197 | 198 | // SolanaTVLStatistics 199 | try { 200 | const responseSolanaTVLStatistics = await SolanaTVLs.findAll({ 201 | where: { 202 | timestamp: { 203 | [Op.gt]: oneYearAgoReferenceTime, 204 | }, 205 | }, 206 | distinct: true, 207 | order: [["timestamp", "DESC"]], 208 | }); 209 | responseCache.set("SolanaTVLStatistics", responseSolanaTVLStatistics); 210 | } catch (err) { 211 | console.log(err); 212 | } 213 | 214 | // SolanaUSDCInsurances 215 | try { 216 | const responseSolanaUSDCInsurances = await SolanaUSDCs.findAll({ 217 | where: { 218 | timestamp: { 219 | [Op.gt]: oneYearAgoReferenceTime, 220 | }, 221 | }, 222 | distinct: true, 223 | order: [["timestamp", "DESC"]], 224 | }); 225 | 226 | responseCache.set("SolanaUSDCInsurances", responseSolanaUSDCInsurances); 227 | } catch (err) { 228 | console.log(err); 229 | } 230 | 231 | // SolanaTransactions 232 | try { 233 | const responseSolanaTransactions = await SolanaTransactions.findAll({ 234 | where: { 235 | timestamp: { 236 | [Op.gt]: oneYearAgoReferenceTime, 237 | }, 238 | }, 239 | distinct: true, 240 | order: [["timestamp", "DESC"]], 241 | }); 242 | 243 | responseCache.set("SolanaTransactions", responseSolanaTransactions); 244 | } catch (err) { 245 | console.log(err); 246 | } 247 | 248 | // SolanaNodes 249 | try { 250 | const rowsSolanaNodes = await SolanaNodes.findAll({ 251 | where: { 252 | timestamp: { 253 | [Op.gt]: oneYearAgoReferenceTime, 254 | }, 255 | }, 256 | distinct: true, 257 | order: [["timestamp", "DESC"]], 258 | }); 259 | 260 | const responseSolanaNodes = rowsSolanaNodes.map((item) => ({ 261 | xData: item.timestamp, 262 | yData: item.counts, 263 | })); 264 | 265 | responseCache.set("SolanaNodes", responseSolanaNodes); 266 | } catch (err) { 267 | console.log(err); 268 | } 269 | 270 | // SolanaStackoverflowActivity 271 | const responseSolanaStackoverflowActivity = await Promise.all(Array.from( 272 | Array(53).keys() 273 | ).map(async (index) => { 274 | const referenceTime = currentTime - week * index; 275 | try { 276 | const data = await StackOverflow.findAndCountAll({ 277 | where: { 278 | date: { 279 | [Op.and]: { 280 | [Op.lt]: referenceTime, 281 | [Op.gt]: referenceTime - 4 * week, 282 | }, 283 | }, 284 | }, 285 | distinct: true, 286 | col: "questionId", 287 | limit: 1, 288 | }); 289 | return { 290 | xData: referenceTime, 291 | yData: data.count, 292 | }; 293 | } catch (err) { 294 | console.log(err); 295 | } 296 | })); 297 | responseCache.set( 298 | "SolanaStackoverflowActivity", 299 | responseSolanaStackoverflowActivity 300 | ); 301 | 302 | // ProgramsDeployed 303 | const responseProgramsDeployed = await Promise.all(Array.from(Array(53).keys()).map( 304 | async (index) => { 305 | const referenceTime = currentTime - week * index; 306 | try { 307 | const data = await ProgramsDeployed.findAndCountAll({ 308 | where: { 309 | date: { 310 | [Op.and]: { 311 | [Op.lt]: referenceTime, 312 | [Op.gt]: referenceTime - 4 * week, 313 | }, 314 | }, 315 | }, 316 | distinct: true, 317 | col: "program", 318 | limit: 1, 319 | }); 320 | return { 321 | xData: referenceTime, 322 | yData: data.count, 323 | }; 324 | } catch (err) { 325 | console.log(err); 326 | } 327 | } 328 | )); 329 | responseCache.set("ProgramsDeployed", responseProgramsDeployed); 330 | 331 | // AnchorVsNative 332 | const responseAnchorVsNative = await Promise.all(Array.from(Array(53).keys()).map( 333 | async (index) => { 334 | const referenceTime = currentTime - week * index; 335 | try { 336 | const anchorData = await SolanaGithubRepos.findAndCountAll({ 337 | where: { 338 | started: { 339 | [Op.lt]: referenceTime, 340 | }, 341 | ecosystem: "solana", 342 | }, 343 | distinct: true, 344 | include: [ 345 | { 346 | model: RepoTypes, 347 | attributes: ["type"], 348 | where: { 349 | type: "anchor", 350 | }, 351 | }, 352 | ], 353 | limit: 1, 354 | }); 355 | 356 | const nativeData = await SolanaGithubRepos.findAndCountAll({ 357 | where: { 358 | started: { 359 | [Op.lt]: referenceTime, 360 | }, 361 | ecosystem: "solana", 362 | }, 363 | distinct: true, 364 | include: [ 365 | { 366 | model: RepoTypes, 367 | attributes: ["type"], 368 | where: { 369 | [Op.and]: [ 370 | { 371 | type: { 372 | [Op.not]: "anchor", 373 | }, 374 | }, 375 | { 376 | type: "native", 377 | }, 378 | ], 379 | }, 380 | }, 381 | ], 382 | limit: 1, 383 | }); 384 | 385 | return { 386 | time: referenceTime, 387 | anchor: anchorData.count, 388 | native: nativeData.count, 389 | }; 390 | } catch (err) { 391 | console.log(err); 392 | } 393 | } 394 | )); 395 | responseCache.set("AnchorVsNative", responseAnchorVsNative); 396 | 397 | // NftVsOther 398 | const responseNftVsOther = await Promise.all(Array.from(Array(53).keys()).map( 399 | async (index) => { 400 | const referenceTime = currentTime - week * index; 401 | try { 402 | const nftData = await SolanaGithubRepos.findAndCountAll({ 403 | where: { 404 | started: { 405 | [Op.lt]: referenceTime, 406 | }, 407 | ecosystem: "solana", 408 | }, 409 | distinct: true, 410 | include: [ 411 | { 412 | model: RepoTypes, 413 | attributes: ["type"], 414 | where: { 415 | type: "nft", 416 | }, 417 | }, 418 | ], 419 | limit: 1, 420 | }); 421 | 422 | const otherData = await SolanaGithubRepos.findAndCountAll({ 423 | where: { 424 | started: { 425 | [Op.lt]: referenceTime, 426 | }, 427 | ecosystem: "solana", 428 | }, 429 | distinct: true, 430 | include: [ 431 | { 432 | model: RepoTypes, 433 | attributes: ["type"], 434 | where: { 435 | type: { 436 | [Op.not]: "nft", 437 | }, 438 | }, 439 | }, 440 | ], 441 | limit: 1, 442 | }); 443 | return { 444 | time: referenceTime, 445 | nft: nftData.count, 446 | other: otherData.count, 447 | }; 448 | } catch (err) { 449 | console.log(err); 450 | } 451 | } 452 | )); 453 | responseCache.set("NftVsOther", responseNftVsOther); 454 | 455 | // JsVsGoVsSolnetVsRust 456 | const responseJsVsGoVsSolnetVsRust = await Promise.all(Array.from(Array(53).keys()).map( 457 | async (index) => { 458 | const referenceTime = currentTime - week * index; 459 | try { 460 | const jsData = await SolanaGithubRepos.findAndCountAll({ 461 | where: { 462 | started: { 463 | [Op.lt]: referenceTime, 464 | }, 465 | ecosystem: "solana", 466 | }, 467 | distinct: true, 468 | include: [ 469 | { 470 | model: RepoTypes, 471 | attributes: ["type"], 472 | where: { 473 | [Op.or]: [ 474 | { 475 | type: "web3.js", 476 | }, 477 | { 478 | type: "anchor.js", 479 | }, 480 | ], 481 | }, 482 | }, 483 | ], 484 | limit: 1, 485 | }); 486 | 487 | const goData = await SolanaGithubRepos.findAndCountAll({ 488 | where: { 489 | started: { 490 | [Op.lt]: referenceTime, 491 | }, 492 | ecosystem: "solana", 493 | }, 494 | distinct: true, 495 | include: [ 496 | { 497 | model: RepoTypes, 498 | attributes: ["type"], 499 | where: { 500 | type: "go", 501 | }, 502 | }, 503 | ], 504 | limit: 1, 505 | }); 506 | 507 | const solnetData = await SolanaGithubRepos.findAndCountAll({ 508 | where: { 509 | started: { 510 | [Op.lt]: referenceTime, 511 | }, 512 | ecosystem: "solana", 513 | }, 514 | distinct: true, 515 | include: [ 516 | { 517 | model: RepoTypes, 518 | attributes: ["type"], 519 | where: { 520 | type: "dotnet", 521 | }, 522 | }, 523 | ], 524 | limit: 1, 525 | }); 526 | 527 | const rustData = await SolanaGithubRepos.findAndCountAll({ 528 | where: { 529 | started: { 530 | [Op.lt]: referenceTime, 531 | ecosystem: "solana", 532 | }, 533 | }, 534 | distinct: true, 535 | include: [ 536 | { 537 | model: RepoTypes, 538 | where: { 539 | [Op.or]: [ 540 | { 541 | type: "anchor", 542 | }, 543 | { 544 | type: "native", 545 | }, 546 | ], 547 | }, 548 | }, 549 | ], 550 | limit: 1, 551 | }); 552 | 553 | return { 554 | time: referenceTime, 555 | js: jsData.count, 556 | go: goData.count, 557 | solnet: solnetData.count, 558 | rust: rustData.count, 559 | }; 560 | } catch (err) { 561 | console.log(err); 562 | } 563 | } 564 | )); 565 | responseCache.set("JsVsGoVsSolnetVsRust", responseJsVsGoVsSolnetVsRust); 566 | 567 | // SolanaMonthlyRetention 568 | let oneMonthData = []; 569 | let twoMonthData = []; 570 | let threeMonthData = []; 571 | const dataSolanaMonthlyRetention = await db.sequelize.query( 572 | common.constant.SOLANA_MONTHLY_RETENTION, 573 | { 574 | type: QueryTypes.SELECT, 575 | } 576 | ); 577 | await dataSolanaMonthlyRetention.forEach((monthData) => { 578 | const currentTime = new Date(); 579 | const time = Math.trunc( 580 | currentTime.setMonth(currentTime.getMonth() - (monthData.first % 12)) / 581 | 1000 582 | ); 583 | const monthOne = Math.trunc((100 * monthData.month_1) / monthData.month_0); 584 | const monthTwo = Math.trunc((100 * monthData.month_2) / monthData.month_0); 585 | const monthThree = Math.trunc( 586 | (100 * monthData.month_3) / monthData.month_0 587 | ); 588 | if (monthOne > 0) { 589 | oneMonthData.push({ 590 | xData: time, 591 | yData: monthOne, 592 | }); 593 | } 594 | if (monthTwo > 0) { 595 | twoMonthData.push({ 596 | xData: time, 597 | yData: monthTwo, 598 | }); 599 | } 600 | if (monthThree > 0) { 601 | threeMonthData.push({ 602 | xData: time, 603 | yData: monthThree, 604 | }); 605 | } 606 | }); 607 | responseCache.set("SolanaMonthlyRetention", [ 608 | { 609 | monthOne: oneMonthData, 610 | monthTwo: twoMonthData, 611 | monthThree: threeMonthData, 612 | }, 613 | ]); 614 | 615 | // SolanaWeeklyRetention 616 | let oneWeekData = []; 617 | let twoWeekData = []; 618 | let threeWeekData = []; 619 | let fourWeekData = []; 620 | let fiveWeekData = []; 621 | let sixWeekData = []; 622 | const dataSolanaWeeklyRetention = await db.sequelize.query( 623 | common.constant.SOLANA_WEEKLY_RETENTION, 624 | { 625 | type: QueryTypes.SELECT, 626 | } 627 | ); 628 | await dataSolanaWeeklyRetention.forEach((weekData) => { 629 | const currentTime = new Date(); 630 | const time = Math.trunc( 631 | currentTime.setDate(currentTime.getDate() - (weekData.first % 52) * 7) / 632 | 1000 633 | ); 634 | const weekOne = Math.trunc((100 * weekData.week_1) / weekData.week_0); 635 | const weekTwo = Math.trunc((100 * weekData.week_2) / weekData.week_0); 636 | const weekThree = Math.trunc((100 * weekData.week_3) / weekData.week_0); 637 | const weekfour = Math.trunc((100 * weekData.week_4) / weekData.week_0); 638 | const weekfive = Math.trunc((100 * weekData.week_5) / weekData.week_0); 639 | const weeksix = Math.trunc((100 * weekData.week_6) / weekData.week_0); 640 | if (weekOne > 0) { 641 | oneWeekData.push({ 642 | xData: time, 643 | yData: weekOne, 644 | }); 645 | } 646 | if (weekTwo > 0) { 647 | twoWeekData.push({ 648 | xData: time, 649 | yData: weekTwo, 650 | }); 651 | } 652 | if (weekThree > 0) { 653 | threeWeekData.push({ 654 | xData: time, 655 | yData: weekThree, 656 | }); 657 | } 658 | if (weekfour > 0) { 659 | fourWeekData.push({ 660 | xData: time, 661 | yData: weekfour, 662 | }); 663 | } 664 | if (weekfive > 0) { 665 | fiveWeekData.push({ 666 | xData: time, 667 | yData: weekfive, 668 | }); 669 | } 670 | if (weeksix > 0) { 671 | sixWeekData.push({ 672 | xData: time, 673 | yData: weeksix, 674 | }); 675 | } 676 | }); 677 | responseCache.set("SolanaWeeklyRetention", [ 678 | { 679 | weekOne: oneWeekData, 680 | weekTwo: twoWeekData, 681 | weekThree: threeWeekData, 682 | weekFour: fourWeekData, 683 | weekFive: fiveWeekData, 684 | weekSix: sixWeekData, 685 | }, 686 | ]); 687 | }); 688 | -------------------------------------------------------------------------------- /src/db/config/config.js: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | const common = require("../../common"); 3 | const { 4 | DB_HOST, 5 | DB_PORT, 6 | DB_USERNAME, 7 | DB_PASSWORD, 8 | DB_NAME, 9 | DB_DIALECT, 10 | PROD_DB_HOST, 11 | PROD_DB_PORT, 12 | PROD_DB_USERNAME, 13 | PROD_DB_PASSWORD, 14 | PROD_DB_NAME, 15 | } = process.env; 16 | 17 | module.exports = { 18 | development: { 19 | username: DB_USERNAME, 20 | password: DB_PASSWORD, 21 | database: DB_NAME || "solana_analytics", 22 | host: DB_HOST, 23 | port: DB_PORT, 24 | dialect: DB_DIALECT || common.constant.DB, 25 | }, 26 | }; 27 | -------------------------------------------------------------------------------- /src/db/models/activities.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const { Model } = require("sequelize"); 3 | module.exports = (sequelize, DataTypes) => { 4 | class Activities extends Model { 5 | /** 6 | * Helper method for defining associations. 7 | * This method is not a part of Sequelize lifecycle. 8 | * The `models/index` file will call this method automatically. 9 | */ 10 | static associate(models) { 11 | // define association here 12 | Activities.belongsTo(models.Developers, { 13 | foreignKey: "developerId", 14 | onDelete: "CASCADE", 15 | }); 16 | Activities.belongsTo(models.SolanaGithubRepos, { 17 | foreignKey: "repositoryId", 18 | targetKey: "id", 19 | onDelete: "CASCADE", 20 | }); 21 | } 22 | } 23 | Activities.init( 24 | { 25 | commits: DataTypes.INTEGER, 26 | additions: DataTypes.INTEGER, 27 | deletions: DataTypes.INTEGER, 28 | date: DataTypes.INTEGER, 29 | repositoryId: DataTypes.INTEGER, 30 | developerId: DataTypes.INTEGER, 31 | }, 32 | { 33 | sequelize, 34 | modelName: "Activities", 35 | } 36 | ); 37 | return Activities; 38 | }; 39 | -------------------------------------------------------------------------------- /src/db/models/developers.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const { Model } = require("sequelize"); 3 | module.exports = (sequelize, DataTypes) => { 4 | class Developers extends Model { 5 | /** 6 | * Helper method for defining associations. 7 | * This method is not a part of Sequelize lifecycle. 8 | * The `models/index` file will call this method automatically. 9 | */ 10 | static associate(models) { 11 | // define association here 12 | Developers.hasMany(models.Activities, { 13 | foreignKey: "developerId", 14 | }); 15 | } 16 | } 17 | Developers.init( 18 | { 19 | username: DataTypes.STRING, 20 | name: DataTypes.STRING, 21 | gitUrl: DataTypes.STRING, 22 | avatar: DataTypes.STRING, 23 | location: DataTypes.STRING, 24 | twitter: DataTypes.STRING, 25 | }, 26 | { 27 | sequelize, 28 | modelName: "Developers", 29 | } 30 | ); 31 | return Developers; 32 | }; 33 | -------------------------------------------------------------------------------- /src/db/models/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const path = require("path"); 5 | const Sequelize = require("sequelize"); 6 | const basename = path.basename(__filename); 7 | const env = process.env.NODE_ENV || "development"; 8 | const config = require(__dirname + "/../config/config.js")[env]; 9 | const db = {}; 10 | 11 | let sequelize; 12 | if (config.use_env_variable) { 13 | sequelize = new Sequelize(process.env[config.use_env_variable], config); 14 | } else { 15 | sequelize = new Sequelize( 16 | config.database, 17 | config.username, 18 | config.password, 19 | config 20 | ); 21 | } 22 | 23 | fs.readdirSync(__dirname) 24 | .filter((file) => { 25 | return ( 26 | file.indexOf(".") !== 0 && file !== basename && file.slice(-3) === ".js" 27 | ); 28 | }) 29 | .forEach((file) => { 30 | const model = require(path.join(__dirname, file))( 31 | sequelize, 32 | Sequelize.DataTypes 33 | ); 34 | db[model.name] = model; 35 | }); 36 | 37 | Object.keys(db).forEach((modelName) => { 38 | if (db[modelName].associate) { 39 | db[modelName].associate(db); 40 | } 41 | }); 42 | 43 | db.sequelize = sequelize; 44 | db.Sequelize = Sequelize; 45 | 46 | module.exports = db; 47 | -------------------------------------------------------------------------------- /src/db/models/repotypes.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const { Model } = require("sequelize"); 3 | module.exports = (sequelize, DataTypes) => { 4 | class RepoTypes extends Model { 5 | static associate(models) { 6 | RepoTypes.belongsTo(models.SolanaGithubRepos, { 7 | targetKey: "repoId", 8 | foreignKey: "repoId", 9 | }); 10 | } 11 | } 12 | RepoTypes.init( 13 | { 14 | repoId: DataTypes.STRING, 15 | type: DataTypes.STRING, 16 | }, 17 | { 18 | sequelize, 19 | modelName: "RepoTypes", 20 | } 21 | ); 22 | RepoTypes.removeAttribute("id"); 23 | return RepoTypes; 24 | }; 25 | -------------------------------------------------------------------------------- /src/db/models/solanagithubrepos.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const { Model } = require("sequelize"); 3 | module.exports = (sequelize, DataTypes) => { 4 | class SolanaGithubRepos extends Model { 5 | /** 6 | * Helper method for defining associations. 7 | * This method is not a part of Sequelize lifecycle. 8 | * The `models/index` file will call this method automatically. 9 | */ 10 | static associate(models) { 11 | SolanaGithubRepos.hasMany(models.RepoTypes, { 12 | foreignKey: "repoId", 13 | sourceKey: "repoId", 14 | onDelete: "CASCADE", 15 | }); 16 | SolanaGithubRepos.hasMany(models.Activities, { 17 | foreignKey: "repositoryId", 18 | sourceKey: "id", 19 | onDelete: "CASCADE", 20 | }); 21 | } 22 | } 23 | SolanaGithubRepos.init( 24 | { 25 | repoId: { 26 | type: DataTypes.STRING, 27 | unique: true, 28 | }, 29 | name: DataTypes.STRING, 30 | url: DataTypes.STRING, 31 | owner: DataTypes.STRING, 32 | started: DataTypes.INTEGER, 33 | ecosystem: DataTypes.STRING, 34 | }, 35 | { 36 | sequelize, 37 | modelName: "SolanaGithubRepos", 38 | } 39 | ); 40 | return SolanaGithubRepos; 41 | }; 42 | -------------------------------------------------------------------------------- /src/middleware/githubError.js: -------------------------------------------------------------------------------- 1 | module.exports = function githubError(error, req, res, next) { 2 | if (error.type === "github") { 3 | return res.status(error.status || 500).json({ 4 | message: "Github Error!", 5 | error, 6 | }); 7 | } else { 8 | return next(error); 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /src/middleware/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | githubError: require("./githubError"), 3 | postgresError: require("./postgresError"), 4 | unknownError: require("./unknownError"), 5 | }; 6 | -------------------------------------------------------------------------------- /src/middleware/postgresError.js: -------------------------------------------------------------------------------- 1 | module.exports = function postgresError(error, req, res, next) { 2 | if (error.type === "postgres") { 3 | return res.status(error.status || 500).json({ 4 | message: "Postgres Error!", 5 | error, 6 | }); 7 | } else { 8 | return next(error); 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /src/middleware/unknownError.js: -------------------------------------------------------------------------------- 1 | module.exports = function unknownError(error, req, res, next) { 2 | return res.status(500).json({ 3 | message: "UnknownError!", 4 | error, 5 | }); 6 | }; 7 | --------------------------------------------------------------------------------