├── .github └── CODEOWNERS ├── .gitignore ├── .nvmrc ├── .prettierrc ├── README.md ├── client ├── .env.development ├── .gitignore ├── .nvmrc ├── README.md ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ ├── index.html │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── robots.txt ├── src │ ├── App.css │ ├── App.js │ ├── App.test.js │ ├── index.css │ ├── index.js │ ├── logo.svg │ └── setupTests.js └── yarn.lock ├── file_types.d.ts ├── package.json ├── public └── welcome.html ├── src ├── caching.ts ├── compute │ ├── brackets.ts │ ├── brackets_pipelines.ts │ ├── choices_over_years_graph.ts │ ├── common.ts │ ├── demographics.ts │ ├── experience.ts │ ├── generic.ts │ ├── happiness.ts │ ├── index.ts │ ├── matrices.ts │ └── tools.ts ├── config.ts ├── data │ ├── bestofjs.yml │ ├── ids.yml │ └── locales.yml ├── debug.ts ├── entities.ts ├── external_apis │ ├── github.ts │ ├── index.ts │ ├── mdn.ts │ └── twitter.ts ├── filters.ts ├── helpers.ts ├── i18n.ts ├── resolvers │ ├── brackets.ts │ ├── categories.ts │ ├── demographics.ts │ ├── entities.ts │ ├── environments.ts │ ├── features.ts │ ├── features_others.ts │ ├── happiness.ts │ ├── index.ts │ ├── matrices.ts │ ├── opinions.ts │ ├── proficiency.ts │ ├── query.ts │ ├── resources.ts │ ├── surveys.ts │ ├── tools.ts │ ├── tools_others.ts │ └── totals.ts ├── rpcs.ts ├── server.ts ├── standalone.ts ├── type_defs │ ├── bracket.graphql │ ├── categories.graphql │ ├── countries.graphql │ ├── demographics.graphql │ ├── entity.graphql │ ├── environments.graphql │ ├── experience.graphql │ ├── features.graphql │ ├── features_others.graphql │ ├── filters.graphql │ ├── github.graphql │ ├── happiness.graphql │ ├── matrices.graphql │ ├── mdn.graphql │ ├── opinions.graphql │ ├── proficiency.graphql │ ├── resources.graphql │ ├── schema.graphql │ ├── surveys.graphql │ ├── tools.graphql │ ├── tools_cardinality_by_user.graphql │ ├── tools_others.graphql │ ├── totals.graphql │ ├── translations.graphql │ ├── twitter.graphql │ └── usage.graphql └── types │ ├── demographics.ts │ ├── entity.ts │ ├── features.ts │ ├── github.ts │ ├── index.ts │ ├── locale.ts │ ├── mdn.ts │ ├── schema.ts │ ├── surveys.ts │ ├── tools.ts │ └── twitter.ts ├── stateofjs-api.code-workspace ├── tsconfig.json ├── webpack.common.js ├── webpack.development.js ├── webpack.production.js └── yarn.lock /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | /src/i18n/ca-ES @socunanena 2 | 3 | /src/i18n/cs-CZ @adamkudrna 4 | 5 | /src/i18n/de-DE @abaldeweg 6 | 7 | /src/i18n/es-ES @timbergus @ezkato 8 | 9 | /src/i18n/fr-FR @arnauddrain 10 | 11 | /src/i18n/hi-IN @jaideepghosh 12 | 13 | /src/i18n/it-IT @polettoweb 14 | 15 | /src/i18n/pt-PT @danisal 16 | 17 | /src/i18n/ru-RU @lex111 @Omhet @shramkoweb 18 | 19 | /src/i18n/ua-UA @shramkoweb 20 | 21 | /src/i18n/sv-SE @m-hagberg 22 | 23 | /src/i18n/tr-TR @berkayyildiz 24 | 25 | /src/i18n/id-ID @ervinismu 26 | 27 | /src/i18n/zh-Hant @ymcheung 28 | 29 | /src/i18n/ja-JP @myakura @Spice-Z 30 | 31 | /src/i18n/pl-PL @luk-str 32 | 33 | /src/i18n/nl-NL @MaxAltena 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | node_modules 3 | .DS_Store 4 | .env 5 | test.mjs 6 | dist 7 | .logs -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v14.17.5 -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | printWidth: 100 2 | semi: false 3 | tabWidth: 4 4 | singleQuote: true 5 | arrowParens: avoid 6 | trailingComma: none -------------------------------------------------------------------------------- /client/.env.development: -------------------------------------------------------------------------------- 1 | SKIP_PREFLIGHT_CHECK=true 2 | REACT_APP_ENDPOINT_URL=http://localhost:4000/graphql -------------------------------------------------------------------------------- /client/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /client/.nvmrc: -------------------------------------------------------------------------------- 1 | v14.17.5 -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 2 | 3 | ## Available Scripts 4 | 5 | In the project directory, you can run: 6 | 7 | ### `yarn start` 8 | 9 | Runs the app in the development mode.
10 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 11 | 12 | The page will reload if you make edits.
13 | You will also see any lint errors in the console. 14 | 15 | ### `yarn build` 16 | 17 | Builds the app for production to the `build` folder.
18 | It correctly bundles React in production mode and optimizes the build for the best performance. 19 | 20 | The build is minified and the filenames include the hashes.
21 | Your app is ready to be deployed! 22 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "client", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^5.14.1", 7 | "@testing-library/react": "^12.1.2", 8 | "@testing-library/user-event": "^13.5.0", 9 | "graphiql": "^1.4.2", 10 | "graphiql-explorer": "^0.6.3", 11 | "graphql": "^15.6.1", 12 | "js-yaml": "^4.1.0", 13 | "react": "^17.0.2", 14 | "react-dom": "^17.0.2", 15 | "react-scripts": "4.0.3" 16 | }, 17 | "scripts": { 18 | "dev": "react-scripts start --port 4001", 19 | "start": "react-scripts start --port 4001", 20 | "build": "react-scripts build", 21 | "test": "react-scripts test", 22 | "eject": "react-scripts eject" 23 | }, 24 | "eslintConfig": { 25 | "extends": "react-app" 26 | }, 27 | "browserslist": { 28 | "production": [ 29 | ">0.2%", 30 | "not dead", 31 | "not op_mini all" 32 | ], 33 | "development": [ 34 | "last 1 chrome version", 35 | "last 1 firefox version", 36 | "last 1 safari version" 37 | ] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /client/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Devographics/state-of-js-graphql-results-api/2c1cf0aa6d2b901cba78d1dd566a7c7eb195f1ef/client/public/favicon.ico -------------------------------------------------------------------------------- /client/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | 15 | 24 | State of JS API - GraphiQL 25 | 26 | 27 | 28 |
29 | 39 | 40 | -------------------------------------------------------------------------------- /client/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Devographics/state-of-js-graphql-results-api/2c1cf0aa6d2b901cba78d1dd566a7c7eb195f1ef/client/public/logo192.png -------------------------------------------------------------------------------- /client/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Devographics/state-of-js-graphql-results-api/2c1cf0aa6d2b901cba78d1dd566a7c7eb195f1ef/client/public/logo512.png -------------------------------------------------------------------------------- /client/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /client/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | -------------------------------------------------------------------------------- /client/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #282c34; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } 39 | 40 | .graphiql-container { 41 | height: 100vh; 42 | width: 100vw; 43 | } 44 | -------------------------------------------------------------------------------- /client/src/App.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react' 2 | import GraphiQL from 'graphiql' 3 | import GraphiQLExplorer from 'graphiql-explorer' 4 | import { buildClientSchema, getIntrospectionQuery, parse } from 'graphql' 5 | import dotenv from 'dotenv' 6 | import 'graphiql/graphiql.css' 7 | import './App.css' 8 | 9 | const envFile = process.env.NODE_ENV ? `.env.${process.env.NODE_ENV}` : '.env' 10 | dotenv.config({ path: envFile }) 11 | 12 | function fetcher(params) { 13 | return fetch(process.env.REACT_APP_ENDPOINT_URL, { 14 | method: 'POST', 15 | headers: { 16 | Accept: 'application/json', 17 | 'Content-Type': 'application/json' 18 | }, 19 | body: JSON.stringify(params) 20 | }) 21 | .then(function(response) { 22 | return response.text() 23 | }) 24 | .then(function(responseBody) { 25 | try { 26 | return JSON.parse(responseBody) 27 | } catch (e) { 28 | return responseBody 29 | } 30 | }) 31 | } 32 | 33 | const DEFAULT_QUERY = `# Get started by opening the explorer and running your queries 34 | query ReactHistoricalResults { 35 | survey(survey: js) { 36 | tool(id: react) { 37 | experience { 38 | allYears { 39 | awarenessInterestSatisfaction { 40 | awareness 41 | interest 42 | satisfaction 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | ` 50 | 51 | let defaultQuery = null 52 | try { 53 | defaultQuery = localStorage.getItem('graphiql:query') || DEFAULT_QUERY 54 | } catch (e) { 55 | console.warn('Error getting initial defaultQuery: ', e) 56 | defaultQuery = DEFAULT_QUERY 57 | } 58 | 59 | let defaultExplorerIsOpen = null 60 | try { 61 | defaultExplorerIsOpen = JSON.parse(localStorage.getItem('graphiql:explorerIsOpen') || 'true') 62 | } catch (e) { 63 | console.warn('Error getting initial defaultExplorerIsOpen: ', e) 64 | defaultExplorerIsOpen = true 65 | } 66 | 67 | class App extends Component { 68 | _graphiql: GraphiQL 69 | state = { schema: null, query: defaultQuery, explorerIsOpen: defaultExplorerIsOpen } 70 | 71 | componentDidMount() { 72 | fetcher({ 73 | query: getIntrospectionQuery() 74 | }).then(result => { 75 | const editor = this._graphiql.getQueryEditor() 76 | editor.setOption('extraKeys', { 77 | ...(editor.options.extraKeys || {}), 78 | 'Shift-Alt-LeftClick': this._handleInspectOperation 79 | }) 80 | 81 | this.setState({ schema: buildClientSchema(result.data) }) 82 | }) 83 | } 84 | 85 | _handleInspectOperation = (cm, mousePos) => { 86 | const parsedQuery = parse(this.state.query || '') 87 | 88 | if (!parsedQuery) { 89 | console.error("Couldn't parse query document") 90 | return null 91 | } 92 | 93 | var token = cm.getTokenAt(mousePos) 94 | var start = { line: mousePos.line, ch: token.start } 95 | var end = { line: mousePos.line, ch: token.end } 96 | var relevantMousePos = { 97 | start: cm.indexFromPos(start), 98 | end: cm.indexFromPos(end) 99 | } 100 | 101 | var position = relevantMousePos 102 | 103 | var def = parsedQuery.definitions.find(definition => { 104 | if (!definition.loc) { 105 | console.log('Missing location information for definition') 106 | return false 107 | } 108 | 109 | const { start, end } = definition.loc 110 | return start <= position.start && end >= position.end 111 | }) 112 | 113 | if (!def) { 114 | console.error('Unable to find definition corresponding to mouse position') 115 | return null 116 | } 117 | 118 | var operationKind = 119 | def.kind === 'OperationDefinition' 120 | ? def.operation 121 | : def.kind === 'FragmentDefinition' 122 | ? 'fragment' 123 | : 'unknown' 124 | 125 | var operationName = 126 | def.kind === 'OperationDefinition' && !!def.name 127 | ? def.name.value 128 | : def.kind === 'FragmentDefinition' && !!def.name 129 | ? def.name.value 130 | : 'unknown' 131 | 132 | var selector = `.graphiql-explorer-root #${operationKind}-${operationName}` 133 | 134 | var el = document.querySelector(selector) 135 | el && el.scrollIntoView() 136 | } 137 | 138 | _handleEditQuery = query => { 139 | localStorage.setItem('graphiql:query', query) 140 | this.setState({ query }) 141 | } 142 | 143 | _handleToggleExplorer = () => { 144 | const explorerIsOpen = !this.state.explorerIsOpen 145 | localStorage.setItem('graphiql:explorerIsOpen', JSON.stringify(explorerIsOpen)) 146 | 147 | this.setState({ explorerIsOpen: explorerIsOpen }) 148 | } 149 | 150 | render() { 151 | const { schema, query } = this.state 152 | return ( 153 |
154 | this._graphiql.handleRunQuery(operationName)} 159 | explorerIsOpen={this.state.explorerIsOpen} 160 | onToggleExplorer={this._handleToggleExplorer} 161 | showAttribution={false} 162 | /> 163 | (this._graphiql = ref)} 165 | fetcher={fetcher} 166 | schema={schema} 167 | query={query} 168 | onEditQuery={this._handleEditQuery} 169 | > 170 | State of JS 171 | 172 | this._graphiql.handlePrettifyQuery()} 174 | label="Prettify" 175 | title="Prettify Query (Shift-Ctrl-P)" 176 | /> 177 | this._graphiql.handleToggleHistory()} 179 | label="History" 180 | title="Show History" 181 | /> 182 | 187 | { 189 | if (typeof window === 'undefined') { 190 | console.warn('Clicked in browserless environment') 191 | return 192 | } 193 | 194 | window.open('https://api.stateofjs.com/') 195 | }} 196 | label="Help" 197 | title="Open API help" 198 | /> 199 | 200 | 201 |
202 | ) 203 | } 204 | } 205 | 206 | export default App 207 | -------------------------------------------------------------------------------- /client/src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | const { getByText } = render(); 7 | const linkElement = getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /client/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /client/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom' 3 | import './index.css' 4 | import App from './App' 5 | 6 | ReactDOM.render(, document.getElementById('root')) 7 | -------------------------------------------------------------------------------- /client/src/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /client/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom/extend-expect'; 6 | -------------------------------------------------------------------------------- /file_types.d.ts: -------------------------------------------------------------------------------- 1 | // import { Entity } from './src/types' 2 | 3 | declare module '*.graphql' { 4 | import { DocumentNode } from 'graphql' 5 | const Schema: DocumentNode 6 | 7 | export = Schema 8 | } 9 | 10 | /** 11 | * Define the type for the static features yaml file 12 | */ 13 | declare module 'entities/*.yml' { 14 | interface Entity { 15 | id: string 16 | name: string 17 | mdn?: string 18 | caniuse?: string 19 | } 20 | const content: Entity[] 21 | 22 | export default content 23 | } 24 | 25 | /** 26 | * Define the type for the static projects yaml file 27 | */ 28 | declare module '*projects.yml' { 29 | interface ProjectData { 30 | id: string 31 | name: string 32 | description: string 33 | github: string 34 | stars: number 35 | homepage: string 36 | } 37 | const content: ProjectData[] 38 | 39 | export default content 40 | } 41 | 42 | declare module '*.yml' { 43 | const content: any 44 | export default content 45 | } 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "state-of-js-graphql-results-api", 3 | "version": "1.0.0", 4 | "license": "MIT", 5 | "dependencies": { 6 | "@octokit/core": "^3.5.1", 7 | "@sentry/node": "^6.13.3", 8 | "@sentry/tracing": "^6.13.3", 9 | "apollo-server-express": "^3.3.0", 10 | "apollo-server-plugin-response-cache": "^3.2.0", 11 | "dotenv": "^10.0.0", 12 | "express": "^4.17.1", 13 | "graphql": "^15.6.1", 14 | "js-yaml": "^4.1.0", 15 | "lodash": "^4.17.21", 16 | "markdown-it": "^12.2.0", 17 | "marked": "^3.0.7", 18 | "mongodb": "^4.1.3", 19 | "node-fetch": "^2.6.5", 20 | "twitter-api-v2": "^1.5.2", 21 | "write-file-webpack-plugin": "^4.5.1" 22 | }, 23 | "devDependencies": { 24 | "@types/node-fetch": "^2.5.12", 25 | "@types/js-yaml": "^4.0.3", 26 | "@types/lodash": "^4.14.175", 27 | "@types/marked": "^2.0.4", 28 | "@types/mongodb": "^4.0.7", 29 | "@types/node": "^16.7.0", 30 | "@types/webpack-env": "^1.16.3", 31 | "clean-webpack-plugin": "4.0.0", 32 | "copy-webpack-plugin": "^9.0.1", 33 | "graphql-tag": "^2.12.5", 34 | "js-yaml-loader": "^1.2.2", 35 | "nodemon-webpack-plugin": "^4.5.2", 36 | "npm-run-all": "^4.1.5", 37 | "prettier": "^2.3.2", 38 | "ts-loader": "^9.2.5", 39 | "typescript": "^4.3.5", 40 | "webpack": "^5.58.2", 41 | "webpack-cli": "^4.9.0", 42 | "webpack-merge": "^5.8.0", 43 | "webpack-node-externals": "^3.0.0" 44 | }, 45 | "scripts": { 46 | "build": "NODE_ENV=production webpack --config webpack.production.js", 47 | "dev": "NODE_ENV=development webpack --config webpack.development.js", 48 | "dev:clean": "NODE_ENV=development DISABLE_CACHE=true webpack --config webpack.development.js", 49 | "fmt": "prettier --write \"src/**/*.{ts,js,mjs,yml,graphql}\"", 50 | "heroku-postbuild": "NODE_ENV=production webpack --config webpack.production.js", 51 | "start": "node dist/server.js" 52 | }, 53 | "engines": { 54 | "node": "14.17.5" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /public/welcome.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | 15 | 24 | State of JS API 25 | 69 | 70 | 71 |
72 |

State of JS API

73 |
74 | GraphQL EndpointSend your GraphQL queries to this URL 76 |
77 |
78 | GraphiQL IDEQuery the API using a web IDE 80 |
81 |
82 | API HelpLearn more about the API 84 |
85 |
86 | GitHubView the API code or leave an issue 88 |
89 |
90 | 100 | 101 | -------------------------------------------------------------------------------- /src/caching.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import config from './config' 3 | 4 | type DynamicComputeCall = (db: Db, ...args: any[]) => Promise 5 | 6 | type ArgumentTypes = F extends (db: Db, ...args: infer A) => Promise ? A : never 7 | 8 | type ResultType = F extends (...args: any[]) => infer P 9 | ? P extends Promise 10 | ? R 11 | : never 12 | : never 13 | 14 | /** 15 | * Compute a cache key from a function and its arguments, 16 | * the function should have a name in order to generate a proper key. 17 | */ 18 | export const computeKey = (func: Function, args?: any) => { 19 | const serializedArgs = args 20 | ? args 21 | .map((a: any) => { 22 | return JSON.stringify(a) 23 | }) 24 | .join(', ') 25 | : '' 26 | 27 | if (func.name === '') { 28 | // enforce regular function usage over arrow functions, to have a proper cache key 29 | // console.trace is used to be able to know where the call comes from 30 | console.trace( 31 | `found a function without name, please consider using a regular function instead of an arrow function to solve this issue as it can lead to cache mismatch` 32 | ) 33 | } 34 | 35 | return `${func.name}(${serializedArgs})` 36 | } 37 | 38 | /** 39 | * Cache results in a dedicated mongo collection to improve performance, 40 | * if the result isn't already available in the collection, it will be created. 41 | */ 42 | export const useCache = async ( 43 | func: F, 44 | db: Db, 45 | args: ArgumentTypes 46 | ): Promise> => { 47 | const key = computeKey(func, args) 48 | 49 | const collection = db.collection(config.mongo.cache_collection) 50 | const existingResult = await collection.findOne({ key }) 51 | if (existingResult && !process.env.DISABLE_CACHE) { 52 | console.log(`< using result from cache for: ${key}`) 53 | return existingResult.value 54 | } 55 | 56 | console.log(`> fetching/caching result for: ${key}`) 57 | const value = await func(db, ...(args || [])) 58 | 59 | // in case previous cached entry exists, delete it 60 | await collection.deleteOne({ key }) 61 | await collection.insertOne({ key, value }) 62 | 63 | return value 64 | } 65 | 66 | export const clearCache = async (db: Db) => { 67 | const collection = db.collection(config.mongo.cache_collection) 68 | const result = await collection.deleteMany({}) 69 | return result 70 | } -------------------------------------------------------------------------------- /src/compute/brackets.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { Completion, SurveyConfig } from '../types' 3 | import { TermAggregationOptions, RawResult, groupByYears } from '../compute/generic' 4 | import config from '../config' 5 | import { generateFiltersQuery } from '../filters' 6 | import { inspect } from 'util' 7 | import idsLookupTable from '../data/ids.yml' 8 | import { getWinsPipeline, getMatchupsPipeline } from './brackets_pipelines' 9 | 10 | // Wins 11 | 12 | export interface WinsStats { 13 | count: number 14 | percentage: number 15 | } 16 | 17 | export interface WinsBucket { 18 | id: number | string 19 | round1: WinsStats 20 | round2: WinsStats 21 | round3: WinsStats 22 | combined: WinsStats 23 | year: number 24 | } 25 | 26 | export interface WinsYearAggregations { 27 | year: number 28 | total: number 29 | completion: Completion 30 | buckets: WinsBucket[] 31 | } 32 | 33 | // Matchups 34 | 35 | export interface MatchupAggregationResult { 36 | id: number 37 | year: number 38 | matchups: MatchupStats[] 39 | } 40 | 41 | export interface MatchupStats { 42 | id: number | string 43 | count: number 44 | percentage: number 45 | } 46 | 47 | export interface MatchupBucket { 48 | id: number | string 49 | matchups: MatchupStats[] 50 | } 51 | 52 | export interface MatchupYearAggregations { 53 | year: number 54 | total: number 55 | completion: Completion 56 | buckets: MatchupBucket[] 57 | } 58 | 59 | export const winsAggregationFunction = async ( 60 | db: Db, 61 | survey: SurveyConfig, 62 | key: string, 63 | options: TermAggregationOptions = {} 64 | ) => { 65 | const collection = db.collection(config.mongo.normalized_collection) 66 | 67 | const { filters, sort = 'total', order = -1, year }: TermAggregationOptions = options 68 | 69 | const match: any = { 70 | survey: survey.survey, 71 | [key]: { $nin: [null, '', []] }, 72 | ...generateFiltersQuery(filters) 73 | } 74 | // if year is passed, restrict aggregation to specific year 75 | if (year) { 76 | match.year = year 77 | } 78 | 79 | const winsPipeline = getWinsPipeline(match, key) 80 | 81 | const rawResults = (await collection.aggregate(winsPipeline).toArray()) as WinsBucket[] 82 | 83 | console.log( 84 | inspect( 85 | { 86 | match, 87 | winsPipeline, 88 | rawResults 89 | }, 90 | { colors: true, depth: null } 91 | ) 92 | ) 93 | 94 | // add proper ids 95 | const resultsWithId = rawResults.map(result => ({ 96 | ...result, 97 | id: idsLookupTable[key][result.id] 98 | })) 99 | 100 | // group by years and add counts 101 | const resultsByYear = await groupByYears(resultsWithId, db, survey, match) 102 | 103 | // console.log(JSON.stringify(resultsByYear, '', 2)) 104 | return resultsByYear 105 | } 106 | 107 | export const matchupsAggregationFunction = async ( 108 | db: Db, 109 | survey: SurveyConfig, 110 | key: string, 111 | options: TermAggregationOptions = {} 112 | ) => { 113 | const collection = db.collection(config.mongo.normalized_collection) 114 | 115 | const { filters, sort = 'total', order = -1, year }: TermAggregationOptions = options 116 | 117 | const match: any = { 118 | survey: survey.survey, 119 | [key]: { $nin: [null, '', []] }, 120 | ...generateFiltersQuery(filters) 121 | } 122 | 123 | // if year is passed, restrict aggregation to specific year 124 | if (year) { 125 | match.year = year 126 | } 127 | 128 | const matchupsPipeline = getMatchupsPipeline(match, key) 129 | const rawResults = (await collection.aggregate(matchupsPipeline).toArray()) as MatchupAggregationResult[] 130 | 131 | console.log( 132 | inspect( 133 | { 134 | match, 135 | matchupsPipeline, 136 | rawResults 137 | }, 138 | { colors: true, depth: null } 139 | ) 140 | ) 141 | 142 | // add proper ids 143 | // const resultsWithId = rawResults.map(result => ({ 144 | // ...result, 145 | // id: idsLookupTable[key][result.id] 146 | // })) 147 | rawResults.forEach(result => { 148 | result.id = idsLookupTable[key][result.id] 149 | result.matchups = result.matchups.map(matchup => ({ 150 | ...matchup, 151 | id: idsLookupTable[key][matchup.id] 152 | })) 153 | }) 154 | 155 | // console.log('// resultsWithId') 156 | // console.log(JSON.stringify(rawResults, '', 2)) 157 | 158 | // group by years and add counts 159 | const resultsByYear = await groupByYears(rawResults, db, survey, match) 160 | 161 | // console.log('// resultsByYear') 162 | // console.log(JSON.stringify(resultsByYear, '', 2)) 163 | 164 | return resultsByYear 165 | } 166 | -------------------------------------------------------------------------------- /src/compute/brackets_pipelines.ts: -------------------------------------------------------------------------------- 1 | // see https://github.com/StateOfJS/state-of-js-graphql-results-api/issues/190#issuecomment-952308689 2 | // thanks @thomasheyenbrock!! 3 | export const getWinsPipeline = (match: any, key: String) => [ 4 | { $match: match }, 5 | /** 6 | * Reduce over the bracketResult array and determine the round. (Use $reduce 7 | * instead of $map in order to get a running index.) 8 | */ 9 | { 10 | $project: { 11 | _id: 0, 12 | matches: { 13 | $reduce: { 14 | input: `$${key}`, 15 | initialValue: { acc: [], index: 0 }, 16 | in: { 17 | acc: { 18 | $concatArrays: [ 19 | '$$value.acc', 20 | [ 21 | { 22 | player: { $slice: ['$$this', 2] }, 23 | winner: { $arrayElemAt: ['$$this', 2] }, 24 | round: { 25 | $switch: { 26 | branches: [ 27 | { 28 | case: { $lt: ['$$value.index', 4] }, 29 | then: 1 30 | }, 31 | { case: { $lt: ['$$value.index', 6] }, then: 2 } 32 | ], 33 | default: 3 34 | } 35 | } 36 | } 37 | ] 38 | ] 39 | }, 40 | index: { $add: ['$$value.index', 1] } 41 | } 42 | } 43 | } 44 | } 45 | }, 46 | { $project: { match: '$matches.acc' } }, 47 | /** 48 | * Unwrap the individual matches, and also the players, effectively producing 49 | * two documents for a single match (one will be used for each player). 50 | */ 51 | { $unwind: '$match' }, 52 | { $unwind: '$match.player' }, 53 | /** 54 | * Group by player and round, summing up the totals and wins. 55 | */ 56 | { 57 | $project: { 58 | player: '$match.player', 59 | round: '$match.round', 60 | hasWon: { 61 | $cond: { 62 | if: { $eq: ['$match.player', '$match.winner'] }, 63 | then: 1, 64 | else: 0 65 | } 66 | } 67 | } 68 | }, 69 | { 70 | $group: { 71 | _id: { player: '$player', round: '$round' }, 72 | totalCount: { $sum: 1 }, 73 | count: { $sum: '$hasWon' } 74 | } 75 | }, 76 | /** 77 | * Create the three properties "round1", "round2", and "round3". Only one of 78 | * them will actually contain data at this stage. 79 | */ 80 | { 81 | $project: { 82 | _id: 0, 83 | player: '$_id.player', 84 | round1: { 85 | $cond: { 86 | if: { $eq: ['$_id.round', 1] }, 87 | then: { 88 | totalCount: '$totalCount', 89 | count: '$count', 90 | percentage: { $divide: ['$count', '$totalCount'] } 91 | }, 92 | else: {} 93 | } 94 | }, 95 | round2: { 96 | $cond: { 97 | if: { $eq: ['$_id.round', 2] }, 98 | then: { 99 | totalCount: '$totalCount', 100 | count: '$count', 101 | percentage: { $divide: ['$count', '$totalCount'] } 102 | }, 103 | else: {} 104 | } 105 | }, 106 | round3: { 107 | $cond: { 108 | if: { $eq: ['$_id.round', 3] }, 109 | then: { 110 | totalCount: '$totalCount', 111 | count: '$count', 112 | percentage: { $divide: ['$count', '$totalCount'] } 113 | }, 114 | else: {} 115 | } 116 | } 117 | } 118 | }, 119 | /** 120 | * Group by player and merge together the round-fields created in the 121 | * previous stage. 122 | */ 123 | { 124 | $group: { 125 | _id: '$player', 126 | round1: { $mergeObjects: '$round1' }, 127 | round2: { $mergeObjects: '$round2' }, 128 | round3: { $mergeObjects: '$round3' } 129 | } 130 | }, 131 | /** 132 | * Sum up the totals and wins of all three rounds. 133 | */ 134 | { 135 | $project: { 136 | _id: 0, 137 | id: '$_id', 138 | combined: { 139 | totalCount: { 140 | $sum: ['$round1.totalCount', '$round2.totalCount', '$round3.totalCount'] 141 | }, 142 | count: { 143 | $sum: ['$round1.count', '$round2.count', '$round3.count'] 144 | } 145 | }, 146 | round1: 1, 147 | round2: 1, 148 | round3: 1 149 | } 150 | }, 151 | /** 152 | * Final formatting. 153 | */ 154 | { 155 | $project: { 156 | id: 1, 157 | year: { $literal: 2021 }, 158 | combined: { 159 | count: '$combined.count', 160 | percentage: { 161 | $round: [ 162 | { 163 | $multiply: [ 164 | { $divide: ['$combined.count', '$combined.totalCount'] }, 165 | 100 166 | ] 167 | }, 168 | 1 169 | ] 170 | } 171 | }, 172 | round1: { 173 | count: { $ifNull: ['$round1.count', 0] }, 174 | percentage: { 175 | $round: [{ $multiply: [{ $ifNull: ['$round1.percentage', null] }, 100] }, 1] 176 | } 177 | }, 178 | round2: { 179 | count: { $ifNull: ['$round2.count', 0] }, 180 | percentage: { 181 | $round: [{ $multiply: [{ $ifNull: ['$round2.percentage', null] }, 100] }, 1] 182 | } 183 | }, 184 | round3: { 185 | count: { $ifNull: ['$round3.count', 0] }, 186 | percentage: { 187 | $round: [{ $multiply: [{ $ifNull: ['$round3.percentage', null] }, 100] }, 1] 188 | } 189 | } 190 | } 191 | }, 192 | /** 193 | * remove any item with id: null 194 | */ 195 | { 196 | $match: { 197 | id: { $ne: null } 198 | } 199 | } 200 | ] 201 | 202 | // count how many matches each item won 203 | export const getMatchupsPipeline = (match: any, key: String) => [ 204 | { $match: match }, 205 | /** 206 | * Map over the individual matches and transform the shape. 207 | */ 208 | { 209 | $project: { 210 | _id: 0, 211 | matches: { 212 | $map: { 213 | input: `$${key}`, 214 | in: { 215 | /** 216 | * We store an array here that we'll unwrap later in order to split 217 | * a single match into two documents (one for each player). 218 | */ 219 | players: [ 220 | { 221 | player: { $arrayElemAt: ['$$this', 0] }, 222 | opponent: { $arrayElemAt: ['$$this', 1] } 223 | }, 224 | { 225 | player: { $arrayElemAt: ['$$this', 1] }, 226 | opponent: { $arrayElemAt: ['$$this', 0] } 227 | } 228 | ], 229 | winner: { $arrayElemAt: ['$$this', 2] } 230 | } 231 | } 232 | } 233 | } 234 | }, 235 | /** 236 | * Unwind the individual matches and players. 237 | */ 238 | { $unwind: '$matches' }, 239 | { $unwind: '$matches.players' }, 240 | 241 | /** 242 | * Group by player-opponent-combination and sum up totals and wins. 243 | */ 244 | { 245 | $project: { 246 | player: '$matches.players.player', 247 | opponent: '$matches.players.opponent', 248 | hasWon: { 249 | $cond: { 250 | if: { $eq: ['$matches.players.player', '$matches.winner'] }, 251 | then: 1, 252 | else: 0 253 | } 254 | } 255 | } 256 | }, 257 | { 258 | $group: { 259 | _id: { player: '$player', opponent: '$opponent' }, 260 | totalCount: { $sum: 1 }, 261 | count: { $sum: '$hasWon' } 262 | } 263 | }, 264 | /** 265 | * Calculate the percentage. 266 | */ 267 | { 268 | $project: { 269 | _id: 0, 270 | player: '$_id.player', 271 | opponent: '$_id.opponent', 272 | count: '$count', 273 | percentage: { 274 | $round: [{ $multiply: [{ $divide: ['$count', '$totalCount'] }, 100] }, 1] 275 | } 276 | } 277 | }, 278 | /** 279 | * Sort by percentage descending 280 | */ 281 | { $sort: { percentage: -1 } }, 282 | /** 283 | * Remove any match where the player or opponent is null 284 | */ 285 | { $match: { player: { $ne: null }, opponent: { $ne: null } } }, 286 | /** 287 | * Group by player and push an object for each opponent. 288 | */ 289 | { 290 | $group: { 291 | _id: '$player', 292 | matchups: { 293 | $push: { id: '$opponent', count: '$count', percentage: '$percentage' } 294 | } 295 | } 296 | }, 297 | { 298 | $project: { 299 | _id: 0, 300 | id: '$_id', 301 | matchups: 1, 302 | year: { $literal: 2021 } 303 | } 304 | }, 305 | /** 306 | * Remove any item where id is null 307 | */ 308 | { $match: { id: { $ne: null } } }, 309 | /** 310 | * Sort by id 311 | */ 312 | { $sort: { id: 1 } } 313 | ] 314 | -------------------------------------------------------------------------------- /src/compute/choices_over_years_graph.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash' 2 | import { Db } from 'mongodb' 3 | import config from '../config' 4 | import { SurveyConfig } from '../types' 5 | import { Filters, generateFiltersQuery } from '../filters' 6 | 7 | export async function computeChoicesOverYearsGraph( 8 | db: Db, 9 | survey: SurveyConfig, 10 | field: string, 11 | filters?: Filters 12 | ) { 13 | const collection = db.collection(config.mongo.normalized_collection) 14 | 15 | const results = await collection 16 | .aggregate<{ 17 | years: Array<{ 18 | year: number 19 | choice: string 20 | }> 21 | }>([ 22 | { 23 | $match: generateFiltersQuery(filters) 24 | }, 25 | // only extract the fields we're interested in 26 | { 27 | $project: { 28 | _id: 0, 29 | email: { 30 | $trim: { input: '$user_info.email' } 31 | }, 32 | year: true, 33 | choice: `$${field}` 34 | } 35 | }, 36 | // make sure we do not have empty/null values 37 | { 38 | $match: { 39 | email: { $nin: [null, '', ' '] }, 40 | year: { $nin: [null, '', ' '] }, 41 | choice: { $nin: [null, '', ' '] } 42 | } 43 | }, 44 | // make sure there is a single participation by year 45 | // if that's not the case, last answer will be used 46 | { 47 | $group: { 48 | _id: { 49 | email: '$email', 50 | year: '$year' 51 | }, 52 | email: { $last: '$email' }, 53 | year: { $last: '$year' }, 54 | choice: { $last: '$choice' } 55 | } 56 | }, 57 | // make sure years are in order 58 | { 59 | $sort: { 60 | email: 1, 61 | year: 1 62 | } 63 | }, 64 | // group results by email, and generate an array of responses by year 65 | { 66 | $group: { 67 | _id: '$email', 68 | yearCount: { $sum: 1 }, 69 | years: { 70 | $push: { 71 | year: '$year', 72 | choice: '$choice' 73 | } 74 | } 75 | } 76 | }, 77 | // exclude participation for a single year 78 | { 79 | $match: { 80 | yearCount: { 81 | $gt: 1 82 | } 83 | } 84 | }, 85 | { 86 | $project: { 87 | _id: 0, 88 | years: true 89 | } 90 | } 91 | ]) 92 | .toArray() 93 | 94 | let nodes: Array<{ 95 | id: string 96 | year: number 97 | choice: string 98 | }> = [] 99 | let links: Array<{ 100 | source: string 101 | target: string 102 | count: number 103 | }> = [] 104 | 105 | await results.forEach(item => { 106 | item.years.forEach((current, index, arr) => { 107 | const currentId = `${current.year}.${current.choice}` 108 | if (nodes.find(n => n.id === currentId) === undefined) { 109 | nodes.push({ 110 | id: currentId, 111 | ...current 112 | }) 113 | } 114 | 115 | if (index > 0) { 116 | const previous = arr[index - 1] 117 | // make sure there's only one year between the 2 entries 118 | // otherwise, skip the link 119 | if (current.year - previous.year === 1) { 120 | const previousId = `${previous.year}.${previous.choice}` 121 | let link = links.find(l => { 122 | return l.source === previousId && l.target === currentId 123 | }) 124 | if (!link) { 125 | link = { 126 | source: previousId, 127 | target: currentId, 128 | count: 0 129 | } 130 | links.push(link) 131 | } 132 | link.count += 1 133 | } 134 | } 135 | }) 136 | }) 137 | 138 | nodes = _.orderBy(nodes, 'year') 139 | 140 | links = _.orderBy(links, 'count') 141 | links.reverse() 142 | 143 | return { nodes, links } 144 | } 145 | -------------------------------------------------------------------------------- /src/compute/common.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { getParticipationByYearMap } from './demographics' 3 | import { Completion, SurveyConfig } from '../types' 4 | 5 | /** 6 | * Convert a ratio to percentage, applying a predefined rounding. 7 | */ 8 | export const ratioToPercentage = (ratio: number) => { 9 | return Math.round(ratio * 1000) / 10 10 | } 11 | 12 | /** 13 | * Compute completion percentage. 14 | */ 15 | export const computeCompletion = (answerCount: number, totalCount: number) => { 16 | return ratioToPercentage(answerCount / totalCount) 17 | } 18 | 19 | /** 20 | * Add completion information for yearly buckets. 21 | */ 22 | export const appendCompletionToYearlyResults = async < 23 | T extends { year: number; total: number; completion: Pick } 24 | >( 25 | db: Db, 26 | survey: SurveyConfig, 27 | yearlyResults: T[] 28 | ): Promise & { completion: Completion } 30 | >> => { 31 | const totalRespondentsByYear = await getParticipationByYearMap(db, survey) 32 | 33 | return yearlyResults.map(yearlyResult => { 34 | return { 35 | ...yearlyResult, 36 | completion: { 37 | total: totalRespondentsByYear[yearlyResult.year], 38 | count: yearlyResult.completion.count, 39 | percentage: ratioToPercentage( 40 | yearlyResult.completion.count / totalRespondentsByYear[yearlyResult.year] 41 | ) 42 | } 43 | } as Omit & { completion: Completion } 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /src/compute/demographics.ts: -------------------------------------------------------------------------------- 1 | import orderBy from 'lodash/orderBy' 2 | import { Db } from 'mongodb' 3 | import config from '../config' 4 | import { SurveyConfig, YearParticipation } from '../types' 5 | import { Filters } from '../filters' 6 | 7 | export async function computeParticipationByYear( 8 | db: Db, 9 | survey: SurveyConfig, 10 | filters?: Filters, 11 | year?: number 12 | ): Promise { 13 | const collection = db.collection(config.mongo.normalized_collection) 14 | 15 | const participantsByYear = await collection 16 | .aggregate([ 17 | { 18 | $match: { 19 | survey: survey.survey 20 | } 21 | }, 22 | { 23 | $group: { 24 | _id: { year: '$year' }, 25 | participants: { $sum: 1 } 26 | } 27 | }, 28 | { 29 | $project: { 30 | _id: 0, 31 | year: '$_id.year', 32 | participants: 1 33 | } 34 | } 35 | ]) 36 | .toArray() as YearParticipation[] 37 | 38 | return orderBy(participantsByYear, 'year') 39 | } 40 | 41 | export async function getParticipationByYearMap( 42 | db: Db, 43 | survey: SurveyConfig 44 | ): Promise<{ 45 | [key: number]: number 46 | }> { 47 | const buckets = await computeParticipationByYear(db, survey) 48 | 49 | return buckets.reduce((acc, bucket) => { 50 | return { 51 | ...acc, 52 | [Number(bucket.year)]: bucket.participants 53 | } 54 | }, {}) 55 | } 56 | -------------------------------------------------------------------------------- /src/compute/experience.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash' 2 | import { Db } from 'mongodb' 3 | import config from '../config' 4 | import { ratioToPercentage, appendCompletionToYearlyResults } from './common' 5 | import { getEntity } from '../entities' 6 | import { Completion, SurveyConfig } from '../types' 7 | import { Filters, generateFiltersQuery } from '../filters' 8 | import { computeCompletionByYear } from './generic' 9 | 10 | interface ExperienceBucket { 11 | id: string 12 | count: number 13 | } 14 | 15 | const computeAwareness = (buckets: ExperienceBucket[], total: number) => { 16 | const neverHeard = buckets.find(bucket => bucket.id === 'never_heard') 17 | if (neverHeard === undefined) { 18 | return 0 19 | } 20 | 21 | return ratioToPercentage((total - neverHeard.count) / total) 22 | } 23 | 24 | const computeUsage = (buckets: ExperienceBucket[]) => { 25 | const neverHeardCount = buckets.find(bucket => bucket.id === 'never_heard')?.count ?? 0 26 | const interestedCount = buckets.find(bucket => bucket.id === 'interested')?.count ?? 0 27 | const notInterestedCount = buckets.find(bucket => bucket.id === 'not_interested')?.count ?? 0 28 | const wouldUseCount = buckets.find(bucket => bucket.id === 'would_use')?.count ?? 0 29 | const wouldNotUseCount = buckets.find(bucket => bucket.id === 'would_not_use')?.count ?? 0 30 | 31 | const usageCount = wouldUseCount + wouldNotUseCount 32 | const total = usageCount + interestedCount + notInterestedCount + neverHeardCount 33 | 34 | return ratioToPercentage(usageCount / total) 35 | } 36 | 37 | const computeInterest = (buckets: ExperienceBucket[]) => { 38 | const interested = buckets.find(bucket => bucket.id === 'interested') 39 | const notInterested = buckets.find(bucket => bucket.id === 'not_interested') 40 | if (interested === undefined || notInterested === undefined) { 41 | return 0 42 | } 43 | 44 | return ratioToPercentage(interested.count / (interested.count + notInterested.count)) 45 | } 46 | 47 | const computeSatisfaction = (buckets: ExperienceBucket[]) => { 48 | const wouldUse = buckets.find(bucket => bucket.id === 'would_use') 49 | const wouldNotUse = buckets.find(bucket => bucket.id === 'would_not_use') 50 | if (wouldUse === undefined || wouldNotUse === undefined) { 51 | return 0 52 | } 53 | 54 | return ratioToPercentage(wouldUse.count / (wouldUse.count + wouldNotUse.count)) 55 | } 56 | 57 | export async function computeExperienceOverYears( 58 | db: Db, 59 | survey: SurveyConfig, 60 | tool: string, 61 | filters?: Filters 62 | ) { 63 | const collection = db.collection(config.mongo.normalized_collection) 64 | 65 | const path = `tools.${tool}.experience` 66 | 67 | const match = { 68 | survey: survey.survey, 69 | [path]: { $nin: [null, ''] }, 70 | ...generateFiltersQuery(filters) 71 | } 72 | 73 | const results = await collection 74 | .aggregate([ 75 | { 76 | $match: match 77 | }, 78 | { 79 | $group: { 80 | _id: { 81 | experience: `$${path}`, 82 | year: '$year' 83 | }, 84 | total: { $sum: 1 } 85 | } 86 | }, 87 | { 88 | $project: { 89 | _id: 0, 90 | experience: '$_id.experience', 91 | year: '$_id.year', 92 | total: 1 93 | } 94 | } 95 | ]) 96 | .toArray() 97 | 98 | const completionByYear = await computeCompletionByYear(db, match) 99 | 100 | // group by years and add counts 101 | const experienceByYear = _.orderBy( 102 | results.reduce< 103 | Array<{ 104 | year: number 105 | total: number 106 | completion: Pick 107 | awarenessUsageInterestSatisfaction: { 108 | awareness: number 109 | usage: number 110 | interest: number 111 | satisfaction: number 112 | } 113 | buckets: Array<{ 114 | id: string 115 | count: number 116 | countDelta?: number 117 | percentage: number 118 | percentageDelta?: number 119 | }> 120 | }> 121 | >((acc, result) => { 122 | let yearBucket = acc.find(b => b.year === result.year) 123 | if (yearBucket === undefined) { 124 | yearBucket = { 125 | year: result.year, 126 | total: 0, 127 | completion: { 128 | count: completionByYear[result.year]?.total ?? 0 129 | }, 130 | awarenessUsageInterestSatisfaction: { 131 | awareness: 0, 132 | usage: 0, 133 | interest: 0, 134 | satisfaction: 0 135 | }, 136 | buckets: [] 137 | } 138 | acc.push(yearBucket) 139 | } 140 | 141 | yearBucket.buckets.push({ 142 | id: result.experience, 143 | count: result.total, 144 | percentage: 0 145 | }) 146 | 147 | return acc 148 | }, []), 149 | 'year' 150 | ) 151 | 152 | // compute percentages 153 | experienceByYear.forEach(bucket => { 154 | bucket.total = _.sumBy(bucket.buckets, 'count') 155 | bucket.buckets.forEach(subBucket => { 156 | subBucket.percentage = ratioToPercentage(subBucket.count / bucket.total) 157 | }) 158 | }) 159 | 160 | // compute awareness/interest/satisfaction 161 | experienceByYear.forEach(bucket => { 162 | bucket.awarenessUsageInterestSatisfaction = { 163 | awareness: computeAwareness(bucket.buckets, bucket.total), 164 | usage: computeUsage(bucket.buckets), 165 | interest: computeInterest(bucket.buckets), 166 | satisfaction: computeSatisfaction(bucket.buckets) 167 | } 168 | }) 169 | 170 | // compute deltas 171 | experienceByYear.forEach((year, i) => { 172 | const previousYear = experienceByYear.find(y => y.year === year.year - 1) 173 | if (previousYear) { 174 | year.buckets.forEach(bucket => { 175 | const previousYearBucket = previousYear.buckets.find(b => b.id === bucket.id) 176 | if (previousYearBucket) { 177 | bucket.countDelta = bucket.count - previousYearBucket.count 178 | bucket.percentageDelta = Math.round(100 * (bucket.percentage - previousYearBucket.percentage))/100 179 | } 180 | }) 181 | } 182 | }) 183 | 184 | return appendCompletionToYearlyResults(db, survey, experienceByYear) 185 | } 186 | 187 | const metrics = ['awareness', 'usage', 'interest', 'satisfaction'] 188 | 189 | export async function computeToolsExperienceRanking( 190 | db: Db, 191 | survey: SurveyConfig, 192 | tools: string[], 193 | filters?: Filters 194 | ) { 195 | let availableYears: any[] = [] 196 | const metricByYear: { [key: string]: any } = {} 197 | 198 | for (const tool of tools) { 199 | const toolAllYearsExperience = await computeExperienceOverYears(db, survey, tool, filters) 200 | const toolAwarenessUsageInterestSatisfactionOverYears: any[] = [] 201 | 202 | toolAllYearsExperience.forEach((toolYear: any) => { 203 | availableYears.push(toolYear.year) 204 | 205 | if (metricByYear[toolYear.year] === undefined) { 206 | metricByYear[toolYear.year] = { 207 | awareness: [], 208 | usage: [], 209 | interest: [], 210 | satisfaction: [] 211 | } 212 | } 213 | 214 | metrics.forEach(metric => { 215 | metricByYear[toolYear.year][metric].push({ 216 | tool, 217 | percentage: toolYear.awarenessUsageInterestSatisfaction[metric] 218 | }) 219 | }) 220 | 221 | toolAwarenessUsageInterestSatisfactionOverYears.push({ 222 | year: toolYear.year, 223 | ...toolYear.awarenessUsageInterestSatisfaction 224 | }) 225 | }) 226 | } 227 | 228 | for (const yearMetrics of Object.values(metricByYear)) { 229 | metrics.forEach(metric => { 230 | yearMetrics[metric] = _.sortBy(yearMetrics[metric], 'percentage').reverse() 231 | yearMetrics[metric].forEach((bucket: any, index: number) => { 232 | // make ranking starts at 1 233 | bucket.rank = index + 1 234 | }) 235 | }) 236 | } 237 | 238 | availableYears = _.uniq(availableYears).sort() 239 | 240 | const byTool: any[] = [] 241 | tools.forEach(async tool => { 242 | byTool.push({ 243 | id: tool, 244 | entity: await getEntity({ id: tool }), 245 | ...metrics.reduce((acc, metric) => { 246 | return { 247 | ...acc, 248 | [metric]: availableYears.map(year => { 249 | const toolYearMetric = metricByYear[year][metric].find( 250 | (d: any) => d.tool === tool 251 | ) 252 | let rank = null 253 | let percentage = null 254 | if (toolYearMetric !== undefined) { 255 | rank = toolYearMetric.rank 256 | percentage = toolYearMetric.percentage 257 | } 258 | 259 | return { year, rank, percentage } 260 | }) 261 | } 262 | }, {}) 263 | }) 264 | }) 265 | 266 | return byTool 267 | } 268 | -------------------------------------------------------------------------------- /src/compute/generic.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { inspect } from 'util' 3 | import keyBy from 'lodash/keyBy' 4 | import orderBy from 'lodash/orderBy' 5 | import config from '../config' 6 | import { Completion, SurveyConfig } from '../types' 7 | import { Filters, generateFiltersQuery } from '../filters' 8 | import { ratioToPercentage } from './common' 9 | import { getEntity } from '../entities' 10 | import { getParticipationByYearMap } from './demographics' 11 | import { useCache } from '../caching' 12 | import uniq from 'lodash/uniq' 13 | import { WinsYearAggregations } from './brackets' 14 | 15 | export interface TermAggregationOptions { 16 | // filter aggregations 17 | filters?: Filters 18 | sort?: string 19 | order?: -1 | 1 20 | cutoff?: number 21 | limit?: number 22 | year?: number 23 | } 24 | 25 | export interface RawResult { 26 | id: number | string 27 | entity?: any 28 | year: number 29 | total: number 30 | } 31 | 32 | export interface CompletionResult { 33 | year: number 34 | total: number 35 | } 36 | 37 | export interface TermBucket { 38 | id: number | string 39 | entity?: any 40 | count: number 41 | total: number // alias for count 42 | countDelta?: number 43 | percentage: number 44 | percentageDelta?: number 45 | } 46 | 47 | export interface YearAggregations { 48 | year: number 49 | total: number 50 | completion: Completion 51 | buckets: TermBucket[] 52 | } 53 | 54 | export type AggregationFunction = ( 55 | db: Db, 56 | survey: SurveyConfig, 57 | key: string, 58 | options: TermAggregationOptions 59 | ) => Promise 60 | 61 | export async function getSurveyTotals(db: Db, surveyConfig: SurveyConfig, year?: Number) { 62 | const collection = db.collection(config.mongo.normalized_collection) 63 | let selector: any = { 64 | survey: surveyConfig.survey 65 | } 66 | if (year) { 67 | selector = { ...selector, year } 68 | } 69 | return collection.countDocuments(selector) 70 | } 71 | 72 | export async function computeCompletionByYear( 73 | db: Db, 74 | match: any 75 | ): Promise> { 76 | const collection = db.collection(config.mongo.normalized_collection) 77 | 78 | const aggregationPipeline = [ 79 | { 80 | $match: match 81 | }, 82 | { 83 | $group: { 84 | _id: { year: '$year' }, 85 | total: { 86 | $sum: 1 87 | } 88 | } 89 | }, 90 | { 91 | $project: { 92 | year: '$_id.year', 93 | total: 1 94 | } 95 | } 96 | ] 97 | 98 | const completionResults = (await collection 99 | .aggregate(aggregationPipeline) 100 | .toArray()) as CompletionResult[] 101 | 102 | // console.log( 103 | // inspect( 104 | // { 105 | // aggregationPipeline, 106 | // completionResults, 107 | // }, 108 | // { colors: true, depth: null } 109 | // ) 110 | // ) 111 | 112 | return keyBy(completionResults, 'year') 113 | } 114 | 115 | // no cutoff for now 116 | const addCutoff = false 117 | 118 | export async function computeTermAggregationByYear( 119 | db: Db, 120 | survey: SurveyConfig, 121 | key: string, 122 | options: TermAggregationOptions = {}, 123 | aggregationFunction: AggregationFunction = computeDefaultTermAggregationByYear 124 | ) { 125 | return aggregationFunction(db, survey, key, options) 126 | } 127 | 128 | export async function computeDefaultTermAggregationByYear( 129 | db: Db, 130 | survey: SurveyConfig, 131 | key: string, 132 | options: TermAggregationOptions = {} 133 | ) { 134 | const collection = db.collection(config.mongo.normalized_collection) 135 | 136 | const { 137 | filters, 138 | sort = 'total', 139 | order = -1, 140 | cutoff = 10, 141 | limit = 25, 142 | year 143 | }: TermAggregationOptions = options 144 | 145 | const match: any = { 146 | survey: survey.survey, 147 | [key]: { $nin: [null, '', []] }, 148 | ...generateFiltersQuery(filters) 149 | } 150 | // if year is passed, restrict aggregation to specific year 151 | if (year) { 152 | match.year = year 153 | } 154 | 155 | // console.log(match) 156 | 157 | // generate an aggregation pipeline for all years, or 158 | // optionally restrict it to a specific year of data 159 | const getAggregationPipeline = () => { 160 | const pipeline: any[] = [ 161 | { 162 | $match: match 163 | }, 164 | { 165 | $unwind: { 166 | path: `$${key}` 167 | } 168 | }, 169 | { 170 | $group: { 171 | _id: { 172 | id: `$${key}`, 173 | year: '$year' 174 | }, 175 | total: { $sum: 1 } 176 | } 177 | }, 178 | { 179 | $project: { 180 | _id: 0, 181 | id: '$_id.id', 182 | year: '$_id.year', 183 | total: 1 184 | } 185 | }, 186 | { $sort: { [sort]: order } } 187 | ] 188 | 189 | if (addCutoff) { 190 | pipeline.push({ $match: { total: { $gt: cutoff } } }) 191 | } 192 | 193 | // only add limit if year is specified 194 | if (year) { 195 | pipeline.push({ $limit: limit }) 196 | } 197 | return pipeline 198 | } 199 | 200 | const rawResults = (await collection 201 | .aggregate(getAggregationPipeline()) 202 | .toArray()) as RawResult[] 203 | 204 | // console.log( 205 | // inspect( 206 | // { 207 | // match, 208 | // sampleAggregationPipeline: getAggregationPipeline(), 209 | // rawResults 210 | // }, 211 | // { colors: true, depth: null } 212 | // ) 213 | // ) 214 | 215 | // add entities if applicable 216 | const resultsWithEntity = [] 217 | for (let result of rawResults) { 218 | const entity = await getEntity(result as any) 219 | if (entity) { 220 | result = { ...result, entity } 221 | } 222 | resultsWithEntity.push(result) 223 | } 224 | 225 | // group by years and add counts 226 | const resultsByYear = await groupByYears(resultsWithEntity, db, survey, match) 227 | 228 | // compute percentages 229 | const resultsWithPercentages = computePercentages(resultsByYear) 230 | 231 | // compute deltas 232 | resultsWithPercentages.forEach((year, i) => { 233 | const previousYear = resultsByYear[i - 1] 234 | if (previousYear) { 235 | year.buckets.forEach(bucket => { 236 | const previousYearBucket = previousYear.buckets.find(b => b.id === bucket.id) 237 | if (previousYearBucket) { 238 | bucket.countDelta = bucket.count - previousYearBucket.count 239 | bucket.percentageDelta = 240 | Math.round(100 * (bucket.percentage - previousYearBucket.percentage)) / 100 241 | } 242 | }) 243 | } 244 | }) 245 | 246 | return resultsByYear 247 | } 248 | 249 | interface GroupByYearResult { 250 | id: number | string 251 | year: number 252 | } 253 | 254 | export async function groupByYears( 255 | results: GroupByYearResult[], 256 | db: Db, 257 | survey: SurveyConfig, 258 | match: any, 259 | ) { 260 | const years = uniq(results.map(r => r.year)) 261 | 262 | const totalRespondentsByYear = await getParticipationByYearMap(db, survey) 263 | const completionByYear = await computeCompletionByYear(db, match) 264 | 265 | const resultsWithYears = years.map((year: number) => { 266 | const totalRespondents = totalRespondentsByYear[year] ?? 0 267 | const completionCount = completionByYear[year]?.total ?? 0 268 | 269 | const buckets = results 270 | .filter(r => r.year === year) 271 | 272 | const yearBucket = { 273 | year, 274 | total: totalRespondents, 275 | completion: { 276 | total: totalRespondents, 277 | count: completionCount, 278 | percentage: ratioToPercentage(completionCount / totalRespondents) 279 | }, 280 | buckets 281 | } 282 | return yearBucket 283 | }) 284 | 285 | return orderBy(resultsWithYears, 'year') 286 | } 287 | 288 | export function computePercentages (resultsByYear: YearAggregations[]) { 289 | 290 | resultsByYear.forEach(yearResult => { 291 | yearResult.buckets.forEach(bucket => { 292 | bucket.count = bucket.total 293 | bucket.percentage = ratioToPercentage(bucket.count / yearResult.completion.count) 294 | }) 295 | }) 296 | return resultsByYear 297 | } 298 | 299 | export async function computeTermAggregationAllYearsWithCache( 300 | db: Db, 301 | survey: SurveyConfig, 302 | id: string, 303 | options: TermAggregationOptions = {}, 304 | aggregationFunction?: AggregationFunction 305 | ) { 306 | return useCache(computeTermAggregationByYear, db, [survey, id, options, aggregationFunction]) 307 | } 308 | 309 | export async function computeTermAggregationSingleYear( 310 | db: Db, 311 | survey: SurveyConfig, 312 | key: string, 313 | options: TermAggregationOptions, 314 | aggregationFunction?: AggregationFunction 315 | ) { 316 | const allYears = await computeTermAggregationByYear( 317 | db, 318 | survey, 319 | key, 320 | options, 321 | aggregationFunction 322 | ) 323 | return allYears[0] 324 | } 325 | 326 | export async function computeTermAggregationSingleYearWithCache( 327 | db: Db, 328 | survey: SurveyConfig, 329 | id: string, 330 | options: TermAggregationOptions, 331 | aggregationFunction?: AggregationFunction 332 | ) { 333 | return useCache(computeTermAggregationSingleYear, db, [ 334 | survey, 335 | id, 336 | options, 337 | aggregationFunction 338 | ]) 339 | } 340 | -------------------------------------------------------------------------------- /src/compute/happiness.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { computeTermAggregationByYear } from './generic' 3 | import { SurveyConfig } from '../types' 4 | import { Filters } from '../filters' 5 | 6 | export async function computeHappinessByYear( 7 | db: Db, 8 | survey: SurveyConfig, 9 | id: string, 10 | filters?: Filters 11 | ) { 12 | const happinessByYear = await computeTermAggregationByYear(db, survey, `happiness.${id}`, { 13 | filters, 14 | sort: 'id', 15 | order: 1 16 | }) 17 | 18 | // compute mean for each year 19 | happinessByYear.forEach((bucket: any) => { 20 | const totalScore = bucket.buckets.reduce((acc: any, subBucket: any) => { 21 | return acc + subBucket.id * subBucket.count 22 | }, 0) 23 | bucket.mean = Math.round((totalScore / bucket.total) * 10) / 10 + 1 24 | }) 25 | 26 | return happinessByYear 27 | } 28 | -------------------------------------------------------------------------------- /src/compute/index.ts: -------------------------------------------------------------------------------- 1 | export * from './choices_over_years_graph' 2 | export * from './demographics' 3 | export * from './experience' 4 | export * from './generic' 5 | export * from './happiness' 6 | export * from './matrices' 7 | export * from './tools' 8 | -------------------------------------------------------------------------------- /src/compute/matrices.ts: -------------------------------------------------------------------------------- 1 | import { inspect } from 'util' 2 | import _ from 'lodash' 3 | import { Db } from 'mongodb' 4 | import config from '../config' 5 | import { ratioToPercentage } from './common' 6 | import { getEntity } from '../entities' 7 | import { SurveyConfig } from '../types' 8 | import { ToolExperienceFilterId, toolExperienceConfigById } from './tools' 9 | 10 | export const computeToolMatrixBreakdown = async ( 11 | db: Db, 12 | { 13 | survey, 14 | tool, 15 | experience, 16 | type, 17 | year 18 | }: { 19 | survey: SurveyConfig 20 | tool: string 21 | experience: ToolExperienceFilterId 22 | type: string 23 | year: number 24 | } 25 | ) => { 26 | const collection = db.collection(config.mongo.normalized_collection) 27 | 28 | const experienceKey = `tools.${tool}.experience` 29 | const experienceConfig = toolExperienceConfigById[experience] 30 | const experiencePredicate = experienceConfig.predicate 31 | const experienceComparisonPredicate = experienceConfig.comparisonPredicate 32 | 33 | let dimensionPath = `user_info.${type}` 34 | if (type === 'source') { 35 | dimensionPath = `${dimensionPath}.normalized` 36 | } 37 | 38 | const comparisonRangesAggregationPipeline = [ 39 | { 40 | $match: { 41 | survey: survey.survey, 42 | year, 43 | [experienceKey]: experienceComparisonPredicate, 44 | [dimensionPath]: { 45 | $exists: true, 46 | $nin: [null, ''] 47 | } 48 | } 49 | }, 50 | { 51 | // group by dimension choice 52 | $group: { 53 | _id: { 54 | group_by: `$${dimensionPath}` 55 | }, 56 | count: { $sum: 1 } 57 | } 58 | }, 59 | { 60 | $project: { 61 | _id: 0, 62 | id: '$_id.group_by', 63 | count: 1 64 | } 65 | } 66 | ] 67 | const comparisonRangesResults = await collection 68 | .aggregate(comparisonRangesAggregationPipeline) 69 | .toArray() 70 | 71 | const comparisonTotal = _.sumBy(comparisonRangesResults, 'count') 72 | 73 | const comparisonRangeById = _.keyBy(comparisonRangesResults, 'id') 74 | 75 | const experienceDistributionByRangeAggregationPipeline = [ 76 | { 77 | $match: { 78 | survey: survey.survey, 79 | year, 80 | [experienceKey]: experiencePredicate, 81 | [dimensionPath]: { 82 | $exists: true, 83 | $nin: [null, ''] 84 | } 85 | } 86 | }, 87 | { 88 | $group: { 89 | _id: { 90 | group_by: `$${dimensionPath}` 91 | }, 92 | count: { $sum: 1 } 93 | } 94 | }, 95 | { 96 | $project: { 97 | _id: 0, 98 | id: '$_id.group_by', 99 | count: 1 100 | } 101 | }, 102 | { $sort: { count: -1 } }, 103 | { $limit: 10 } 104 | ] 105 | const experienceDistributionByRangeResults = await collection 106 | .aggregate(experienceDistributionByRangeAggregationPipeline) 107 | .toArray() 108 | 109 | // fetch the total number of respondents having picked 110 | // the given experience, and who also answered the dimension 111 | // question. 112 | const experienceTotalQuery = { 113 | survey: survey.survey, 114 | year, 115 | [experienceKey]: experiencePredicate, 116 | [dimensionPath]: { 117 | $exists: true, 118 | $nin: [null, ''] 119 | } 120 | } 121 | const total = await collection.countDocuments(experienceTotalQuery) 122 | const overallPercentage = ratioToPercentage(total / comparisonTotal) 123 | 124 | experienceDistributionByRangeResults.forEach(bucket => { 125 | bucket.percentage = ratioToPercentage(bucket.count / total) 126 | 127 | // As we're using an intersection, it's safe to assume that 128 | // the dimension item is always available. 129 | const comparisonRange = comparisonRangeById[bucket.id] 130 | 131 | bucket.range_total = comparisonRange.count 132 | bucket.range_percentage = ratioToPercentage(bucket.count / comparisonRange.count) 133 | // how does the distribution for this specific experience/range compare 134 | // to the overall distribution for the range? 135 | bucket.range_percentage_delta = _.round(bucket.range_percentage - overallPercentage, 2) 136 | }) 137 | 138 | // console.log( 139 | // inspect( 140 | // { 141 | // total, 142 | // comparisonTotal, 143 | // comparisonRangesAggregationPipeline, 144 | // experienceDistributionByRangeAggregationPipeline, 145 | // overallPercentage, 146 | // comparisonRangeById, 147 | // id: tool, 148 | // total_in_experience: total, 149 | // ranges: experienceDistributionByRangeResults 150 | // }, 151 | // { colors: true, depth: null } 152 | // ) 153 | // ) 154 | 155 | return { 156 | id: tool, 157 | entity: await getEntity({ id: tool }), 158 | total: comparisonTotal, 159 | count: total, 160 | percentage: overallPercentage, 161 | buckets: experienceDistributionByRangeResults 162 | } 163 | } 164 | 165 | export async function computeToolsMatrix( 166 | db: Db, 167 | { 168 | survey, 169 | tools, 170 | experience, 171 | type, 172 | year 173 | }: { 174 | survey: SurveyConfig 175 | tools: string[] 176 | experience: ToolExperienceFilterId 177 | type: string 178 | year: number 179 | } 180 | ) { 181 | const allTools: any[] = [] 182 | for (const tool of tools) { 183 | allTools.push( 184 | await computeToolMatrixBreakdown(db, { 185 | survey, 186 | tool, 187 | experience, 188 | type, 189 | year 190 | }) 191 | ) 192 | } 193 | 194 | return allTools 195 | } 196 | -------------------------------------------------------------------------------- /src/compute/tools.ts: -------------------------------------------------------------------------------- 1 | import { inspect } from 'util' 2 | import { Db } from 'mongodb' 3 | import { SurveyConfig } from '../types' 4 | import { Filters } from '../filters' 5 | import config from '../config' 6 | import { computeChoicesOverYearsGraph } from './choices_over_years_graph' 7 | import { getParticipationByYearMap } from './demographics' 8 | 9 | export const allToolExperienceIds = [ 10 | 'would_use', 11 | 'would_not_use', 12 | 'interested', 13 | 'not_interested', 14 | 'never_heard' 15 | ] as const 16 | export type ToolExperienceId = typeof allToolExperienceIds[number] 17 | 18 | export const allToolCompoundExperienceIds = [ 19 | // `would_use` + `would_not_use` VS total 20 | 'usage', 21 | // `would_use` + `interested` VS `would_use` + `would_not_use` + `interested` + `not_interested` 22 | 'positive_sentiment', 23 | // `would_not_use` + `not_interested` VS `would_use` + `would_not_use` + `interested` + `not_interested` 24 | 'negative_sentiment', 25 | // `would_use` VS `would_not_use` 26 | 'satisfaction', 27 | // `interested` VS `not_interested` 28 | 'interest', 29 | // `never_heard` VS total (inverted) 30 | 'awareness' 31 | ] as const 32 | export type ToolCompoundExperienceId = typeof allToolCompoundExperienceIds[number] 33 | 34 | export type ToolExperienceFilterId = ToolExperienceId | ToolCompoundExperienceId 35 | 36 | export const toolExperienceConfigById: Record< 37 | ToolExperienceFilterId, 38 | Readonly<{ 39 | predicate: ToolExperienceId | { $in: Readonly } 40 | comparisonPredicate: ToolExperienceId | { $in: Readonly } 41 | }> 42 | > = { 43 | would_use: { 44 | predicate: 'would_use', 45 | comparisonPredicate: { $in: allToolExperienceIds } 46 | }, 47 | would_not_use: { 48 | predicate: 'would_not_use', 49 | comparisonPredicate: { $in: allToolExperienceIds } 50 | }, 51 | interested: { 52 | predicate: 'interested', 53 | comparisonPredicate: { $in: allToolExperienceIds } 54 | }, 55 | not_interested: { 56 | predicate: 'not_interested', 57 | comparisonPredicate: { $in: allToolExperienceIds } 58 | }, 59 | never_heard: { 60 | predicate: 'never_heard', 61 | comparisonPredicate: { $in: allToolExperienceIds } 62 | }, 63 | usage: { 64 | predicate: { $in: ['would_use', 'would_not_use'] }, 65 | comparisonPredicate: { $in: allToolExperienceIds } 66 | }, 67 | positive_sentiment: { 68 | predicate: { $in: ['would_use', 'interested'] }, 69 | comparisonPredicate: { 70 | $in: ['would_use', 'would_not_use', 'interested', 'not_interested'] 71 | } 72 | }, 73 | negative_sentiment: { 74 | predicate: { $in: ['would_not_use', 'not_interested'] }, 75 | comparisonPredicate: { 76 | $in: ['would_use', 'would_not_use', 'interested', 'not_interested'] 77 | } 78 | }, 79 | satisfaction: { 80 | predicate: 'would_use', 81 | comparisonPredicate: { 82 | $in: ['would_use', 'would_not_use'] 83 | } 84 | }, 85 | interest: { 86 | predicate: 'interested', 87 | comparisonPredicate: { 88 | $in: ['interested', 'not_interested'] 89 | } 90 | }, 91 | awareness: { 92 | predicate: { 93 | $in: ['would_use', 'would_not_use', 'interested', 'not_interested'] 94 | }, 95 | comparisonPredicate: { $in: allToolExperienceIds } 96 | } 97 | } as const 98 | 99 | export async function computeToolExperienceGraph( 100 | db: Db, 101 | survey: SurveyConfig, 102 | tool: string, 103 | filters?: Filters 104 | ) { 105 | const field = `tools.${tool}.experience` 106 | 107 | const { nodes, links } = await computeChoicesOverYearsGraph(db, survey, field, filters) 108 | 109 | return { 110 | // remap for experience 111 | nodes: nodes.map(node => ({ 112 | id: node.id, 113 | year: node.year, 114 | experience: node.choice 115 | })), 116 | links 117 | } 118 | } 119 | 120 | export async function computeToolsCardinalityByUser( 121 | db: Db, 122 | survey: SurveyConfig, 123 | year: number, 124 | toolIds: string[], 125 | experienceId: ToolExperienceId 126 | ) { 127 | const pipeline = [ 128 | { 129 | // filter on specific survey and year. 130 | $match: { 131 | survey: survey.survey, 132 | year, 133 | } 134 | }, 135 | { 136 | // for each specified tool ID, convert to 1 137 | // if the experience matches `experienceId`, 138 | // 0 otherwise. 139 | $project: toolIds.reduce((acc, toolId) => { 140 | return { 141 | ...acc, 142 | [toolId]: { 143 | $cond: { 144 | if: { 145 | $eq: [ 146 | `$tools.${toolId}.experience`, 147 | experienceId 148 | ] 149 | }, 150 | then: 1, 151 | else: 0 152 | } 153 | }, 154 | } 155 | }, {}) 156 | }, 157 | { 158 | // compute cardinality for each document 159 | $project: { 160 | cardinality: { 161 | $sum: toolIds.map(toolId => `$${toolId}`) 162 | } 163 | } 164 | }, 165 | { 166 | // aggregate cardinality 167 | $group: { 168 | _id: "$cardinality", 169 | count: { $sum: 1 } 170 | } 171 | }, 172 | { 173 | // rename fields 174 | $project: { 175 | cardinality: '$_id', 176 | count: '$count' 177 | } 178 | }, 179 | { 180 | // exclude 0 cardinality 181 | $match: { 182 | cardinality: { 183 | $gt: 0, 184 | }, 185 | }, 186 | }, 187 | { 188 | // higher cardinality first 189 | $sort: { 190 | cardinality: -1 191 | } 192 | } 193 | ] 194 | 195 | const results = await db.collection(config.mongo.normalized_collection).aggregate<{ 196 | _id: number 197 | cardinality: number 198 | count: number 199 | }>(pipeline).toArray() 200 | 201 | if (results.length === 0) { 202 | return [] 203 | } 204 | 205 | const totalRespondentsByYear = await getParticipationByYearMap(db, survey) 206 | const numberOfRespondents = totalRespondentsByYear[year] 207 | if (numberOfRespondents === undefined) { 208 | throw new Error(`unable to find number of respondents for year: ${year}`) 209 | } 210 | 211 | const resultsWithPercentage = results.map(result => ({ 212 | cardinality: result.cardinality, 213 | count: result.count, 214 | percentage: result.count / numberOfRespondents * 100 215 | })) 216 | 217 | // console.log(inspect({ numberOfRespondents, pipeline, results, resultsWithPercentage }, { colors: true, depth: null })) 218 | 219 | return resultsWithPercentage 220 | } -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | export default { 2 | mongo: { 3 | normalized_collection: 'normalized_responses', 4 | cache_collection: 'cached_results' 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /src/data/ids.yml: -------------------------------------------------------------------------------- 1 | opinions.css_pain_points: 2 | - browser_interoperability 3 | - interactions 4 | - architecture 5 | - layout_positioning 6 | - scoping_specificity 7 | - responsive_design 8 | - form_elements_styling 9 | - performance_issues 10 | 11 | tools_others.tool_evaluation: 12 | - learning_curve_documentation # is a tool is easy to learn? 13 | - momentum_popularity # does it have a lot of hype? 14 | - user_base_size # is it a well-established option? 15 | - developer_experience_tooling # does it provide a good developer experience? 16 | - performance_user_experience # does it provide a good user experience? 17 | - creator_team # is it backed by a well-known/well-funded team? 18 | - accessibility_features # does it implement accessibility features well? 19 | - community_inclusivity # does it have a good community? 20 | 21 | opinions.currently_missing_from_css: 22 | - container_queries 23 | - parent_selector 24 | - nesting 25 | - color_functions 26 | - subgrid 27 | - browser_support 28 | - mixins 29 | - scoping 30 | -------------------------------------------------------------------------------- /src/data/locales.yml: -------------------------------------------------------------------------------- 1 | - id: ca-ES 2 | label: Català 3 | translators: ['socunanena'] 4 | repo: StateOfJS/locale-ca-ES 5 | 6 | - id: cs-CZ 7 | label: Česky 8 | translators: ['adamkudrna'] 9 | repo: StateOfJS/locale-cs-CZ 10 | 11 | - id: de-DE 12 | label: Deutsch 13 | translators: ['abaldeweg', 'Theiaz'] 14 | repo: StateOfJS/locale-de-DE 15 | 16 | - id: en-US 17 | label: English 18 | translators: [] 19 | repo: StateOfJS/locale-en-US 20 | 21 | - id: es-ES 22 | label: Español 23 | translators: [timbergus, ezakto] 24 | repo: StateOfJS/locale-es-ES 25 | 26 | - id: fa-IR 27 | label: فارسی 28 | translators: ['fghamsary'] 29 | repo: StateOfJS/locale-fa-IR 30 | 31 | - id: fr-FR 32 | label: Français 33 | translators: ['arnauddrain', 'AvocadoVenom'] 34 | repo: StateOfJS/locale-fr-FR 35 | 36 | - id: gl-ES 37 | label: Galego 38 | translators: [nunhes] 39 | repo: StateOfJS/locale-gl-ES 40 | 41 | - id: hi-IN 42 | label: Hindi 43 | translators: ['jaideepghosh'] 44 | repo: StateOfJS/locale-hi-IN 45 | 46 | - id: it-IT 47 | label: Italiano 48 | translators: ['polettoweb'] 49 | repo: StateOfJS/locale-it-IT 50 | 51 | - id: pt-PT 52 | label: Português 53 | translators: ['danisal'] 54 | repo: StateOfJS/locale-pt-PT 55 | 56 | - id: ru-RU 57 | label: Русский 58 | translators: ['lex111', 'Omhet', 'shramkoweb'] 59 | repo: StateOfJS/locale-ru-RU 60 | 61 | - id: ua-UA 62 | label: Українська 63 | translators: ['shramkoweb'] 64 | repo: StateOfJS/locale-ua-UA 65 | 66 | - id: sv-SE 67 | label: Svenska 68 | translators: ['m-hagberg'] 69 | repo: StateOfJS/locale-sv-SE 70 | 71 | - id: tr-TR 72 | label: Türkçe 73 | translators: ['berkayyildiz'] 74 | repo: StateOfJS/locale-tr-TR 75 | 76 | - id: id-ID 77 | label: Indonesia 78 | translators: [ervinismu] 79 | repo: StateOfJS/locale-id-ID 80 | 81 | - id: 'zh-Hans' 82 | label: 简体中文 83 | translators: ['TIOvOIT', 'scarsu', 'shadowings-zy'] 84 | repo: StateOfJS/locale-zh-Hans 85 | 86 | - id: 'zh-Hant' 87 | label: 正體中文 88 | translators: ['ymcheung', 'mingjunlu'] 89 | repo: StateOfJS/locale-zh-Hant 90 | 91 | - id: ja-JP 92 | label: 日本語 93 | translators: ['myakura', 'Spice-Z'] 94 | repo: StateOfJS/locale-ja-JP 95 | 96 | - id: pl-PL 97 | label: Polski 98 | translators: ['luk-str'] 99 | repo: StateOfJS/locale-pl-PL 100 | 101 | - id: ko-KR 102 | label: 한국어 103 | translators: ['eterv'] 104 | repo: StateOfJS/locale-ko-KR 105 | 106 | - id: nl-NL 107 | label: Nederlands 108 | translators: ['MaxAltena'] 109 | repo: StateOfJS/locale-nl-NL 110 | 111 | - id: ro-RO 112 | label: Română 113 | translators: ['magdaavram'] 114 | repo: StateOfJS/locale-ro-RO 115 | -------------------------------------------------------------------------------- /src/debug.ts: -------------------------------------------------------------------------------- 1 | import fs, { promises as fsAsync } from 'fs' 2 | 3 | const logsDirectory = '.logs'; 4 | 5 | export const logToFile = async (fileName: string, object: any, options: any = {}) => { 6 | const { mode = 'append', timestamp = false } = options 7 | // __dirname = /Users/sacha/Dev/state-of-js-graphql-results-api/dist 8 | const path = __dirname.split('/').slice(1, -1).join('/') 9 | const logsDirPath = `/${path}/${logsDirectory}` 10 | if (!fs.existsSync(logsDirPath)) { 11 | fs.mkdirSync(logsDirPath, { recursive: true }) 12 | } 13 | const fullPath = `${logsDirPath}/${fileName}` 14 | const contents = typeof object === 'string' ? object : JSON.stringify(object, null, 2) 15 | const now = new Date() 16 | const text = timestamp ? now.toString() + '\n---\n' + contents : contents 17 | if (mode === 'append') { 18 | const stream = fs.createWriteStream(fullPath, { flags: 'a' }) 19 | stream.write(text + '\n') 20 | stream.end() 21 | } else { 22 | fs.readFile(fullPath, (error, data) => { 23 | let shouldWrite = false 24 | if (error && error.code === 'ENOENT') { 25 | // the file just does not exist, ok to write 26 | shouldWrite = true 27 | } else if (error) { 28 | // maybe EACCESS or something wrong with the disk 29 | throw error 30 | } else { 31 | const fileContent = data.toString() 32 | if (fileContent !== text) { 33 | shouldWrite = true 34 | } 35 | } 36 | 37 | if (shouldWrite) { 38 | fs.writeFile(fullPath, text, error => { 39 | // throws an error, you could also catch it here 40 | if (error) throw error 41 | 42 | // eslint-disable-next-line no-console 43 | console.log(`Log saved to ${fullPath}`) 44 | }) 45 | } 46 | }) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/entities.ts: -------------------------------------------------------------------------------- 1 | import { Entity } from './types' 2 | import { Octokit } from '@octokit/core' 3 | import fetch from 'node-fetch' 4 | import yaml from 'js-yaml' 5 | import { readdir, readFile } from 'fs/promises' 6 | import last from 'lodash/last' 7 | import { logToFile } from './debug' 8 | 9 | let entities: Entity[] = [] 10 | 11 | const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }) 12 | 13 | // load locales if not yet loaded 14 | export const loadOrGetEntities = async () => { 15 | if (entities.length === 0) { 16 | entities = await loadEntities() 17 | } 18 | return entities 19 | } 20 | 21 | export const loadFromGitHub = async () => { 22 | const entities: Entity[] = [] 23 | console.log(`-> loading entities repo`) 24 | 25 | const options = { 26 | owner: 'StateOfJS', 27 | repo: 'entities', 28 | path: '' 29 | } 30 | 31 | const contents = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', options) 32 | const files = contents.data as any[] 33 | 34 | // loop over repo contents and fetch raw yaml files 35 | for (const file of files) { 36 | const extension: string = last(file?.name.split('.')) || '' 37 | if (['yml', 'yaml'].includes(extension)) { 38 | const response = await fetch(file.download_url) 39 | const contents = await response.text() 40 | try { 41 | const yamlContents: any = yaml.load(contents) 42 | const category = file.name.replace('./', '').replace('.yml', '') 43 | yamlContents.forEach((entity: Entity) => { 44 | const tags = entity.tags ? [...entity.tags, category] : [category] 45 | entities.push({ 46 | ...entity, 47 | category, 48 | tags 49 | }) 50 | }) 51 | } catch (error) { 52 | console.log(`// Error loading file ${file.name}`) 53 | console.log(error) 54 | } 55 | } 56 | } 57 | return entities 58 | } 59 | 60 | // when developing locally, load from local files 61 | export const loadLocally = async () => { 62 | console.log(`-> loading entities locally`) 63 | 64 | const entities: Entity[] = [] 65 | 66 | const devDir = __dirname.split('/').slice(1, -2).join('/') 67 | const path = `/${devDir}/stateof-entities/` 68 | const files = await readdir(path) 69 | const yamlFiles = files.filter((f: String) => f.includes('.yml')) 70 | 71 | // loop over dir contents and fetch raw yaml files 72 | for (const fileName of yamlFiles) { 73 | const filePath = path + '/' + fileName 74 | const contents = await readFile(filePath, 'utf8') 75 | const yamlContents: any = yaml.load(contents) 76 | const category = fileName.replace('./', '').replace('.yml', '') 77 | yamlContents.forEach((entity: Entity) => { 78 | const tags = entity.tags ? [...entity.tags, category] : [category] 79 | entities.push({ 80 | ...entity, 81 | category, 82 | tags 83 | }) 84 | }) 85 | } 86 | 87 | return entities 88 | } 89 | 90 | // load locales contents through GitHub API or locally 91 | export const loadEntities = async () => { 92 | console.log('// loading entities') 93 | 94 | const entities: Entity[] = 95 | process.env.LOAD_LOCALES === 'local' ? await loadLocally() : await loadFromGitHub() 96 | console.log('// done loading entities') 97 | 98 | return entities 99 | } 100 | 101 | export const initEntities = async () => { 102 | console.log('// initializing locales…') 103 | const entities = await loadOrGetEntities() 104 | logToFile('entities.json', entities, { mode: 'overwrite' }) 105 | } 106 | 107 | export const getEntities = async ({ type, tag, tags }: { type?: string; tag?: string, tags?: string[] }) => { 108 | let entities = await loadOrGetEntities() 109 | if (type) { 110 | entities = entities.filter(e => e.type === type) 111 | } 112 | if (tag) { 113 | entities = entities.filter(e => e.tags && e.tags.includes(tag)) 114 | } 115 | if (tags) { 116 | entities = entities.filter(e => tags.every(t => e.tags && e.tags.includes(t))) 117 | } 118 | return entities 119 | } 120 | 121 | // Look up entities by id, name, or aliases (case-insensitive) 122 | export const getEntity = async ({ id }: { id: string }) => { 123 | const entities = await loadOrGetEntities() 124 | 125 | if (!id || typeof id !== 'string') { 126 | return 127 | } 128 | 129 | const lowerCaseId = id.toLowerCase() 130 | const entity = entities.find(e => { 131 | return ( 132 | (e.id && e.id.toLowerCase() === lowerCaseId) || 133 | (e.id && e.id.toLowerCase().replace(/\-/g, '_') === lowerCaseId) || 134 | (e.name && e.name.toLowerCase() === lowerCaseId) || 135 | (e.aliases && e.aliases.find((a: string) => a.toLowerCase() === lowerCaseId)) 136 | ) 137 | }) 138 | 139 | return entity || {} 140 | } -------------------------------------------------------------------------------- /src/external_apis/github.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch' 2 | 3 | export const normalizeGithubResource = (res: any) => { 4 | return { 5 | name: res.name, 6 | full_name: res.full_name, 7 | description: res.description, 8 | url: res.html_url, 9 | stars: res.stargazers_count, 10 | forks: res.forks, 11 | opened_issues: res.open_issues_count, 12 | homepage: res.homepage 13 | } 14 | } 15 | 16 | export const fetchGithubResource = async (ownerAndRepo: string) => { 17 | try { 18 | const res = await fetch(`https://api.github.com/repos/${ownerAndRepo}`, { 19 | headers: { Authorization: `token ${process.env.GITHUB_TOKEN}` } 20 | }) 21 | const json = await res.json() 22 | const data = normalizeGithubResource(json) 23 | return data 24 | } catch (error) { 25 | console.error(`an error occurred while fetching github resource`, error) 26 | throw error 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/external_apis/index.ts: -------------------------------------------------------------------------------- 1 | export * from './github' 2 | export * from './mdn' 3 | export * from './twitter' -------------------------------------------------------------------------------- /src/external_apis/mdn.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch' 2 | 3 | interface MDNJSONRes { 4 | doc: TranslatedMDNInfo 5 | } 6 | 7 | interface TranslatedMDNInfo { 8 | locale: string 9 | title: string 10 | summary: string 11 | mdn_url?: string 12 | url?: string 13 | } 14 | 15 | interface MDNInfo extends TranslatedMDNInfo { 16 | translations: TranslatedMDNInfo[] 17 | } 18 | 19 | export const normalizeMdnResource = (res: MDNJSONRes): TranslatedMDNInfo[] => { 20 | const { locale, title, summary, mdn_url } = res.doc 21 | return [ 22 | { 23 | locale, 24 | title, 25 | summary, 26 | url: mdn_url 27 | } 28 | ] 29 | } 30 | 31 | export const fetchMdnResource = async (path: string) => { 32 | try { 33 | const url = `https://developer.mozilla.org${path}/index.json` 34 | const res = await fetch(url) 35 | const json = await res.json() as MDNJSONRes 36 | 37 | return normalizeMdnResource(json) 38 | } catch (error) { 39 | // console.error(`an error occurred while fetching mdn resource`, error) 40 | // throw error 41 | return 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/external_apis/twitter.ts: -------------------------------------------------------------------------------- 1 | import TwitterApi, { ApiResponseError, UserFollowingV2Paginator } from 'twitter-api-v2' 2 | import { Db } from 'mongodb' 3 | 4 | // Instanciate with desired auth type (here's Bearer v2 auth) 5 | const twitterClient = new TwitterApi(process.env.TWITTER_BEARER_TOKEN || '') 6 | 7 | // Tell typescript it's a readonly app 8 | const roClient = twitterClient.readOnly 9 | 10 | // https://github.com/PLhery/node-twitter-api-v2/blob/master/doc/v2.md#users-by-usernames 11 | 12 | // export const fetchTwitterResources = async (ids: string[]) => { 13 | // console.log('// fetchTwitterResource') 14 | // try { 15 | // const data = await roClient.v2.usersByUsernames(ids, { 'user.fields': 'profile_image_url' }) 16 | 17 | // console.log(data) 18 | // // const user = data && data.data 19 | // // const avatarUrl = user?.profile_image_url?.replace('_normal', '') 20 | // return [] 21 | // } catch (error) { 22 | // console.log(error) 23 | // return 24 | // } 25 | // } 26 | 27 | // https://github.com/PLhery/node-twitter-api-v2/blob/master/doc/v2.md#single-user-by-username 28 | 29 | export const fetchTwitterUser = async (db: Db, twitterName: string) => { 30 | const user = await getTwitterUser(twitterName) 31 | if (!user) { 32 | return 33 | } 34 | const avatarUrl = user?.profile_image_url?.replace('_normal', '') 35 | const { id, description, public_metrics = {} } = user 36 | const { followers_count, following_count, tweet_count, listed_count } = public_metrics 37 | const publicMetrics = { 38 | followers: followers_count, 39 | following: following_count, 40 | tweet: tweet_count, 41 | listed: listed_count 42 | } 43 | return { twitterName, avatarUrl, id, description, publicMetrics } 44 | } 45 | 46 | export const getTwitterUser = async (twitterName: string) => { 47 | try { 48 | const data = await roClient.v2.userByUsername(twitterName, { 49 | 'user.fields': ['public_metrics', 'profile_image_url', 'description'] 50 | }) 51 | const user = data && data.data 52 | return user 53 | } catch (error: any) { 54 | console.log('// getTwitterUser error') 55 | // console.log(error) 56 | console.log(error.rateLimit) 57 | const resetTime = new Date(error.rateLimit.reset * 1000) 58 | console.log(resetTime) 59 | console.log(error.data) 60 | return 61 | } 62 | } 63 | 64 | export const getTwitterFollowings = async (twitterId: string) => { 65 | let followings = [] 66 | try { 67 | const result = await roClient.v2.following(twitterId, { 68 | asPaginator: true, 69 | max_results: 1000 70 | }) 71 | // see https://github.com/PLhery/node-twitter-api-v2/blob/master/doc/paginators.md#fetch-until-rate-limit-hits 72 | for await (const following of result) { 73 | followings.push(following) 74 | } 75 | // followings = await result.fetchLast(9999) 76 | // console.log(followings) 77 | } catch (error: any) { 78 | console.log('// getTwitterFollowings error') 79 | if (error instanceof ApiResponseError && error.rateLimitError && error.rateLimit) { 80 | console.log( 81 | `You just hit the rate limit! Limit for this endpoint is ${error.rateLimit.limit} requests!` 82 | ) 83 | console.log(`Request counter will reset at timestamp ${error.rateLimit.reset}.`) 84 | } else { 85 | console.log(error) 86 | console.log(error?.data?.errors) 87 | } 88 | } 89 | // console.log(`// @${twitterName}: fetched ${followings.length} followings`) 90 | const followingsUsernames = followings.map(f => f.username) 91 | return followingsUsernames 92 | } 93 | -------------------------------------------------------------------------------- /src/filters.ts: -------------------------------------------------------------------------------- 1 | export interface Filter { 2 | // must equal value 3 | eq?: T 4 | // must be one of given values 5 | in?: T[] 6 | // must not be one of given values 7 | nin?: T[] 8 | } 9 | 10 | export interface Filters { 11 | gender?: Filter 12 | country?: Filter 13 | race_ethnicity?: Filter 14 | industry_sector?: Filter 15 | yearly_salary?: Filter 16 | company_size?: Filter 17 | years_of_experience?: Filter 18 | source?: Filter 19 | } 20 | 21 | export interface FilterQuery { 22 | // must equal value 23 | $eq?: T 24 | // must be one of given values 25 | $in?: T[] 26 | // must not be one of given values 27 | $nin?: T[] 28 | } 29 | 30 | export interface FiltersQuery { 31 | 'user_info.gender'?: FilterQuery 32 | 'user_info.country_alpha3'?: FilterQuery 33 | 'user_info.race_ethnicity.choices'?: FilterQuery 34 | 'user_info.industry_sector.choices'?: FilterQuery 35 | 'user_info.company_size'?: FilterQuery 36 | 'user_info.yearly_salary'?: FilterQuery 37 | 'user_info.years_of_experience'?: FilterQuery 38 | 'user_info.source.normalized'?: FilterQuery 39 | } 40 | 41 | /** 42 | * Map natural operators (exposed by the API), to MongoDB operators. 43 | */ 44 | const mapFilter = (filter: Filter): FilterQuery => { 45 | const q: FilterQuery = {} 46 | if (filter.eq !== undefined) { 47 | q['$eq'] = filter.eq 48 | } 49 | if (filter.in !== undefined) { 50 | if (!Array.isArray(filter.in)) { 51 | throw new Error(`'in' operator only supports arrays`) 52 | } 53 | q['$in'] = filter.in 54 | } 55 | if (filter.nin !== undefined) { 56 | if (!Array.isArray(filter.nin)) { 57 | throw new Error(`'nin' operator only supports arrays`) 58 | } 59 | q['$nin'] = filter.nin 60 | } 61 | 62 | return q 63 | } 64 | 65 | /** 66 | * Generate a MongoDB $match query from filters object. 67 | */ 68 | export const generateFiltersQuery = (filters?: Filters): FiltersQuery => { 69 | const match: FiltersQuery = {} 70 | if (filters !== undefined) { 71 | if (filters.gender !== undefined) { 72 | match['user_info.gender'] = mapFilter(filters.gender) 73 | } 74 | if (filters.country !== undefined) { 75 | match['user_info.country_alpha3'] = mapFilter(filters.country) 76 | } 77 | if (filters.race_ethnicity !== undefined) { 78 | match['user_info.race_ethnicity.choices'] = mapFilter(filters.race_ethnicity) 79 | } 80 | if (filters.industry_sector !== undefined) { 81 | match['user_info.industry_sector.choices'] = mapFilter(filters.industry_sector) 82 | } 83 | if (filters.company_size !== undefined) { 84 | match['user_info.company_size'] = mapFilter(filters.company_size) 85 | } 86 | if (filters.yearly_salary !== undefined) { 87 | match['user_info.yearly_salary'] = mapFilter(filters.yearly_salary) 88 | } 89 | if (filters.years_of_experience !== undefined) { 90 | match['user_info.years_of_experience'] = mapFilter(filters.years_of_experience) 91 | } 92 | if (filters.source !== undefined) { 93 | match['user_info.source.normalized'] = mapFilter(filters.source) 94 | } 95 | } 96 | 97 | return match 98 | } 99 | -------------------------------------------------------------------------------- /src/helpers.ts: -------------------------------------------------------------------------------- 1 | import { EnumTypeDefinitionNode } from 'graphql' 2 | import typeDefs from './type_defs/schema.graphql' 3 | // import allEntities from './data/entities/index' 4 | import { RequestContext, ResolverDynamicConfig, SurveyConfig } from './types' 5 | import { 6 | computeTermAggregationAllYearsWithCache, 7 | computeTermAggregationSingleYearWithCache 8 | } from './compute' 9 | import { Filters } from './filters' 10 | import { loadOrGetEntities } from './entities' 11 | import { TermAggregationOptions, AggregationFunction } from './compute/generic' 12 | 13 | /** 14 | * Return either e.g. other_tools.browsers.choices or other_tools.browsers.others_normalized 15 | */ 16 | export const getOtherKey = (id: string) => 17 | id.includes('_others') ? `${id.replace('_others', '')}.others.normalized` : `${id}.choices` 18 | 19 | export const getGraphQLEnumValues = (name: string): string[] => { 20 | const enumDef = typeDefs.definitions.find(def => { 21 | return def.kind === 'EnumTypeDefinition' && def.name.value === name 22 | }) as EnumTypeDefinitionNode 23 | 24 | if (enumDef === undefined) { 25 | throw new Error(`No enum found matching name: ${name}`) 26 | } 27 | 28 | return enumDef.values!.map(v => v.name.value) 29 | } 30 | 31 | /** 32 | * Get resolvers when the db key is the same as the field id 33 | * 34 | * @param id the field's GraphQL id 35 | * @param options options 36 | */ 37 | export const getStaticResolvers = (id: string, options: TermAggregationOptions = {}, aggregationFunction?: AggregationFunction) => ({ 38 | all_years: async ( 39 | { survey, filters }: ResolverDynamicConfig, 40 | args: any, 41 | { db }: RequestContext 42 | ) => computeTermAggregationAllYearsWithCache(db, survey, id, { ...options, filters }, aggregationFunction), 43 | year: async ( 44 | { survey, filters }: ResolverDynamicConfig, 45 | { year }: { year: number }, 46 | { db }: RequestContext 47 | ) => computeTermAggregationSingleYearWithCache(db, survey, id, { ...options, filters, year }, aggregationFunction) 48 | }) 49 | 50 | /** 51 | * Get resolvers when the db key is *not* the same as the field id 52 | * 53 | * @param getId a function that takes the field's GraphQL id and returns the db key 54 | * @param options options 55 | */ 56 | export const getDynamicResolvers = ( 57 | getId: (id: string) => string, 58 | options: TermAggregationOptions = {}, 59 | aggregationFunction?: AggregationFunction 60 | ) => ({ 61 | all_years: async ( 62 | { survey, id, filters }: ResolverDynamicConfig, 63 | args: any, 64 | { db }: RequestContext 65 | ) => 66 | computeTermAggregationAllYearsWithCache( 67 | db, 68 | survey, 69 | getId(id), 70 | { ...options, filters }, 71 | aggregationFunction 72 | ), 73 | year: async ( 74 | { survey, id, filters }: ResolverDynamicConfig, 75 | { year }: { year: number }, 76 | { db }: RequestContext 77 | ) => 78 | computeTermAggregationSingleYearWithCache( 79 | db, 80 | survey, 81 | getId(id), 82 | { 83 | ...options, 84 | filters, 85 | year 86 | }, 87 | aggregationFunction 88 | ) 89 | }) 90 | 91 | const demographicsFields = [ 92 | 'age', 93 | 'country', 94 | 'locale', 95 | 'source', 96 | 'gender', 97 | 'race_ethnicity', 98 | 'yearly_salary', 99 | 'company_size', 100 | 'years_of_experience', 101 | 'job_title', 102 | 'industry_sector', 103 | 'industry_sector_others', 104 | 'knowledge_score', 105 | 'higher_education_degree', 106 | 'disability_status', 107 | 'disability_status_other' 108 | ] 109 | 110 | /** 111 | * Generic resolvers for passing down arguments for demographic fields 112 | * 113 | * @param survey current survey 114 | */ 115 | export const getDemographicsResolvers = (survey: SurveyConfig) => { 116 | const resolvers: any = {} 117 | demographicsFields.forEach(field => { 118 | resolvers[field] = ({ filters }: { filters: Filters }) => ({ 119 | survey, 120 | filters 121 | }) 122 | }) 123 | return resolvers 124 | } 125 | -------------------------------------------------------------------------------- /src/i18n.ts: -------------------------------------------------------------------------------- 1 | import { EnumTypeDefinitionNode } from 'graphql' 2 | import { Entity, StringFile, Locale, TranslationStringObject } from './types' 3 | import typeDefs from './type_defs/schema.graphql' 4 | import { Octokit } from '@octokit/core' 5 | import fetch from 'node-fetch' 6 | import localesYAML from './data/locales.yml' 7 | import yaml from 'js-yaml' 8 | import marked from 'marked' 9 | import { logToFile } from './debug' 10 | import { readdir, readFile } from 'fs/promises' 11 | import last from 'lodash/last' 12 | import { loadOrGetEntities } from './entities' 13 | 14 | let locales: Locale[] = [] 15 | 16 | const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }) 17 | 18 | // load locales if not yet loaded 19 | export const loadOrGetLocales = async () => { 20 | if (locales.length === 0) { 21 | locales = await loadLocales() 22 | } 23 | return locales 24 | } 25 | 26 | export const loadFromGitHub = async (localesWithRepos: any) => { 27 | let locales: Locale[] = [] 28 | let i = 0 29 | 30 | for (const locale of localesWithRepos) { 31 | i++ 32 | console.log(`-> loading repo ${locale.repo} (${i}/${localesWithRepos.length})`) 33 | 34 | locale.stringFiles = [] 35 | 36 | const [owner, repo] = locale.repo.split('/') 37 | const options = { 38 | owner, 39 | repo, 40 | path: '' 41 | } 42 | 43 | const contents = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', options) 44 | const files = contents.data as any[] 45 | 46 | // loop over repo contents and fetch raw yaml files 47 | for (const file of files) { 48 | const extension: string = last(file?.name.split('.')) || '' 49 | if (['yml', 'yaml'].includes(extension)) { 50 | const response = await fetch(file.download_url) 51 | const contents = await response.text() 52 | try { 53 | const yamlContents: any = yaml.load(contents) 54 | const strings = yamlContents.translations 55 | const context = file.name.replace('./', '').replace('.yml', '') 56 | locale.stringFiles.push({ 57 | strings, 58 | url: file.download_url, 59 | context 60 | }) 61 | } catch (error) { 62 | console.log(`// Error loading file ${file.name}`) 63 | console.log(error) 64 | } 65 | } 66 | } 67 | locales.push(locale) 68 | } 69 | return locales 70 | } 71 | 72 | // when developing locally, load from local files 73 | export const loadLocally = async (localesWithRepos: any) => { 74 | let i = 0 75 | 76 | for (const locale of localesWithRepos) { 77 | i++ 78 | console.log(`-> loading directory ${locale.repo} locally (${i}/${localesWithRepos.length})`) 79 | 80 | locale.stringFiles = [] 81 | 82 | const [owner, repo] = locale.repo.split('/') 83 | 84 | // __dirname = /Users/sacha/Dev/state-of-js-graphql-results-api/dist 85 | const devDir = __dirname.split('/').slice(1, -2).join('/') 86 | const path = `/${devDir}/stateof-locales/${repo}` 87 | const files = await readdir(path) 88 | const yamlFiles = files.filter((f: String) => f.includes('.yml')) 89 | 90 | // loop over repo contents and fetch raw yaml files 91 | for (const fileName of yamlFiles) { 92 | const filePath = path + '/' + fileName 93 | const contents = await readFile(filePath, 'utf8') 94 | const yamlContents: any = yaml.load(contents) 95 | const strings = yamlContents.translations 96 | const context = fileName.replace('./', '').replace('.yml', '') 97 | locale.stringFiles.push({ 98 | strings, 99 | url: filePath, 100 | context 101 | }) 102 | } 103 | locales.push(locale) 104 | } 105 | return locales 106 | } 107 | 108 | // load locales contents through GitHub API or locally 109 | export const loadLocales = async () => { 110 | console.log('// loading locales…') 111 | // only keep locales which have a repo defined 112 | const localesWithRepos = localesYAML.filter((locale: Locale) => !!locale.repo) 113 | 114 | const locales: Locale[] = 115 | process.env.LOAD_LOCALES === 'local' 116 | ? await loadLocally(localesWithRepos) 117 | : await loadFromGitHub(localesWithRepos) 118 | console.log('// done loading locales') 119 | 120 | return locales 121 | } 122 | 123 | /** 124 | * Return either e.g. other_tools.browsers.choices or other_tools.browsers.others_normalized 125 | */ 126 | export const getOtherKey = (id: string) => 127 | id.includes('_others') ? `${id.replace('_others', '')}.others_normalized` : `${id}.choices` 128 | 129 | export const getGraphQLEnumValues = (name: string): string[] => { 130 | const enumDef = typeDefs.definitions.find(def => { 131 | return def.kind === 'EnumTypeDefinition' && def.name.value === name 132 | }) as EnumTypeDefinitionNode 133 | 134 | if (enumDef === undefined) { 135 | throw new Error(`No enum found matching name: ${name}`) 136 | } 137 | 138 | return enumDef.values!.map(v => v.name.value) 139 | } 140 | 141 | /* 142 | 143 | For a given locale id, get closest existing key. 144 | 145 | Ex: 146 | 147 | en-US -> en-US 148 | en-us -> en-US 149 | en-gb -> en-US 150 | etc. 151 | 152 | */ 153 | export const truncateKey = (key: string) => key.split('-')[0] 154 | 155 | export const getValidLocale = async (localeId: string) => { 156 | const locales = await loadOrGetLocales() 157 | const exactLocale = locales.find( 158 | (locale: Locale) => locale.id.toLowerCase() === localeId.toLowerCase() 159 | ) 160 | const similarLocale = locales.find( 161 | (locale: Locale) => truncateKey(locale.id) === truncateKey(localeId) 162 | ) 163 | return exactLocale || similarLocale 164 | } 165 | 166 | /* 167 | 168 | Get locale strings for a specific locale 169 | 170 | */ 171 | export const getLocaleStrings = (locale: Locale, contexts?: string[]) => { 172 | let stringFiles = locale.stringFiles 173 | 174 | // if contexts are specified, filter strings by them 175 | if (contexts) { 176 | stringFiles = stringFiles.filter((sf: StringFile) => { 177 | return contexts.includes(sf.context) 178 | }) 179 | } 180 | 181 | // flatten all stringFiles together 182 | const strings = stringFiles 183 | .map((sf: StringFile) => { 184 | let { strings, prefix, context } = sf 185 | if (strings === null) { 186 | return [] 187 | } 188 | 189 | // if strings need to be prefixed, do it now 190 | if (prefix) { 191 | strings = strings.map((s: TranslationStringObject) => ({ 192 | ...s, 193 | key: `${prefix}.${s.key}` 194 | })) 195 | } 196 | // add context to all strings just in case 197 | strings = strings.map((s: TranslationStringObject) => ({ ...s, context })) 198 | // add HTML version in case string is markdown 199 | strings = strings.map((s: TranslationStringObject) => ({ 200 | ...s, 201 | tHtml: marked(String(s.t)) 202 | })) 203 | return strings 204 | }) 205 | .flat() 206 | 207 | return { strings } 208 | } 209 | 210 | /* 211 | 212 | Get locale strings with en-US strings as fallback 213 | 214 | */ 215 | export const getLocaleStringsWithFallback = async (locale: Locale, contexts?: string[]) => { 216 | let localeStrings: TranslationStringObject[] = [], 217 | translatedCount: number = 0, 218 | totalCount: number = 0, 219 | untranslatedKeys: string[] = [] 220 | 221 | const enLocale = await getValidLocale('en-US') 222 | if (enLocale) { 223 | const enStrings = getLocaleStrings(enLocale, contexts).strings 224 | 225 | // handle en-US locale separetely first 226 | if (locale.id === 'en-US') { 227 | return { 228 | strings: enStrings.map((t: TranslationStringObject) => ({ ...t, fallback: false })), 229 | translatedCount: enStrings.length, 230 | totalCount: enStrings.length, 231 | completion: 100, 232 | untranslatedKeys 233 | } 234 | } 235 | 236 | localeStrings = getLocaleStrings(locale, contexts).strings 237 | 238 | enStrings.forEach((enTranslation: TranslationStringObject) => { 239 | totalCount++ 240 | // note: exclude fallback strings that might have been added during 241 | // a previous iteration of the current loop 242 | const localeTranslationIndex = localeStrings.findIndex( 243 | t => t.key === enTranslation.key && !t.fallback 244 | ) 245 | 246 | if ( 247 | localeTranslationIndex === -1 || 248 | localeStrings[localeTranslationIndex].t === enTranslation.t || 249 | (localeStrings[localeTranslationIndex].t && 250 | localeStrings[localeTranslationIndex].t.trim() === 'TODO') 251 | ) { 252 | // en-US key doesn't exist in current locale file 253 | // OR current locale file's translation is same as en-US (untranslated) 254 | // OR is "TODO" 255 | localeStrings.push({ 256 | ...enTranslation, 257 | fallback: true 258 | }) 259 | untranslatedKeys.push(enTranslation.key) 260 | } else { 261 | // current locale has key, no fallback needed 262 | translatedCount++ 263 | localeStrings[localeTranslationIndex].fallback = false 264 | } 265 | }) 266 | } 267 | return { 268 | strings: localeStrings, 269 | translatedCount, 270 | totalCount, 271 | completion: Math.round((translatedCount * 100) / totalCount), 272 | untranslatedKeys 273 | } 274 | } 275 | 276 | /* 277 | 278 | Get a specific locale object with properly parsed strings 279 | 280 | */ 281 | export const getLocaleObject = async ( 282 | localeId: string, 283 | contexts?: string[], 284 | enableFallbacks: boolean = true 285 | ) => { 286 | const validLocale = await getValidLocale(localeId) 287 | if (!validLocale) { 288 | throw new Error(`No locale found for key ${localeId}`) 289 | } 290 | const localeData = enableFallbacks 291 | ? await getLocaleStringsWithFallback(validLocale, contexts) 292 | : await getLocaleStrings(validLocale, contexts) 293 | 294 | return { 295 | ...validLocale, 296 | ...localeData 297 | } 298 | } 299 | 300 | /* 301 | 302 | Get all locales 303 | 304 | */ 305 | export const getLocales = async (contexts?: string[], enableFallbacks?: boolean) => { 306 | const rawLocales = await loadOrGetLocales() 307 | const locales = [] 308 | for (const locale of rawLocales) { 309 | const localeObject = await getLocaleObject(locale.id, contexts, enableFallbacks) 310 | locales.push(localeObject) 311 | } 312 | return locales 313 | } 314 | 315 | /* 316 | 317 | Get a specific translation 318 | 319 | Reverse array first so that strings added last take priority 320 | 321 | */ 322 | export const getTranslation = async (key: string, localeId: string) => { 323 | const locale = await getLocaleObject(localeId) 324 | return locale.strings.reverse().find((s: any) => s.key === key) 325 | } 326 | 327 | export const initLocales = async () => { 328 | console.log('// initializing locales…') 329 | const rawLocales = await loadOrGetLocales() 330 | logToFile('raw_locales.json', rawLocales, { mode: 'overwrite' }) 331 | const parsedLocales = await getLocales() 332 | logToFile('parsed_locales.json', parsedLocales, { mode: 'overwrite' }) 333 | } 334 | -------------------------------------------------------------------------------- /src/resolvers/brackets.ts: -------------------------------------------------------------------------------- 1 | import { getDynamicResolvers } from '../helpers' 2 | import { winsAggregationFunction, matchupsAggregationFunction } from '../compute/brackets' 3 | 4 | export default { 5 | BracketWins: getDynamicResolvers( 6 | id => { 7 | const fullPath = id.replace('__', '.') 8 | return fullPath 9 | }, 10 | {}, 11 | winsAggregationFunction 12 | ), 13 | 14 | BracketMatchups: getDynamicResolvers( 15 | id => { 16 | const fullPath = id.replace('__', '.') 17 | return fullPath 18 | }, 19 | {}, 20 | matchupsAggregationFunction 21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /src/resolvers/categories.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { computeHappinessByYear, computeTermAggregationByYear } from '../compute' 3 | import { useCache } from '../caching' 4 | import { RequestContext, SurveyConfig } from '../types' 5 | import { Filters } from '../filters' 6 | import { YearAggregations } from '../compute/generic' 7 | 8 | interface CategoryConfig { 9 | survey: SurveyConfig 10 | id: string 11 | filters?: Filters 12 | } 13 | 14 | const computeOtherTools = async (db: Db, survey: SurveyConfig, id: string, filters?: Filters) => 15 | useCache(computeTermAggregationByYear, db, [ 16 | survey, 17 | `tools_others.${id}.others.normalized`, 18 | { filters } 19 | ]) 20 | 21 | export default { 22 | CategoryOtherTools: { 23 | all_years: async ( 24 | { survey, id, filters }: CategoryConfig, 25 | args: any, 26 | { db }: RequestContext 27 | ) => computeOtherTools(db, survey, id, filters), 28 | year: async ( 29 | { survey, id, filters }: CategoryConfig, 30 | { year }: { year: number }, 31 | { db }: RequestContext 32 | ) => { 33 | const allYears = await computeOtherTools(db, survey, id, filters) 34 | return allYears.find((yearItem: YearAggregations) => yearItem.year === year) 35 | } 36 | }, 37 | CategoryHappiness: { 38 | all_years: async ( 39 | { survey, id, filters }: CategoryConfig, 40 | args: any, 41 | { db }: RequestContext 42 | ) => useCache(computeHappinessByYear, db, [survey, id, filters]), 43 | year: async ( 44 | { survey, id, filters }: CategoryConfig, 45 | { year }: { year: number }, 46 | { db }: RequestContext 47 | ) => { 48 | const allYears = await useCache(computeHappinessByYear, db, [survey, id, filters]) 49 | return allYears.find((yearItem: YearAggregations) => yearItem.year === year) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/resolvers/demographics.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { RequestContext, SurveyConfig, ResolverStaticConfig } from '../types' 3 | import { Filters } from '../filters' 4 | import { useCache } from '../caching' 5 | import { computeParticipationByYear } from '../compute' 6 | import { getStaticResolvers } from '../helpers' 7 | 8 | const computeParticipation = async ( 9 | db: Db, 10 | survey: SurveyConfig, 11 | filters?: Filters, 12 | year?: number 13 | ) => useCache(computeParticipationByYear, db, [survey, filters, year]) 14 | 15 | export default { 16 | Participation: { 17 | all_years: async ( 18 | { survey, filters }: ResolverStaticConfig, 19 | args: any, 20 | { db }: RequestContext 21 | ) => computeParticipation(db, survey, filters), 22 | year: async ( 23 | { survey, filters }: ResolverStaticConfig, 24 | { year }: { year: number }, 25 | { db }: RequestContext 26 | ) => { 27 | const allYears = await computeParticipation(db, survey, filters) 28 | return allYears.find(y => y.year === year) 29 | } 30 | }, 31 | 32 | Country: getStaticResolvers('user_info.country_alpha3', { 33 | sort: 'id', 34 | limit: 999, 35 | cutoff: 1 36 | }), 37 | 38 | LocaleStats: getStaticResolvers('user_info.locale', { 39 | sort: 'id', 40 | limit: 100, 41 | cutoff: 1 42 | }), 43 | 44 | Source: getStaticResolvers('user_info.source.normalized'), 45 | 46 | Gender: getStaticResolvers('user_info.gender', { cutoff: 1 }), 47 | 48 | RaceEthnicity: getStaticResolvers('user_info.race_ethnicity.choices', { cutoff: 1 }), 49 | 50 | Age: getStaticResolvers('user_info.age', { limit: 100, cutoff: 1 }), 51 | 52 | Salary: getStaticResolvers('user_info.yearly_salary', { limit: 100, cutoff: 1 }), 53 | 54 | CompanySize: getStaticResolvers('user_info.company_size', { limit: 100, cutoff: 1 }), 55 | 56 | WorkExperience: getStaticResolvers('user_info.years_of_experience', { 57 | limit: 100, 58 | cutoff: 1 59 | }), 60 | 61 | JobTitle: getStaticResolvers('user_info.job_title', { 62 | cutoff: 1 63 | }), 64 | 65 | IndustrySector: getStaticResolvers('user_info.industry_sector.choices', { 66 | cutoff: 1 67 | }), 68 | 69 | KnowledgeScore: getStaticResolvers('user_info.knowledge_score', { limit: 100, cutoff: 1 }), 70 | 71 | HigherEducationDegree: getStaticResolvers('user_info.higher_education_degree', { 72 | cutoff: 1 73 | }), 74 | 75 | DisabilityStatus: getStaticResolvers('user_info.disability_status.choices', { 76 | cutoff: 1 77 | }), 78 | 79 | OtherDisabilityStatus: getStaticResolvers('user_info.disability_status.others.normalized', { 80 | cutoff: 1 81 | }), 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/resolvers/entities.ts: -------------------------------------------------------------------------------- 1 | import { GitHub, SurveyConfig, Entity, RequestContext } from '../types' 2 | import projects from '../data/bestofjs.yml' 3 | import { fetchMdnResource, fetchTwitterUser } from '../external_apis' 4 | import { useCache } from '../caching' 5 | 6 | const getSimulatedGithub = (id: string): GitHub | null => { 7 | const project = projects.find((p: Entity) => p.id === id) 8 | 9 | if (project !== undefined) { 10 | const { name, description, github, stars, homepage } = project 11 | 12 | return { 13 | id, 14 | name, 15 | description, 16 | url: `https://github.com/${github}`, 17 | stars, 18 | homepage 19 | } 20 | } else { 21 | return null 22 | } 23 | } 24 | 25 | export default { 26 | Entity: { 27 | github: async ({ id }: { survey: SurveyConfig; id: string }) => { 28 | // note: for now just get local data from projects.yml 29 | // instead of actually querying github 30 | return getSimulatedGithub(id) 31 | // const projectObject = projects.find(p => p.id === entity.id) 32 | // return { 33 | // ...projectObject 34 | // } 35 | // if (!projectObject || !projectObject.github) { 36 | // return 37 | // } 38 | // const github = await fetchGithubResource(projectObject.github) 39 | // return github 40 | }, 41 | mdn: async (entity: Entity) => { 42 | if (!entity || !entity.mdn) { 43 | return 44 | } 45 | 46 | const mdn = await fetchMdnResource(entity.mdn) 47 | 48 | if (mdn) { 49 | return mdn.find((t: any) => t.locale === 'en-US') 50 | } else { 51 | return 52 | } 53 | }, 54 | twitter: async (entity: Entity, args: any, { db }: RequestContext) => { 55 | const twitter = entity.twitterName && useCache(fetchTwitterUser, db, [entity.twitterName]) 56 | 57 | // const twitter = await fetchTwitterResource(entity.id) 58 | return twitter 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/resolvers/environments.ts: -------------------------------------------------------------------------------- 1 | import { getOtherKey } from '../helpers' 2 | import { getDynamicResolvers } from '../helpers' 3 | 4 | export default { 5 | // minimal cutoff because we don't have many 6 | // freeform extra choices for now (others). 7 | // it's used more to check the volume of data 8 | // rather than actually using it. 9 | Environments: getDynamicResolvers(id => `environments.${getOtherKey(id)}`, { 10 | cutoff: 0 11 | }), 12 | 13 | EnvironmentsRatings: getDynamicResolvers(id => `environments.${id}`, { 14 | cutoff: 0 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /src/resolvers/features.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { useCache } from '../caching' 3 | import { fetchMdnResource } from '../external_apis' 4 | import { RequestContext, SurveyConfig } from '../types' 5 | import { computeTermAggregationByYear } from '../compute' 6 | import { Filters } from '../filters' 7 | import { Entity } from '../types' 8 | import { getEntities } from '../entities' 9 | import { YearAggregations } from '../compute/generic' 10 | 11 | const computeFeatureExperience = async ( 12 | db: Db, 13 | survey: SurveyConfig, 14 | id: string, 15 | filters?: Filters 16 | ) => useCache(computeTermAggregationByYear, db, [survey, `features.${id}.experience`, { filters }]) 17 | 18 | export default { 19 | FeatureExperience: { 20 | all_years: async ( 21 | { survey, id, filters }: { survey: SurveyConfig; id: string; filters?: Filters }, 22 | args: any, 23 | { db }: RequestContext 24 | ) => computeFeatureExperience(db, survey, id, filters), 25 | year: async ( 26 | { survey, id, filters }: { survey: SurveyConfig; id: string; filters?: Filters }, 27 | { year }: { year: number }, 28 | { db }: RequestContext 29 | ) => { 30 | const allYears = await computeFeatureExperience(db, survey, id, filters) 31 | return allYears.find((yearItem: YearAggregations) => yearItem.year === year) 32 | } 33 | }, 34 | Feature: { 35 | name: async ({ id }: { id: string }) => { 36 | const features = await getEntities({ tag: 'feature' }) 37 | const feature = features.find((f: Entity) => f.id === id) 38 | 39 | return feature && feature.name 40 | }, 41 | mdn: async ({ id }: { id: string }) => { 42 | const features = await getEntities({ tag: 'feature' }) 43 | const feature = features.find((f: Entity) => f.id === id) 44 | if (!feature || !feature.mdn) { 45 | return 46 | } 47 | 48 | const mdn = await fetchMdnResource(feature.mdn) 49 | 50 | if (mdn) { 51 | return mdn.find(t => t.locale === 'en-US') 52 | } else { 53 | return 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/resolvers/features_others.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { useCache } from '../caching' 3 | import { computeTermAggregationByYear } from '../compute' 4 | import { getOtherKey } from '../helpers' 5 | import { RequestContext, SurveyConfig } from '../types' 6 | import { Filters } from '../filters' 7 | import { Entity } from '../types' 8 | import { getEntities } from '../entities' 9 | import { YearAggregations } from '../compute/generic' 10 | 11 | interface OtherFeaturesConfig { 12 | survey: SurveyConfig 13 | id: string 14 | filters?: Filters 15 | } 16 | 17 | const computeOtherFeatures = async ( 18 | db: Db, 19 | survey: SurveyConfig, 20 | id: string, 21 | filters?: Filters 22 | ) => { 23 | const features = await getEntities({ tag: 'feature'}) 24 | 25 | const otherFeaturesByYear = await useCache(computeTermAggregationByYear, db, [ 26 | survey, 27 | `features_others.${getOtherKey(id)}`, 28 | { filters } 29 | ]) 30 | 31 | return otherFeaturesByYear.map((yearOtherFeatures: YearAggregations) => { 32 | return { 33 | ...yearOtherFeatures, 34 | buckets: yearOtherFeatures.buckets.map(bucket => { 35 | const feature = features.find((f: Entity) => f.id === bucket.id) 36 | 37 | return { 38 | ...bucket, 39 | name: feature ? feature.name : bucket.id 40 | } 41 | }) 42 | } 43 | }) 44 | } 45 | 46 | export default { 47 | OtherFeatures: { 48 | all_years: async ( 49 | { survey, id, filters }: OtherFeaturesConfig, 50 | args: any, 51 | { db }: RequestContext 52 | ) => computeOtherFeatures(db, survey, id, filters), 53 | year: async ( 54 | { survey, id, filters }: OtherFeaturesConfig, 55 | { year }: { year: number }, 56 | { db }: RequestContext 57 | ) => { 58 | const allYears = await computeOtherFeatures(db, survey, id, filters) 59 | return allYears.find((yearItem: YearAggregations) => yearItem.year === year) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/resolvers/happiness.ts: -------------------------------------------------------------------------------- 1 | import { computeHappinessByYear } from '../compute' 2 | import { useCache } from '../caching' 3 | import { RequestContext, SurveyConfig } from '../types' 4 | import { Filters } from '../filters' 5 | import { YearAggregations } from '../compute/generic' 6 | 7 | interface HappinessConfig { 8 | survey: SurveyConfig 9 | id: string 10 | filters?: Filters 11 | } 12 | 13 | export default { 14 | Happiness: { 15 | all_years: async ( 16 | { survey, id, filters }: HappinessConfig, 17 | args: any, 18 | { db }: RequestContext 19 | ) => useCache(computeHappinessByYear, db, [survey, id, filters]), 20 | year: async ( 21 | { survey, id, filters }: HappinessConfig, 22 | { year }: { year: number }, 23 | { db }: RequestContext 24 | ) => { 25 | const allYears = await useCache(computeHappinessByYear, db, [survey, id, filters]) 26 | return allYears.find((yearItem: YearAggregations) => yearItem.year === year) 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/resolvers/index.ts: -------------------------------------------------------------------------------- 1 | import categories from './categories' 2 | import demographics from './demographics' 3 | import entities from './entities' 4 | import environments from './environments' 5 | import features from './features' 6 | import otherFeatures from './features_others' 7 | import happiness from './happiness' 8 | import matrices from './matrices' 9 | import opinions from './opinions' 10 | import proficiency from './proficiency' 11 | import query from './query' 12 | import resources from './resources' 13 | import surveys from './surveys' 14 | import totals from './totals' 15 | import tools from './tools' 16 | import otherTools from './tools_others' 17 | import brackets from './brackets' 18 | 19 | export default { 20 | ...surveys, 21 | ...totals, 22 | ...demographics, 23 | ...categories, 24 | ...opinions, 25 | ...features, 26 | ...matrices, 27 | ...tools, 28 | ...otherFeatures, 29 | ...otherTools, 30 | ...resources, 31 | ...entities, 32 | ...environments, 33 | ...proficiency, 34 | ...happiness, 35 | ...query, 36 | ...brackets, 37 | } 38 | -------------------------------------------------------------------------------- /src/resolvers/matrices.ts: -------------------------------------------------------------------------------- 1 | import { useCache } from '../caching' 2 | import { computeToolsMatrix, ToolExperienceFilterId } from '../compute' 3 | import { SurveyConfig, RequestContext } from '../types' 4 | 5 | export default { 6 | Matrices: { 7 | tools: async ( 8 | { survey }: { survey: SurveyConfig }, 9 | { 10 | year, 11 | ids, 12 | experiences, 13 | dimensions, 14 | }: { 15 | year: number 16 | ids: string[] 17 | experiences: ToolExperienceFilterId[] 18 | dimensions: string[] 19 | }, 20 | { db }: RequestContext 21 | ) => { 22 | const result = [] 23 | for (const experience of experiences) { 24 | const by_dimension = [] 25 | for (const dimension of dimensions) { 26 | const tools = await useCache(computeToolsMatrix, db, [ 27 | { 28 | survey, 29 | tools: ids, 30 | experience, 31 | type: dimension, 32 | year, 33 | } 34 | ]) 35 | 36 | by_dimension.push({ 37 | dimension, 38 | tools, 39 | }) 40 | } 41 | 42 | result.push({ 43 | experience, 44 | dimensions: by_dimension, 45 | }) 46 | } 47 | 48 | return result 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/resolvers/opinions.ts: -------------------------------------------------------------------------------- 1 | import { getDynamicResolvers } from '../helpers' 2 | 3 | export default { 4 | Opinion: getDynamicResolvers(id => `opinions.${id}`, { sort: 'id', order: 1 }), 5 | OtherOpinions: getDynamicResolvers(id => `opinions_others.${id}.others.normalized`, { 6 | sort: 'id', 7 | order: 1 8 | }) 9 | } 10 | -------------------------------------------------------------------------------- /src/resolvers/proficiency.ts: -------------------------------------------------------------------------------- 1 | import { getDynamicResolvers } from '../helpers' 2 | 3 | export default { 4 | Proficiency: getDynamicResolvers(id => `user_info.${id}`) 5 | } 6 | -------------------------------------------------------------------------------- /src/resolvers/query.ts: -------------------------------------------------------------------------------- 1 | import { SurveyType } from '../types' 2 | import { getEntities, getEntity } from '../entities' 3 | import { getLocales, getLocaleObject, getTranslation } from '../i18n' 4 | import { SurveyConfig } from '../types' 5 | 6 | export default { 7 | Query: { 8 | survey: (parent: any, { survey }: { survey: SurveyType }) => ({ 9 | survey 10 | }), 11 | entity: async (survey: SurveyConfig, { id }: { id: string }) => ({ 12 | survey, 13 | ...(await getEntity({ id })) 14 | }), 15 | entities: ( 16 | parent: any, 17 | { type, tag, tags }: { type: string; tag: string, tags: string[] } 18 | ) => getEntities({ type, tag, tags }), 19 | translation: (parent: any, { key, localeId }: { key: string; localeId: string }) => 20 | getTranslation(key, localeId), 21 | locale: (parent: any, { localeId, contexts, enableFallbacks }: { localeId: string; contexts: string[], enableFallbacks?: boolean }) => 22 | getLocaleObject(localeId, contexts, enableFallbacks), 23 | locales: (parent: any, { contexts, enableFallbacks }: { contexts: string[], enableFallbacks?: boolean }) => getLocales(contexts, enableFallbacks) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/resolvers/resources.ts: -------------------------------------------------------------------------------- 1 | import { getDynamicResolvers, getOtherKey } from '../helpers' 2 | 3 | export default { 4 | Resources: getDynamicResolvers(id => `resources.${getOtherKey(id)}`) 5 | } -------------------------------------------------------------------------------- /src/resolvers/surveys.ts: -------------------------------------------------------------------------------- 1 | import { getGraphQLEnumValues, getDemographicsResolvers } from '../helpers' 2 | import { getEntity } from '../entities' 3 | import { RequestContext, SurveyConfig } from '../types' 4 | import { Filters } from '../filters' 5 | import { 6 | computeToolExperienceGraph, 7 | computeToolsCardinalityByUser, 8 | ToolExperienceId 9 | } from '../compute' 10 | import { useCache } from '../caching' 11 | 12 | const toolIds = getGraphQLEnumValues('ToolID') 13 | const featureIds = getGraphQLEnumValues('FeatureID') 14 | 15 | /** 16 | * Please maintain the same order as the one shown in the explorer, 17 | * it makes it easier to find a specific query and ensures consistency. 18 | */ 19 | export default { 20 | Survey: { 21 | surveyName: (survey: SurveyConfig) => { 22 | return survey.survey 23 | }, 24 | bracketWins: ( 25 | survey: SurveyConfig, 26 | { id, filters }: { id: string; filters?: Filters } 27 | ) => ({ 28 | survey, 29 | id, 30 | filters 31 | }), 32 | bracketMatchups: ( 33 | survey: SurveyConfig, 34 | { id, filters }: { id: string; filters?: Filters } 35 | ) => ({ 36 | survey, 37 | id, 38 | filters 39 | }), 40 | category: (survey: SurveyConfig, { id }: { id: string }) => ({ 41 | survey, 42 | id, 43 | happiness: ({ filters }: { filters: Filters }) => ({ 44 | survey, 45 | id, 46 | filters 47 | }), 48 | otherTools: ({ filters }: { filters: Filters }) => ({ 49 | survey, 50 | id, 51 | filters 52 | }) 53 | }), 54 | demographics: (survey: SurveyConfig) => ({ 55 | participation: { survey }, 56 | ...getDemographicsResolvers(survey) 57 | }), 58 | environments: ( 59 | survey: SurveyConfig, 60 | { id, filters }: { id: string; filters?: Filters } 61 | ) => ({ 62 | survey, 63 | id, 64 | filters 65 | }), 66 | environments_ratings: ( 67 | survey: SurveyConfig, 68 | { id, filters }: { id: string; filters?: Filters } 69 | ) => ({ 70 | survey, 71 | id, 72 | filters 73 | }), 74 | feature: (survey: SurveyConfig, { id }: { id: string }) => ({ 75 | survey, 76 | id, 77 | experience: ({ filters }: { filters?: Filters }) => ({ 78 | survey, 79 | id, 80 | filters 81 | }) 82 | }), 83 | features: (survey: SurveyConfig, { ids = featureIds }: { ids: string[] }) => 84 | ids.map(id => ({ 85 | survey, 86 | id, 87 | experience: ({ filters }: { filters?: Filters }) => ({ 88 | survey, 89 | id, 90 | filters 91 | }) 92 | })), 93 | features_others: ( 94 | survey: SurveyConfig, 95 | { id, filters }: { id: string; filters?: Filters } 96 | ) => ({ 97 | survey, 98 | id, 99 | filters 100 | }), 101 | happiness: (survey: SurveyConfig, { id, filters }: { id: string; filters?: Filters }) => ({ 102 | survey, 103 | id, 104 | filters 105 | }), 106 | matrices: (survey: SurveyConfig) => ({ 107 | survey 108 | }), 109 | opinion: (survey: SurveyConfig, { id, filters }: { id: string; filters?: Filters }) => ({ 110 | survey, 111 | id, 112 | filters 113 | }), 114 | opinions_others: ( 115 | survey: SurveyConfig, 116 | { id, filters }: { id: string; filters?: Filters } 117 | ) => ({ 118 | survey, 119 | id, 120 | filters 121 | }), 122 | proficiency: ( 123 | survey: SurveyConfig, 124 | { id, filters }: { id: string; filters?: Filters } 125 | ) => ({ 126 | survey, 127 | id, 128 | filters 129 | }), 130 | resources: (survey: SurveyConfig, { id, filters }: { id: string; filters?: Filters }) => ({ 131 | survey, 132 | id, 133 | filters 134 | }), 135 | tool: async (survey: SurveyConfig, { id }: { id: string }) => ({ 136 | survey, 137 | id, 138 | entity: await getEntity({ id }), 139 | experience: ({ filters }: { filters?: Filters }) => ({ 140 | survey, 141 | id, 142 | filters 143 | }), 144 | experienceGraph: async ({ filters }: { filters?: Filters }, { db }: RequestContext) => 145 | useCache(computeToolExperienceGraph, db, [survey, id, filters]) 146 | }), 147 | tools: async (survey: SurveyConfig, { ids = toolIds }: { ids?: string[] }) => 148 | ids.map(async id => ({ 149 | survey, 150 | id, 151 | entity: await getEntity({ id }), 152 | experience: ({ filters }: { filters?: Filters }) => ({ 153 | survey, 154 | id, 155 | filters 156 | }), 157 | experienceGraph: async ( 158 | { filters }: { filters?: Filters }, 159 | { db }: RequestContext 160 | ) => useCache(computeToolExperienceGraph, db, [survey, id, filters]) 161 | })), 162 | tools_cardinality_by_user: ( 163 | survey: SurveyConfig, 164 | { 165 | year, 166 | // tool IDs 167 | ids, 168 | experienceId 169 | }: { 170 | year: number 171 | ids: string[] 172 | experienceId: ToolExperienceId 173 | }, 174 | context: RequestContext 175 | ) => useCache(computeToolsCardinalityByUser, context.db, [survey, year, ids, experienceId]), 176 | tools_others: ( 177 | survey: SurveyConfig, 178 | { id, filters }: { id: string; filters?: Filters } 179 | ) => ({ 180 | survey, 181 | id, 182 | filters 183 | }), 184 | tools_rankings: ( 185 | survey: SurveyConfig, 186 | { ids, filters }: { ids: string[]; filters: Filters } 187 | ) => ({ 188 | survey, 189 | ids, 190 | filters 191 | }), 192 | totals: (survey: SurveyConfig) => survey 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/resolvers/tools.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { computeExperienceOverYears, computeToolsExperienceRanking } from '../compute' 3 | import { useCache } from '../caching' 4 | import { SurveyConfig, RequestContext } from '../types' 5 | import { Filters } from '../filters' 6 | import { YearAggregations } from '../compute/generic' 7 | 8 | interface ToolConfig { 9 | survey: SurveyConfig 10 | id: string 11 | filters?: Filters 12 | } 13 | 14 | const computeToolExperience = async (db: Db, survey: SurveyConfig, id: string, filters?: Filters) => 15 | useCache(computeExperienceOverYears, db, [survey, id, filters]) 16 | 17 | export default { 18 | ToolsRankings: { 19 | experience: async ( 20 | { survey, ids, filters }: { survey: SurveyConfig; ids: string[]; filters?: Filters }, 21 | args: any, 22 | { db }: RequestContext 23 | ) => useCache(computeToolsExperienceRanking, db, [survey, ids, filters]) 24 | }, 25 | ToolExperience: { 26 | all_years: async ({ survey, id, filters }: ToolConfig, args: any, { db }: RequestContext) => 27 | computeToolExperience(db, survey, id, filters), 28 | year: async ( 29 | { survey, id, filters }: ToolConfig, 30 | { year }: { year: number }, 31 | { db }: RequestContext 32 | ) => { 33 | const allYears = await computeToolExperience(db, survey, id, filters) 34 | return allYears.find(yearItem => yearItem.year === year) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/resolvers/tools_others.ts: -------------------------------------------------------------------------------- 1 | import { getDynamicResolvers, getOtherKey } from '../helpers' 2 | 3 | export default { 4 | OtherTools: getDynamicResolvers(id => `tools_others.${getOtherKey(id)}`) 5 | } 6 | -------------------------------------------------------------------------------- /src/resolvers/totals.ts: -------------------------------------------------------------------------------- 1 | import { useCache } from '../caching' 2 | import { RequestContext, SurveyConfig } from '../types' 3 | import { getSurveyTotals } from '../compute/generic' 4 | 5 | export default { 6 | Totals: { 7 | all_years: async (survey: SurveyConfig, args: any, { db }: RequestContext) => 8 | useCache(getSurveyTotals, db, [survey]), 9 | year: async (survey: SurveyConfig, { year }: { year: number }, { db }: RequestContext) => 10 | useCache(getSurveyTotals, db, [survey, year]) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/rpcs.ts: -------------------------------------------------------------------------------- 1 | import { TwitterStat } from './types/twitter' 2 | import { MongoClient } from 'mongodb' 3 | import config from './config' 4 | import { getEntities } from './entities' 5 | import { getTwitterUser, getTwitterFollowings } from './external_apis/twitter' 6 | 7 | function sleep(ms: number) { 8 | return new Promise(resolve => { 9 | setTimeout(resolve, ms) 10 | }) 11 | } 12 | 13 | export const analyzeTwitterFollowings = async () => { 14 | console.log('// Running analyzeTwitterFollowings…') 15 | let count = 0 16 | 17 | // 0. Mongo setup 18 | const mongoClient = new MongoClient(process.env!.MONGO_URI!, { connectTimeoutMS: 1000 }) 19 | await mongoClient.connect() 20 | const db = mongoClient.db(process.env.MONGO_DB_NAME) 21 | const resultsCollection = db.collection(config.mongo.normalized_collection) 22 | const twitterStatsCollection = db.collection('twitterStats') 23 | 24 | // 1. get all results documents with a twitter username associated 25 | const results = resultsCollection.find({ 'user_info.twitter_username': { $exists: true } }) 26 | console.log(`// Found ${await results.count()} results with associated Twitter info`) 27 | 28 | // 2. loop over results 29 | for await (const result of results) { 30 | count++ 31 | console.log(`${count}. @${result.user_info.twitter_username}`) 32 | 33 | const { surveySlug, user_info } = result 34 | const twitterName = user_info.twitter_username 35 | 36 | // 3. for each result, check if its twitter stats have already been calculated or not 37 | const existingStat = await twitterStatsCollection.findOne({ twitterName, surveySlug }) 38 | 39 | if (existingStat) { 40 | console.log(` -> Found existing stat, skipping`) 41 | console.log(existingStat) 42 | } 43 | 44 | // 4. if not, get twitter stats and insert them 45 | if (!existingStat) { 46 | const twitterUser = await getTwitterUser(twitterName) 47 | if (!twitterUser) { 48 | return 49 | } 50 | const twitterId = twitterUser?.id 51 | const followings = await getTwitterFollowings(twitterId) 52 | const entities = await getEntities({ tags: ['people', 'css'] }) 53 | const peopleUsernames = entities.map(e => e.twitterName) 54 | const followingsSubset = followings.filter(f => peopleUsernames.includes(f)) 55 | 56 | console.log( 57 | ` -> Stats: ${followings.length} followings; ${followingsSubset.length} people` 58 | ) 59 | 60 | const stat: TwitterStat = { 61 | twitterId, 62 | twitterName, 63 | surveySlug, 64 | followings, 65 | followingsSubset, 66 | followersCount: twitterUser?.public_metrics?.followers_count, 67 | followingCount: twitterUser?.public_metrics?.following_count, 68 | tweetCount: twitterUser?.public_metrics?.tweet_count, 69 | listedCount: twitterUser?.public_metrics?.listed_count 70 | } 71 | 72 | twitterStatsCollection.insertOne(stat) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | dotenv.config() 3 | import { ApolloServer } from 'apollo-server-express' 4 | import { MongoClient } from 'mongodb' 5 | import responseCachePlugin from 'apollo-server-plugin-response-cache' 6 | import typeDefs from './type_defs/schema.graphql' 7 | import { RequestContext } from './types' 8 | import resolvers from './resolvers' 9 | import express from 'express' 10 | import { initLocales } from './i18n' 11 | import { initEntities } from './entities' 12 | import { analyzeTwitterFollowings } from './rpcs' 13 | import { clearCache } from './caching' 14 | 15 | // import Sentry from '@sentry/node' 16 | // import Tracing from '@sentry/tracing' 17 | 18 | import path from 'path' 19 | 20 | const Sentry = require('@sentry/node') 21 | const Tracing = require('@sentry/tracing') 22 | 23 | const app = express() 24 | 25 | const environment = process.env.ENVIRONMENT || process.env.NODE_ENV; 26 | 27 | Sentry.init({ 28 | dsn: process.env.SENTRY_DSN, 29 | integrations: [ 30 | // enable HTTP calls tracing 31 | new Sentry.Integrations.Http({ tracing: true }), 32 | // enable Express.js middleware tracing 33 | // new Tracing.Integrations.Express({ app }), 34 | ], 35 | // We recommend adjusting this value in production, or using tracesSampler 36 | // for finer control 37 | tracesSampleRate: 1.0, 38 | environment, 39 | }); 40 | 41 | // const path = require('path') 42 | 43 | const isDev = process.env.NODE_ENV === 'development' 44 | 45 | const checkSecretKey = (req: any) => { 46 | if (req?.query?.key !== process.env.SECRET_KEY) { 47 | throw new Error('Authorization error') 48 | } 49 | } 50 | 51 | const start = async () => { 52 | const mongoClient = new MongoClient(process.env!.MONGO_URI!, { 53 | // useNewUrlParser: true, 54 | // useUnifiedTopology: true, 55 | connectTimeoutMS: 10000 56 | }) 57 | await mongoClient.connect() 58 | const db = mongoClient.db(process.env.MONGO_DB_NAME) 59 | 60 | const server = new ApolloServer({ 61 | typeDefs, 62 | resolvers: resolvers as any, 63 | debug: isDev, 64 | // tracing: isDev, 65 | // cacheControl: true, 66 | introspection: true, 67 | // playground: false, 68 | plugins: [responseCachePlugin()], 69 | // engine: { 70 | // debugPrintReports: true 71 | // }, 72 | context: (): RequestContext => ({ 73 | db 74 | }) 75 | }) 76 | 77 | app.use(Sentry.Handlers.requestHandler()); 78 | // TracingHandler creates a trace for every incoming request 79 | // app.use(Sentry.Handlers.tracingHandler()); 80 | 81 | await server.start() 82 | 83 | server.applyMiddleware({ app }) 84 | 85 | app.get('/', function (req, res) { 86 | res.sendFile(path.join(__dirname + '/public/welcome.html')) 87 | }) 88 | 89 | app.get('/debug-sentry', function mainHandler(req, res) { 90 | throw new Error('My first Sentry error!'); 91 | }); 92 | 93 | app.get('/analyze-twitter', async function (req, res) { 94 | checkSecretKey(req) 95 | analyzeTwitterFollowings() 96 | res.status(200).send('Analyzing…') 97 | }) 98 | 99 | app.get('/clear-cache', async function (req, res) { 100 | checkSecretKey(req) 101 | clearCache(db) 102 | res.status(200).send('Cache cleared') 103 | }) 104 | 105 | app.use(Sentry.Handlers.errorHandler()); 106 | 107 | const port = process.env.PORT || 4000 108 | 109 | await initLocales() 110 | await initEntities() 111 | 112 | app.listen({ port: port }, () => 113 | console.log(`🚀 Server ready at http://localhost:${port}${server.graphqlPath}`) 114 | ) 115 | 116 | 117 | } 118 | 119 | start() 120 | -------------------------------------------------------------------------------- /src/standalone.ts: -------------------------------------------------------------------------------- 1 | import dotenv from 'dotenv' 2 | dotenv.config() 3 | import { MongoClient } from 'mongodb' 4 | import { inspect } from 'util' 5 | import { 6 | computeChoicesOverYearsGraph 7 | } from './compute' 8 | 9 | const run = async () => { 10 | const mongoClient = new MongoClient(process.env!.MONGO_URI!, { 11 | // useNewUrlParser: true, 12 | // useUnifiedTopology: true, 13 | connectTimeoutMS: 1000 14 | }) 15 | await mongoClient.connect() 16 | const db = mongoClient.db(process.env.MONGO_DB_NAME) 17 | 18 | const res = await computeChoicesOverYearsGraph( 19 | db, 20 | { survey: 'js' }, 21 | 'tools.typescript.experience' 22 | ) 23 | 24 | console.log(inspect(res, { depth: null, colors: true })) 25 | 26 | await mongoClient.close() 27 | } 28 | 29 | run() 30 | -------------------------------------------------------------------------------- /src/type_defs/bracket.graphql: -------------------------------------------------------------------------------- 1 | enum BracketID { 2 | tools_others__tool_evaluation 3 | opinions__css_pain_points 4 | opinions__currently_missing_from_css 5 | } 6 | 7 | """ 8 | Bracket Wins 9 | (how many wins a player has accumulated) 10 | """ 11 | 12 | type BracketWinsStats { 13 | count: Int # number of time that bracket item won a match 14 | percentage: Float # percentage of all matches won 15 | } 16 | 17 | type BracketWinsBucket { 18 | id: String 19 | round1: BracketWinsStats 20 | round2: BracketWinsStats 21 | round3: BracketWinsStats 22 | combined: BracketWinsStats 23 | } 24 | 25 | type YearBracketWins { 26 | year: Int 27 | total: Int 28 | completion: Completion 29 | buckets: [BracketWinsBucket] 30 | } 31 | 32 | type BracketWins { 33 | id: BracketID! 34 | all_years: [YearBracketWins] 35 | year(year: Int!): YearBracketWins 36 | } 37 | 38 | """ 39 | Bracket Matchups 40 | (how a player fared against other players) 41 | """ 42 | 43 | type BracketMatchupStats { 44 | id: String # id of the player 45 | count: Int # number of time player won against root player 46 | percentage: Float # percentage player won against root player 47 | } 48 | 49 | type BracketMatchupsBucket { 50 | id: String # id of the "root" player 51 | matchups: [BracketMatchupStats] 52 | } 53 | 54 | type YearBracketMatchups { 55 | year: Int 56 | total: Int 57 | completion: Completion 58 | buckets: [BracketMatchupsBucket] 59 | } 60 | 61 | type BracketMatchups { 62 | id: BracketID! 63 | all_years: [YearBracketMatchups] 64 | year(year: Int!): YearBracketMatchups 65 | } -------------------------------------------------------------------------------- /src/type_defs/categories.graphql: -------------------------------------------------------------------------------- 1 | enum CategoryID { 2 | # JS 3 | javascript_flavors 4 | front_end_frameworks 5 | data_layer 6 | back_end_frameworks 7 | testing 8 | mobile_desktop 9 | # CSS 10 | pre_post_processors 11 | css_frameworks 12 | css_methodologies 13 | css_in_js 14 | } 15 | 16 | """ 17 | Happiness 18 | """ 19 | type CategoryHappiness { 20 | id: CategoryID! 21 | all_years: [YearHappiness] 22 | year(year: Int!): YearHappiness 23 | } 24 | 25 | """ 26 | Other Tools 27 | """ 28 | type CategoryOtherTools { 29 | id: CategoryID! 30 | all_years: [YearOtherTools] 31 | year(year: Int!): YearOtherTools 32 | } 33 | 34 | """ 35 | Category 36 | """ 37 | type Category { 38 | happiness(filters: Filters): CategoryHappiness 39 | tools_others(filters: Filters): CategoryOtherTools 40 | } 41 | -------------------------------------------------------------------------------- /src/type_defs/countries.graphql: -------------------------------------------------------------------------------- 1 | enum CountryID { 2 | AFG 3 | ALA 4 | ALB 5 | DZA 6 | ASM 7 | AND 8 | AGO 9 | AIA 10 | ATA 11 | ATG 12 | ARG 13 | ARM 14 | ABW 15 | AUS 16 | AUT 17 | AZE 18 | BHS 19 | BHR 20 | BGD 21 | BRB 22 | BLR 23 | BEL 24 | BLZ 25 | BEN 26 | BMU 27 | BTN 28 | BOL 29 | BES 30 | BIH 31 | BWA 32 | BVT 33 | BRA 34 | IOT 35 | BRN 36 | BGR 37 | BFA 38 | BDI 39 | CPV 40 | KHM 41 | CMR 42 | CAN 43 | CYM 44 | CAF 45 | TCD 46 | CHL 47 | CHN 48 | CXR 49 | CCK 50 | COL 51 | COM 52 | COG 53 | COD 54 | COK 55 | CRI 56 | CIV 57 | HRV 58 | CUB 59 | CUW 60 | CYP 61 | CZE 62 | DNK 63 | DJI 64 | DMA 65 | DOM 66 | ECU 67 | EGY 68 | SLV 69 | GNQ 70 | ERI 71 | EST 72 | SWZ 73 | ETH 74 | FLK 75 | FRO 76 | FJI 77 | FIN 78 | FRA 79 | GUF 80 | PYF 81 | ATF 82 | GAB 83 | GMB 84 | GEO 85 | DEU 86 | GHA 87 | GIB 88 | GRC 89 | GRL 90 | GRD 91 | GLP 92 | GUM 93 | GTM 94 | GGY 95 | GIN 96 | GNB 97 | GUY 98 | HTI 99 | HMD 100 | VAT 101 | HND 102 | HKG 103 | HUN 104 | ISL 105 | IND 106 | IDN 107 | IRN 108 | IRQ 109 | IRL 110 | IMN 111 | ISR 112 | ITA 113 | JAM 114 | JPN 115 | JEY 116 | JOR 117 | KAZ 118 | KEN 119 | KIR 120 | PRK 121 | KOR 122 | KWT 123 | KGZ 124 | LAO 125 | LVA 126 | LBN 127 | LSO 128 | LBR 129 | LBY 130 | LIE 131 | LTU 132 | LUX 133 | MAC 134 | MDG 135 | MWI 136 | MYS 137 | MDV 138 | MLI 139 | MLT 140 | MHL 141 | MTQ 142 | MRT 143 | MUS 144 | MYT 145 | MEX 146 | FSM 147 | MDA 148 | MCO 149 | MNG 150 | MNE 151 | MSR 152 | MAR 153 | MOZ 154 | MMR 155 | NAM 156 | NRU 157 | NPL 158 | NLD 159 | NCL 160 | NZL 161 | NIC 162 | NER 163 | NGA 164 | NIU 165 | NFK 166 | MKD 167 | MNP 168 | NOR 169 | OMN 170 | PAK 171 | PLW 172 | PSE 173 | PAN 174 | PNG 175 | PRY 176 | PER 177 | PHL 178 | PCN 179 | POL 180 | PRT 181 | PRI 182 | QAT 183 | REU 184 | ROU 185 | RUS 186 | RWA 187 | BLM 188 | SHN 189 | KNA 190 | LCA 191 | MAF 192 | SPM 193 | VCT 194 | WSM 195 | SMR 196 | STP 197 | SAU 198 | SEN 199 | SRB 200 | SYC 201 | SLE 202 | SGP 203 | SXM 204 | SVK 205 | SVN 206 | SLB 207 | SOM 208 | ZAF 209 | SGS 210 | SSD 211 | ESP 212 | LKA 213 | SDN 214 | SUR 215 | SJM 216 | SWE 217 | CHE 218 | SYR 219 | TWN 220 | TJK 221 | TZA 222 | THA 223 | TLS 224 | TGO 225 | TKL 226 | TON 227 | TTO 228 | TUN 229 | TUR 230 | TKM 231 | TCA 232 | TUV 233 | UGA 234 | UKR 235 | ARE 236 | GBR 237 | USA 238 | UMI 239 | URY 240 | UZB 241 | VUT 242 | VEN 243 | VNM 244 | VGB 245 | VIR 246 | WLF 247 | ESH 248 | YEM 249 | ZMB 250 | ZWE 251 | } 252 | -------------------------------------------------------------------------------- /src/type_defs/demographics.graphql: -------------------------------------------------------------------------------- 1 | """ 2 | Participation 3 | """ 4 | type YearParticipation { 5 | year: Int 6 | participants: Int 7 | } 8 | 9 | type Participation { 10 | all_years: [YearParticipation] 11 | year(year: Int!): YearParticipation 12 | } 13 | 14 | """ 15 | Age 16 | """ 17 | enum AgeRange { 18 | range_less_than_10 19 | range_10_18 20 | range_18_24 21 | range_25_34 22 | range_35_44 23 | range_45_54 24 | range_55_64 25 | range_more_than_65 26 | } 27 | 28 | type AgeBucket { 29 | id: AgeRange 30 | count: Int 31 | percentage: Float 32 | } 33 | 34 | type YearAge { 35 | year: Int 36 | """ 37 | Total number of respondents who have answered this specific question. 38 | """ 39 | total: Int 40 | completion: Completion 41 | buckets: [AgeBucket] 42 | } 43 | 44 | type Age { 45 | all_years: [YearAge] 46 | year(year: Int!): YearAge 47 | } 48 | 49 | """ 50 | Country 51 | """ 52 | type CountryBucket { 53 | id: CountryID 54 | count: Int 55 | percentage: Float 56 | } 57 | 58 | type YearCountry { 59 | year: Int 60 | total: Int 61 | completion: Completion 62 | buckets: [CountryBucket] 63 | } 64 | 65 | type Country { 66 | all_years: [YearCountry] 67 | year(year: Int!): YearCountry 68 | } 69 | 70 | """ 71 | Locale 72 | """ 73 | type LocaleBucket { 74 | id: String 75 | count: Int 76 | percentage: Float 77 | } 78 | 79 | type YearLocale { 80 | year: Int 81 | total: Int 82 | completion: Completion 83 | buckets: [LocaleBucket] 84 | } 85 | 86 | type LocaleStats { 87 | all_years: [YearLocale] 88 | year(year: Int!): YearLocale 89 | } 90 | 91 | """ 92 | Source 93 | """ 94 | type SourceBucket { 95 | id: String 96 | count: Int 97 | percentage: Float 98 | entity: Entity 99 | } 100 | 101 | type YearSource { 102 | year: Int 103 | total: Int 104 | completion: Completion 105 | buckets: [SourceBucket] 106 | } 107 | 108 | type Source { 109 | all_years: [YearSource] 110 | year(year: Int!): YearSource 111 | } 112 | 113 | """ 114 | Gender 115 | """ 116 | enum GenderID { 117 | male 118 | female 119 | prefer_not_to_say 120 | non_binary 121 | not_listed 122 | } 123 | 124 | type GenderBucket { 125 | id: GenderID 126 | count: Int 127 | percentage: Float 128 | } 129 | 130 | type YearGender { 131 | year: Int 132 | """ 133 | Total number of respondents who have answered this specific question. 134 | """ 135 | total: Int 136 | completion: Completion 137 | buckets: [GenderBucket] 138 | } 139 | 140 | type Gender { 141 | all_years: [YearGender] 142 | year(year: Int!): YearGender 143 | } 144 | 145 | """ 146 | Race & Ethnicity 147 | """ 148 | enum RaceEthnicityID { 149 | biracial 150 | black_african 151 | east_asian 152 | hispanic_latin 153 | middle_eastern 154 | multiracial 155 | native_american_islander_australian 156 | south_asian 157 | white_european 158 | } 159 | 160 | type RaceEthnicityBucket { 161 | id: RaceEthnicityID 162 | count: Int 163 | percentage: Float 164 | } 165 | 166 | type YearRaceEthnicity { 167 | year: Int 168 | """ 169 | Total number of respondents who have answered this specific question. 170 | """ 171 | total: Int 172 | completion: Completion 173 | buckets: [RaceEthnicityBucket] 174 | } 175 | 176 | type RaceEthnicity { 177 | all_years: [YearRaceEthnicity] 178 | year(year: Int!): YearRaceEthnicity 179 | } 180 | 181 | """ 182 | Salary 183 | """ 184 | enum SalaryRange { 185 | range_work_for_free 186 | range_0_10 187 | range_10_30 188 | range_30_50 189 | range_50_100 190 | range_100_200 191 | range_more_than_200 192 | } 193 | 194 | type SalaryBucket { 195 | id: SalaryRange 196 | count: Int 197 | percentage: Float 198 | } 199 | 200 | type YearSalary { 201 | year: Int 202 | """ 203 | Total number of respondents who have answered this specific question. 204 | """ 205 | total: Int 206 | completion: Completion 207 | buckets: [SalaryBucket] 208 | } 209 | 210 | type Salary { 211 | all_years: [YearSalary] 212 | year(year: Int!): YearSalary 213 | } 214 | 215 | """ 216 | Company Size 217 | """ 218 | enum CompanySizeRange { 219 | range_1 220 | range_1_5 221 | range_5_10 222 | range_10_20 223 | range_20_50 224 | range_50_100 225 | range_100_1000 226 | range_more_than_1000 227 | } 228 | 229 | type CompanySizeBucket { 230 | id: CompanySizeRange 231 | count: Int 232 | percentage: Float 233 | } 234 | 235 | type YearCompanySize { 236 | year: Int 237 | """ 238 | Total number of respondents who have answered this specific question. 239 | """ 240 | total: Int 241 | completion: Completion 242 | buckets: [CompanySizeBucket] 243 | } 244 | 245 | type CompanySize { 246 | all_years: [YearCompanySize] 247 | year(year: Int!): YearCompanySize 248 | } 249 | 250 | """ 251 | Experience 252 | """ 253 | enum WorkExperienceRange { 254 | range_less_than_1 255 | range_1_2 256 | range_2_5 257 | range_5_10 258 | range_10_20 259 | range_more_than_20 260 | } 261 | 262 | type WorkExperienceBucket { 263 | id: WorkExperienceRange 264 | count: Int 265 | percentage: Float 266 | } 267 | 268 | type YearWorkExperience { 269 | year: Int 270 | """ 271 | Total number of respondents who have answered this specific question. 272 | """ 273 | total: Int 274 | completion: Completion 275 | buckets: [WorkExperienceBucket] 276 | } 277 | 278 | type WorkExperience { 279 | all_years: [YearWorkExperience] 280 | year(year: Int!): YearWorkExperience 281 | } 282 | 283 | """ 284 | Job Title 285 | """ 286 | enum JobTitleID { 287 | full_stack_developer 288 | front_end_developer 289 | web_developer 290 | back_end_developer 291 | web_designer 292 | ui_designer 293 | ux_designer 294 | cto 295 | } 296 | 297 | type JobTitleBucket { 298 | id: JobTitleID 299 | count: Int 300 | percentage: Float 301 | } 302 | 303 | type YearJobTitle { 304 | year: Int 305 | total: Int 306 | completion: Completion 307 | buckets: [JobTitleBucket] 308 | } 309 | 310 | type JobTitle { 311 | all_years: [YearJobTitle] 312 | year(year: Int!): YearJobTitle 313 | } 314 | 315 | """ 316 | Industry Sector 317 | """ 318 | enum IndustrySectorID { 319 | ecommerce 320 | news_media 321 | healthcare 322 | finance 323 | programming_tools 324 | socialmedia 325 | marketing_tools 326 | education 327 | real_estate 328 | entertainment 329 | government 330 | consulting 331 | } 332 | 333 | type IndustrySectorBucket { 334 | id: IndustrySectorID 335 | count: Int 336 | percentage: Float 337 | } 338 | 339 | type YearIndustrySector { 340 | year: Int 341 | total: Int 342 | completion: Completion 343 | buckets: [IndustrySectorBucket] 344 | } 345 | 346 | type IndustrySector { 347 | all_years: [YearIndustrySector] 348 | year(year: Int!): YearIndustrySector 349 | } 350 | 351 | """ 352 | Knowledge Score 353 | """ 354 | type KnowledgeScoreBucket { 355 | id: Int 356 | count: Int 357 | percentage: Float 358 | } 359 | 360 | type YearKnowledgeScore { 361 | year: Int 362 | buckets: [KnowledgeScoreBucket] 363 | } 364 | 365 | type KnowledgeScore { 366 | all_years: [YearKnowledgeScore] 367 | year(year: Int!): YearKnowledgeScore 368 | } 369 | 370 | """ 371 | Higher Education Degree 372 | """ 373 | enum HigherEducationDegreeID { 374 | no_degree 375 | yes_related 376 | yes_unrelated 377 | } 378 | 379 | type HigherEducationDegreeBucket { 380 | id: HigherEducationDegreeID 381 | count: Int 382 | percentage: Float 383 | } 384 | 385 | type YearHigherEducationDegree { 386 | year: Int 387 | """ 388 | Total number of respondents who have answered this specific question. 389 | """ 390 | total: Int 391 | completion: Completion 392 | buckets: [HigherEducationDegreeBucket] 393 | } 394 | 395 | type HigherEducationDegree { 396 | all_years: [YearHigherEducationDegree] 397 | year(year: Int!): YearHigherEducationDegree 398 | } 399 | 400 | """ 401 | Disability Status 402 | """ 403 | enum DisabilityStatusID { 404 | visual_impairments 405 | hearing_impairments 406 | mobility_impairments 407 | cognitive_impairments 408 | not_listed 409 | } 410 | 411 | type DisabilityStatusBucket { 412 | id: DisabilityStatusID 413 | count: Int 414 | percentage: Float 415 | } 416 | 417 | type YearDisabilityStatus { 418 | year: Int 419 | """ 420 | Total number of respondents who have answered this specific question. 421 | """ 422 | total: Int 423 | completion: Completion 424 | buckets: [DisabilityStatusBucket] 425 | } 426 | 427 | type DisabilityStatus { 428 | all_years: [YearDisabilityStatus] 429 | year(year: Int!): YearDisabilityStatus 430 | } 431 | 432 | type YearOtherDisabilityStatus { 433 | year: Int 434 | """ 435 | Total number of respondents who have answered this specific question. 436 | """ 437 | total: Int 438 | completion: Completion 439 | buckets: [EntityBucket] 440 | } 441 | 442 | type OtherDisabilityStatus { 443 | all_years: [YearOtherDisabilityStatus] 444 | year(year: Int!): YearOtherDisabilityStatus 445 | } 446 | 447 | """ 448 | Information about particpants: 449 | - overall participation 450 | - gender 451 | - salary 452 | - company size 453 | - … 454 | """ 455 | type Demographics { 456 | """ 457 | Age 458 | """ 459 | age(filters: Filters): Age 460 | """ 461 | Country 462 | """ 463 | country(filters: Filters): Country 464 | """ 465 | Locale 466 | """ 467 | locale(filters: Filters): LocaleStats 468 | """ 469 | How respondents found the survey 470 | """ 471 | source(filters: Filters): Source 472 | """ 473 | Participants count 474 | """ 475 | participation(filters: Filters): Participation 476 | """ 477 | Gender 478 | """ 479 | gender(filters: Filters): Gender 480 | """ 481 | Race & Ethnicity 482 | """ 483 | race_ethnicity(filters: Filters): RaceEthnicity 484 | """ 485 | Salary 486 | """ 487 | yearly_salary(filters: Filters): Salary 488 | """ 489 | Company size 490 | """ 491 | company_size(filters: Filters): CompanySize 492 | """ 493 | Work experience as a developer 494 | """ 495 | years_of_experience(filters: Filters): WorkExperience 496 | """ 497 | Job title 498 | """ 499 | job_title(filters: Filters): JobTitle 500 | """ 501 | Industry Sector 502 | """ 503 | industry_sector(filters: Filters): IndustrySector 504 | """ 505 | Knowledge Score 506 | """ 507 | knowledge_score(filters: Filters): KnowledgeScore 508 | """ 509 | Higher Education Degree 510 | """ 511 | higher_education_degree(filters: Filters): HigherEducationDegree 512 | """ 513 | Disability Status 514 | """ 515 | disability_status(filters: Filters): DisabilityStatus 516 | """ 517 | Disability Status (Other) 518 | """ 519 | disability_status_other(filters: Filters): OtherDisabilityStatus 520 | } 521 | -------------------------------------------------------------------------------- /src/type_defs/entity.graphql: -------------------------------------------------------------------------------- 1 | """ 2 | An entity is any object that can have associated metadata 3 | (such as a homepage, github repo, description). 4 | For example: a library, a podcast, a blog, a framework… 5 | """ 6 | type Entity { 7 | id: String 8 | name: String 9 | otherName: String 10 | twitterName: String 11 | homepage: String 12 | category: String 13 | github: GitHub 14 | npm: String 15 | mdn: MDN 16 | description: String 17 | type: String 18 | tags: [String] 19 | patterns: [String] 20 | twitter: Twitter 21 | youtubeName: String 22 | blog: String 23 | rss: String 24 | related: [Entity] 25 | } 26 | 27 | """ 28 | A datapoint associated with a given entity. 29 | """ 30 | type EntityBucket { 31 | id: String 32 | count: Int 33 | percentage: Float 34 | entity: Entity 35 | } 36 | -------------------------------------------------------------------------------- /src/type_defs/environments.graphql: -------------------------------------------------------------------------------- 1 | enum EnvironmentID { 2 | # choices based 3 | browsers 4 | browsers_others 5 | form_factors 6 | form_factors_others 7 | accessibility_features 8 | accessibility_features_others 9 | what_do_you_use_css_for 10 | what_do_you_use_css_for_others 11 | # rating based 12 | css_for_print 13 | css_for_email 14 | } 15 | 16 | type YearEnvironments { 17 | year: Int 18 | """ 19 | Total number of respondents who have answered this specific question. 20 | """ 21 | total: Int 22 | completion: Completion 23 | buckets: [EntityBucket] 24 | } 25 | 26 | """ 27 | An environment, based on multiple choices (e.g. browsers, form factors, etc.) 28 | """ 29 | type Environments { 30 | id: EnvironmentID! 31 | year(year: Int!): YearEnvironments 32 | all_years: [YearEnvironments] 33 | } 34 | 35 | type EnvironmentRatingBucket { 36 | id: Int 37 | count: Int 38 | percentage: Float 39 | } 40 | 41 | type YearEnvironmentsRatings { 42 | year: Int 43 | """ 44 | Total number of respondents who have answered this specific question. 45 | """ 46 | total: Int 47 | completion: Completion 48 | buckets: [EnvironmentRatingBucket] 49 | } 50 | 51 | """ 52 | An environment-based rating (e.g. css for emails, css for print, etc.) 53 | """ 54 | type EnvironmentsRatings { 55 | id: EnvironmentID! 56 | year(year: Int!): YearEnvironmentsRatings 57 | all_years: [YearEnvironmentsRatings] 58 | } 59 | -------------------------------------------------------------------------------- /src/type_defs/experience.graphql: -------------------------------------------------------------------------------- 1 | enum ExperienceID { 2 | would_use 3 | would_not_use 4 | interested 5 | not_interested 6 | never_heard 7 | } 8 | 9 | enum FeatureExperienceID { 10 | never_heard 11 | heard 12 | used 13 | } 14 | -------------------------------------------------------------------------------- /src/type_defs/features.graphql: -------------------------------------------------------------------------------- 1 | enum FeatureID { 2 | # JS syntax 3 | destructuring 4 | spread_operator 5 | arrow_functions 6 | nullish_coalescing 7 | optional_chaining 8 | private_fields 9 | # JS language 10 | proxies 11 | async_await 12 | promises 13 | decorators 14 | promise_all_settled 15 | dynamic_import 16 | # JS data structures 17 | maps 18 | sets 19 | typed_arrays 20 | array_prototype_flat 21 | big_int 22 | # JS browser apis 23 | service_workers 24 | local_storage 25 | intl 26 | web_audio 27 | webgl 28 | web_animations 29 | webrtc 30 | web_speech 31 | webvr 32 | websocket 33 | fetch 34 | custom_elements 35 | shadow_dom 36 | i18n 37 | web_components 38 | # JS other features 39 | pwa 40 | wasm 41 | 42 | # CSS layout features 43 | grid 44 | subgrid 45 | regions 46 | flexbox 47 | multi_column 48 | writing_modes 49 | exclusions 50 | position_sticky 51 | logical_properties 52 | aspect_ratio 53 | content_visibility 54 | flexbox_gap 55 | break_rules 56 | at_container 57 | # CSS shapes & graphics features 58 | shapes 59 | object_fit 60 | clip_path 61 | masks 62 | blend_modes 63 | filter_effects 64 | backdrop_filter 65 | intrinsic_sizing 66 | conic_gradient 67 | color_function 68 | accent_color 69 | color_gamut 70 | # CSS interactions features 71 | scroll_snap 72 | overscroll_behavior 73 | overflow_anchor 74 | touch_action 75 | pointer_events 76 | scroll_timeline 77 | # CSS typography features 78 | web_fonts 79 | line_breaking 80 | font_variant 81 | initial_letter 82 | font_variant_numeric 83 | font_display 84 | line_clamp 85 | leading_trim 86 | direction 87 | variable_fonts 88 | # CSS animations & transforms features 89 | transitions 90 | transforms 91 | animations 92 | perspective 93 | # CSS accessibility features 94 | prefers_reduced_data 95 | prefers_reduced_motion 96 | prefers_color_scheme 97 | color_contrast 98 | color_scheme 99 | tabindex 100 | aria_attributes 101 | # CSS other features 102 | variables 103 | feature_support_queries 104 | containment 105 | will_change 106 | calc 107 | houdini 108 | comparison_functions 109 | at_property 110 | marker 111 | # units_selectors 112 | px 113 | pt 114 | percent 115 | em 116 | rem 117 | vh_vw 118 | vmin_vmax 119 | ch 120 | ex 121 | mm 122 | cm 123 | in 124 | # pseudo_elements 125 | before 126 | after 127 | first_line 128 | first_letter 129 | selection 130 | placeholder 131 | backdrop 132 | # combinators 133 | descendant 134 | child 135 | next_sibling 136 | subsequent_sibling 137 | # tree_document_structure 138 | root 139 | empty 140 | not 141 | nth_child 142 | nth_last_child 143 | first_child 144 | last_child 145 | only_child 146 | nth_of_type 147 | nth_last_of_type 148 | first_of_type 149 | last_of_type 150 | only_of_type 151 | lang 152 | is 153 | where 154 | # attributes 155 | presence 156 | equality 157 | starts_with 158 | ends_with 159 | contains_word 160 | contains_substring 161 | # links_urls 162 | any_link 163 | link_visited 164 | local_link 165 | target 166 | # interaction 167 | hover 168 | active 169 | focus 170 | focus_within 171 | focus_visible 172 | # form_controls 173 | enabled_disabled 174 | read_only_write 175 | placeholder_shown 176 | default 177 | checked 178 | indeterminate 179 | valid_invalid 180 | user_invalid 181 | in_out_range 182 | required_optional 183 | } 184 | 185 | """ 186 | A feature experience datapoint 187 | """ 188 | type FeatureExperienceBucket { 189 | id: FeatureExperienceID 190 | count: Int 191 | # difference with previous year 192 | countDelta: Int 193 | percentage: Float 194 | # difference with previous year 195 | percentageDelta: Float 196 | } 197 | 198 | """ 199 | Feature data for a specific year 200 | """ 201 | type YearFeature { 202 | year: Int 203 | """ 204 | Total number of respondents who have answered this specific question. 205 | """ 206 | total: Int 207 | completion: Completion 208 | buckets: [FeatureExperienceBucket] 209 | } 210 | 211 | type FeatureExperience { 212 | all_years: [YearFeature] 213 | year(year: Int!): YearFeature 214 | } 215 | 216 | """ 217 | A feature (e.g. arrow functions, websocket, etc.) 218 | """ 219 | type Feature { 220 | id: FeatureID! 221 | name: String 222 | mdn: MDN 223 | experience(filters: Filters): FeatureExperience 224 | } 225 | -------------------------------------------------------------------------------- /src/type_defs/features_others.graphql: -------------------------------------------------------------------------------- 1 | enum OtherFeaturesID { 2 | units 3 | pseudo_elements 4 | combinators 5 | tree_document_structure 6 | attributes 7 | links_urls 8 | interaction 9 | form_controls 10 | } 11 | 12 | type FeatureBucket { 13 | id: FeatureID 14 | name: String 15 | count: Int 16 | percentage: Float 17 | } 18 | 19 | type YearOtherFeatures { 20 | year: Int 21 | """ 22 | Total number of respondents who have answered this specific question. 23 | """ 24 | total: Int 25 | completion: Completion 26 | buckets: [FeatureBucket] 27 | } 28 | 29 | type OtherFeatures { 30 | id: OtherFeaturesID! 31 | all_years: [YearOtherFeatures] 32 | year(year: Int!): YearOtherFeatures 33 | } 34 | -------------------------------------------------------------------------------- /src/type_defs/filters.graphql: -------------------------------------------------------------------------------- 1 | input GenderFilter { 2 | eq: GenderID 3 | in: [GenderID] 4 | nin: [GenderID] 5 | } 6 | 7 | input CountryFilter { 8 | eq: CountryID 9 | in: [CountryID] 10 | nin: [CountryID] 11 | } 12 | 13 | input RaceEthnicityFilter { 14 | eq: RaceEthnicityID 15 | # in: [RaceEthnicityID] 16 | # nin: [RaceEthnicityID] 17 | } 18 | 19 | input IndustrySectorFilter { 20 | eq: IndustrySectorID 21 | # in: [IndustrySectorID] 22 | # nin: [IndustrySectorID] 23 | } 24 | 25 | input YearlySalaryRangeFilter { 26 | eq: SalaryRange 27 | in: [SalaryRange] 28 | nin: [SalaryRange] 29 | } 30 | 31 | input CompanySizeFilter { 32 | eq: CompanySizeRange 33 | in: [CompanySizeRange] 34 | nin: [CompanySizeRange] 35 | } 36 | 37 | input YearsOfExperienceFilter { 38 | eq: WorkExperienceRange 39 | in: [WorkExperienceRange] 40 | nin: [WorkExperienceRange] 41 | } 42 | 43 | input SourceFilter { 44 | eq: String 45 | in: [String] 46 | nin: [String] 47 | } 48 | 49 | input Filters { 50 | gender: GenderFilter 51 | country: CountryFilter 52 | race_ethnicity: RaceEthnicityFilter 53 | yearly_salary: YearlySalaryRangeFilter 54 | company_size: CompanySizeFilter 55 | years_of_experience: YearsOfExperienceFilter 56 | source: SourceFilter 57 | industry_sector: IndustrySectorFilter 58 | } 59 | -------------------------------------------------------------------------------- /src/type_defs/github.graphql: -------------------------------------------------------------------------------- 1 | type GitHub { 2 | name: String 3 | full_name: String 4 | description: String 5 | url: String 6 | stars: Int 7 | forks: Int 8 | opened_issues: Int 9 | homepage: String 10 | } 11 | -------------------------------------------------------------------------------- /src/type_defs/happiness.graphql: -------------------------------------------------------------------------------- 1 | enum HappinessID { 2 | # JS 3 | javascript_flavors 4 | front_end_frameworks 5 | datalayer # data_layer? 6 | back_end_frameworks 7 | testing 8 | build_tools 9 | mobile_desktop 10 | state_of_js 11 | # CSS 12 | pre_post_processors 13 | css_frameworks 14 | css_methodologies 15 | css_in_js 16 | state_of_css 17 | # Other 18 | state_of_the_web 19 | } 20 | 21 | """ 22 | Happiness 23 | """ 24 | type HappinessBucket { 25 | id: Int 26 | count: Int 27 | percentage: Float 28 | } 29 | 30 | type YearHappiness { 31 | year: Int 32 | """ 33 | Total number of respondents who have answered this specific question. 34 | """ 35 | total: Int 36 | """ 37 | Mean happiness score for the year, please note that despite the 38 | happiness indices starts at 0, the average is computed from 1. 39 | """ 40 | mean: Float 41 | completion: Completion 42 | buckets: [HappinessBucket] 43 | } 44 | 45 | type Happiness { 46 | id: HappinessID! 47 | all_years: [YearHappiness] 48 | year(year: Int!): YearHappiness 49 | } 50 | -------------------------------------------------------------------------------- /src/type_defs/matrices.graphql: -------------------------------------------------------------------------------- 1 | #import "./experience.graphql" 2 | 3 | enum ToolMatrixExperienceID { 4 | would_use 5 | would_not_use 6 | interested 7 | not_interested 8 | never_heard 9 | """ 10 | `would_use` + `would_not_use` VS total 11 | """ 12 | usage 13 | """ 14 | `would_use` + `interested` VS `would_use` + `would_not_use` + `interested` + `not_interested` 15 | """ 16 | positive_sentiment 17 | """ 18 | `would_not_use` + `not_interested` VS `would_use` + `would_not_use` + `interested` + `not_interested` 19 | """ 20 | negative_sentiment 21 | """ 22 | `would_use` VS `would_not_use` 23 | """ 24 | satisfaction 25 | """ 26 | `interested` VS `not_interested` 27 | """ 28 | interest 29 | """ 30 | `never_heard` VS total (inverted) 31 | """ 32 | awareness 33 | } 34 | 35 | enum MatrixDimensionID { 36 | company_size 37 | source 38 | yearly_salary 39 | years_of_experience 40 | } 41 | 42 | type MatrixBucket { 43 | """ 44 | Id of the bucket dimension range, e.g. `range_50_100` 45 | for `company_size`. 46 | """ 47 | id: String 48 | """ 49 | Number of responses for a given tool/feature in a specific range. 50 | e.g. users who picked `range_50_100` for `company_size` and also 51 | picked `would_use` for experience with `tailwind_css`. 52 | """ 53 | count: Int 54 | """ 55 | Ratio from all respondents who picked a specific experience 56 | for the current tool and also answered to the question related 57 | to the dimension, e.g. `yearly_salary`. 58 | `count` VS `total`. 59 | """ 60 | percentage: Float 61 | """ 62 | Total number of respondents for this specific range, 63 | e.g. number of users who selected `range_50_100` 64 | for the `company_size` question and also answered 65 | the experience question. 66 | """ 67 | range_total: Int 68 | """ 69 | Ratio of experience in this specific range, 70 | `count` VS `range_total`. 71 | """ 72 | range_percentage: Float 73 | """ 74 | Delta between the overall percentage of responses 75 | for the selected experience filter compared 76 | to the percentage in this range. 77 | `range_percentage` VS overall percentage. 78 | """ 79 | range_percentage_delta: Float 80 | } 81 | 82 | type ToolMatrix { 83 | id: ToolID 84 | entity: Entity 85 | count: Int 86 | total: Int 87 | percentage: Float 88 | buckets: [MatrixBucket] 89 | } 90 | 91 | type ToolsExperienceDimensionMatrix { 92 | dimension: MatrixDimensionID 93 | tools: [ToolMatrix] 94 | } 95 | 96 | type ToolsExperienceMatrices { 97 | experience: ToolMatrixExperienceID 98 | dimensions: [ToolsExperienceDimensionMatrix] 99 | } 100 | 101 | type Matrices { 102 | tools( 103 | year: Int!, 104 | ids: [ToolID]!, 105 | experiences: [ToolMatrixExperienceID]!, 106 | dimensions: [MatrixDimensionID]! 107 | ): [ToolsExperienceMatrices] 108 | } 109 | -------------------------------------------------------------------------------- /src/type_defs/mdn.graphql: -------------------------------------------------------------------------------- 1 | type MDN { 2 | locale: String 3 | url: String 4 | title: String 5 | summary: String 6 | } 7 | -------------------------------------------------------------------------------- /src/type_defs/opinions.graphql: -------------------------------------------------------------------------------- 1 | """ 2 | Opinions 3 | """ 4 | 5 | enum OpinionID { 6 | js_moving_in_right_direction 7 | building_js_apps_overly_complex 8 | js_over_used_online 9 | enjoy_building_js_apps 10 | would_like_js_to_be_main_lang 11 | js_ecosystem_changing_to_fast 12 | 13 | css_easy_to_learn 14 | css_evolving_slowly 15 | utility_classes_to_be_avoided 16 | selector_nesting_to_be_avoided 17 | css_is_programming_language 18 | enjoy_writing_css 19 | } 20 | 21 | type OpinionBucket { 22 | id: Int 23 | count: Int 24 | percentage: Float 25 | } 26 | 27 | type YearOpinion { 28 | year: Int 29 | """ 30 | Total number of respondents who have answered this specific question. 31 | """ 32 | total: Int 33 | completion: Completion 34 | buckets: [OpinionBucket] 35 | } 36 | 37 | type Opinion { 38 | id: OpinionID! 39 | all_years: [YearOpinion] 40 | year(year: Int!): YearOpinion 41 | } 42 | 43 | """ 44 | Other Opinions 45 | """ 46 | 47 | enum OtherOpinionsID { 48 | missing_from_js 49 | 50 | currently_missing_from_css 51 | browser_interoperability_features 52 | css_pain_points 53 | } 54 | 55 | type YearOtherOpinions { 56 | year: Int 57 | """ 58 | Total number of respondents who have answered this specific question. 59 | """ 60 | total: Int 61 | completion: Completion 62 | buckets: [EntityBucket] 63 | } 64 | 65 | type OtherOpinions { 66 | id: OtherOpinionsID! 67 | all_years: [YearOtherOpinions] 68 | year(year: Int!): YearOtherOpinions 69 | } 70 | -------------------------------------------------------------------------------- /src/type_defs/proficiency.graphql: -------------------------------------------------------------------------------- 1 | enum ProficiencyID { 2 | backend_proficiency 3 | javascript_proficiency 4 | css_proficiency 5 | } 6 | 7 | type ProficiencyBucket { 8 | id: Int 9 | count: Int 10 | percentage: Float 11 | } 12 | 13 | type YearProficiency { 14 | year: Int 15 | """ 16 | Total number of respondents who have answered this specific question. 17 | """ 18 | total: Int 19 | completion: Completion 20 | buckets: [ProficiencyBucket] 21 | } 22 | 23 | type Proficiency { 24 | id: ProficiencyID! 25 | all_years: [YearProficiency] 26 | year(year: Int!): YearProficiency 27 | } 28 | -------------------------------------------------------------------------------- /src/type_defs/resources.graphql: -------------------------------------------------------------------------------- 1 | enum ResourcesID { 2 | blogs_news_magazines 3 | blogs_news_magazines_others 4 | sites_courses 5 | sites_courses_others 6 | podcasts 7 | podcasts_others 8 | first_steps 9 | first_steps_others 10 | people_others 11 | } 12 | 13 | type YearResources { 14 | year: Int 15 | """ 16 | Total number of respondents who have answered this specific question. 17 | """ 18 | total: Int 19 | completion: Completion 20 | buckets: [EntityBucket] 21 | } 22 | 23 | type Resources { 24 | id: ResourcesID! 25 | all_years: [YearResources] 26 | year(year: Int!): YearResources 27 | } 28 | -------------------------------------------------------------------------------- /src/type_defs/schema.graphql: -------------------------------------------------------------------------------- 1 | #import "./surveys.graphql" 2 | 3 | """ 4 | Completion ratio and count 5 | """ 6 | type Completion { 7 | total: Int 8 | count: Int 9 | percentage: Float 10 | } 11 | 12 | type Query { 13 | """ 14 | Data for a specific survey. 15 | """ 16 | survey(survey: SurveyType!): Survey 17 | """ 18 | Data about a specific entity (tool, library, framework, features, etc.) 19 | """ 20 | entity(id: ID!): Entity 21 | """ 22 | Get all entities (tools, libraries, frameworks, features, etc.) 23 | """ 24 | entities(type: String, context: String, tag: String, tags: [String]): [Entity] 25 | """ 26 | Translate a string 27 | """ 28 | translation(key: String!, localeId: String!): TranslationString 29 | """ 30 | Get all of a locale's translations 31 | """ 32 | locale(localeId: String!, contexts: [Contexts], enableFallbacks: Boolean): Locale 33 | """ 34 | Get all locales 35 | """ 36 | locales(contexts: [Contexts], enableFallbacks: Boolean): [Locale] 37 | } 38 | -------------------------------------------------------------------------------- /src/type_defs/surveys.graphql: -------------------------------------------------------------------------------- 1 | #import "./filters.graphql" 2 | #import "./categories.graphql" 3 | #import "./demographics.graphql" 4 | #import "./countries.graphql" 5 | #import "./entity.graphql" 6 | #import "./features.graphql" 7 | #import "./features_others.graphql" 8 | #import "./github.graphql" 9 | #import "./twitter.graphql" 10 | #import "./matrices.graphql" 11 | #import "./mdn.graphql" 12 | #import "./opinions.graphql" 13 | #import "./resources.graphql" 14 | #import "./tools.graphql" 15 | #import "./tools_others.graphql" 16 | #import "./tools_cardinality_by_user.graphql" 17 | #import "./translations.graphql" 18 | #import "./environments.graphql" 19 | #import "./proficiency.graphql" 20 | #import "./totals.graphql" 21 | #import "./happiness.graphql" 22 | #import "./bracket.graphql" 23 | 24 | enum SurveyType { 25 | state_of_js 26 | state_of_css 27 | } 28 | 29 | 30 | """ 31 | All surveys 32 | """ 33 | type Surveys { 34 | state_of_js: StateOfJSSurvey 35 | state_of_css: StateOfCSSSurvey 36 | } 37 | 38 | type StateOfJSSurvey { 39 | foobar: String 40 | } 41 | 42 | type StateOfCSSSurvey { 43 | foobar: String 44 | } 45 | 46 | """ 47 | A survey 48 | """ 49 | type Survey { 50 | """ 51 | The survey's name 52 | """ 53 | surveyName: SurveyType 54 | """ 55 | Total responses 56 | """ 57 | totals(filters: Filters): Totals 58 | """ 59 | Experience results for a specific tool 60 | """ 61 | tool(id: ToolID!): Tool 62 | """ 63 | Experience results for a range of tools 64 | """ 65 | tools(ids: [ToolID]): [Tool] 66 | """ 67 | Other tools (browsers, editors, etc.) 68 | """ 69 | tools_others(id: OtherToolsID!, filters: Filters): OtherTools 70 | """ 71 | Rankings (awareness, interest, satisfaction) for a range of tools 72 | """ 73 | tools_rankings(ids: [ToolID]!): ToolsRankings 74 | """ 75 | Cardinality By User (by-users tool count breakdown for a specific set of tools and specific criteria) 76 | """ 77 | tools_cardinality_by_user( 78 | year: Int!, 79 | ids: [ToolID]!, 80 | experienceId: ExperienceID! 81 | ): [ToolsCardinalityByUser] 82 | """ 83 | Usage results for a specific feature 84 | """ 85 | feature(id: FeatureID!): Feature 86 | """ 87 | Usage results for a range of features 88 | """ 89 | features(ids: [FeatureID]): [Feature] 90 | """ 91 | Choice based features 92 | """ 93 | features_others(id: OtherFeaturesID!, filters: Filters): OtherFeatures 94 | """ 95 | Demographics data (gender, company size, salary, etc.) 96 | """ 97 | demographics: Demographics 98 | """ 99 | Opinions data 100 | """ 101 | opinion(id: OpinionID!, filters: Filters): Opinion 102 | """ 103 | Opinions data 104 | """ 105 | opinions_others(id: OtherOpinionsID!, filters: Filters): OtherOpinions 106 | """ 107 | Resources (sites, blogs, podcasts, etc.) 108 | """ 109 | resources(id: ResourcesID!, filters: Filters): Resources 110 | """ 111 | Data about a specific tool category 112 | """ 113 | category(id: CategoryID!): Category 114 | """ 115 | Matrices data (used for cross-referencing heatmap charts) 116 | """ 117 | matrices: Matrices 118 | """ 119 | Environments data, for those based on multiple choices, 120 | such as browsers, form factors... Only contain predifined 121 | choices, freeform answers are stored in `environmentsOthers`. 122 | """ 123 | environments(id: EnvironmentID!, filters: Filters): Environments 124 | """ 125 | Environments data, for those based on rating, such as css for emails... 126 | """ 127 | environments_ratings(id: EnvironmentID!, filters: Filters): EnvironmentsRatings 128 | """ 129 | Proficiency data, such as backend proficiency, javascript... 130 | """ 131 | proficiency(id: ProficiencyID!, filters: Filters): Proficiency 132 | """ 133 | Happiness data, either for a specific category or more generally 134 | """ 135 | happiness(id: HappinessID!, filters: Filters): Happiness 136 | """ 137 | Brackets 138 | """ 139 | bracketWins(id: BracketID!, filters: Filters): BracketWins 140 | bracketMatchups(id: BracketID!, filters: Filters): BracketMatchups 141 | } 142 | -------------------------------------------------------------------------------- /src/type_defs/tools.graphql: -------------------------------------------------------------------------------- 1 | enum ToolID { 2 | # JS flavors 3 | typescript 4 | reason 5 | elm 6 | clojurescript 7 | purescript 8 | # JS front end frameworks 9 | react 10 | vuejs 11 | angular 12 | preact 13 | ember 14 | svelte 15 | alpinejs 16 | litelement 17 | stimulus 18 | # JS data layer 19 | redux 20 | apollo 21 | graphql 22 | mobx 23 | relay 24 | xstate 25 | vuex 26 | # JS back end frameworks 27 | express 28 | nextjs 29 | koa 30 | meteor 31 | sails 32 | feathers 33 | nuxt 34 | gatsby 35 | nest 36 | strapi 37 | fastify 38 | hapi 39 | # JS testing 40 | jest 41 | mocha 42 | storybook 43 | cypress 44 | enzyme 45 | ava 46 | jasmine 47 | puppeteer 48 | testing_library 49 | playwright 50 | webdriverio 51 | # JS build tools 52 | webpack 53 | parcel 54 | gulp 55 | rollup 56 | browserify 57 | tsc 58 | rome 59 | snowpack 60 | swc 61 | esbuild 62 | # JS mobile desktop 63 | electron 64 | reactnative 65 | nativeapps 66 | cordova 67 | ionic 68 | nwjs 69 | expo 70 | capacitor 71 | quasar 72 | # CSS Pre-/Post-processors 73 | sass 74 | less 75 | post_css 76 | stylus 77 | assembler_css 78 | # CSS frameworks 79 | bootstrap 80 | materialize_css 81 | ant_design 82 | semantic_ui 83 | bulma 84 | foundation 85 | ui_kit 86 | tachyons 87 | primer 88 | tailwind_css 89 | pure_css 90 | skeleton 91 | spectre_css 92 | halfmoon 93 | # CSS methodologies 94 | bem 95 | atomic_css 96 | oocss 97 | smacss 98 | it_css 99 | cube_css 100 | # CSS in JS 101 | styled_components 102 | jss 103 | styled_jsx 104 | radium 105 | emotion 106 | css_modules 107 | styled_system 108 | stitches 109 | styletron 110 | fela 111 | linaria 112 | astroturf 113 | twin 114 | theme_ui 115 | vanilla_extract 116 | windi_css 117 | } 118 | 119 | """ 120 | An aggregation bucket for tool experience containing both an absolute count 121 | for the parent year, and the percentage it corresponds to regarding 122 | the total number of respondents who have answered the question 123 | in this particular year. 124 | """ 125 | type ToolExperienceBucket { 126 | id: ExperienceID 127 | count: Int 128 | countDelta: Int 129 | percentage: Float 130 | percentageDelta: Float 131 | } 132 | 133 | """ 134 | Experience ranking for a tool in a specific year, even if the data 135 | is computed at the same point in time, we estimate that there is a logical 136 | progression in this: 137 | 138 | awareness > usage > interest > satisfaction 139 | """ 140 | type ToolAwarenessUsageInterestSatisfaction { 141 | """ 142 | Awareness is the total number of participants who answered to 143 | the experience question VS those who never heard of a tool. 144 | 145 | This value is expressed as a percentage. 146 | """ 147 | awareness: Float 148 | """ 149 | Usage is the total number of participants who used the tool, 150 | include both users willing to use it again and those who wouldn't. 151 | 152 | This value is expressed as a percentage. 153 | """ 154 | usage: Float 155 | """ 156 | Interest is the ratio of participants who heard of tool and 157 | are interested/not interested VS those who are only interested in it. 158 | 159 | This value is expressed as a percentage. 160 | """ 161 | interest: Float 162 | """ 163 | Satisfaction is the ratio of participants who used of tool and 164 | are satisfied/not satisfied VS those who are willing to use it again. 165 | 166 | This value is expressed as a percentage. 167 | """ 168 | satisfaction: Float 169 | } 170 | 171 | type ToolYearExperience { 172 | year: Int 173 | """ 174 | Total number of respondents who have answered this specific question. 175 | """ 176 | total: Int 177 | completion: Completion 178 | buckets: [ToolExperienceBucket] 179 | awarenessUsageInterestSatisfaction: ToolAwarenessUsageInterestSatisfaction 180 | } 181 | 182 | type ToolExperience { 183 | all_years: [ToolYearExperience] 184 | year(year: Int!): ToolYearExperience 185 | } 186 | 187 | type ToolExperienceGraphNode { 188 | id: String 189 | year: Int 190 | experience: ExperienceID 191 | } 192 | 193 | """ 194 | Track number of connections between 2 nodes, 195 | for example number of user who were interested in React in 206 196 | and are willing to use it in 2017, connections are only established 197 | for consecutive years. 198 | """ 199 | type ToolExperienceGraphLink { 200 | source: String 201 | target: String 202 | count: Int 203 | } 204 | 205 | """ 206 | A graph of users' experience over years, compared to just computing 207 | the overall choice count for each year, this keeps continuity for each user. 208 | """ 209 | type ToolExperienceGraph { 210 | nodes: [ToolExperienceGraphNode] 211 | links: [ToolExperienceGraphLink] 212 | } 213 | 214 | type Tool { 215 | id: ToolID! 216 | experience(filters: Filters): ToolExperience 217 | experienceGraph(filters: Filters): ToolExperienceGraph 218 | entity: Entity 219 | } 220 | 221 | type ToolExperienceRankingYearMetric { 222 | year: Int 223 | rank: Int 224 | percentage: Float 225 | } 226 | 227 | """ 228 | Used to represent the ranking of a tool compared to others 229 | for awareness/interest and stisfaction. 230 | """ 231 | type ToolExperienceRanking { 232 | id: ToolID 233 | entity: Entity 234 | awareness: [ToolExperienceRankingYearMetric] 235 | usage: [ToolExperienceRankingYearMetric] 236 | interest: [ToolExperienceRankingYearMetric] 237 | satisfaction: [ToolExperienceRankingYearMetric] 238 | } 239 | 240 | """ 241 | Contains various rankings for a set of tools. 242 | """ 243 | type ToolsRankings { 244 | ids: [ToolID]! 245 | experience(filters: Filters): [ToolExperienceRanking] 246 | } 247 | -------------------------------------------------------------------------------- /src/type_defs/tools_cardinality_by_user.graphql: -------------------------------------------------------------------------------- 1 | type ToolsCardinalityByUser { 2 | cardinality: Int 3 | count: Int 4 | """ 5 | Percentage against number of respondents for the related year. 6 | """ 7 | percentage: Float 8 | } -------------------------------------------------------------------------------- /src/type_defs/tools_others.graphql: -------------------------------------------------------------------------------- 1 | enum OtherToolsID { 2 | pre_post_processors_others 3 | css_frameworks_others 4 | css_methodologies_others 5 | css_in_js_others 6 | utilities 7 | utilities_others 8 | text_editors 9 | text_editors_others 10 | browsers 11 | browsers_others 12 | build_tools 13 | build_tools_others 14 | non_js_languages 15 | non_js_languages_others 16 | javascript_flavors_others 17 | front_end_frameworks_others 18 | datalayer_others 19 | back_end_frameworks_others 20 | testing_others 21 | mobile_desktop_others 22 | libraries 23 | libraries_others 24 | runtimes 25 | runtimes_others 26 | } 27 | 28 | type YearOtherTools { 29 | year: Int 30 | """ 31 | Total number of respondents who have answered this specific question. 32 | """ 33 | total: Int 34 | completion: Completion 35 | buckets: [EntityBucket] 36 | } 37 | 38 | type OtherTools { 39 | id: OtherToolsID! 40 | # byYear: [YearResource] 41 | year(year: Int!): YearOtherTools 42 | all_years: [YearOtherTools] 43 | } 44 | -------------------------------------------------------------------------------- /src/type_defs/totals.graphql: -------------------------------------------------------------------------------- 1 | type Totals { 2 | all_years: Int 3 | year(year: Int!): Int 4 | } 5 | -------------------------------------------------------------------------------- /src/type_defs/translations.graphql: -------------------------------------------------------------------------------- 1 | # enum LocaleID { 2 | # en_US 3 | # it_IT 4 | # zh_Hans 5 | # } 6 | 7 | enum Contexts { 8 | projects 9 | features 10 | entities 11 | common 12 | homepage 13 | results 14 | accounts 15 | surveys 16 | 17 | state_of_js 18 | state_of_js_2020 19 | state_of_js_2020_survey 20 | state_of_js_2021 21 | state_of_js_2021_survey 22 | 23 | state_of_css 24 | state_of_css_2020 25 | state_of_css_2020_survey 26 | state_of_css_2021 27 | state_of_css_2021_survey 28 | } 29 | 30 | type TranslationString { 31 | key: String 32 | t: String 33 | tHtml: String 34 | context: String 35 | fallback: Boolean 36 | } 37 | 38 | type Locale { 39 | id: String 40 | label: String 41 | translators: [String] 42 | repo: String 43 | strings: [TranslationString] 44 | translatedCount: Int 45 | totalCount: Int 46 | completion: Int 47 | untranslatedKeys: [String] 48 | } 49 | -------------------------------------------------------------------------------- /src/type_defs/twitter.graphql: -------------------------------------------------------------------------------- 1 | type Twitter { 2 | userName: String 3 | avatarUrl: String 4 | id: Int 5 | description: String 6 | publicMetrics: TwittterPublicMetrics 7 | } 8 | 9 | type TwittterPublicMetrics { 10 | followers: Int 11 | following: Int 12 | tweet: Int 13 | listed: Int 14 | } 15 | -------------------------------------------------------------------------------- /src/type_defs/usage.graphql: -------------------------------------------------------------------------------- 1 | enum UsageID { 2 | used_it 3 | know_not_used 4 | never_heard_not_sure 5 | } 6 | -------------------------------------------------------------------------------- /src/types/demographics.ts: -------------------------------------------------------------------------------- 1 | export interface YearParticipation { 2 | year: number 3 | participants: number 4 | } 5 | 6 | export interface Participation { 7 | all_years: YearParticipation[] 8 | year: YearParticipation 9 | } 10 | -------------------------------------------------------------------------------- /src/types/entity.ts: -------------------------------------------------------------------------------- 1 | import { GitHub } from './github' 2 | import { MDN } from './mdn' 3 | 4 | export interface Entity { 5 | id: string 6 | aliases?: string[] 7 | name: string 8 | otherName: string 9 | twitterName: string 10 | homepage?: string 11 | category?: string 12 | description?: string 13 | tags?: string[] 14 | match?: string[] 15 | github?: GitHub 16 | npm?: string 17 | type?: string 18 | mdn?: string 19 | patterns?: string[] 20 | } 21 | 22 | export interface EntityBucket { 23 | id: string 24 | count: number 25 | percentage: number 26 | entity: Entity 27 | } 28 | -------------------------------------------------------------------------------- /src/types/features.ts: -------------------------------------------------------------------------------- 1 | import { Completion } from './index' 2 | import { MDN } from './mdn' 3 | 4 | export interface FeatureBucket { 5 | id: string 6 | name: string 7 | count: number 8 | percentage: number 9 | } 10 | 11 | export interface YearFeature { 12 | year: number 13 | total: number 14 | completion: Completion 15 | buckets: FeatureBucket[] 16 | } 17 | 18 | export interface FeatureExperienceBucket { 19 | id: string 20 | count: number 21 | percentage: number 22 | } 23 | 24 | export interface FeatureExperience { 25 | all_years: YearFeature[] 26 | year: YearFeature 27 | } 28 | 29 | export interface Feature { 30 | id: string 31 | name: string 32 | mdn: MDN 33 | experience: FeatureExperience 34 | } 35 | -------------------------------------------------------------------------------- /src/types/github.ts: -------------------------------------------------------------------------------- 1 | export interface GitHub { 2 | id: string 3 | name: string 4 | full_name?: string 5 | description: string 6 | url: string 7 | stars: number 8 | forks?: number 9 | opened_issues?: number 10 | homepage: string 11 | } 12 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | import { Db } from 'mongodb' 2 | import { SurveyType } from './surveys' 3 | import { Filters } from '../filters' 4 | 5 | /** 6 | * This context is injected in each and every requests. 7 | */ 8 | export interface RequestContext { 9 | db: Db 10 | } 11 | 12 | export interface SurveyConfig { 13 | survey: SurveyType 14 | } 15 | 16 | export interface ResolverStaticConfig { 17 | survey: SurveyConfig 18 | filters?: Filters 19 | } 20 | 21 | export interface ResolverDynamicConfig { 22 | survey: SurveyConfig 23 | id: string 24 | filters?: Filters 25 | } 26 | 27 | export * from './demographics' 28 | export * from './entity' 29 | export * from './features' 30 | export * from './github' 31 | export * from './schema' 32 | export * from './surveys' 33 | export * from './tools' 34 | export * from './locale' 35 | -------------------------------------------------------------------------------- /src/types/locale.ts: -------------------------------------------------------------------------------- 1 | export interface Locale { 2 | id: string 3 | label: string 4 | stringFiles: StringFile[] 5 | translators?: string[] 6 | repo: string 7 | } 8 | 9 | export interface StringFile { 10 | strings: TranslationStringObject[] 11 | context: string 12 | prefix?: string 13 | } 14 | 15 | export interface TranslationStringObject { 16 | key: string 17 | t: string 18 | tHtml: string 19 | context: string 20 | fallback: Boolean 21 | } 22 | -------------------------------------------------------------------------------- /src/types/mdn.ts: -------------------------------------------------------------------------------- 1 | export interface MDN { 2 | locale: string 3 | url: string 4 | title: string 5 | summary: string 6 | } 7 | -------------------------------------------------------------------------------- /src/types/schema.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Used to represent survey question completion. 3 | */ 4 | export interface Completion { 5 | // total number of participants 6 | total: number 7 | // current number of respondents 8 | count: number 9 | // percentage of respondents for a question 10 | // compared to the total number of participants 11 | percentage: number 12 | } 13 | -------------------------------------------------------------------------------- /src/types/surveys.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The various types of surveys supported by the API 3 | */ 4 | export type SurveyType = 'js' | 'css' 5 | 6 | export type SurveyYear = 2016 | 2017 | 2018 | 2019 | 2020 | 2021 7 | 8 | export type SurveySlug = 9 | | 'css2019' 10 | | 'css2020' 11 | | 'css2021' 12 | | 'js2016' 13 | | 'js2017' 14 | | 'js2018' 15 | | 'js2019' 16 | | 'js2020' 17 | | 'js2021' 18 | -------------------------------------------------------------------------------- /src/types/tools.ts: -------------------------------------------------------------------------------- 1 | import { Completion } from './index' 2 | import { Entity } from './entity' 3 | 4 | export interface ToolExperienceBucket { 5 | id: string 6 | count: number 7 | percentage: number 8 | } 9 | 10 | export interface ToolAwarenessUsageInterestSatisfaction { 11 | awareness: number 12 | usage: number 13 | interest: number 14 | satisfaction: number 15 | } 16 | 17 | export interface ToolYearExperience { 18 | year: number 19 | total: number 20 | completion: Completion 21 | buckets: ToolExperienceBucket[] 22 | awarenessUsageInterestSatisfaction: ToolAwarenessUsageInterestSatisfaction 23 | } 24 | 25 | export interface ToolExperience { 26 | all_years: ToolYearExperience[] 27 | year: ToolYearExperience 28 | } 29 | 30 | export interface Tool { 31 | id: string 32 | experience: ToolExperience 33 | entity: Entity 34 | } 35 | 36 | export interface ToolExperienceRankingYearMetric { 37 | year: number 38 | rank: number 39 | percentage: number 40 | } 41 | 42 | export interface ToolExperienceRanking { 43 | id: string 44 | entity: Entity 45 | awareness: ToolExperienceRankingYearMetric[] 46 | interest: ToolExperienceRankingYearMetric[] 47 | satisfaction: ToolExperienceRankingYearMetric[] 48 | } 49 | 50 | export interface ToolsRankings { 51 | ids: string[] 52 | experience: ToolExperienceRanking[] 53 | } 54 | -------------------------------------------------------------------------------- /src/types/twitter.ts: -------------------------------------------------------------------------------- 1 | import { SurveySlug } from './surveys' 2 | 3 | export interface TwitterStat { 4 | twitterId: string 5 | twitterName: string 6 | surveySlug: SurveySlug 7 | followings: string[] 8 | followingsSubset: string[] 9 | followersCount?: number 10 | followingCount?: number 11 | tweetCount?: number 12 | listedCount?: number 13 | } 14 | -------------------------------------------------------------------------------- /stateofjs-api.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | } 6 | ], 7 | "settings": { 8 | } 9 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2017", 4 | "module": "commonjs", 5 | "lib": ["es7", "es2019"], 6 | "outDir": "dist", 7 | "rootDir": "src", 8 | "removeComments": true, 9 | "strict": true, 10 | "esModuleInterop": true, 11 | "experimentalDecorators": true, 12 | "noImplicitAny": true 13 | }, 14 | "files": ["file_types.d.ts"], 15 | "exclude": ["/node_modules/"], 16 | "include": ["src/**/*"] 17 | } -------------------------------------------------------------------------------- /webpack.common.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const nodeExternals = require('webpack-node-externals') 3 | const WriteFilePlugin = require('write-file-webpack-plugin') 4 | const CopyWebpackPlugin = require('copy-webpack-plugin') 5 | 6 | module.exports = { 7 | entry: { 8 | server: path.join(__dirname, 'src/server.ts'), 9 | standalone: path.join(__dirname, 'src/standalone.ts') 10 | }, 11 | module: { 12 | rules: [ 13 | { 14 | test: /\.yml$/, 15 | exclude: /node_modules/, 16 | use: 'js-yaml-loader' 17 | }, 18 | { 19 | test: /\.graphql$/, 20 | exclude: /node_modules/, 21 | use: 'graphql-tag/loader' 22 | }, 23 | { 24 | test: /\.ts$/, 25 | exclude: /node_modules/, 26 | use: 'ts-loader' 27 | } 28 | ] 29 | }, 30 | output: { 31 | filename: '[name].js', 32 | path: path.resolve(__dirname, 'dist') 33 | }, 34 | resolve: { 35 | extensions: ['.ts', '.js', '.yml', '.graphql'] 36 | }, 37 | externals: [nodeExternals({})], 38 | target: 'node', 39 | node: false, 40 | plugins: [ 41 | new WriteFilePlugin(), 42 | new CopyWebpackPlugin({ patterns: [{ from: path.join(__dirname, 'public'), to: path.join(__dirname, 'dist/public') }] }) 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /webpack.development.js: -------------------------------------------------------------------------------- 1 | const { CleanWebpackPlugin } = require('clean-webpack-plugin') 2 | const NodemonPlugin = require('nodemon-webpack-plugin') 3 | 4 | const common = require('./webpack.common.js') 5 | 6 | module.exports = { 7 | ...common, 8 | devtool: 'inline-source-map', 9 | mode: 'development', 10 | plugins: [new CleanWebpackPlugin(), new NodemonPlugin()], 11 | watch: true, 12 | } 13 | -------------------------------------------------------------------------------- /webpack.production.js: -------------------------------------------------------------------------------- 1 | const { CleanWebpackPlugin } = require('clean-webpack-plugin') 2 | 3 | const common = require('./webpack.common.js') 4 | 5 | module.exports = { 6 | ...common, 7 | devtool: 'source-map', 8 | mode: 'production', 9 | plugins: [new CleanWebpackPlugin()], 10 | } 11 | --------------------------------------------------------------------------------