├── client ├── .eslintignore ├── config │ ├── spa.config.d.ts │ ├── verifySpaParameters.js │ └── spa.config.js ├── .vscode │ ├── settings.json │ ├── tasks.json │ └── launch.json ├── src │ ├── entrypoints │ │ └── app.tsx │ ├── utils │ │ ├── logger.ts │ │ ├── typeguards.ts │ │ ├── error.ts │ │ ├── csv.ts │ │ ├── cache.ts │ │ ├── fetch.ts │ │ └── filesystem.ts │ ├── state │ │ ├── store.ts │ │ ├── reducers.ts │ │ └── actions.ts │ ├── components │ │ ├── QueryPageContainer.tsx │ │ ├── QueryPaginationContainer.tsx │ │ ├── QueryStatus.tsx │ │ ├── ErrorBoundary.tsx │ │ ├── QueryPagination.tsx │ │ ├── QueryTable.tsx │ │ ├── QueryPage.tsx │ │ ├── ExportCsv.tsx │ │ └── QueryInput.tsx │ ├── test │ │ ├── BackendRequest.test.ts │ │ └── QueryPage.test.tsx │ └── api │ │ ├── BackendRequest.ts │ │ └── BackendManager.ts ├── .eslintrc.js ├── jest.config.js ├── tsconfig.json ├── package.json └── webpack.config.js ├── server ├── .eslintignore ├── pub │ ├── fav.ico │ └── apple-touch-icon.png ├── .env ├── jest.config.js ├── .vscode │ ├── settings.json │ ├── tasks.json │ └── launch.json ├── .eslintrc.js ├── src │ ├── srv │ │ ├── main.ts │ │ └── server.ts │ ├── utils │ │ ├── logger.ts │ │ ├── storage.ts │ │ ├── misc.ts │ │ └── error.ts │ ├── test │ │ ├── BigQueryModel.mock.test.ts │ │ ├── TestConfig.ts │ │ ├── server.test.ts │ │ ├── BigQueryTypes.test.ts │ │ └── BigQueryModel.test.ts │ └── api │ │ ├── controllers │ │ └── BigQueryController.ts │ │ └── types │ │ └── BigQueryTypes.ts ├── tsconfig.json └── package.json ├── key.json.enc ├── yarn.lock ├── docs ├── images │ ├── crisp-bigquery.png │ └── crisp-bigquery-article.jpg └── screenshots │ ├── screenshot1.jpg │ ├── screenshot2.jpg │ ├── screenshot3.jpg │ ├── screenshot4.jpg │ └── screenshot5.jpg ├── .dockerignore ├── .travis.yml ├── crisp-bigquery.code-workspace ├── .github └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── start-container.sh ├── start-container.cmd ├── Dockerfile ├── LICENSE ├── .gitignore └── package.json /client/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /server/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | pub 4 | -------------------------------------------------------------------------------- /key.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/key.json.enc -------------------------------------------------------------------------------- /server/pub/fav.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/server/pub/fav.ico -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/images/crisp-bigquery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/images/crisp-bigquery.png -------------------------------------------------------------------------------- /docs/screenshots/screenshot1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/screenshots/screenshot1.jpg -------------------------------------------------------------------------------- /docs/screenshots/screenshot2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/screenshots/screenshot2.jpg -------------------------------------------------------------------------------- /docs/screenshots/screenshot3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/screenshots/screenshot3.jpg -------------------------------------------------------------------------------- /docs/screenshots/screenshot4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/screenshots/screenshot4.jpg -------------------------------------------------------------------------------- /docs/screenshots/screenshot5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/screenshots/screenshot5.jpg -------------------------------------------------------------------------------- /server/.env: -------------------------------------------------------------------------------- 1 | GCP_PROJECT_ID= 2 | BIGQUERY_DATASET_NAME=samples 3 | BIGQUERY_TABLE_NAME=github 4 | KEY_FILE_PATH=../key.json -------------------------------------------------------------------------------- /server/pub/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/server/pub/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/images/crisp-bigquery-article.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winwiz1/crisp-bigquery/HEAD/docs/images/crisp-bigquery-article.jpg -------------------------------------------------------------------------------- /client/config/spa.config.d.ts: -------------------------------------------------------------------------------- 1 | export const appTitle: string 2 | export function getRedirectName(): string 3 | export function getNames(): ReadonlyArray 4 | 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | .github/ 3 | .vscode/ 4 | docs/ 5 | build/ 6 | dist/ 7 | server/config/ 8 | node_modules/ 9 | README.md 10 | .gitignore 11 | .travis.yml 12 | crisp-react.code-workspace 13 | start-container-windows.cmd 14 | bookmarks.json 15 | logs/ 16 | *.log 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | branches: 2 | only: 3 | - master 4 | cache: 5 | directories: 6 | - client/node_modules 7 | - server/node_modules 8 | yarn: true 9 | dist: bionic 10 | git: 11 | depth: 3 12 | language: node_js 13 | node_js: 14 | - 14.15.0 15 | before_install: 16 | - openssl aes-256-cbc -K $encrypted_81dd31eb889f_key -iv $encrypted_81dd31eb889f_iv 17 | -in key.json.enc -out ./key.json -d 18 | -------------------------------------------------------------------------------- /client/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules\\typescript\\lib", 3 | "typescript.surveys.enabled": false, 4 | "workbench.colorCustomizations": { 5 | "titleBar.activeBackground": "#0a4705", 6 | "titleBar.activeForeground": "#ffffff96", 7 | "titleBar.inactiveBackground": "#3d3939", 8 | "titleBar.inactiveForeground": "#cccccc49" 9 | }, 10 | "editor.tabSize": 2, 11 | } 12 | -------------------------------------------------------------------------------- /server/jest.config.js: -------------------------------------------------------------------------------- 1 | const {defaults} = require('jest-config'); 2 | 3 | module.exports = { 4 | preset: 'ts-jest', 5 | testEnvironment: 'node', 6 | transform: { 7 | '^.+\\.ts$': 'ts-jest', 8 | }, 9 | testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.ts$', 10 | moduleFileExtensions: [...defaults.moduleFileExtensions, 'ts'], 11 | roots: [ "/src" ], 12 | setupFiles: ["dotenv/config"], 13 | }; -------------------------------------------------------------------------------- /server/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "explorer.confirmDelete": false, 3 | "typescript.tsdk": "node_modules\\typescript\\lib", 4 | "typescript.surveys.enabled": false, 5 | "workbench.colorCustomizations": { 6 | "titleBar.activeBackground": "#16075c", 7 | "titleBar.activeForeground": "#ffffff96", 8 | "titleBar.inactiveBackground": "#3d3939", 9 | "titleBar.inactiveForeground": "#cccccc49" 10 | }, 11 | "editor.tabSize": 2, 12 | } 13 | -------------------------------------------------------------------------------- /crisp-bigquery.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "server" 5 | }, 6 | { 7 | "path": "client" 8 | } 9 | ], 10 | "settings": { 11 | 12 | }, 13 | "launch" : { 14 | "configurations": [ ], 15 | "compounds": [{ 16 | "name": "Debug Client and Backend", 17 | "configurations": [ 18 | "Launch Chrome Connected to Backend", 19 | "Debug Backend Connected To Devserver" 20 | ] 21 | }, 22 | { 23 | "name": "Debug Production Client and Backend", 24 | "configurations": [ 25 | "Launch Chrome in Production", 26 | "Debug Backend" 27 | ] 28 | }, 29 | ] 30 | } 31 | } -------------------------------------------------------------------------------- /server/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": false, 4 | "amd": false, 5 | "node": true 6 | }, 7 | root: true, 8 | parser: "@typescript-eslint/parser", 9 | plugins: [ 10 | "@typescript-eslint", 11 | ], 12 | extends: [ 13 | "eslint:recommended", 14 | "plugin:@typescript-eslint/recommended", 15 | ], 16 | rules: { 17 | "@typescript-eslint/no-var-requires": 0, 18 | "@typescript-eslint/no-non-null-assertion": 0, 19 | "@typescript-eslint/no-unused-vars": 0, 20 | "@typescript-eslint/no-explicit-any": 0, 21 | "@typescript-eslint/no-inferrable-types": 0, 22 | } 23 | }; 24 | -------------------------------------------------------------------------------- /server/src/srv/main.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Code responsible for Express instantiation. 3 | */ 4 | import Server from "./server"; 5 | import { logger } from "../utils/logger"; 6 | import { getListeningPort } from "../utils/misc"; 7 | 8 | const port = getListeningPort(); 9 | const server = Server(); 10 | 11 | const instance = server.listen(port, () => { 12 | // Use warning to make restarts visible 13 | logger.warn({ message: `Server started, port: ${port}` }); 14 | }); 15 | 16 | process.on("SIGTERM", function () { 17 | instance.close(function () { 18 | logger.warn({ message: "Server terminated" }); 19 | process.exit(0); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /client/src/entrypoints/app.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import * as ReactDOM from "react-dom"; 3 | import { Provider } from "react-redux"; 4 | import { Helmet } from "react-helmet"; 5 | import { QueryPageContainer } from "../components/QueryPageContainer"; 6 | import { rootStore } from "../state/store"; 7 | import { ErrorBoundary } from "../components/ErrorBoundary"; 8 | import * as SPAs from "../../config/spa.config"; 9 | 10 | ReactDOM.render( 11 | 12 | 13 | 14 | 15 | 16 | , 17 | document.getElementById("app-root") 18 | ); 19 | -------------------------------------------------------------------------------- /client/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": true, 4 | "amd": false, 5 | "node": false 6 | }, 7 | root: true, 8 | parser: "@typescript-eslint/parser", 9 | plugins: [ 10 | "@typescript-eslint", 11 | ], 12 | extends: [ 13 | "eslint:recommended", 14 | "plugin:@typescript-eslint/recommended", 15 | ], 16 | rules: { 17 | "@typescript-eslint/no-var-requires": 0, 18 | "@typescript-eslint/no-non-null-assertion": 0, 19 | "@typescript-eslint/no-unused-vars": 0, 20 | "@typescript-eslint/no-explicit-any": 0, 21 | "@typescript-eslint/no-inferrable-types": 0, 22 | "@typescript-eslint/ban-ts-comment": 0, 23 | } 24 | }; 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: winwiz1 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /start-container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Uncomment the next two lines if running this script from empty directory 3 | # git clone https://github.com/winwiz1/crisp-bigquery.git 4 | # cd crisp-bigquery 5 | HOST_PORT=3000 6 | HOST_ADDRESS=127.0.0.1 7 | docker rmi crisp-bigquery:localbuild 2>/dev/null 8 | docker build -t crisp-bigquery:localbuild . || { echo 'Failed to build image' ; exit 2; } 9 | docker stop crisp-bigquery 2>/dev/null 10 | docker rm crisp-bigquery 2>/dev/null 11 | docker run -d --name=crisp-bigquery -p ${HOST_PORT}:3000 --env-file ./server/.env crisp-bigquery:localbuild || { echo 'Failed to run container' ; exit 1; } 12 | echo 'Finished' && docker ps -f name=crisp-bigquery 13 | # xdg-open http://${HOST_ADDRESS}:${HOST_PORT} & 14 | -------------------------------------------------------------------------------- /client/jest.config.js: -------------------------------------------------------------------------------- 1 | const { defaults } = require("jest-config"); 2 | 3 | module.exports = { 4 | preset: "ts-jest", 5 | testEnvironment: "jsdom", 6 | transform: { 7 | "^.+\\.tsx?$": "ts-jest", 8 | }, 9 | testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$", 10 | moduleFileExtensions: [...defaults.moduleFileExtensions, "ts", "tsx"], 11 | roots: ["/src"], 12 | setupFilesAfterEnv: [ 13 | // not needed anymore 14 | //"@testing-library/react/cleanup-after-each", 15 | "@testing-library/jest-dom/extend-expect" 16 | ], 17 | moduleNameMapper: { 18 | "\\.(css|less|scss|sss|styl)$": "/node_modules/jest-css-modules", 19 | '^@backend/(.*)$': '/../server/src/api/$1', 20 | }, 21 | }; 22 | -------------------------------------------------------------------------------- /client/src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import * as log from "loglevel"; 2 | import * as SPAs from "../../config/spa.config"; 3 | 4 | export class Log { 5 | private readonly m_title = SPAs.appTitle; 6 | private readonly m_logger = log.getLogger(this.m_title); 7 | 8 | public trace(message: string): void { 9 | return this.m_logger.trace(message); 10 | } 11 | 12 | public debug(message: string): void { 13 | return this.m_logger.debug(message); 14 | } 15 | 16 | public info(message: string): void { 17 | return this.m_logger.info(message); 18 | } 19 | 20 | public warn(message: string): void { 21 | return this.m_logger.warn(message); 22 | } 23 | 24 | public error(message: string): void { 25 | return this.m_logger.error(message); 26 | } 27 | } 28 | 29 | export default new Log(); 30 | -------------------------------------------------------------------------------- /start-container.cmd: -------------------------------------------------------------------------------- 1 | @echo OFF 2 | rem Uncomment the next two lines if running this file from empty directory 3 | rem git clone https://github.com/winwiz1/crisp-bigquery.git 4 | rem cd crisp-bigquery 5 | setlocal 6 | set HOST_PORT=3000 7 | set HOST_ADDRESS=127.0.0.1 8 | docker rmi crisp-bigquery:localbuild 2>nul 9 | docker build -t crisp-bigquery:localbuild . 10 | if ERRORLEVEL 1 echo Failed to build image && exit /b 2 11 | docker stop crisp-bigquery 2>nul 12 | docker rm crisp-bigquery 2>nul 13 | docker run -d --name=crisp-bigquery -p %HOST_PORT%:3000 --env-file ./server/.env crisp-bigquery:localbuild 14 | if ERRORLEVEL 1 echo Failed to run container && exit /b 1 15 | echo Finished && docker ps -f name=crisp-bigquery 16 | rem Uncomment the next line if Chrome is installed 17 | rem start chrome http://%HOST_ADDRESS%:%HOST_PORT% 18 | -------------------------------------------------------------------------------- /client/src/utils/typeguards.ts: -------------------------------------------------------------------------------- 1 | import { CustomError } from "./error"; 2 | import { BigQueryRetrievalResult } from "../api/BackendManager"; 3 | 4 | // eslint-disable-next-line 5 | export function isError(err: any): err is Error { 6 | return !!err && err instanceof Error && err.constructor !== CustomError; 7 | } 8 | 9 | // eslint-disable-next-line 10 | export function isCustomError(err: any): err is CustomError { 11 | return !!err && err.constructor === CustomError; 12 | } 13 | 14 | // eslint-disable-next-line 15 | export function isDOMException(err: any): err is DOMException { 16 | return !!err && err.constructor === DOMException; 17 | } 18 | 19 | // eslint-disable-next-line 20 | export function isBigQueryRetrievalResult(x: any): x is BigQueryRetrievalResult { 21 | return !!x && x instanceof BigQueryRetrievalResult; 22 | } 23 | 24 | // eslint-disable-next-line 25 | export function isString(x: any): x is string { 26 | return typeof x === "string" && x.length > 0; 27 | } 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14.15.0-slim as build 2 | RUN apt-get update -qq && apt-get upgrade -qq \ 3 | && apt-get clean autoclean && apt-get autoremove -y \ 4 | && rm -rf \ 5 | /var/lib/cache \ 6 | /var/lib/log 7 | 8 | WORKDIR /crisp-bigquery/server 9 | COPY --chown=node:node ./server/ . 10 | RUN yarn 11 | WORKDIR /crisp-bigquery/client 12 | COPY --chown=node:node ./client/ . 13 | COPY --chown=node:node ./.env . 14 | RUN yarn && yarn build:prod 15 | 16 | FROM build as prod 17 | 18 | WORKDIR /crisp-bigquery 19 | COPY --chown=node:node ./key.json . 20 | WORKDIR /crisp-bigquery/server 21 | COPY --chown=node:node ./server/ . 22 | COPY --from=build --chown=node:node /crisp-bigquery/client/config/ /crisp-bigquery/server/config/ 23 | RUN yarn && yarn compile 24 | 25 | COPY --from=build --chown=node:node /crisp-bigquery/client/dist/ /crisp-bigquery/server/build/client/static/ 26 | 27 | EXPOSE 3000 28 | ENV NODE_ENV=production 29 | STOPSIGNAL SIGTERM 30 | 31 | USER node 32 | CMD ["node", "./build/srv/main.js"] 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: winwiz1 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | - NodeJS version 31 | 32 | **Smartphone (please complete the following information):** 33 | - Device: [e.g. iPhone6] 34 | - OS: [e.g. iOS8.1] 35 | - Browser [e.g. stock browser, safari] 36 | - Version [e.g. 22] 37 | 38 | **Additional context** 39 | Add any other context about the problem here. 40 | -------------------------------------------------------------------------------- /server/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "tsc", 8 | "type": "typescript", 9 | "tsconfig": "tsconfig.json", 10 | "problemMatcher": [ 11 | "$tsc" 12 | ], 13 | "group": { 14 | "kind": "build", 15 | "isDefault": true 16 | } 17 | }, 18 | { 19 | "label": "tsc-watch", 20 | "type": "typescript", 21 | "tsconfig": "tsconfig.json", 22 | "option": "watch", 23 | "problemMatcher": [ 24 | "$tsc-watch" 25 | ], 26 | "isBackground": true, 27 | "presentation": { 28 | "echo": true, 29 | "reveal": "always", 30 | "focus": false, 31 | "panel": "new" 32 | }, 33 | }, 34 | { 35 | "label": "kill process in terminal", 36 | "type": "process", 37 | "command": "${command:workbench.action.terminal.kill}" 38 | }, 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 winwiz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /server/src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The Logger. 3 | * Prints logs (errors only by default) on console and optionally 4 | * into a file. 5 | */ 6 | import * as Winston from "winston"; 7 | import { isGoogleCloudRun, isDocker } from "./misc"; 8 | 9 | const logFileName = "server.log"; 10 | const isCloudRun = isGoogleCloudRun(); 11 | const isContainer = isDocker(); 12 | const logDestinations: Winston.LoggerOptions["transports"] = 13 | [ 14 | // On Cloud Run the console output can go into Stackdriver 15 | // Logging that could be automatically exported to BigQuery 16 | new (Winston.transports.Console)(), 17 | ]; 18 | 19 | if (!isCloudRun && !isContainer) { 20 | // On Cloud Run writing into a disk file reduces the available memory 21 | logDestinations.push(new (Winston.transports.File)({ filename: logFileName })); 22 | } 23 | 24 | export const logger = Winston.createLogger({ 25 | format: (isCloudRun || isContainer)? Winston.format.combine( 26 | Winston.format.splat(), 27 | Winston.format.simple() 28 | ) : Winston.format.json({ replacer: undefined, space: 3 }), 29 | level: "warn", 30 | transports: logDestinations 31 | }); 32 | -------------------------------------------------------------------------------- /client/config/verifySpaParameters.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | // Verify SPA configuration options 4 | function verifySpaParameters(parameters, index) { 5 | const params = parameters || {}; 6 | 7 | function invalidParameter(paramName, missing) { 8 | throw new Error( 9 | `SPA#${index} configuration parameter '${paramName}' is ${missing ? 10 | "missing or has incorrect type" : "invalid"}` 11 | ); 12 | } 13 | 14 | const ret = { 15 | /* eslint-disable */ 16 | name, 17 | entryPoint, 18 | redirect 19 | /* eslint-enable */ 20 | } = params; 21 | 22 | if (typeof ret.name !== "string") { 23 | invalidParameter("name", true); 24 | } 25 | 26 | if (!/^\w+$/.test(ret.name)) { 27 | invalidParameter("name", false); 28 | } 29 | 30 | if (typeof ret.entryPoint !== "string") { 31 | invalidParameter("entryPoint", true); 32 | } 33 | 34 | if (!fs.existsSync(ret.entryPoint)) { 35 | invalidParameter("entryPoint", false); 36 | } 37 | 38 | if (typeof ret.redirect !== "boolean") { 39 | invalidParameter("redirect", true); 40 | } 41 | 42 | return ret; 43 | } 44 | 45 | module.exports = verifySpaParameters; 46 | -------------------------------------------------------------------------------- /server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": [ 3 | "src/**/*.ts", 4 | "src/**/*.tsx" 5 | ], 6 | "exclude": [ 7 | "node_modules", 8 | "**/*.spec.ts" 9 | ], 10 | "compilerOptions": { 11 | "outDir": "./build/", 12 | "allowSyntheticDefaultImports": false, 13 | "target": "es2017", 14 | "downlevelIteration": true, 15 | "importHelpers": true, 16 | "sourceMap": true, 17 | "module": "commonjs", 18 | "moduleResolution": "node", 19 | "esModuleInterop": false, 20 | "jsx": "preserve", 21 | "allowJs": false, 22 | "checkJs": false, 23 | "lib": [ 24 | "es2017" 25 | ], 26 | "forceConsistentCasingInFileNames": true, 27 | "experimentalDecorators": true, 28 | "noImplicitAny": true, 29 | "noUnusedParameters": true, 30 | "noImplicitReturns": true, 31 | "noImplicitThis": true, 32 | "noUnusedLocals": true, 33 | "skipLibCheck": true, 34 | "strictNullChecks": true, 35 | "strictPropertyInitialization": true, 36 | "suppressImplicitAnyIndexErrors": false, 37 | "removeComments": true, 38 | "typeRoots" : ["./node_modules/@types"] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | key.json 8 | 9 | # Runtime data 10 | pids 11 | *.pid 12 | *.seed 13 | *.pid.lock 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # nyc test coverage 22 | .nyc_output 23 | 24 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # Bower dependency directory (https://bower.io/) 28 | bower_components 29 | 30 | # node-waf configuration 31 | .lock-wscript 32 | 33 | # Build artefacts 34 | build/ 35 | dist/ 36 | server/config/ 37 | 38 | # Dependency directories 39 | node_modules/ 40 | jspm_packages/ 41 | 42 | # TypeScript v1 declaration files 43 | typings/ 44 | 45 | # Optional npm cache directory 46 | .npm 47 | 48 | # Optional eslint cache 49 | .eslintcache 50 | 51 | # Optional REPL history 52 | .node_repl_history 53 | 54 | # Output of 'npm pack' 55 | *.tgz 56 | 57 | # Yarn Integrity file 58 | .yarn-integrity 59 | 60 | # dotenv environment variables file 61 | # .env 62 | 63 | # next.js build output 64 | .next 65 | 66 | #VS Code Bookmarks extension 67 | bookmarks.json 68 | -------------------------------------------------------------------------------- /client/src/state/store.ts: -------------------------------------------------------------------------------- 1 | /* 2 | The Redux store. Also called state tree. 3 | */ 4 | 5 | import { combineReducers, createStore } from "redux"; 6 | import { allReducers } from "./reducers"; 7 | 8 | // Slice of state tree related to data fetching from the server 9 | export interface IFetchState { 10 | // If 'true' then backend data request is inflight 11 | inFlight: boolean; 12 | // Current cache page 13 | currentPage: number; 14 | } 15 | 16 | // Slice of state tree reserved for future 17 | // eslint-disable-next-line @typescript-eslint/no-empty-interface 18 | export interface IOtherState { 19 | } 20 | 21 | // The initial object for the fetch slice of the state tree 22 | export const initialFetchState: IFetchState = { 23 | currentPage: 0, 24 | inFlight: false 25 | }; 26 | 27 | // The initial object for the other slice of the state tree 28 | export const ininitialOtherState: IOtherState = { 29 | }; 30 | 31 | // The root reducer used to create Redux store 32 | const rootReducer = combineReducers({ 33 | fetch: allReducers.fetchReducer, 34 | other: allReducers.otherReducer 35 | }); 36 | export type RootState = ReturnType; 37 | 38 | // Our final goal: the store 39 | export const rootStore = createStore(rootReducer); 40 | -------------------------------------------------------------------------------- /client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": [ 3 | "src/**/*.ts", 4 | "src/**/*.tsx" 5 | ], 6 | "exclude": [ 7 | "node_modules", 8 | "dist", 9 | "**/*.spec.ts" 10 | ], 11 | "compilerOptions": { 12 | "outDir": "./dist/js/", 13 | "baseUrl": ".", 14 | "paths": { 15 | "@backend/*": ["../server/src/api/*"], 16 | }, 17 | "allowSyntheticDefaultImports": false, 18 | "target": "es5", 19 | "downlevelIteration": true, 20 | "importHelpers": true, 21 | "sourceMap": true, 22 | "module": "esnext", 23 | "moduleResolution": "node", 24 | "jsx": "react", 25 | "allowJs": false, 26 | "checkJs": false, 27 | "lib": [ 28 | "dom", 29 | "esnext" 30 | ], 31 | "forceConsistentCasingInFileNames": true, 32 | "experimentalDecorators": true, 33 | "noImplicitAny": true, 34 | "noUnusedParameters": true, 35 | "noImplicitReturns": true, 36 | "noImplicitThis": true, 37 | "noUnusedLocals": true, 38 | "skipLibCheck": true, 39 | "strictNullChecks": true, 40 | "suppressImplicitAnyIndexErrors": false, 41 | "removeComments": false, 42 | "typeRoots": [ 43 | "./node_modules/@types" 44 | ], 45 | "esModuleInterop": false 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /client/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "npm build", 8 | "type": "npm", 9 | "script": "build", 10 | "problemMatcher": [] 11 | }, 12 | { 13 | "label": "npm build:prod-debug", 14 | "type": "npm", 15 | "script": "build:prod-debug", 16 | "problemMatcher": [] 17 | }, 18 | { 19 | "label": "npm compile", 20 | "type": "npm", 21 | "script": "compile", 22 | "problemMatcher": [] 23 | }, 24 | { 25 | "label": "npm dev", 26 | "type": "npm", 27 | "script": "dev", 28 | "isBackground": true, 29 | "problemMatcher": { 30 | "owner": "custom", 31 | "pattern": { 32 | "regexp": "^$" 33 | }, 34 | "background": { 35 | "activeOnStart": true, 36 | "beginsPattern": "Using\\s+\\d+\\s+worker.*]", 37 | "endsPattern": "Time:\\s+\\d+ms" 38 | } 39 | } 40 | }, 41 | { 42 | "label": "kill process in terminal", 43 | "type": "process", 44 | "command": "${command:workbench.action.terminal.kill}" 45 | }, 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /client/src/state/reducers.ts: -------------------------------------------------------------------------------- 1 | import { Reducer } from "redux"; 2 | import { ActionTypes, AllActions } from "./actions"; 3 | import { 4 | IFetchState, 5 | IOtherState, 6 | initialFetchState, 7 | ininitialOtherState 8 | } from "./store"; 9 | 10 | /* 11 | The reducer 12 | */ 13 | const fetchReducer: Reducer = 14 | (state: IFetchState = initialFetchState, action: AllActions) => { 15 | switch (action.type) { 16 | case ActionTypes.FetchStart: { 17 | return { 18 | ...state, 19 | inFlight: true, 20 | }; 21 | } 22 | case ActionTypes.FetchEnd: { 23 | return { 24 | currentPage: action.currentPage || state.currentPage, 25 | inFlight: false, 26 | }; 27 | } 28 | case ActionTypes.SetPage: { 29 | return { 30 | ...state, 31 | currentPage: action.currentPage 32 | }; 33 | } 34 | default: { 35 | return state; 36 | } 37 | } 38 | }; 39 | 40 | /* 41 | Placeholder for future functionality 42 | */ 43 | const otherReducer: Reducer = 44 | (_state: IOtherState = ininitialOtherState, action: AllActions) => { 45 | switch (action.type) { 46 | default: { 47 | return {}; 48 | } 49 | } 50 | }; 51 | 52 | export const allReducers = { fetchReducer, otherReducer }; 53 | -------------------------------------------------------------------------------- /server/src/test/BigQueryModel.mock.test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Demonstates mocking 3 | */ 4 | import { BigQueryModel } from "../api/models/BigQueryModel"; 5 | import { BigQueryRetrieval } from "../api/types/BigQueryTypes"; 6 | import { TestConfig } from "./TestConfig"; 7 | 8 | const mockMessage = "test-mock"; 9 | let spyInstance: jest.SpyInstance|undefined; 10 | 11 | beforeAll(() => { 12 | jest.spyOn(BigQueryModel.prototype, "fetch").mockImplementation(async (_params: any) => { 13 | return Promise.resolve(); 14 | }); 15 | spyInstance = jest.spyOn(BigQueryModel.prototype, "getData").mockImplementation(() => { 16 | return new Error(mockMessage); 17 | }); 18 | }); 19 | 20 | afterAll(() => { 21 | expect(spyInstance).toBeDefined(); 22 | expect(spyInstance).toHaveBeenCalledTimes(1); 23 | jest.restoreAllMocks(); 24 | }); 25 | 26 | describe("Test mocking", () => { 27 | const config = TestConfig.getModelConfig(); 28 | BigQueryModel.Config = config; 29 | 30 | it("Mocks selected methods of BigQueryModel class", async () => { 31 | const bqRequest = TestConfig.getRequest(TestConfig.getStockTestRequest()); 32 | expect(bqRequest).toBeDefined(); 33 | 34 | const model = BigQueryModel.Factory; 35 | await model.fetch(bqRequest!); 36 | const data: BigQueryRetrieval = model.getData(); 37 | 38 | expect(data).toBeInstanceOf(Error); 39 | const err = data as Error; 40 | expect(err.message).toContain(mockMessage); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /server/src/utils/storage.ts: -------------------------------------------------------------------------------- 1 | /* 2 | The class PersistentStorageManager is responsible for 3 | database operations handling 4 | */ 5 | import * as NodeCache from "node-cache"; 6 | 7 | export class PersistentStorageManager { 8 | public static ReadLimitCounters(_cache: NodeCache): void { 9 | // TODO 10 | // Provide an implmentation to restore all the data limit counters from 11 | // the database of your choice. If a counter was updated >24 hours ago 12 | // then do not restore this counter and delete it from the database 13 | // (or archive/inactivate its record for history or audit purposes). 14 | 15 | } 16 | 17 | public static WriteLimitCounters( 18 | _counterNames: [string, string?], 19 | _counterValue: number): void { 20 | // TODO 21 | // If the counter value '_counterValue' is greater than zero then: 22 | // - Throw an exception if the tuple contains less than two names. 23 | // - Provide an implmentation to update the database by creating the records 24 | // for the two new counters (with names taken from the tuple) and set both 25 | // their values to the counter value. If one or both counters already 26 | // exist then increase their value by the counter value. 27 | // If the counter value '_counterValue' is zero then: 28 | // - Verify that tuple contains one name only and throw an exception 29 | // otherwise. 30 | // - Delete the counter from the database (or archive/inactivate its record 31 | // for history or audit purposes). 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /client/src/components/QueryPageContainer.tsx: -------------------------------------------------------------------------------- 1 | /* 2 | Connect QueryPage to Redux store. 3 | */ 4 | import { Dispatch, bindActionCreators } from "redux"; 5 | import { 6 | MapStateToProps, 7 | MapDispatchToProps, 8 | connect 9 | } from "react-redux"; 10 | import { RootState } from "../state/store"; 11 | import { AllActions, actionCreators } from "../state/actions"; 12 | import { 13 | QueryPage, 14 | IQueryPageProps_dataSlice, 15 | IQueryPageProps_actionSlice 16 | } from "./QueryPage"; 17 | 18 | // Helper function used by connect() 19 | const mapStateToProps: MapStateToProps = 20 | (state: RootState, _ownProps: IQueryPageContainerProps) => ({ 21 | fetch: state.fetch 22 | }); 23 | 24 | // Helper function used by connect() 25 | const mapDispatchToProps: MapDispatchToProps = 26 | (dispatch: Dispatch) => ({ 27 | actions: bindActionCreators(actionCreators, dispatch) 28 | }); 29 | 30 | // eslint-disable-next-line @typescript-eslint/no-empty-interface 31 | interface IQueryPageContainerProps { 32 | } 33 | 34 | // Container component (effectively a wrapper) that connects QueryPage to the Redux store 35 | export const QueryPageContainer = 36 | connect< 37 | // The type of props injected by mapStateToProps 38 | IQueryPageProps_dataSlice, 39 | // The type of props injected by mapDispatchToProps 40 | ReturnType, 41 | // The type of props to be taken by the container component we are about to create 42 | IQueryPageContainerProps, 43 | // The type of Redux state tree 44 | RootState> 45 | (mapStateToProps, mapDispatchToProps)(QueryPage); 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crisp-bigquery", 3 | "version": "1.1.1", 4 | "description": "Full stack BigQuery with React and Express in TypeScript", 5 | "author": "winwiz1 (https://github.com/winwiz1/)", 6 | "contributors": [ 7 | "winwiz1 (https://github.com/winwiz1/)" 8 | ], 9 | "license": "MIT", 10 | "homepage": "https://winwiz1.github.io/crisp-bigquery/", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/winwiz1/crisp-bigquery.git" 14 | }, 15 | "keywords": [ 16 | "bigquery", 17 | "react", 18 | "redux", 19 | "typescript", 20 | "nodejs", 21 | "express", 22 | "visual-studio-code", 23 | "chrome-devtools", 24 | "webpack", 25 | "webpack-dev-server", 26 | "typestyle", 27 | "react-testing-library", 28 | "supertest", 29 | "jest", 30 | "semantic-ui" 31 | ], 32 | "files": [ 33 | "./client/*", 34 | "./server/*" 35 | ], 36 | "scripts": { 37 | "install": "cd client && yarn", 38 | "postinstall": "cd server && yarn", 39 | "pretest": "cd client && yarn build", 40 | "test": "cd client && yarn test", 41 | "posttest": "cd server && yarn copy && yarn test", 42 | "compile": "cd client && yarn compile", 43 | "postcompile": "cd server && yarn compile", 44 | "build": "cd client && yarn build", 45 | "postbuild": "cd server && yarn prestart", 46 | "build:prod": "cd client && yarn build:prod", 47 | "postbuild:prod": "cd server && yarn prestart", 48 | "start:prod": "cd client && yarn build:prod", 49 | "poststart:prod": "cd server && yarn start:prod", 50 | "run:prod": "cd server && yarn run:prod" 51 | }, 52 | "engines" : { 53 | "node" : ">=10.17" 54 | } 55 | } -------------------------------------------------------------------------------- /client/src/utils/error.ts: -------------------------------------------------------------------------------- 1 | import logger from "./logger"; 2 | 3 | /* 4 | This class allows to have two different error messages describing the same error. 5 | One user friendly, is meant to be understood by end users. Another detailed and 6 | possibly containing technical jargon. Support can ask a user to copy the latter 7 | from the log (currently browser's JS console) and paste into an email to be sent 8 | to the Support for further troubleshooting. 9 | */ 10 | export class CustomError extends Error { 11 | constructor( 12 | message: string, // error description for end user 13 | readonly detailMessage?: string 14 | ) { // troubleshooting info for Support 15 | super(message); 16 | // http://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget 17 | Object.setPrototypeOf(this, new.target.prototype); 18 | this.detailMessage = this.detailMessage ?? ""; 19 | 20 | let errStr = message; 21 | detailMessage && (errStr += `\nInformation for Support:\n${detailMessage || ""}`); 22 | logger.error(errStr); 23 | } 24 | } 25 | 26 | /* 27 | Utility function used for error handling. 28 | */ 29 | export function domErrorToString(err: DOMError | undefined): string { 30 | // eslint-disable-next-line no-extra-boolean-cast 31 | return !!err ? 32 | `Error: ${err.name}, description: ${err.toString?.() || ""}` : 33 | "No error details available"; 34 | } 35 | 36 | /* 37 | Utility function used for error handling. 38 | */ 39 | export function domExceptionToString(ex: DOMException | undefined): string { 40 | // eslint-disable-next-line no-extra-boolean-cast 41 | return !!ex ? 42 | `Exception: ${ex.code}, description: ${ex.message || ""}` : 43 | "No exception details available"; 44 | } 45 | -------------------------------------------------------------------------------- /client/src/components/QueryPaginationContainer.tsx: -------------------------------------------------------------------------------- 1 | /* 2 | Connect QueryPagination to Redux store. 3 | */ 4 | import { Dispatch, bindActionCreators } from "redux"; 5 | import { 6 | MapStateToProps, 7 | MapDispatchToProps, 8 | connect 9 | } from "react-redux"; 10 | import { RootState } from "../state/store"; 11 | import { AllActions, actionCreators } from "../state/actions"; 12 | import { 13 | QueryPagination, 14 | IQueryFetchProps, 15 | IQueryActionProps, 16 | IPaginationRenderProps 17 | } from "./QueryPagination"; 18 | import { IQueryStatusProps } from "./QueryStatus"; 19 | 20 | // Helper function used by connect() 21 | const mapStateToProps: MapStateToProps = 22 | (state: RootState, _ownProps: IQueryPaginationContainerProps) => { 23 | const ret: IQueryFetchProps = { 24 | fetch: state.fetch 25 | }; 26 | return ret; 27 | }; 28 | 29 | // Helper function used by connect() 30 | const mapDispatchToProps: MapDispatchToProps = 31 | (dispatch: Dispatch) => { 32 | const ret: IQueryActionProps = { 33 | actions: bindActionCreators(actionCreators, dispatch) 34 | }; 35 | return ret; 36 | }; 37 | 38 | export interface IQueryPaginationContainerProps extends IQueryStatusProps, IPaginationRenderProps { 39 | } 40 | 41 | // Container component that connects QueryPagination to the Redux store 42 | export const QueryPaginationContainer = 43 | connect< 44 | // The type of props injected by mapStateToProps 45 | IQueryFetchProps, 46 | // The type of props injected by mapDispatchToProps 47 | ReturnType, 48 | // The type of props to be taken by container component we are about to create 49 | IQueryPaginationContainerProps, 50 | // The type of Redux state tree 51 | RootState> 52 | (mapStateToProps, mapDispatchToProps)(QueryPagination); 53 | -------------------------------------------------------------------------------- /client/src/state/actions.ts: -------------------------------------------------------------------------------- 1 | import { Action, AnyAction, Dispatch } from "redux"; 2 | 3 | /* 4 | Action types 5 | */ 6 | export enum ActionTypes { 7 | FetchStart = "FetchStart", 8 | FetchEnd = "FetchEnd", 9 | SetPage = "SetPage", 10 | } 11 | 12 | /* 13 | Actions: one action for each action type 14 | */ 15 | 16 | // eslint-disable-next-line @typescript-eslint/no-empty-interface 17 | interface IActionFetchStart extends Action { 18 | } 19 | 20 | interface IActionFetchEnd extends Action { 21 | readonly currentPage?: number; 22 | readonly pageCount?: number; 23 | } 24 | 25 | interface IActionSetPage extends Action { 26 | readonly currentPage: number; 27 | } 28 | 29 | const actionFetchStartCreator = (): IActionFetchStart => { 30 | return { 31 | type: ActionTypes.FetchStart 32 | }; 33 | }; 34 | 35 | /* 36 | Action creators 37 | */ 38 | 39 | const actionFetchEndCreator = ( 40 | page: number | undefined = undefined): IActionFetchEnd => { 41 | return { 42 | type: ActionTypes.FetchEnd, 43 | ...(page && { currentPage: page }), 44 | }; 45 | }; 46 | 47 | const actionSetPageCreator = ( 48 | page: number): IActionSetPage => { 49 | return { 50 | currentPage: page, 51 | type: ActionTypes.SetPage, 52 | }; 53 | }; 54 | 55 | export type AllActions = 56 | | IActionFetchStart 57 | | IActionFetchEnd 58 | | IActionSetPage; 59 | 60 | export const actionCreators = { 61 | actionFetchEnd: actionFetchEndCreator, 62 | actionFetchStart: actionFetchStartCreator, 63 | actionSetPage: actionSetPageCreator, 64 | }; 65 | 66 | // Currently unused as we are not injecting dispatch() directly into 67 | // the props of container components created by connect(). We inject 68 | // dispatch() indirectly via action creators bound to dispatch. 69 | export interface IDispatchProps { 70 | dispatch: Dispatch; 71 | } 72 | -------------------------------------------------------------------------------- /server/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Debug Backend", 11 | "program": "${workspaceFolder}/src/srv/main.ts", 12 | "preLaunchTask": "tsc-watch", 13 | "postDebugTask": "kill process in terminal", 14 | "outFiles": [ 15 | "${workspaceFolder}/build/**/*.js" 16 | ], 17 | "sourceMaps": true, 18 | "restart": true, 19 | }, 20 | { 21 | "type": "node", 22 | "request": "launch", 23 | "name": "Debug Backend Connected To Devserver", 24 | "program": "${workspaceFolder}/src/srv/main.ts", 25 | "preLaunchTask": "tsc-watch", 26 | "postDebugTask": "kill process in terminal", 27 | "outFiles": [ 28 | "${workspaceFolder}/build/**/*.js" 29 | ], 30 | "sourceMaps": true, 31 | "restart": true, 32 | "env": { 33 | "USE_DEV_WEBSERVER": "true", 34 | "GCP_PROJECT_ID": "", 35 | "BIGQUERY_DATASET_NAME": "samples", 36 | "BIGQUERY_TABLE_NAME": "github", 37 | "KEY_FILE_PATH": "../key.json" 38 | }, 39 | }, 40 | { 41 | "name": "Debug Jest Tests", 42 | "type": "node", 43 | "request": "launch", 44 | "runtimeArgs": [ 45 | "--inspect-brk", 46 | "${workspaceRoot}/node_modules/jest/bin/jest.js", 47 | "--runInBand" 48 | ], 49 | "console": "integratedTerminal", 50 | "internalConsoleOptions": "neverOpen", 51 | "port": 9229, 52 | "env": { 53 | "NODE_ENV": "test", 54 | "GCP_PROJECT_ID": "", 55 | "BIGQUERY_DATASET_NAME": "samples", 56 | "BIGQUERY_TABLE_NAME": "github", 57 | "KEY_FILE_PATH": "../key.json" 58 | }, 59 | }, 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /client/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "chrome", 9 | "request": "launch", 10 | "name": "Launch Chrome in Production", 11 | "url": "http://localhost:3000/app.html", 12 | "sourceMaps": true, 13 | "preLaunchTask": "npm build:prod-debug", 14 | "trace": "verbose", 15 | "webRoot": "${workspaceFolder}", 16 | "breakOnLoad": true, 17 | "disableNetworkCache": true, 18 | }, 19 | { 20 | "type": "chrome", 21 | "request": "launch", 22 | "name": "Launch Chrome Connected to Devserver", 23 | "url": "http://localhost:8080/app.html", 24 | "sourceMaps": true, 25 | "preLaunchTask": "npm dev", 26 | "postDebugTask": "kill process in terminal", 27 | "trace": "verbose", 28 | "webRoot": "${workspaceFolder}", 29 | "breakOnLoad": true, 30 | "disableNetworkCache": true, 31 | }, 32 | { 33 | "type": "chrome", 34 | "request": "launch", 35 | "name": "Launch Chrome Connected to Backend", 36 | "url": "http://localhost:3000/app.html", 37 | "sourceMaps": true, 38 | "preLaunchTask": "npm dev", 39 | "postDebugTask": "kill process in terminal", 40 | "trace": "verbose", 41 | "webRoot": "${workspaceFolder}", 42 | "breakOnLoad": true, 43 | "disableNetworkCache": true, 44 | }, 45 | { 46 | "name": "Debug Jest Tests", 47 | "type": "node", 48 | "request": "launch", 49 | "runtimeArgs": [ 50 | "--inspect-brk", 51 | "${workspaceRoot}/node_modules/jest/bin/jest.js", 52 | "--runInBand" 53 | ], 54 | "console": "integratedTerminal", 55 | "internalConsoleOptions": "neverOpen", 56 | "port": 9229, 57 | "env": { 58 | "NODE_ENV": "test" 59 | }, 60 | }, 61 | ] 62 | } 63 | -------------------------------------------------------------------------------- /client/src/utils/csv.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Class CsvConverter. 3 | Converts BigQuery tabular data into CSV format using D3 library. 4 | Since CSV export is optional, the class and its imports are 5 | included into a separate script bundle. The bundle is downloaded 6 | via dynamic import when it's needed. 7 | */ 8 | import * as d3 from "d3-dsv"; 9 | import logger from "./logger"; 10 | import { QueryCache } from "./cache"; 11 | 12 | export class CsvConverter { 13 | public constructor(private config: { 14 | cache: QueryCache, 15 | progressRoutine: (percent: number) => void, 16 | appendRoutine: (blob: Blob) => Promise 17 | }) { 18 | 19 | } 20 | 21 | public readonly performConversion = async (): Promise => { 22 | const pageCount = this.config.cache.getPageCount(); 23 | 24 | for (let currentPage = 0; currentPage < pageCount; ++currentPage) { 25 | const str = this.convertPage(currentPage); 26 | const blob = new Blob([str], { type: "text/csv" }); 27 | const ret = await this.config.appendRoutine(blob); 28 | 29 | if (!ret) { 30 | logger.error("Data conversion failure due to file appending error."); 31 | return Promise.resolve(false); 32 | } 33 | 34 | this.config.progressRoutine(((currentPage + 1) * 100) / pageCount); 35 | await new Promise((resolve) => setTimeout(resolve, 10)); 36 | } 37 | 38 | return Promise.resolve(true); 39 | } 40 | 41 | public readonly convertPage = (ind: number): string => { 42 | const page = this.config.cache.getPage(ind); 43 | 44 | if (!page) { 45 | // Can only happen if code is incorrectly modified 46 | throw new Error("Internal error: no page to convert. Please contact Support."); 47 | } 48 | 49 | const firstRow = ind > 0 ? [] : CsvConverter.s_columns; 50 | 51 | const ret = d3.csvFormatRows([firstRow].concat(page.data.map((d, _i) => { 52 | return [ 53 | d.DateTime ?? "", 54 | d.Name ?? "", 55 | d.Language ?? "", 56 | d.Size ?? "", 57 | d.Homepage ?? "", 58 | d.Login ?? "", 59 | d.Owner ?? "" 60 | ]; 61 | }))); 62 | 63 | return ret; 64 | } 65 | 66 | private static readonly s_columns = [ 67 | "DateTime", 68 | "Name", 69 | "Language", 70 | "Size", 71 | "Homepage", 72 | "Login", 73 | "Owner" 74 | ]; 75 | } 76 | 77 | export { CsvConverter as default } from "./csv"; 78 | -------------------------------------------------------------------------------- /client/config/spa.config.js: -------------------------------------------------------------------------------- 1 | /* 2 | Crisp React supports splitting React application into several SPAs. 3 | Since this solution is based on Crisp React, please see: 4 | https://winwiz1.github.io/crisp-react/#spa-configuration 5 | https://github.com/winwiz1/crisp-react/blob/master/client/config/spa.config.js 6 | */ 7 | 8 | var ConfiguredSPAs = function() { 9 | function SPA(params) { 10 | this.params = params; 11 | } 12 | 13 | /****************** Start SPA Configuration ******************/ 14 | var SPAs = [ 15 | new SPA({ 16 | name: "app", 17 | entryPoint: "./src/entrypoints/app.tsx", 18 | redirect: true 19 | }) 20 | ]; 21 | SPAs.appTitle = "Crisp BigQuery"; 22 | /****************** End SPA Configuration ******************/ 23 | 24 | SPAs.verifyParameters = function(verifier) { 25 | if (SPAs.length === 0) { 26 | throw new RangeError("At least one SPA needs to be configured"); 27 | } 28 | 29 | SPAs.forEach(function(spa, idx) { 30 | spa.params = verifier(spa.params, idx); 31 | }); 32 | 33 | var num = SPAs.reduce(function(acc, item) { 34 | return item.params.redirect ? acc + 1 : acc; 35 | }, 0); 36 | 37 | if (num !== 1) { 38 | throw new RangeError("One and only one SPA must have 'redirect: true'"); 39 | } 40 | }; 41 | 42 | SPAs.getEntrypoints = function() { 43 | var entryPoints = new Object(); 44 | SPAs.forEach(function(spa) { 45 | entryPoints[spa.params.name] = spa.params.entryPoint; 46 | }); 47 | return entryPoints; 48 | }; 49 | 50 | SPAs.getRedirectName = function() { 51 | return SPAs.find(function(spa) { 52 | return spa.params.redirect; 53 | }).params.name; 54 | }; 55 | 56 | SPAs.getNames = function() { 57 | var spaNames = []; 58 | SPAs.forEach(function(spa) { 59 | spaNames.push(spa.params.name); 60 | }); 61 | return spaNames; 62 | }; 63 | 64 | SPAs.getRewriteRules = function() { 65 | var ret = []; 66 | SPAs.forEach(function(spa) { 67 | var rule = new Object(); 68 | rule.from = new RegExp("^/" + spa.params.name + "(\\.html)?$"); 69 | rule.to = spa.params.name + ".html"; 70 | ret.push(rule); 71 | }); 72 | ret.push({ 73 | from: new RegExp("^.*$"), 74 | to: "/" + SPAs.getRedirectName() + ".html" 75 | }); 76 | return ret; 77 | }; 78 | 79 | return SPAs; 80 | }; 81 | 82 | module.exports = ConfiguredSPAs(); 83 | -------------------------------------------------------------------------------- /client/src/test/BackendRequest.test.ts: -------------------------------------------------------------------------------- 1 | import * as moment from "moment"; 2 | import { BackendRequest, IBackendRequestData } from "../api/BackendRequest"; 3 | 4 | describe("Testing BackendRequest with invalid data", () => { 5 | it("should reject endDate past startDate", () => { 6 | const data: IBackendRequestData = { 7 | endDate: moment().add(-1, "days").toDate(), 8 | rowCount: 1, 9 | startDate: moment().toDate(), 10 | }; 11 | function testCreate() { 12 | new BackendRequest(data); 13 | } 14 | expect(testCreate).toThrowError(/invalid/i); 15 | }); 16 | 17 | it("should reject query timeframe > 8 days", () => { 18 | const data: IBackendRequestData = { 19 | endDate: moment().add(9, "days").toDate(), 20 | rowCount: 1, 21 | startDate: moment().toDate(), 22 | }; 23 | function testCreate() { 24 | new BackendRequest(data); 25 | } 26 | expect(testCreate).toThrowError(/cannot exceed/); 27 | expect(testCreate).toThrowError(RangeError); 28 | }); 29 | 30 | it("should reject invalid rowCount", () => { 31 | const data: IBackendRequestData = { 32 | rowCount: 1, 33 | startDate: moment().toDate(), 34 | }; 35 | const invalidCounts = [0, 2001, 1000000, -1]; 36 | 37 | for (const cnt of invalidCounts) { 38 | const testCreate = () => { 39 | new BackendRequest({ ...data, rowCount: cnt }); 40 | }; 41 | expect(testCreate).toThrowError(/invalid.*?rowCount/i); 42 | } 43 | }); 44 | 45 | it("should reject inconsistent pagination data", () => { 46 | const date = moment().toDate(); 47 | const data: IBackendRequestData = { 48 | endDate: date, 49 | jobId: "abc", 50 | pageToken: undefined, 51 | rowCount: 1, 52 | startDate: date, 53 | }; 54 | 55 | function testCreate1() { 56 | new BackendRequest(data); 57 | } 58 | 59 | expect(testCreate1).toThrowError(/inconsistent/i); 60 | 61 | function testCreate2() { 62 | new BackendRequest({ ...data, jobId: undefined, pageToken: "abc" }); 63 | } 64 | 65 | expect(testCreate2).toThrowError(/inconsistent/i); 66 | }); 67 | }); 68 | 69 | describe("Testing BackendRequest with valid data", () => { 70 | it("should accept valid data", () => { 71 | const data: IBackendRequestData = { 72 | endDate: moment().add(7, "days").toDate(), 73 | jobId: "abc", 74 | language: "C++", 75 | name: "c", 76 | pageToken: "123", 77 | rowCount: 1000, 78 | startDate: moment().toDate(), 79 | }; 80 | const request = new BackendRequest(data); 81 | expect(request).toBeDefined(); 82 | }); 83 | }); 84 | -------------------------------------------------------------------------------- /client/src/utils/cache.ts: -------------------------------------------------------------------------------- 1 | /* 2 | QueryCache provides client-side caching for the data received from 3 | the backend. Currently there is no limit on the amount of data held 4 | in memory which is only sustainable assuming other components limit 5 | this amount. 6 | */ 7 | import { BigQueryRetrievalResult } from "../api/BackendManager"; 8 | 9 | export type QueryCachePage = { 10 | index: number 11 | data: BigQueryRetrievalResult["rows"] 12 | token: BigQueryRetrievalResult["pageToken"] 13 | job: BigQueryRetrievalResult["jobId"] 14 | totalRows: BigQueryRetrievalResult["totalRows"] 15 | rows: BigQueryRetrievalResult["returnedRows"] 16 | }; 17 | 18 | export class QueryCache { 19 | // Returns 'true' unless the limit on row count is reached in which 20 | // case returns 'false'. 21 | public readonly addPage = (retrieval: BigQueryRetrievalResult): boolean => { 22 | if (!retrieval) { 23 | return true; 24 | } 25 | 26 | if (!retrieval.jobComplete) { 27 | throw new Error(QueryCache.s_errMsgIncompleteJob); 28 | } 29 | 30 | if (this.isOverLimit()) { 31 | return false; 32 | } 33 | 34 | const pageCount = this.getPageCount(); 35 | this.m_pages.push({ 36 | data: retrieval.rows, 37 | index: pageCount, 38 | job: (retrieval.jobId || undefined), 39 | rows: retrieval.returnedRows, 40 | token: (retrieval.pageToken || undefined), 41 | totalRows: retrieval.totalRows, 42 | }); 43 | 44 | this.m_rowCount += retrieval.rows.length; 45 | return true; 46 | } 47 | 48 | public readonly getPage = (idx: number): QueryCachePage | undefined => { 49 | return (idx < 0 || this.m_pages.length === 0 || idx >= this.m_pages.length) ? 50 | undefined : this.m_pages[idx]; 51 | } 52 | 53 | public readonly getLastPage = (): QueryCachePage | undefined => { 54 | const count = this.getPageCount(); 55 | 56 | return count > 0 ? this.m_pages[count - 1] : undefined; 57 | } 58 | 59 | public readonly getPageCount = (): number => { 60 | return this.m_pages.length; 61 | } 62 | 63 | public readonly getRowCount = (): number => { 64 | return this.m_rowCount; 65 | } 66 | 67 | public readonly clear = (): void => { 68 | this.m_pages.length = 0; 69 | this.m_rowCount = 0; 70 | } 71 | 72 | public readonly isOverLimit = (): boolean => { 73 | return this.m_rowCount > QueryCache.s_rowCountLimit; 74 | } 75 | 76 | private readonly m_pages: Array = []; 77 | private m_rowCount: number = 0; 78 | private static readonly s_rowCountLimit = 200000; 79 | private static readonly s_errMsgIncompleteJob = 80 | "Cannot cache response data (incomplete job). Please contact Support."; 81 | } 82 | -------------------------------------------------------------------------------- /server/src/test/TestConfig.ts: -------------------------------------------------------------------------------- 1 | import * as moment from "moment"; 2 | import { BigQueryModelConfig } from "../api/models/BigQueryModel"; 3 | import { JsonParsingError, BigQueryRequest } from "../api/types/BigQueryTypes"; 4 | 5 | type TestRequest = { 6 | startDate: string 7 | endDate: string 8 | rowCount: number 9 | name: string | undefined 10 | language: string | undefined 11 | pageToken: string | undefined 12 | jobId: string | undefined 13 | }; 14 | 15 | export class TestConfig { 16 | static readonly getValidNames = (): ReadonlyArray => { 17 | const names = [ 18 | "myrepo", 19 | "Repo1234567890", 20 | "name.repo.v12", 21 | undefined 22 | ]; 23 | 24 | return names; 25 | } 26 | 27 | static readonly getInvalidNames = (): ReadonlyArray => { 28 | const names = [ 29 | "repo#", 30 | "Repo { 41 | return { 42 | endDate: "2012-03-25", 43 | jobId: undefined, 44 | language: undefined, 45 | name: undefined, 46 | pageToken: undefined, 47 | rowCount: 100, 48 | startDate: "2012-03-25", 49 | }; 50 | } 51 | 52 | static readonly getRequestAsString = (req: TestRequest): string => { 53 | const obj = JSON.stringify({ 54 | endDate: moment.utc(req.endDate).utc().toDate(), 55 | jobId: req.jobId, 56 | language: req.language, 57 | name: req.name, 58 | pageToken: req.pageToken, 59 | rowCount: req.rowCount, 60 | startDate: moment.utc(req.startDate).utc().toDate(), 61 | 62 | }); 63 | return obj; 64 | } 65 | 66 | static readonly getRequestAsJson = (req: TestRequest): any | undefined => { 67 | const ret = JSON.parse(TestConfig.getRequestAsString(req)); 68 | return ret; 69 | } 70 | 71 | static readonly getRequest = ( 72 | req: TestRequest, 73 | clientAddress = "10.10.11.12", 74 | useQueryCache = true): BigQueryRequest | undefined => { 75 | const ret = TestConfig.getRequestAsJson(req); 76 | const errInfo: JsonParsingError = { message: undefined }; 77 | return ret ? BigQueryRequest.fromJson(ret, clientAddress, errInfo, useQueryCache) : undefined; 78 | } 79 | 80 | static readonly getModelConfig = ( 81 | dataLimitClient: number | undefined = undefined, 82 | dataLimitInstance: number | undefined = undefined): BigQueryModelConfig => { 83 | return new BigQueryModelConfig( 84 | dataLimitClient, 85 | dataLimitInstance 86 | ); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crisp-bigquery-backend", 3 | "version": "1.0.0", 4 | "description": "Backend for the Crisp BigQuery project", 5 | "author": "winwiz1 (https://github.com/winwiz1/)", 6 | "contributors": [ 7 | "winwiz1 (https://github.com/winwiz1/)" 8 | ], 9 | "license": "MIT", 10 | "homepage": "https://winwiz1.github.io/crisp-bigquery/", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/winwiz1/crisp-bigquery.git" 14 | }, 15 | "keywords": [ 16 | "bigquery", 17 | "typescript", 18 | "express", 19 | "visual-studio-code", 20 | "supertest", 21 | "jest" 22 | ], 23 | "scripts": { 24 | "lint": "eslint . --ext .js,.ts", 25 | "compile": "tsc -p .", 26 | "compile:watch": "tsc -w -p .", 27 | "precopy": "rimraf build/client && mkdirp build/client/static", 28 | "copy": "copyfiles -f ../client/dist/* build/client/static/ && copyfiles -f ../client/config/* config/", 29 | "prestart": "yarn copy && yarn compile", 30 | "start": "echo-cli Starting the backend... && node -r dotenv/config ./build/srv/main.js", 31 | "prestart:prod": "yarn prestart", 32 | "start:prod": "echo-cli Starting the backend... && cross-env NODE_ENV=production node -r dotenv/config ./build/srv/main.js", 33 | "prod": "yarn start:prod", 34 | "prestart:dev": "yarn prestart", 35 | "start:dev": "echo-cli Starting the backend... && cross-env USE_DEV_WEBSERVER=true node -r dotenv/config ./build/srv/main.js", 36 | "dev": "yarn start:dev", 37 | "test": "cross-env NODE_ENV=test jest", 38 | "run:prod": "echo-cli Starting the backend... && cross-env NODE_ENV=production node -r dotenv/config ./build/srv/main.js" 39 | }, 40 | "dependencies": { 41 | "@google-cloud/bigquery": "^5.4.0", 42 | "dotenv": "^8.2.0", 43 | "express": "4.17.1", 44 | "express-rate-limit": "^5.1.3", 45 | "express-static-gzip": "^2.1.0", 46 | "helmet": "^4.2.0", 47 | "http-proxy-middleware": "^1.0.6", 48 | "moment": "^2.29.1", 49 | "nocache": "^2.1.0", 50 | "node-cache": "^5.1.2", 51 | "node-fetch": "^2.6.1", 52 | "serve-favicon": "^2.5.0", 53 | "winston": "3.3.3" 54 | }, 55 | "devDependencies": { 56 | "@types/express": "4.17.8", 57 | "@types/express-rate-limit": "^5.1.0", 58 | "@types/helmet": "^4.0.0", 59 | "@types/http-proxy-middleware": "^0.19.3", 60 | "@types/jest": "26.0.15", 61 | "@types/node": "14.14.6", 62 | "@types/node-fetch": "2.5.7", 63 | "@types/serve-favicon": "^2.5.1", 64 | "@types/supertest": "^2.0.10", 65 | "@typescript-eslint/eslint-plugin": "^4.6.1", 66 | "@typescript-eslint/parser": "^4.6.1", 67 | "copyfiles": "^2.4.0", 68 | "cross-env": "^7.0.2", 69 | "echo-cli": "^1.0.8", 70 | "eslint": "^7.12.1", 71 | "jest": "26.6.3", 72 | "mkdirp": "^1.0.3", 73 | "rimraf": "^3.0.2", 74 | "supertest": "^6.0.1", 75 | "ts-jest": "26.4.3", 76 | "tslib": "2.0.3", 77 | "typescript": "4.0.5" 78 | }, 79 | "resolutions": { 80 | "minimist": "^1.2.2" 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /client/src/test/QueryPage.test.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * QueryPage tests using React Testing Library 3 | * with Jest adaptors. 4 | */ 5 | import * as React from "react"; 6 | import { Provider } from "react-redux"; 7 | import { render, fireEvent, act } from "@testing-library/react"; 8 | import "@testing-library/jest-dom/extend-expect"; 9 | import { rootStore } from "../state/store"; 10 | import { QueryPageContainer } from "../components/QueryPageContainer"; 11 | 12 | describe("Testing QueryPageContainer", () => { 13 | // Demonstrates how to test static rendering 14 | test("Basic tests", async () => { 15 | const { getByText, getAllByText, queryByText } = render( 16 | 17 | 18 | 19 | ); 20 | 21 | getByText(content => content.startsWith("Query Options")); 22 | expect(queryByText("Not there", { exact: true, selector: "div" })).toBeNull(); 23 | expect(getAllByText("Start Date", { exact: false, selector: "div" })[0]).toBeVisible(); 24 | expect(getAllByText("End Date", { exact: false, selector: "div" })[0]).toBeVisible(); 25 | expect(getAllByText("Repository Name", { exact: false, selector: "div" })[0]).toBeVisible(); 26 | expect(getAllByText("Repository Language", { exact: false, selector: "div" })[0]).toBeVisible(); 27 | expect(getByText("Paginated Results Table", { exact: false, selector: "div" })).toBeVisible(); 28 | expect(getByText("Query Data", { exact: true, selector: "div" })).toBeVisible(); 29 | }); 30 | 31 | // Demonstrates how to test dynamic rendering, for example a toggle performed by the accordion. 32 | test("Test accordion", async () => { 33 | const { container } = render( 34 | 35 | 36 | 37 | ); 38 | 39 | // Find the accordion. See comments in 40 | // https://github.com/winwiz1/crisp-react/blob/master/client/src/test/Overview.test.tsx 41 | const cssAccordion = "main section div.accordion.ui div.active.title div"; 42 | const accordion = container.querySelector(cssAccordion); 43 | // Check we found the correct one 44 | expect(accordion).toHaveTextContent("Query Options"); 45 | 46 | // Find the 'Start Date' label 47 | const cssStartDateVisible = "main section div.accordion.ui div.content.active" + 48 | " div.ui.raised.segment.blurring.dimmable section div.ui.basic.segment div"; 49 | let labelStartDate = container.querySelector(cssStartDateVisible); 50 | // Check we found the correct one 51 | expect(labelStartDate).toHaveTextContent("Start Date"); 52 | 53 | // Toggle the accordion 54 | const leftClick = { button: 0 }; 55 | act(() => { 56 | fireEvent.click(accordion!, leftClick); 57 | }); 58 | 59 | // Check the toggle 60 | labelStartDate = container.querySelector(cssStartDateVisible); 61 | expect(labelStartDate).toBeNull(); 62 | const cssStartDateHidden = cssStartDateVisible.replace(/\.active/, ""); 63 | labelStartDate = container.querySelector(cssStartDateHidden); 64 | expect(labelStartDate).toHaveTextContent("Start Date"); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /client/src/components/QueryStatus.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { style, media } from "typestyle"; 3 | import { Message } from "semantic-ui-react"; 4 | import { QueryCache } from "../utils/cache"; 5 | import logger from "../utils/logger"; 6 | 7 | export interface IQueryStatusProps { 8 | status: { 9 | currentPage: number; 10 | cache: QueryCache; 11 | err?: Error; 12 | clearError: () => void; 13 | }; 14 | } 15 | 16 | // When Message.onDismiss is set, the component behaves as a block element. 17 | // We limit he width it takes, otherwise the error message can become too elongated. 18 | const cssMaxWidth = style( 19 | { transition: "all .5s" }, 20 | media({ maxWidth: 1024 }, { maxWidth: "50vw" }), 21 | media({ minWidth: 1025 }, { maxWidth: "35vw" }), 22 | ); 23 | 24 | export const QueryStatus: React.FunctionComponent = props => { 25 | const pageCnt = props.status.cache.getPageCount(); 26 | const moreData = !!props.status.cache.getLastPage()?.token && !props.status.err; 27 | const hasData = props.status.cache.getRowCount() > 0; 28 | const finished = !moreData && hasData; 29 | let header: string | undefined; 30 | let message: string | undefined; 31 | 32 | if (props.status.err) { 33 | header = "Error"; 34 | const errTime = new Date().toLocaleString("en-US", { hour12: false }); 35 | message = `Time: ${errTime} Description: ${props.status.err.message}`; 36 | } else if (moreData) { 37 | header = "More data available"; 38 | message = `Switch to the page ${pageCnt + 1} to request it.`; 39 | } else if (finished) { 40 | header = "Query finished"; 41 | message = "All query data has been received"; 42 | } 43 | 44 | const statusData = { 45 | error: !!props.status.err, 46 | header, 47 | message, 48 | moreData, 49 | queryFinished: finished 50 | }; 51 | 52 | const bothFlagsSet = statusData.moreData && statusData.queryFinished; 53 | const eitherFlagSetNoMessage = (statusData.error || statusData.moreData) && 54 | !statusData.message; 55 | 56 | if (bothFlagsSet || eitherFlagSetNoMessage) { 57 | // Can only happen if code is incorrectly modified 58 | const errMsg = "Inconsistent query status data. Please contact Support."; 59 | logger.error(`QueryStatus props are inconsistent: 60 | error: ${statusData.error}, moreData: ${statusData.moreData}, 61 | queryFinished: ${statusData.queryFinished}, message: ${statusData.message ?? ""}`); 62 | throw new Error(errMsg); 63 | } 64 | 65 | let iconName = ""; 66 | 67 | if (statusData.error) { 68 | iconName = "exclamation circle"; 69 | } else if (statusData.moreData) { 70 | iconName = "copy outline"; 71 | } else if (statusData.queryFinished) { 72 | iconName = "check circle green"; 73 | } 74 | 75 | const onDismiss = (_event: React.MouseEvent, _data: any): void => { 76 | props.status.clearError(); 77 | }; 78 | 79 | return ( 80 |
81 |
92 | ); 93 | }; 94 | -------------------------------------------------------------------------------- /client/src/api/BackendRequest.ts: -------------------------------------------------------------------------------- 1 | import * as moment from "moment"; 2 | 3 | /* 4 | Define the data and functionality used to create 5 | a request for the backend data. 6 | */ 7 | export interface IBackendRequestData { 8 | // Support days (and not hours) as the query granularity. 9 | // The start-of-day time (GMT timezone) applies. 10 | readonly startDate: Date; 11 | // Support days (and not hours) as the query granularity. 12 | // The end-of-day time (GMT timezone) applies. 13 | // If missing then defaults to the same day as startDate. 14 | endDate?: Date; 15 | // Support up to 2000 rows. 16 | readonly rowCount: number; 17 | // Repository name 18 | readonly name?: string; 19 | // Repository language 20 | readonly language?: string; 21 | // Opaque pagination token 22 | readonly jobId?: string; 23 | // Another opaque pagination token 24 | readonly pageToken?: string; 25 | // TODO 26 | readonly log?: () => void; 27 | } 28 | 29 | export class BackendRequest { 30 | constructor( 31 | readonly requestData: IBackendRequestData = { 32 | endDate: undefined, 33 | jobId: undefined, 34 | language: undefined, 35 | name: undefined, 36 | pageToken: undefined, 37 | rowCount: 100, 38 | startDate: moment.utc().toDate(), 39 | } 40 | ) { 41 | if (!this.requestData.endDate) { 42 | this.requestData.endDate = this.requestData.startDate; 43 | } 44 | // Validation 45 | const startMoment = moment.utc(this.requestData.startDate); 46 | const endMoment = moment.utc(this.requestData.endDate); 47 | 48 | if (endMoment.diff(startMoment, "days", true) > 49 | (BackendRequest.s_maxQueryDurationDays + 1)) { 50 | // Can only happen if code is incorrectly modified 51 | throw new RangeError("Query timeframe cannot exceed 1 week"); 52 | } 53 | 54 | if (startMoment.isAfter(endMoment)) { 55 | // Can only happen if code is incorrectly modified 56 | throw new Error("Invalid query timeframe, please contact Support."); 57 | } 58 | 59 | if (this.requestData.rowCount <= 0 || this.requestData.rowCount > 2000) { 60 | // Can only happen if code is incorrectly modified 61 | throw new RangeError("Invalid rowCount"); 62 | } 63 | 64 | if (!!this.requestData.jobId !== !!this.requestData.pageToken) { 65 | throw new Error("Inconsistent pagination data, please contact Support."); 66 | } 67 | } 68 | 69 | static get MaxQueryDuration(): number { 70 | return BackendRequest.s_maxQueryDurationDays; 71 | } 72 | 73 | public readonly toString = (): string => { 74 | const startDate = moment(this.requestData.startDate); 75 | const endDate = moment(this.requestData.endDate); 76 | return JSON.stringify({ 77 | endDate: moment(endDate).utc().add(endDate.utcOffset(), "m").toDate(), 78 | rowCount: this.requestData.rowCount, 79 | startDate: moment(startDate).utc().add(startDate.utcOffset(), "m").toDate(), 80 | ...(this.requestData.name && { name: this.requestData.name }), 81 | ...(this.requestData.language && { language: this.requestData.language }), 82 | ...(this.requestData.jobId && { jobId: this.requestData.jobId }), 83 | ...(this.requestData.pageToken && { pageToken: this.requestData.pageToken }) 84 | }); 85 | } 86 | 87 | private static readonly s_maxQueryDurationDays = 7; 88 | } 89 | -------------------------------------------------------------------------------- /server/src/test/server.test.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Express tests using Supertest Library and 3 | * Jest as the testing framework. 4 | */ 5 | 6 | import * as request from "supertest"; 7 | import Server, { StaticAssetPath } from "../srv/server"; 8 | import * as SPAs from "../../config/spa.config"; 9 | import { BigQueryModel } from "../api/models/BigQueryModel"; 10 | import { TestConfig } from "./TestConfig"; 11 | import { 12 | BigQueryRequest, 13 | BigQueryRetrievalResult, 14 | BigQueryRetrievalRow 15 | } from "../api/types/BigQueryTypes"; 16 | 17 | const server = Server(StaticAssetPath.SOURCE); 18 | const regexResponse = new RegExp(SPAs.appTitle); 19 | const totalRowsMocked = 123; 20 | const totalBytesMocked = 1024; 21 | 22 | beforeAll(() => { 23 | jest.spyOn(BigQueryModel.prototype, "fetch").mockImplementation(async (_params: any) => { 24 | return Promise.resolve(); 25 | }); 26 | jest.spyOn(BigQueryModel.prototype, "Data", "get").mockImplementation(() => { 27 | const ret = new BigQueryRetrievalResult ( 28 | new Array(1), 29 | true, // jobComplete 30 | totalRowsMocked, // totalRows 31 | 0, // returnedRows 32 | totalBytesMocked, // totalBytesProcessed 33 | undefined, // pageToken 34 | undefined // jobId 35 | ); 36 | return ret; 37 | }); 38 | }); 39 | 40 | // Test that webserver does serve SPA landing pages. 41 | // If there are two SPAs in spa.config.js called 'first and 'second', 42 | // then set the array to: ["/", "/first", "/second"] 43 | const statusCode200path = SPAs.getNames().map(name => "/" + name); 44 | statusCode200path.push("/"); 45 | 46 | // Test that webserver implements fallback to the SPA landing page for 47 | // unknown (and presumably internal to SPA) pages. This is required from 48 | // any webserver that supports an SPA. 49 | const statusCode303path = [ 50 | "/a", "/b", "/ABC" 51 | ]; 52 | 53 | // Test that the fallback tolerance does have its limits. 54 | const statusCode404path = [ 55 | "/abc%xyz;", "/images/logo123.png", "/static/invalid" 56 | ]; 57 | 58 | describe("Test Express routes", () => { 59 | it("test URLs returning HTTP status 200", () => { 60 | statusCode200path.forEach(async path => { 61 | const response = await request(server).get(path); 62 | expect(response.status).toBe(200); 63 | expect(response.text).toMatch(regexResponse); 64 | }); 65 | }); 66 | 67 | it("test URLs causing fallback with HTTP status 303", () => { 68 | statusCode303path.forEach(async (path) => { 69 | const response = await request(server).get(path); 70 | expect(response.status).toBe(303); 71 | expect(response.get("Location")).toBe("/"); 72 | }); 73 | }); 74 | 75 | it("test invalid URLs causing HTTP status 404", () => { 76 | statusCode404path.forEach(async path => { 77 | const response = await request(server).get(path); 78 | expect(response.status).toBe(404); 79 | }); 80 | }); 81 | }); 82 | 83 | describe("Test API route", () => { 84 | it("test delivery of fetched data", async () => { 85 | const strRequest = TestConfig.getRequestAsString(TestConfig.getStockTestRequest()); 86 | 87 | const response = await request(server) 88 | .post(BigQueryRequest.Path) 89 | .set("Content-Type", "application/json") 90 | .send(strRequest); 91 | 92 | const obj: BigQueryRetrievalResult = Object.create(BigQueryRetrievalResult.prototype); 93 | const data = Object.assign(obj, response.body); 94 | expect(data.totalRows).toBe(totalRowsMocked); 95 | expect(data.totalBytesProcessed).toBe(totalBytesMocked); 96 | }); 97 | }); 98 | -------------------------------------------------------------------------------- /server/src/api/controllers/BigQueryController.ts: -------------------------------------------------------------------------------- 1 | import * as express from "express"; 2 | import * as RateLimit from "express-rate-limit"; 3 | import { BigQueryModel, BigQueryModelConfig } from "../models/BigQueryModel"; 4 | import { 5 | BigQueryRequest, 6 | BigQueryRetrievalResult, JsonParsingError 7 | } from "../types/BigQueryTypes"; 8 | import { 9 | CustomError, 10 | isError, 11 | isCustomError, 12 | } from "../../utils/error"; 13 | 14 | const jsonParser = express.json({ 15 | inflate: true, 16 | limit: "1kb", 17 | strict: true, 18 | type: "application/json" 19 | }); 20 | 21 | // Allow 20 requests (one data page/screen each) plus one 22 | // auto-pagination request (fetching 100 data pages) every 3 minutes. 23 | const rateLimiter = RateLimit({ 24 | windowMs: 3 * 60 * 1000, // 3 minutes 25 | max: 120 // limit each IP to 120 requests per windowMs 26 | }); 27 | 28 | /* 29 | API route handler 30 | */ 31 | export class BigQueryController { 32 | static readonly addRoute = (app: express.Application): void => { 33 | app.post(BigQueryRequest.Path, 34 | rateLimiter, 35 | jsonParser, 36 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { 37 | try { 38 | if (!BigQueryController.s_configSet) { 39 | const config = new BigQueryModelConfig(); 40 | BigQueryModel.Config = config; 41 | BigQueryController.s_configSet = true; 42 | } 43 | 44 | const errInfo: JsonParsingError = { message: undefined }; 45 | const bqRequest = BigQueryRequest.fromJson(req.body, req.ip, errInfo, true); 46 | 47 | if (!bqRequest) { 48 | const err = new CustomError(400, BigQueryController.s_ErrMsgParams, true); 49 | err.unobscuredMessage = `Invalid request from ${req.ip} with hostname ${req.hostname} using path ${req.originalUrl}. `; 50 | !!errInfo.message && (err.unobscuredMessage += errInfo.message); 51 | return next(err); 52 | } 53 | 54 | const model = BigQueryModel.Factory; 55 | await model.fetch(bqRequest); 56 | const data = model.Data; 57 | 58 | if (data instanceof Error) { 59 | if (isCustomError(data)) { 60 | return next(data as CustomError); 61 | } 62 | const error = new CustomError(500, BigQueryController.s_ErrMsgBigQuery, true, true); 63 | // Can only be set to "" if code is incorrectly modified 64 | error.unobscuredMessage = (data as Error).message ?? ""; 65 | return next(error); 66 | } else { 67 | res.status(200).json(data as BigQueryRetrievalResult); 68 | } 69 | } catch (err) { 70 | if (isCustomError(err)) { 71 | return next(err); 72 | } 73 | const error = new CustomError(500, BigQueryController.s_ErrMsgBigQuery, true, true); 74 | const errMsg: string = isError(err) ? err.message : ( 75 | "Exception: <" + 76 | Object.keys(err).map((key) => `${key}: ${err[key] ?? "no data"}`).join("\n") + 77 | ">" 78 | ); 79 | error.unobscuredMessage = errMsg; 80 | return next(error); 81 | } 82 | }); 83 | } 84 | 85 | /********************** private data ************************/ 86 | 87 | private static s_configSet = false; 88 | private static readonly s_ErrMsgBigQuery = "Could not query the backend database. Please retry later. If the problem persists contact Support"; 89 | private static readonly s_ErrMsgParams = "Invalid data retrieval parameter(s). Please notify Support"; 90 | } 91 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crisp-bigquery-client", 3 | "version": "1.0.0", 4 | "description": "Client for the Crisp BigQuery project", 5 | "author": "winwiz1 (https://github.com/winwiz1/)", 6 | "contributors": [ 7 | "winwiz1 (https://github.com/winwiz1/)" 8 | ], 9 | "license": "MIT", 10 | "homepage": "https://winwiz1.github.io/crisp-bigquery/", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/winwiz1/crisp-bigquery.git" 14 | }, 15 | "keywords": [ 16 | "react", 17 | "typescript", 18 | "bigquery", 19 | "visual-studio-code", 20 | "chrome-devtools", 21 | "webpack", 22 | "webpack-dev-server", 23 | "typestyle", 24 | "react-testing-library", 25 | "jest", 26 | "semantic-ui" 27 | ], 28 | "scripts": { 29 | "build": "webpack", 30 | "postbuild": "yarn prettier", 31 | "build:prod": "webpack --env.prod", 32 | "postbuild:prod": "yarn prettier", 33 | "compile": "tsc -p .", 34 | "lint": "eslint . --ext .ts,.tsx", 35 | "dev": "webpack-dev-server --config webpack.config.js", 36 | "test": "cross-env NODE_ENV=test jest", 37 | "build:prod-debug": "webpack --env.prod", 38 | "postbuild:prod-debug": "yarn copy", 39 | "precopy": "rimraf ../server/build/client && mkdirp ../server/build/client", 40 | "copy": "copyfiles -f dist/* ../server/build/client/", 41 | "prettier": "prettier --no-config --write ./dist/*.html" 42 | }, 43 | "dependencies": { 44 | "d3-dsv": "^2.0.0", 45 | "loglevel": "^1.7.0", 46 | "moment": "^2.29.1", 47 | "react": "17.0.1", 48 | "react-day-picker": "^7.4.0", 49 | "react-dom": "17.0.1", 50 | "react-helmet": "6.1.0", 51 | "react-redux": "^7.2.2", 52 | "react-router-dom": "^5.2.0", 53 | "redux": "^4.0.5", 54 | "semantic-ui-react": "2.0.1", 55 | "typestyle": "^2.0.4" 56 | }, 57 | "devDependencies": { 58 | "@testing-library/jest-dom": "^5.11.5", 59 | "@testing-library/react": "^11.1.1", 60 | "@types/d3-dsv": "^2.0.0", 61 | "@types/debug": "^4.1.5", 62 | "@types/filesystem": "^0.0.29", 63 | "@types/jest": "^26.0.15", 64 | "@types/react": "^16.9.56", 65 | "@types/react-day-picker": "^5.3.0", 66 | "@types/react-dom": "16.9.9", 67 | "@types/react-helmet": "6.1.0", 68 | "@types/react-redux": "^7.1.11", 69 | "@types/react-router-dom": "^5.1.6", 70 | "@types/semantic-ui": "2.2.7", 71 | "@types/webpack-dev-server": "^3.11.1", 72 | "@typescript-eslint/eslint-plugin": "^4.6.1", 73 | "@typescript-eslint/parser": "^4.6.1", 74 | "clean-webpack-plugin": "3.0.0", 75 | "compression-webpack-plugin": "6.0.5", 76 | "copyfiles": "^2.4.0", 77 | "cross-env": "^7.0.2", 78 | "css-loader": "5.0.1", 79 | "eslint": "^7.13.0", 80 | "fork-ts-checker-webpack-plugin": "6.0.0", 81 | "html-webpack-brotli-plugin": "0.0.4", 82 | "html-webpack-harddisk-plugin": "1.0.2", 83 | "html-webpack-plugin": "4.5.0", 84 | "html-webpack-template": "6.2.0", 85 | "jest": "^26.6.3", 86 | "jest-css-modules": "^2.1.0", 87 | "mkdirp": "^1.0.4", 88 | "prettier": "^2.1.2", 89 | "rimraf": "^3.0.2", 90 | "style-loader": "2.0.0", 91 | "ts-jest": "^26.4.3", 92 | "ts-loader": "8.0.10", 93 | "tsconfig-paths-webpack-plugin": "^3.3.0", 94 | "tslib": "2.0.3", 95 | "typescript": "4.0.5", 96 | "webpack": "4.44.2", 97 | "webpack-cli": "3.3.12", 98 | "webpack-dev-server": "^3.11.0", 99 | "webpack-subresource-integrity": "^1.5.2" 100 | }, 101 | "resolutions": { 102 | "minimist": "^1.2.2" 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /client/src/utils/fetch.ts: -------------------------------------------------------------------------------- 1 | /* 2 | The implementation of low level fetchAdapter() function. 3 | Provides extended error handling. 4 | */ 5 | import { isError, isDOMException } from "./typeguards"; 6 | import { CustomError } from "./error"; 7 | 8 | export interface IFetch { 9 | targetPath: string; 10 | method: "POST" | "GET"; 11 | isJson?: boolean; 12 | body?: string; 13 | abortSignal: AbortSignal; 14 | successHandler: (data: any) => void; 15 | errorHandler: (err: CustomError) => void; 16 | // User oriented error message for resp.ok === false 17 | errorMessage?: string; 18 | // User oriented error message for no response due to exception 19 | exceptionMessage?: string; 20 | // An additional suggestion to the above e.g. to imply the 21 | // exception is network related. 22 | exceptionExtraMessageNetwork?: string; 23 | } 24 | 25 | export const fetchAdapter = async (props: IFetch): Promise => { 26 | let isResponseJson: boolean | undefined; 27 | let isResponseText: boolean | undefined; 28 | let isResponseOk: boolean | undefined; 29 | let responseStatus: number | undefined; 30 | 31 | await fetch(props.targetPath, { 32 | credentials: "same-origin", 33 | method: props.method, 34 | mode: "same-origin", 35 | ...(props.isJson && { headers: { "Content-Type": "application/json" } }), 36 | signal: props.abortSignal, 37 | ...(props.body && { body: props.body }), 38 | }) 39 | .then(resp => { 40 | const contentType = resp.headers.get("Content-Type"); 41 | isResponseJson = (contentType?.includes("application/json")) ? true : false; 42 | isResponseText = (contentType?.includes("text/plain") || contentType?.includes("text/html")) ? true : false; 43 | const ret = isResponseJson ? resp.json() : resp.text(); 44 | isResponseOk = resp.ok; 45 | responseStatus = resp.status; 46 | return ret; 47 | }) 48 | .then(respData => { 49 | if (isResponseOk) { 50 | props.successHandler(respData); 51 | } else { 52 | let errMsg = props.errorMessage ?? "Could not get data from the backend."; 53 | if (isResponseText === true && isResponseJson === false && 54 | responseStatus && responseStatus >= 400) { 55 | errMsg += ` Error: ${respData}`; 56 | } 57 | const detailMsg = `Fetch error ${responseStatus} for URL ${props.targetPath}, details: ${respData}`; 58 | props.errorHandler(new CustomError(errMsg, detailMsg)); 59 | } 60 | }) 61 | .catch((err: any) => { 62 | if (isDOMException(err)) { 63 | if (err.code === 20 || err.name === "AbortError") { 64 | props.errorHandler(new CustomError( 65 | "Data retrieval in progress failed due to user initiated cancellation", 66 | `Fetch aborted for URL ${props.targetPath}` 67 | )); 68 | } else { 69 | props.errorHandler(new CustomError( 70 | `Fetch exception: ${err.message}`, 71 | `DOMException for URL ${props.targetPath}: code ${err.code}, name ${err.name}, message ${err.message}`)); 72 | } 73 | } else { 74 | let errMsg = props.exceptionMessage ?? "Failed to get data from the backend."; 75 | if (isResponseOk === undefined) { 76 | errMsg += (props.exceptionExtraMessageNetwork ?? " Please check the Internet connection."); 77 | const detailMsg = `Fetch exception for URL ${props.targetPath} likely due to network connectivity`; 78 | props.errorHandler(new CustomError(errMsg, detailMsg)); 79 | } else { 80 | let detailMsg: string; 81 | if (isError(err)) { 82 | detailMsg = `Fetch exception for URL ${props.targetPath}, details: ${err.message ?? ""}`; 83 | } else { 84 | const msg = Object.getOwnPropertyNames(err).map(key => `${key}: ${err[key] ?? "no data"}`).join("\n"); 85 | detailMsg = `Fetch exception for URL ${props.targetPath}, details: ${msg}`; 86 | } 87 | props.errorHandler(new CustomError(errMsg, detailMsg)); 88 | } 89 | } 90 | }); 91 | }; 92 | -------------------------------------------------------------------------------- /server/src/utils/misc.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | 3 | // Get the port that Express should be listening on 4 | export function getListeningPort(): number { 5 | const port = parseInt(process.env.PORT || "3000", 10); 6 | return port; 7 | } 8 | 9 | // Get the limit for cache size 10 | export function getCacheLimit(): number { 11 | const limit = parseInt(process.env.CACHE_LIMIT || "5000", 10); 12 | return limit; 13 | } 14 | 15 | // Get the daily data usage limit for the backend instance 16 | export function getDataLimit(): number { 17 | const limit = parseInt(process.env.DATA_LIMIT || "100000", 10); // ~100 GB 18 | return limit; 19 | } 20 | 21 | // Get the daily data usage limit for a single client 22 | export function getDataLimitClient(): number { 23 | const limit = parseInt(process.env.DATA_LIMIT_CLIENT || "2500", 10); // 50 queries 50 MB each 24 | return limit; 25 | } 26 | 27 | export function useProxy(): boolean { 28 | const ret = parseInt(process.env.BEHIND_PROXY || "0", 10); 29 | return ret === 1; 30 | } 31 | 32 | // Returns true if running on Google Cloud Run. 33 | // Assumption: the port 8080 is reserved for Cloud Run. 34 | export function isGoogleCloudRun(): boolean { 35 | return getListeningPort() === 8080; 36 | } 37 | 38 | export function isTest(): boolean { 39 | return process.env.NODE_ENV === "test"; 40 | } 41 | 42 | export function isProduction(): boolean { 43 | return process.env.NODE_ENV === "production"; 44 | } 45 | 46 | export function isDocker(): boolean { 47 | try { 48 | fs.statSync("/.dockerenv"); 49 | return true; 50 | } catch { 51 | try { 52 | fs.statSync("/.dockerinit"); 53 | return true; 54 | } catch { 55 | return false; 56 | } 57 | } 58 | } 59 | 60 | export type EnvVariables = { 61 | // GCP Project ID 62 | gcpProjectId: string, 63 | // BigQuery dataset name 64 | bqDatasetName: string, 65 | // BigQuery table name 66 | bqTableName: string, 67 | // Path to JSON file with service account credentials including private key 68 | keyFilePath: string 69 | }; 70 | 71 | export class EnvConfig { 72 | public static readonly getVariables = (): EnvVariables => { 73 | if (EnvConfig.s_checkDone) { 74 | if (!EnvConfig.s_envVariables) { 75 | throw new Error(EnvConfig.s_errMsg ?? "Invalid environment setup"); 76 | } 77 | return EnvConfig.s_envVariables!; 78 | } 79 | const gcpProjectId = process.env["GCP_PROJECT_ID"] || (process.env["PROJECT_ID"] ?? ""); 80 | const bqDatasetName = process.env["BIGQUERY_DATASET_NAME"] ?? ""; 81 | const bqTableName = process.env["BIGQUERY_TABLE_NAME"] ?? ""; 82 | const keyFilePath = process.env["KEY_FILE_PATH"] ?? ""; 83 | EnvConfig.s_checkDone = true; 84 | 85 | if (!gcpProjectId.length) { 86 | EnvConfig.s_errMsg = "Neither GCP_PROJECT_ID nor PROJECT_ID variable is set."; 87 | } 88 | if (!bqDatasetName.length) { 89 | EnvConfig.s_errMsg += " BIGQUERY_DATASET_NAME variable not set."; 90 | } 91 | if (!bqTableName.length) { 92 | EnvConfig.s_errMsg += " BIGQUERY_TABLE_NAME variable not set."; 93 | } 94 | if (!keyFilePath.length) { 95 | EnvConfig.s_errMsg += " KEY_FILE_PATH variable not set."; 96 | } 97 | if (EnvConfig.s_errMsg) { 98 | throw new Error(EnvConfig.s_errMsg); 99 | } 100 | 101 | let bqKeyFileReadable: boolean | undefined; 102 | 103 | try { 104 | fs.accessSync(keyFilePath, fs.constants.R_OK); 105 | bqKeyFileReadable = true; 106 | } catch { 107 | bqKeyFileReadable = false; 108 | } 109 | 110 | if (!bqKeyFileReadable) { 111 | EnvConfig.s_errMsg = `The key file '${keyFilePath}' cannot be read`; 112 | throw new Error(EnvConfig.s_errMsg); 113 | } 114 | 115 | EnvConfig.s_checkDone = true; 116 | EnvConfig.s_envVariables = { 117 | gcpProjectId, 118 | bqDatasetName, 119 | bqTableName, 120 | keyFilePath 121 | }; 122 | 123 | return EnvConfig.s_envVariables; 124 | } 125 | 126 | private static s_errMsg?: string = undefined; 127 | private static s_checkDone?: boolean = false; 128 | private static s_envVariables?: EnvVariables = undefined; 129 | } 130 | -------------------------------------------------------------------------------- /client/src/components/ErrorBoundary.tsx: -------------------------------------------------------------------------------- 1 | /* 2 | React error boundary. Catches and displays unexpected exceptions 3 | using a popup. Many UI design guidelines stipulate popups are 4 | meant to be used sparingly. Hence using this component to process 5 | unexpected and hopefully rare exceptions. 6 | */ 7 | import * as React from "react"; 8 | import * as ReactDOM from "react-dom"; 9 | import { style } from "typestyle"; 10 | import { isCustomError } from "../utils/typeguards"; 11 | import logger from "../utils/logger"; 12 | 13 | type ErrorBoundaryState = { 14 | hasError: boolean 15 | errDescription?: string 16 | }; 17 | 18 | // eslint-disable-next-line 19 | export class ErrorBoundary extends React.PureComponent<{}, ErrorBoundaryState> { 20 | public readonly state: ErrorBoundaryState = { hasError: false }; 21 | 22 | public static getDerivedStateFromError(err: Error): ErrorBoundaryState { 23 | const ret: ErrorBoundaryState = { 24 | errDescription: ErrorBoundary.extractErrorInfo(err), 25 | hasError: true, 26 | }; 27 | return ret; 28 | } 29 | 30 | public componentDidCatch(_err: Error, errInfo: React.ErrorInfo): void { 31 | if (errInfo.componentStack) { 32 | const errMsg = this.state.errDescription + "\n" + errInfo.componentStack; 33 | logger.error(errMsg); 34 | this.setState(prevState => 35 | ({ ...prevState, errDescription: errMsg }) 36 | ); 37 | } 38 | } 39 | 40 | public readonly onClick = (_evt: React.MouseEvent): void => { 41 | this.setState(prevState => { 42 | return prevState.hasError ? { ...prevState, hasError: false } : prevState; 43 | }); 44 | } 45 | 46 | // eslint-disable-next-line 47 | public render() { 48 | if (this.state.hasError) { 49 | return ( 50 | 55 | ); 56 | } else { 57 | return this.props.children; 58 | } 59 | } 60 | 61 | private static extractErrorInfo = (err: Error): string => { 62 | let errStr = "Something went wrong. If the issue persists please contact Support.\n\n"; 63 | errStr += err.message; 64 | if (isCustomError(err)) { 65 | errStr += `\n${err.detailMessage ?? ""}`; 66 | } 67 | 68 | return errStr; 69 | } 70 | } 71 | 72 | interface IPortalCreatorProps { 73 | errorText: string; 74 | errorHeader: string; 75 | onClose: (event: React.MouseEvent) => void; 76 | } 77 | 78 | // eslint-disable-next-line 79 | class PortalCreator extends React.Component { 80 | public componentDidMount() { 81 | document.body.appendChild(this.m_overlayContainer); 82 | } 83 | 84 | public componentWillUnmount() { 85 | document.body.removeChild(this.m_overlayContainer); 86 | } 87 | 88 | public render() { 89 | return ReactDOM.createPortal( 90 |
91 |

{this.props.errorHeader}

92 |
{this.props.errorText}
93 |
94 | 95 |
96 |
, 97 | this.m_overlayContainer 98 | ); 99 | } 100 | 101 | private readonly m_overlayContainer: HTMLDivElement = document.createElement("div"); 102 | public static readonly s_cssOverlay: string = style({ 103 | $debugName: "overlayErrorBox", 104 | $nest: { 105 | "& button": { 106 | padding: "0.3em 0.8em 0.3em 0.8em" 107 | }, 108 | "& div.buttonWrapper": { 109 | margin: "1em", 110 | textAlign: "center" 111 | 112 | }, 113 | "& div.textWrapper": { 114 | whiteSpace: "pre-wrap" 115 | }, 116 | "& h3": { 117 | textAlign: "center", 118 | } 119 | }, 120 | backgroundColor: "linen", 121 | border: "thick solid darkred", 122 | left: "50%", 123 | maxWidth: "80%", 124 | outline: "9999px solid rgba(0,0,0,0.5)", 125 | padding: "0.5em 1em 0.1em 1em", 126 | position: "fixed", 127 | top: "50%", 128 | transform: "translate(-50%, -50%)", 129 | width: "500px", 130 | zIndex: 10 131 | }); 132 | } 133 | -------------------------------------------------------------------------------- /server/src/test/BigQueryTypes.test.ts: -------------------------------------------------------------------------------- 1 | import * as moment from "moment"; 2 | import { JsonParsingError, BigQueryRequest } from "../api/types/BigQueryTypes"; 3 | import { TestConfig } from "./TestConfig"; 4 | 5 | const validAddress = "10.10.10.10"; 6 | 7 | describe("Testing BigQueryRequest with invalid data", () => { 8 | const errInfo: JsonParsingError = { message: undefined }; 9 | 10 | it("should reject empty JSON object", () => { 11 | const obj = {}; 12 | expect(BigQueryRequest.fromJson(obj, validAddress, errInfo, true)).not.toBeDefined(); 13 | }); 14 | 15 | it("should reject invalid JSON object", () => { 16 | const obj = { 17 | count: 100, 18 | endTime: "bye", 19 | startTime: "hello", 20 | }; 21 | expect(BigQueryRequest.fromJson(obj, validAddress, errInfo, true)).not.toBeDefined(); 22 | }); 23 | 24 | it("should reject invalid names", () => { 25 | const reqJson = TestConfig.getRequestAsJson(TestConfig.getStockTestRequest()); 26 | TestConfig.getInvalidNames().forEach(name => { 27 | reqJson.name = name; 28 | expect(TestConfig.getRequest(reqJson)).not.toBeDefined(); 29 | }); 30 | }); 31 | 32 | it("should reject invalid startDate", () => { 33 | const testReq = TestConfig.getStockTestRequest(); 34 | testReq.startDate = "2019-05-10T10:27:55.512Z**bad***"; 35 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).not.toBeDefined(); 36 | }); 37 | 38 | it("should reject invalid endDate", () => { 39 | const testReq = TestConfig.getStockTestRequest(); 40 | testReq.endDate = "2019-05-10T10:27:55.512Z**bad***"; 41 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).not.toBeDefined(); 42 | }); 43 | 44 | it("should reject invalid rowCount", () => { 45 | const testReq = TestConfig.getStockTestRequest(); 46 | testReq.rowCount = -1; 47 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).not.toBeDefined(); 48 | }); 49 | 50 | it("should reject query timeframe longer than 8 days", () => { 51 | const testReq = TestConfig.getStockTestRequest(); 52 | testReq.startDate = moment.utc().toDate().toDateString(); 53 | testReq.endDate = moment.utc().add(9, "days").utc().toDate().toDateString(); 54 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).not.toBeDefined(); 55 | 56 | }); 57 | 58 | it("should reject startTime post endTime", () => { 59 | const testReq = TestConfig.getStockTestRequest(); 60 | testReq.startDate = moment.utc().add(1, "days").utc().toDate().toDateString(); 61 | testReq.endDate = moment.utc().toDate().toDateString(); 62 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).not.toBeDefined(); 63 | }); 64 | }); 65 | 66 | describe("Testing BigQueryRequest with valid data", () => { 67 | const errInfo: JsonParsingError = { message: undefined }; 68 | 69 | it("should accept valid dates, name and language", () => { 70 | const testReq = TestConfig.getStockTestRequest(); 71 | testReq.name = "abc_XYZ.123-v"; 72 | testReq.language = "C++"; 73 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).toBeDefined(); 74 | }); 75 | 76 | it("should accept valid names", () => { 77 | const req = TestConfig.getStockTestRequest(); 78 | TestConfig.getValidNames().forEach(name => { 79 | req.name = name; 80 | expect(TestConfig.getRequest(req)).toBeDefined(); 81 | }); 82 | }); 83 | 84 | it("should accept startDate and endDate 1 week apart", () => { 85 | const testReq = TestConfig.getStockTestRequest(); 86 | testReq.startDate = moment.utc().toDate().toDateString(); 87 | testReq.endDate = moment.utc().add(7, "days").utc().toDate().toDateString(); 88 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).toBeDefined(); 89 | }); 90 | 91 | it("should accept valid rowCount", () => { 92 | const testReq = TestConfig.getStockTestRequest(); 93 | testReq.rowCount = 10000; 94 | expect(BigQueryRequest.fromJson(testReq, validAddress, errInfo, true)).toBeDefined(); 95 | }); 96 | 97 | it("should generate SQL clause", () => { 98 | const testReq = TestConfig.getStockTestRequest(); 99 | const bqRequest = TestConfig.getRequest(testReq); 100 | expect(bqRequest).toBeDefined(); 101 | const sql = bqRequest!.SqlTimeClause(); 102 | const result = /^BETWEEN\s'\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}'\sAND\s'\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}'$/ 103 | .test(sql); 104 | expect(result).toBeTruthy(); 105 | }); 106 | }); 107 | -------------------------------------------------------------------------------- /client/src/api/BackendManager.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Backend API client. 3 | Calls the API endpoint exposed by the backend and not the actual BigQuery API 4 | provided by Google. This arrangement ensures: 5 | - There is no room for CORS security violations (and therefore no need to 6 | relax the security by using CORS HTTP headers) because the script bundle 7 | with this code was downloaded from the same backend. 8 | - The client doesn't have the credentials required by BigQuery API. 9 | - The limit on client's data usage can be implemented independently from client. 10 | */ 11 | import { fetchAdapter, IFetch } from "../utils/fetch"; 12 | import { CustomError } from "../utils/error"; 13 | import { isError, isCustomError, isString } from "../utils/typeguards"; 14 | import { IBackendRequestData, BackendRequest } from "./BackendRequest"; 15 | import { 16 | IBigQueryData, 17 | BigQueryRetrieval, 18 | BigQueryRetrievalRow, 19 | BigQueryRetrievalResult, 20 | BigQueryRequest 21 | } from "@backend/types/BigQueryTypes"; 22 | 23 | export { 24 | BigQueryRetrieval, 25 | BigQueryRetrievalRow, 26 | BigQueryRetrievalResult 27 | }; 28 | 29 | // Extends data storage interface and adds data fetching capability. 30 | interface IBackendClient extends IBigQueryData { 31 | readonly fetch: (request: IBackendRequestData) => Promise; 32 | } 33 | 34 | // Implements data fetching and storage capabilities. 35 | export class BackendManager implements IBackendClient { 36 | constructor(signal: AbortSignal) { 37 | this.m_signal = signal; 38 | } 39 | 40 | get Data(): BigQueryRetrieval { 41 | return this.m_queryResult; 42 | } 43 | 44 | public readonly fetch = async (request: IBackendRequestData): Promise => { 45 | try { 46 | this.m_queryParams = new BackendRequest(request); 47 | await this.fetchData(); 48 | } catch (err) { 49 | if (isError(err)) { 50 | this.m_queryResult = err; 51 | } else { 52 | throw err; 53 | } 54 | } 55 | // Returns 'true' to facilitate timeout handling implemented as 56 | // racing with another Promise that returns 'false'. 57 | return true; 58 | } 59 | 60 | static get RegexName(): RegExp { 61 | return BigQueryRequest.RegexName; 62 | } 63 | 64 | static get RegexLanguage(): RegExp { 65 | return BigQueryRequest.RegexLanguage; 66 | } 67 | 68 | /********************** private methods and data ************************/ 69 | 70 | private fetchData = async (): Promise => { 71 | const fetchProps: IFetch = { 72 | abortSignal: this.m_signal, 73 | body: this.m_queryParams.toString(), 74 | errorHandler: this.errorHandler, 75 | isJson: true, 76 | method: "POST", 77 | successHandler: this.successHandler, 78 | targetPath: BackendManager.s_targetPath, 79 | }; 80 | 81 | await fetchAdapter(fetchProps); 82 | } 83 | 84 | private successHandler = (data: any): void => { 85 | const result = BackendManager.parser(data); 86 | 87 | if (result) { 88 | this.m_queryResult = result; 89 | } else if (isError(data)) { 90 | this.m_queryResult = data as Error; 91 | } else if (isString(data)) { 92 | this.m_queryResult = new Error(data as string); 93 | } else { 94 | this.m_queryResult = new CustomError( 95 | "Unexpected backend responce data, please contact Support.", 96 | "Details: " + Object.getOwnPropertyNames(data).map(key => `${key}: ${data[key] || "no data"}`).join("\n") 97 | ); 98 | } 99 | } 100 | 101 | private errorHandler = (err: CustomError): void => { 102 | if (isCustomError(err)) { 103 | this.m_queryResult = err; 104 | } else { 105 | this.m_queryResult = new CustomError( 106 | "Unexpected backend error data, please contact Support.", 107 | "Details: " + Object.getOwnPropertyNames(err).map(key => `${key}: ${err[key] || "no data"}`).join("\n") 108 | ); 109 | } 110 | } 111 | 112 | private static parser = (inputObj: Record): BigQueryRetrievalResult | undefined => { 113 | const obj: BigQueryRetrievalResult = Object.create(BigQueryRetrievalResult.prototype); 114 | const ret = Object.assign(obj, inputObj); 115 | const propNames = ["rows", "jobComplete", "totalRows", "returnedRows", "totalBytesProcessed"]; 116 | // eslint-disable-next-line no-prototype-builtins 117 | const hasProps = propNames.every(prop => ret.hasOwnProperty(prop)); 118 | 119 | return hasProps ? ret : undefined; 120 | } 121 | 122 | private m_queryParams: BackendRequest; 123 | private m_queryResult: BigQueryRetrieval = new Error(""); 124 | private readonly m_signal: AbortSignal; 125 | private static readonly s_targetPath = BigQueryRequest.Path; 126 | } 127 | -------------------------------------------------------------------------------- /client/src/components/QueryPagination.tsx: -------------------------------------------------------------------------------- 1 | /* 2 | Pagination component renders query status and manages pagination. 3 | */ 4 | import * as React from "react"; 5 | import { style } from "typestyle"; 6 | import { 7 | Button, 8 | Icon, 9 | Label, 10 | Pagination as SemanticPagination, 11 | PaginationProps as SemanticPaginationProps, 12 | Popup 13 | } from "semantic-ui-react"; 14 | import { IFetchState } from "../state/store"; 15 | import { actionCreators } from "../state/actions"; 16 | import { 17 | // This part/slice of QueryPaginationProps contains data and a callback 18 | // supplied by the parent component. Used to render query status. 19 | IQueryStatusProps, 20 | QueryStatus 21 | } from "./QueryStatus"; 22 | import { ExportCsv } from "./ExportCsv"; 23 | 24 | // This part/slice of QueryPaginationProps contains data supplied by 25 | // Redux store and related to fetching result. Used to manage pagination. 26 | export interface IQueryFetchProps { 27 | fetch: IFetchState; 28 | } 29 | 30 | // This part/slice of QueryPaginationProps contains functions that 31 | // create and dispatch Redux actions. Used to manage pagination. 32 | export interface IQueryActionProps { 33 | actions: typeof actionCreators; 34 | } 35 | 36 | // This part/slice of QueryPaginationProps contains data supplied by 37 | // the parent. Used to manage rendering. 38 | export interface IPaginationRenderProps { 39 | render: { 40 | scrollFun: () => void; 41 | scrollAid: boolean; 42 | autoPaginate: () => void; 43 | }; 44 | } 45 | 46 | // The type of props accepted by Pagination component 47 | // is the intersection of the above slices. 48 | type QueryPaginationProps = 49 | IQueryStatusProps & 50 | IQueryFetchProps & 51 | IQueryActionProps & 52 | IPaginationRenderProps; 53 | 54 | // Styles used by the Pagination component. 55 | 56 | const cssFlexContainer = style({ 57 | display: "flex", 58 | flexDirection: "row", 59 | flexWrap: "nowrap", 60 | margin: "1em" 61 | }); 62 | 63 | const cssStatus = style({ 64 | flex: "1 1 auto", 65 | margin: "1em", 66 | }); 67 | 68 | const cssPagination = style({ 69 | flex: "0 1 auto", 70 | margin: "1em" 71 | }); 72 | 73 | const cssButtonPaginate = style({ 74 | height: "2.5em" 75 | }); 76 | 77 | /* 78 | QueryPagination component 79 | */ 80 | export const QueryPagination = (props: QueryPaginationProps): JSX.Element => { 81 | const pageCount = props.status.cache.getPageCount(); 82 | const disabled = pageCount === 0; 83 | const dataPage = props.status.cache.getPage(props.status.currentPage); 84 | const moreData = !!props.status.cache.getLastPage()?.token && !props.status.err; 85 | const scrollAid = pageCount > 0 && dataPage!.rows > 10; 86 | const onPageChange = (_evt: React.MouseEvent, data: SemanticPaginationProps): void => { 87 | data.activePage && props.actions.actionSetPage(data.activePage as number - 1); 88 | }; 89 | const onAutoPagination = () => { 90 | props.render.autoPaginate(); 91 | }; 92 | 93 | return ( 94 |
95 |
96 | 97 |
98 | 140 |
141 | ); 142 | }; 143 | -------------------------------------------------------------------------------- /server/src/test/BigQueryModel.test.ts: -------------------------------------------------------------------------------- 1 | 2 | import { BigQueryModel } from "../api/models/BigQueryModel"; 3 | import { BigQueryRetrieval, BigQueryRetrievalResult } from "../api/types/BigQueryTypes"; 4 | import { TestConfig } from "./TestConfig"; 5 | 6 | describe("Testing BigQueryModel", () => { 7 | const timeoutPaginatingQuery = 120000; // 120 sec 8 | const timeoutNonPaginatingQuery = 10000; // 10 sec 9 | const config = TestConfig.getModelConfig(); 10 | BigQueryModel.Config = config; 11 | 12 | it("test non-paginating query", async () => { 13 | const testReq = TestConfig.getStockTestRequest(); 14 | testReq.name = "C"; 15 | const bqRequest = TestConfig.getRequest(testReq); 16 | const model = BigQueryModel.Factory; 17 | await model.fetch(bqRequest!); 18 | const data: BigQueryRetrieval = model.Data; 19 | expect(data).toBeInstanceOf(BigQueryRetrievalResult); 20 | 21 | const result = data as BigQueryRetrievalResult; 22 | expect(result.jobComplete).toBeTruthy(); 23 | expect(+result.totalRows).toBeGreaterThan(result.rows.length); 24 | }, timeoutNonPaginatingQuery 25 | ); 26 | 27 | it("test paginating query", async () => { 28 | const testReq = TestConfig.getStockTestRequest(); 29 | // pagination size 30 | testReq.rowCount = 500; 31 | // apply filter which yields 1458 records 32 | testReq.name = "C"; 33 | // 1458 records require 3 pages of 500 records 34 | const expectedPagingCount = 3; 35 | let bqRequest = TestConfig.getRequest(testReq); 36 | const model = BigQueryModel.Factory; 37 | await model.fetch(bqRequest!); 38 | const data1: BigQueryRetrieval = model.Data; 39 | expect(data1).toBeInstanceOf(BigQueryRetrievalResult); 40 | 41 | const result1 = data1 as BigQueryRetrievalResult; 42 | expect(result1.jobComplete).toBeTruthy(); 43 | expect(+result1.totalRows).toBeGreaterThan(result1.rows.length); 44 | 45 | let pageToken: string | undefined = result1.pageToken; 46 | let jobId: string | undefined = result1.jobId; 47 | let cnt = 0; 48 | 49 | // eslint-disable-next-line no-extra-boolean-cast 50 | while (!!pageToken) { 51 | testReq.pageToken = pageToken; 52 | testReq.jobId = jobId; 53 | bqRequest = TestConfig.getRequest(testReq); 54 | expect(bqRequest).toBeDefined(); 55 | 56 | await model.fetch(bqRequest!); 57 | const data2: BigQueryRetrieval = model.Data; 58 | expect(data2).toBeInstanceOf(BigQueryRetrievalResult); 59 | 60 | const result2 = data2 as BigQueryRetrievalResult; 61 | expect(result2.jobComplete).toBeTruthy(); 62 | 63 | pageToken = result2.pageToken; 64 | jobId = result2.jobId; 65 | 66 | console.log(`cnt = ${cnt++}`); 67 | expect(cnt).not.toBeGreaterThan(expectedPagingCount); 68 | } 69 | expect(cnt).toEqual(expectedPagingCount - 1); 70 | 71 | }, timeoutPaginatingQuery 72 | ); 73 | 74 | it("tests the data limit imposed on a client", async () => { 75 | const clientAddress = "test.com.local"; 76 | const configLimit = TestConfig.getModelConfig(); 77 | configLimit.setClientDailyLimit(30); // 30 MB 78 | BigQueryModel.Config = configLimit; 79 | // one query takes 10 MB 80 | const expectedPagingCount = 4; 81 | const testReq = TestConfig.getStockTestRequest(); 82 | const bqRequest = TestConfig.getRequest( 83 | testReq, 84 | clientAddress, 85 | false // do not use cache 86 | ); 87 | const model = BigQueryModel.Factory; 88 | await model.fetch(bqRequest!); 89 | const data: BigQueryRetrieval = model.Data; 90 | expect(data).toBeInstanceOf(BigQueryRetrievalResult); 91 | 92 | const result = data as BigQueryRetrievalResult; 93 | expect(result.jobComplete).toBeTruthy(); 94 | expect(+result.totalRows).toBeGreaterThan(result.rows.length); 95 | 96 | let pageToken: string | undefined = result.pageToken; 97 | let jobId: string | undefined = result.jobId; 98 | let cnt = 0; 99 | 100 | // eslint-disable-next-line no-extra-boolean-cast 101 | while (!!pageToken) { 102 | testReq.pageToken = pageToken; 103 | testReq.jobId = jobId; 104 | const bqRequest = TestConfig.getRequest( 105 | testReq, 106 | clientAddress, 107 | false 108 | ); 109 | expect(bqRequest).toBeDefined(); 110 | 111 | await model.fetch(bqRequest!); 112 | const data: BigQueryRetrieval = model.Data; 113 | expect(data).toBeInstanceOf( 114 | cnt === expectedPagingCount - 1 ? Error : BigQueryRetrievalResult 115 | ); 116 | if (data instanceof Error) { 117 | expect(data.message).toContain("limit"); 118 | break; 119 | } 120 | 121 | const result = data as BigQueryRetrievalResult; 122 | expect(result.jobComplete).toBeTruthy(); 123 | 124 | pageToken = result.pageToken; 125 | jobId = result.jobId; 126 | 127 | console.log(`cnt = ${cnt++}`); 128 | expect(cnt).not.toBeGreaterThan(expectedPagingCount); 129 | } 130 | expect(cnt).toEqual(expectedPagingCount - 1); 131 | 132 | }, timeoutPaginatingQuery 133 | ); 134 | }); 135 | -------------------------------------------------------------------------------- /server/src/utils/error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The class CustomError and Express error-handling 3 | * middleware that uses this class. 4 | */ 5 | import * as Express from "express"; 6 | import { logger } from "./logger"; 7 | import { isTest } from "./misc"; 8 | 9 | /* 10 | The class allows to have two different error messages describing the same error. 11 | On the one hand we would like to log all the error details for troubleshooting, 12 | auditing etc. On the other hand the error description also needs to be sent to 13 | the user. Having two wordings for the same issue allows to avoid a possible XSS 14 | reflection. The added benefit of this approach is the ability to keep logs 15 | detailed while sparing the end users from seeing the technical details and 16 | unfamiliar terminology. 17 | */ 18 | export class CustomError extends Error { 19 | constructor( 20 | // HTTP status code 21 | readonly status: number, 22 | // Error message sent to client and obscured/sanitised to avoid possible XSS 23 | message: string, 24 | // Optional logging tuple. The first boolean tells the error-handling 25 | // middleware if the Request needs to be logged. The second boolean controls 26 | // Response logging. 27 | ...loggingTuple: [boolean?, boolean?]) { 28 | super(message); 29 | // http://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget 30 | Object.setPrototypeOf(this, new.target.prototype); 31 | if (status <= 200) { 32 | throw new RangeError(`Invalid CustomError.status: ${status}`); 33 | } 34 | // save the tuple that comes as rest parameter 35 | loggingTuple.length > 0 && (this.m_loggingTuple[0] = loggingTuple[0]!); 36 | loggingTuple.length === 2 && (this.m_loggingTuple[1] = loggingTuple[1]!); 37 | } 38 | 39 | get unobscuredMessage(): string | undefined { 40 | return this.m_unobscuredMsg; 41 | } 42 | set unobscuredMessage(val: string | undefined) { 43 | this.m_unobscuredMsg = val; 44 | } 45 | 46 | get logOnly(): boolean { 47 | return this.m_logOnly; 48 | } 49 | set logOnly(val: boolean) { 50 | this.m_logOnly = val; 51 | } 52 | 53 | // Unobscured error message, might contain error details reflecting 54 | // attempted XSS, used for logging 55 | private m_unobscuredMsg: string | undefined = undefined; 56 | 57 | // If true then no response is sent by error-handling middleware 58 | private m_logOnly: boolean = false; 59 | 60 | // By default do log the request and do not log the response 61 | readonly m_loggingTuple: [boolean, boolean] = [true, false]; 62 | } 63 | 64 | // eslint-disable-next-line 65 | export function isError(err: any): err is Error { 66 | return !!err && err instanceof Error && err.constructor !== CustomError; 67 | } 68 | 69 | // eslint-disable-next-line 70 | export function isCustomError(err: any): err is CustomError { 71 | return !!err && err.constructor === CustomError; 72 | } 73 | 74 | /* 75 | Error-handling middleware 76 | */ 77 | function errorMiddleware( 78 | err: any, 79 | request: Express.Request, 80 | response: Express.Response, 81 | next: Express.NextFunction) { 82 | 83 | if (response.headersSent) { 84 | return next(err); 85 | } 86 | 87 | if (isCustomError(err)) { 88 | const status = err.status; 89 | const logMsg = augmentLogMessage( 90 | err.unobscuredMessage ?? err.message, 91 | err.m_loggingTuple[0]? request: undefined, 92 | err.m_loggingTuple[1]? response: undefined 93 | ); 94 | 95 | isTest() || logger.warn({ 96 | message: `statusCode: '${err.status}' description: ${logMsg}` 97 | }); 98 | if (!err.logOnly) { 99 | response.status(status).type("txt").send(err.message); 100 | } 101 | } else if (isError(err)) { 102 | const logMsg = augmentLogMessage(err.message, request, response); 103 | 104 | logger.warn({ 105 | message: logMsg 106 | }); 107 | response.status(500).type("txt").send(err.message); 108 | } else { 109 | const logMsg = augmentLogMessage(`Unexpected error type: ${typeof err}`, request, response); 110 | 111 | logger.error({ 112 | message: logMsg 113 | }); 114 | next(err); 115 | } 116 | } 117 | 118 | export function handleErrors(app: Express.Application): void { 119 | app.use(errorMiddleware); 120 | } 121 | 122 | // Utility function 123 | function augmentLogMessage(msg: string, req?: Express.Request, resp?: Express.Response): string { 124 | let ret = `'${msg}'`; 125 | 126 | if (req) { 127 | ret += ` req.method: '${req.method}'`; 128 | ret += ` req.remoteIp: '${req.ip.indexOf(":") >= 0 ? req.ip.substring(req.ip.lastIndexOf(":") + 1) : req.ip}'`; 129 | ret += ` req.url: '${req.protocol}` + "://" + `${req.get("host")}${req.originalUrl}'`; 130 | ret += ` req.referrer: '${req.get("Referrer") ?? "no data"}'`; 131 | ret += ` req.size: '${req.socket.bytesRead}'`; 132 | ret += ` req.userAgent: '${req.get("User-Agent") ?? "no data"}'`; 133 | } 134 | 135 | if (resp) { 136 | ret += ` res.statusCode: '${resp.statusCode ?? "not yet set"}'`; 137 | 138 | let respSize: number|undefined; 139 | 140 | if ((resp as any).body) { 141 | if (typeof (resp as any).body === 'object') { 142 | respSize = JSON.stringify((resp as any).body).length; 143 | } else if (typeof (resp as any).body === 'string') { 144 | respSize = (resp as any).body.length; 145 | } 146 | } 147 | 148 | !!respSize && (ret += ` res.size: '${respSize}'`); 149 | } 150 | 151 | return ret; 152 | } 153 | -------------------------------------------------------------------------------- /client/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const fs = require("fs"); 3 | const webpack = require("webpack") 4 | const { CleanWebpackPlugin } = require("clean-webpack-plugin"); 5 | const ForkTsCheckerWebpackPlugin = require("fork-ts-checker-webpack-plugin"); 6 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 7 | const HtmlWebpackHarddiskPlugin = require("html-webpack-harddisk-plugin"); 8 | const TerserPlugin = require("terser-webpack-plugin"); 9 | const configuredSPAs = require("./config/spa.config"); 10 | const verifier = require("./config/verifySpaParameters"); 11 | const TsconfigPathsPlugin = require("tsconfig-paths-webpack-plugin"); 12 | const headHtmlSnippetPath = path.join("src", "entrypoints", "head-snippet.html"); 13 | const headHtmlSnippet = fs.existsSync(headHtmlSnippetPath) ? 14 | fs.readFileSync(headHtmlSnippetPath, "utf8") : undefined; 15 | const metaDescription = "Build a custom website hydrated by BigQuery data."; 16 | const metaKeywords = "BigQuery, NodeJS, React"; 17 | const metaOwnUrl = "https://your.website.com/"; 18 | 19 | configuredSPAs.verifyParameters(verifier); 20 | 21 | // Supress "Failed to load tsconfig.json: Missing baseUrl in compilerOptions" error message. 22 | // Details: https://github.com/dividab/tsconfig-paths-webpack-plugin/issues/17 23 | delete process.env.TS_NODE_PROJECT; 24 | 25 | const getWebpackConfig = (env, argv) => { 26 | const isProduction = (env && env.prod) ? true : false; 27 | 28 | const config = { 29 | mode: isProduction ? "production" : "development", 30 | devtool: "source-map", 31 | entry: configuredSPAs.getEntrypoints(), 32 | module: { 33 | rules: [ 34 | { 35 | test: /\.tsx?$/, 36 | exclude: /node_modules/, 37 | use: [ 38 | { 39 | loader: "ts-loader", 40 | options: { 41 | transpileOnly: true, 42 | happyPackMode: true, 43 | configFile: path.resolve(__dirname, "tsconfig.json"), 44 | }, 45 | } 46 | ], 47 | }, 48 | { 49 | test: /\.css$/, 50 | use: ["style-loader", "css-loader"], 51 | }, 52 | ] 53 | }, 54 | resolve: { 55 | extensions: [".tsx", ".ts", ".js"], 56 | plugins: [ 57 | new TsconfigPathsPlugin() 58 | ] 59 | }, 60 | output: { 61 | filename: "[name].[hash].bundle.js", 62 | chunkFilename: "[name].[hash].bundle.js", 63 | path: path.resolve(__dirname, "dist"), 64 | publicPath: "/static/", 65 | crossOriginLoading: "anonymous", 66 | }, 67 | optimization: { 68 | splitChunks: { 69 | cacheGroups: { 70 | vendor: { 71 | test: /node_modules/, 72 | chunks: "initial", 73 | name: "vendor", 74 | enforce: true 75 | }, 76 | }, 77 | }, 78 | runtimeChunk: "single", 79 | ...(isProduction && { 80 | minimizer: [ 81 | new TerserPlugin({ 82 | cache: false, 83 | parallel: true, 84 | sourceMap: false, // set to true if debugging of production build needed 85 | terserOptions: { 86 | keep_classnames: false, 87 | mangle: true, 88 | compress: false, 89 | keep_fnames: false, 90 | output: { 91 | comments: false, 92 | } 93 | } 94 | }) 95 | ] 96 | }), 97 | }, 98 | plugins: [ 99 | new CleanWebpackPlugin(), 100 | new webpack.DefinePlugin({ 101 | "process.env.DEVELOPMENT": JSON.stringify(isProduction === false) 102 | }), 103 | new ForkTsCheckerWebpackPlugin({ 104 | typescript: true, 105 | eslint: undefined, 106 | logger: { infrastructure: "console", issues: "console" } 107 | }) 108 | ], 109 | devServer: { 110 | index: `/${configuredSPAs.getRedirectName()}.html`, 111 | publicPath: "/static/", 112 | contentBase: path.join(__dirname, "dist"), 113 | compress: false, 114 | hot: true, 115 | inline: true, 116 | port: 8080, 117 | writeToDisk: true, 118 | historyApiFallback: { 119 | index: `${configuredSPAs.getRedirectName()}.html`, 120 | rewrites: configuredSPAs.getRewriteRules() 121 | } 122 | }, 123 | context: path.resolve(__dirname), 124 | }; 125 | 126 | configuredSPAs.getNames().forEach((entryPoint) => { 127 | config.plugins.push( 128 | new HtmlWebpackPlugin({ 129 | template: require("html-webpack-template"), 130 | inject: true, 131 | title: configuredSPAs.appTitle, 132 | appMountId: "app-root", 133 | alwaysWriteToDisk: true, 134 | filename: `${entryPoint}.html`, 135 | chunks: [`${entryPoint}`, "vendor", "runtime"], 136 | headHtmlSnippet, 137 | links: [ 138 | { 139 | rel: "dns-prefetch", 140 | href: "//fonts.gstatic.com/" 141 | }, 142 | { 143 | rel: "dns-prefetch", 144 | href: "//fonts.googleapis.com/" 145 | }, 146 | { 147 | rel: "stylesheet", 148 | href: "https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.css", 149 | integrity: "sha384-JKIDqM48bt14NZpzl9v0AP36VK2C/X6RuSPfimxpoWdSANUXblZUX1cgdQw8cZUK", 150 | crossorigin: "anonymous" 151 | }, 152 | { 153 | href: metaOwnUrl, 154 | rel: "canonical" 155 | }, 156 | { 157 | href: "/apple-touch-icon.png", 158 | rel: "apple-touch-icon", 159 | sizes: "180x180" 160 | }, 161 | ], 162 | meta: [ 163 | { name: "viewport", content: "width=device-width, initial-scale=1.0, shrink-to-fit=no" }, 164 | { name: "description", content: metaDescription }, 165 | { name: "keywords", content: metaKeywords }, 166 | { name: "robots", content: "index, follow" }, 167 | { property: "og:title", content: configuredSPAs.appTitle }, 168 | { property: "og:type", content: "website" }, 169 | { property: "og:url", content: metaOwnUrl }, 170 | { property: "og:description", content: metaDescription }, 171 | { property: "twitter:title", content: configuredSPAs.appTitle }, 172 | { property: "twitter:description", content: metaDescription }, 173 | ], 174 | minify: false, 175 | }) 176 | ); 177 | }) 178 | 179 | config.plugins.push(new HtmlWebpackHarddiskPlugin()); 180 | 181 | if (isProduction) { 182 | const CompressionPlugin = require("compression-webpack-plugin"); 183 | const SriPlugin = require("webpack-subresource-integrity"); 184 | 185 | config.plugins.push( 186 | new webpack.DefinePlugin({ 187 | "process.env.NODE_ENV": JSON.stringify("production") 188 | })); 189 | config.plugins.push( 190 | new SriPlugin({ 191 | hashFuncNames: ["sha384"] 192 | })); 193 | config.plugins.push( 194 | new CompressionPlugin({ 195 | filename: "[path][base].br", 196 | algorithm: "brotliCompress", 197 | test: /\.js$|\.css$|\.html$/, 198 | compressionOptions: { 199 | level: 11, 200 | }, 201 | threshold: 10240, 202 | minRatio: 0.8, 203 | deleteOriginalAssets: false, 204 | })); 205 | config.plugins.push( 206 | new CompressionPlugin({ 207 | filename: "[path][base].gz", 208 | algorithm: "gzip", 209 | test: /\.js$|\.css$|\.html$/, 210 | threshold: 10240, 211 | minRatio: 0.8 212 | })); 213 | } 214 | 215 | return config; 216 | } 217 | 218 | module.exports = getWebpackConfig; 219 | -------------------------------------------------------------------------------- /server/src/srv/server.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The class Server is responsible for Express configuration. 3 | * Express is configured to serve the build artefacts produced 4 | * by the sibling 'client' sub-project (script bundles, HTML files, 5 | * source maps) either from disk or from webpack-dev-server 6 | * acting as a reverse proxy for the latter. 7 | */ 8 | import * as path from "path"; 9 | import * as express from "express"; 10 | import nodeFetch from "node-fetch"; 11 | import * as helmet from "helmet"; 12 | import * as nocache from "nocache"; 13 | import * as expressStaticGzip from "express-static-gzip"; 14 | import favicon = require("serve-favicon"); 15 | import * as SPAs from "../../config/spa.config"; 16 | import { CustomError, handleErrors } from "../utils/error"; 17 | import { logger } from "../utils/logger"; 18 | import { 19 | useProxy, 20 | isProduction, 21 | } from "../utils/misc" 22 | import { BigQueryController } from "../api/controllers/BigQueryController"; 23 | const { createProxyMiddleware } = require("http-proxy-middleware"); 24 | 25 | export enum StaticAssetPath { 26 | // The path to static assets served by Express needs to be 27 | // resolved starting from the source .ts files located 28 | // under src/srv 29 | SOURCE = 0, 30 | // The path to static assets served by Express needs to be 31 | // resolved starting from the transpiled .js files located 32 | // under build/ 33 | TRANSPILED = 1, 34 | } 35 | 36 | class Server { 37 | constructor() { 38 | this.m_app = express(); 39 | this.config(); 40 | this.addRoutes(); 41 | handleErrors(this.m_app); 42 | } 43 | 44 | public readonly getApp = (assetPath = StaticAssetPath.TRANSPILED) => { 45 | this.m_assetPath = assetPath; 46 | this.m_expressStaticMiddleware = 47 | expressStaticGzip(this.getAssetPath() + "/../", Server.s_expressStaticConfig); 48 | return this.m_app; 49 | } 50 | 51 | /********************** private methods and data ************************/ 52 | 53 | private config(): void { 54 | this.m_app.use([ 55 | helmet(isProduction()? { 56 | contentSecurityPolicy: { 57 | directives: { 58 | frameSrc: ["'self'"], 59 | defaultSrc: ["'self'"], 60 | // CSP can be tested by removing the 'nomodule' attribute from inline