├── .dockerignore ├── .eslintrc.cjs ├── .github └── workflows │ └── containerize.yaml ├── .gitignore ├── .npmrc ├── .prettierrc ├── Dockerfile ├── README.md ├── docker-compose.yaml ├── env ├── .env-example ├── allProjectIDs.json ├── campusIDs.json └── projectIDs.json ├── hooks └── pre-commit ├── package-lock.json ├── package.json ├── public ├── new.svg └── placeholder.png ├── src ├── app.ts ├── authentication.ts ├── db.ts ├── env.ts ├── express.ts ├── logger.ts ├── metrics.ts ├── statsd.ts ├── types.ts └── util.ts ├── sync-production.sh ├── tsconfig.json └── views ├── error.ejs └── index.ejs /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store* 2 | **/node_modules 3 | **/README.md 4 | **/.dockerignore 5 | **/README.* 6 | 7 | database 8 | /build 9 | /.git 10 | /Dockerfile* 11 | /docker-compose.yml 12 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: false, 4 | es2021: true, 5 | }, 6 | extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'], 7 | overrides: [], 8 | parser: '@typescript-eslint/parser', 9 | parserOptions: { 10 | ecmaVersion: 'latest', 11 | sourceType: 'module', 12 | }, 13 | plugins: ['@typescript-eslint', 'prettier'], 14 | rules: { 15 | 'prettier/prettier': ['error'], 16 | 17 | 'array-callback-return': 'error', 18 | curly: ['error', 'all'], 19 | 'no-array-constructor': 'error', 20 | 'no-duplicate-imports': ['error', { includeExports: true }], 21 | 'no-extend-native': 'error', 22 | 'no-nested-ternary': 'error', 23 | 'no-return-assign': 'error', 24 | 'no-return-await': 'error', 25 | 'no-throw-literal': 'error', 26 | 'no-unreachable-loop': 'error', 27 | 'no-unused-private-class-members': 'error', 28 | 'no-useless-catch': 'error', 29 | 'no-useless-escape': 'error', 30 | 'no-useless-return': 'error', 31 | 'no-var': 'error', 32 | 'prefer-arrow-callback': 'error', 33 | 'prefer-const': 'error', 34 | 'prefer-template': 'error', 35 | 'require-await': 'error', 36 | 'no-constant-condition': ['error', { checkLoops: false }], 37 | eqeqeq: ['error', 'smart'], 38 | '@typescript-eslint/ban-ts-comment': 'off', 39 | 40 | // managed by prettier 41 | // indent: ['error', 'tab'], 42 | // 'linebreak-style': ['error', 'unix'], 43 | // quotes: ['error', 'single'], 44 | // semi: ['error', 'never'], 45 | // 'max-len': ['error', { code: 120, ignoreUrls: true }], 46 | // 'arrow-parens': ['error', 'as-needed'], 47 | // 'brace-style': ['error', 'stroustrup'], 48 | // 'comma-dangle': ['error', 'always-multiline'], 49 | // curly: ['error', 'multi-or-nest'], 50 | }, 51 | } 52 | -------------------------------------------------------------------------------- /.github/workflows/containerize.yaml: -------------------------------------------------------------------------------- 1 | name: Containerize 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build-and-publish: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v2 15 | 16 | - name: Login to GitHub Package Registry 17 | uses: docker/login-action@v1 18 | with: 19 | registry: docker.pkg.github.com 20 | username: ${{ github.actor }} 21 | password: ${{ secrets.GITHUB_TOKEN }} 22 | 23 | - name: Build Docker image 24 | run: docker build -t docker.pkg.github.com/codam-coding-college/find-peers/find-peers:latest . 25 | 26 | - name: Push Docker image to GitHub Package Registry 27 | run: docker push docker.pkg.github.com/codam-coding-college/find-peers/find-peers:latest 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | .DS_Store 3 | /test.js 4 | /node_modules/ 5 | /token*.json 6 | /env/.env 7 | /database*/ 8 | /test.ts 9 | /sessions/ 10 | *.swp 11 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "bracketSameLine": true, 5 | "printWidth": 180, 6 | "useTabs": true, 7 | "arrowParens": "avoid" 8 | } 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18-alpine 2 | 3 | WORKDIR /app 4 | 5 | HEALTHCHECK --interval=5s --timeout=10s --start-period=5s --retries=1 CMD wget -q -O - http://localhost:8080/robots.txt 6 | EXPOSE 8080 7 | 8 | ENV NODE_ENV=production 9 | COPY package.json package-lock.json ./ 10 | RUN npm ci --omit=dev 11 | COPY . . 12 | RUN npm run build 13 | 14 | RUN rm -rf src 15 | COPY views ./build/views 16 | 17 | ENTRYPOINT [ "npm", "start" ] 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Find peers --> [find-peers.codam.nl](https://find-peers.codam.nl) 2 | 3 | This website is meant to help students to find peers that are working on the same project 4 | 5 | ## Developing 6 | ## Setup & configuration 7 | - Create a Oauth application on [intra](https://profile.intra.42.fr/oauth/applications) 8 | - Copy the file `./env/.env-example` to `./env/.env` and fill out the (secret) data 9 | 10 | Also see `./src/env.ts` for more configuration 11 | 12 | ## Changing listed projects 13 | - The projects shown on the front page are listed in `./env/projectIDs.json`. Should the curriculum change, you can edit that file. Remember to restart the server and wait for the server to pull all the data from the intra api. 14 | - A list of all the projects and their corresponding ID in the 42 network (as of march 2022) can be found in `./env/allProjectIDs.json` 15 | 16 | ## Updating the secrets / API tokens 17 | ```shell 18 | cd find-peers 19 | vim env/.env 20 | # make changes 21 | docker compose down 22 | docker compose up -d 23 | 24 | # To get logs 25 | docker logs --tail 10000 -f find-peers 26 | ``` 27 | 28 | ## Monitoring 29 | At the (unauthenticated) route `/status/pull` you can see a summary of the pull status of every campus 30 | It contains the key `hoursAgo` for every campus, which is the amount of hours since the last successful pull (syncing the 42 DB of the users' completed projects) of that campus 31 | This value should not be higher than 2 * the pull timeout (currently 24 hours) 32 | 33 | ## Configuration files 34 | | File path | Description | Managed by server | 35 | |----------------------------------------------|---------------------------------------------------------------------------------------|-------------------| 36 | | `./env/projectIDs.json` | List of all the projects and their corresponding ID to be displayed on the front page | no | 37 | | `./env/allProjectIDs.json` | List of all projects in the 42 network (as of march 2022) | no | 38 | | `./env/.env-example` | Example file for api tokens, rename to `.env` to activate | no | 39 | | `./env/campusIDs.json` | List of all campuses and their corresponding ID that are fetched from the 42 API | no | 40 | | `./database/` | All database files, mount this when running in a docker container | yes | 41 | | `./database/sessions/` | All session files currently active | yes | 42 | | `./database/users.json` | Userdata associated with session | yes | 43 | | `./database//lastpull.txt` | Unix timestamp when the project users of that campus were last successfully updated | yes | 44 | | `./database//projectUsers.json` | Status of users for each project | yes | 45 | 46 | ## Running 47 | The 'database' of this project is a folder called 'database' at the root of the project. 48 | 49 | ### Docker and Docker-compose 50 | This is in production 51 | ```shell 52 | git clone https://github.com/codam-coding-college/find-peers.git 53 | cd find-peers 54 | docker compose up -d 55 | 56 | # To get logs 57 | docker logs --tail 10000 -f find-peers 58 | ``` 59 | 60 | ### Locally 61 | - Install Nodejs >= 18.x 62 | - Install dependencies\ 63 | `npm install` 64 | - Start development server\ 65 | `npm run dev` 66 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | datadog-agent: 5 | container_name: datadog-agent 6 | image: gcr.io/datadoghq/agent:latest 7 | pid: host 8 | volumes: 9 | - /var/run/docker.sock:/var/run/docker.sock:ro 10 | - /proc/:/host/proc/:ro 11 | - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro 12 | env_file: ./env/.env 13 | environment: 14 | - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true 15 | ports: 16 | - 8125:8125/udp 17 | 18 | find-peers: 19 | container_name: find-peers 20 | restart: unless-stopped 21 | image: ghcr.io/codam-coding-college/find-peers/find-peers:latest 22 | volumes: 23 | - $HOME/find-peers/database:/app/database 24 | env_file: ./env/.env 25 | environment: 26 | - PORT=8080 27 | ports: 28 | - 80:8080 29 | # - 8080:8080 30 | 31 | watchtower: 32 | container_name: watchtower 33 | image: containrrr/watchtower 34 | restart: unless-stopped 35 | volumes: 36 | - /var/run/docker.sock:/var/run/docker.sock 37 | command: find-peers 38 | environment: 39 | - WATCHTOWER_CLEANUP=true 40 | - WATCHTOWER_INCLUDE_RESTARTING=true 41 | - WATCHTOWER_POLL_INTERVAL=30 42 | - WATCHTOWER_ROLLING_RESTART=true 43 | -------------------------------------------------------------------------------- /env/.env-example: -------------------------------------------------------------------------------- 1 | # How to install: 2 | # 1. copy this file and rename to .env 3 | # 2. fill in empty values 4 | # 3. remove all comments (the lines starting with #) 5 | 6 | # this app needs 2 keys: 7 | # 1. For user authentiction - only needs the 'public' scope 8 | # 2. For syncing the database - can have more permissions for faster syncing 9 | 10 | # This is because syncing the database might reach the request quota while syncing the database 11 | # and when users try to login they can't because there is a oauth request to validate their login 12 | 13 | # Salt used for hashing username based metrics, can be anything as long as it stays the same 14 | METRICS_SALT= 15 | 16 | USERAUTH_UID= 17 | USERAUTH_SECRET= 18 | 19 | # the path must be exactly `/auth/42/callback` 20 | USERAUTH_CALLBACK_URL=https://example.com/auth/42/callback 21 | 22 | SYNC_UID= 23 | SYNC_SECRET= 24 | 25 | # Rate limiter, you can find this number on in the line "Your application has a secondly rate limit set at " on the application page 26 | SYNC_MAX_REQUESTS_PER_SECOND= 27 | 28 | # DataDog API key, used for logging metrics 29 | DD_API_KEY= 30 | -------------------------------------------------------------------------------- /env/campusIDs.json: -------------------------------------------------------------------------------- 1 | { 2 | "Amsterdam": 14, 3 | 4 | "Paris": 1, 5 | "Lyon": 9, 6 | "Brussels": 12, 7 | "Helsinki": 13, 8 | "Khouribga": 16, 9 | "Moscow": 17, 10 | "São-Paulo": 20, 11 | "Benguerir": 21, 12 | "Madrid": 22, 13 | "Kazan": 23, 14 | "Quebec": 25, 15 | "Tokyo": 26, 16 | "Rio de Janeiro": 28, 17 | "Seoul": 29, 18 | "Rome": 30, 19 | "Angouleme": 31, 20 | "Yerevan": 32, 21 | "Bangkok": 33, 22 | "Kuala Lumpur": 34, 23 | "Adelaide": 36, 24 | "Malaga": 37, 25 | "Lisboa": 38, 26 | "Heilbronn": 39, 27 | "Urduliz": 40, 28 | "Nice": 41, 29 | "42Network": 42, 30 | "Abu Dhabi": 43, 31 | "Wolfsburg": 44, 32 | "Alicante": 45, 33 | "Barcelona": 46, 34 | "Lausanne": 47, 35 | "Mulhouse": 48, 36 | "Istanbul": 49, 37 | "Kocaeli": 50, 38 | "Berlin": 51, 39 | "Florence": 52, 40 | "Vienna": 53, 41 | "Tétouan": 55, 42 | "Prague": 56, 43 | "London": 57, 44 | "Porto": 58, 45 | "Le Havre": 62, 46 | "Singapore": 64, 47 | "Antananarivo": 65, 48 | "Warsaw": 67, 49 | "Luanda":68, 50 | "Gyeongsan":69 51 | } 52 | -------------------------------------------------------------------------------- /env/projectIDs.json: -------------------------------------------------------------------------------- 1 | { 2 | "libft": 1314, 3 | "Born2beroot": 1994, 4 | "ft_printf": 1316, 5 | "get_next_line": 1327, 6 | "push_swap": 1471, 7 | "minitalk": 2005, 8 | "pipex": 2004, 9 | "so_long": 2009, 10 | "FdF": 2008, 11 | "fract-ol": 1476, 12 | "minishell": 1331, 13 | "Philosophers": 1334, 14 | "miniRT": 1315, 15 | "cub3d": 1326, 16 | "CPP module 00": 1338, 17 | "CPP module 01": 1339, 18 | "CPP module 02": 1340, 19 | "CPP module 03": 1341, 20 | "CPP module 04": 1342, 21 | "CPP module 05": 1343, 22 | "CPP module 06": 1344, 23 | "CPP module 07": 1345, 24 | "CPP module 08": 1346, 25 | "CPP module 09": 2309, 26 | 27 | "NetPractice": 2007, 28 | "Inception": 1983, 29 | "ft_irc": 1336, 30 | "webserv": 1332, 31 | "ft_containers": 1335, 32 | "ft_transcendence": 1337, 33 | 34 | "Internship I": 1638, 35 | "Internship I - Contract Upload": 1640, 36 | "Internship I - Duration": 1639, 37 | "Internship I - Company Mid Evaluation": 1641, 38 | "Internship I - Company Final Evaluation": 1642, 39 | "Internship I - Peer Video": 1643, 40 | 41 | "startup internship": 1662, 42 | "startup internship - Contract Upload": 1663, 43 | "startup internship - Duration": 1664, 44 | "startup internship - Company Mid Evaluation": 1665, 45 | "startup internship - Company Final Evaluation": 1666, 46 | "startup internship - Peer Video": 1667, 47 | 48 | "Piscine Python Django": 1483, 49 | "camagru": 1396, 50 | "darkly":1405 , 51 | "Piscine Swift iOS":1486 , 52 | "swifty-proteins": 1406, 53 | "ft_hangouts": 1379, 54 | "matcha": 1401, 55 | "swifty-companion": 1395, 56 | "swingy": 1436, 57 | "red-tetris": 1428, 58 | "music-room": 1427, 59 | "hypertube": 1402, 60 | 61 | "rt": 1855, 62 | "scop": 1390, 63 | "zappy": 1463, 64 | "doom_nukem": 1853, 65 | "abstract-vm": 1461, 66 | "humangl": 1394, 67 | "guimp": 1455, 68 | "nibbler": 1386, 69 | "42run": 1387, 70 | "Piscine Unity": 1485, 71 | "gbmu": 1411, 72 | "bomberman": 1389, 73 | "particle-system": 1410, 74 | "in-the-shadows": 1409, 75 | "ft_newton": 1962, 76 | "ft_vox": 1449, 77 | "xv": 1408, 78 | "shaderpixel": 1454, 79 | 80 | "ft_ping": 1397, 81 | "libasm": 1330, 82 | "malloc": 1468, 83 | "nm": 1467, 84 | "strace": 1388, 85 | "ft_traceroute": 1399, 86 | "dr-quine": 1418, 87 | "ft_ssl_md5": 1451, 88 | "snow-crash": 1404, 89 | "ft_nmap": 1400, 90 | "woody-woodpacker": 1419, 91 | "ft_ssl_des": 1452, 92 | "rainfall": 1417, 93 | "ft_malcolm": 1840, 94 | "famine": 1430, 95 | "ft_ssl_rsa": 1450, 96 | "boot2root": 1446, 97 | "matt-daemon": 1420, 98 | "pestilence": 1443, 99 | "override": 1448, 100 | "war": 1444, 101 | "death": 1445, 102 | 103 | "lem_in": 1470, 104 | "computorv1": 1382, 105 | "Piscine OCaml": 1484, 106 | "n-puzzle": 1385, 107 | "ready set boole": 2076, 108 | "computorv2": 1433, 109 | "expert-system": 1384, 110 | "rubik": 1393, 111 | "ft_turing": 1403, 112 | "h42n42": 1429, 113 | "matrix": 2077, 114 | "mod1": 1462, 115 | "gomoku": 1383, 116 | "ft_ality": 1407, 117 | "krpsim": 1392, 118 | "fix-me": 1437, 119 | "ft_linear_regression": 1391, 120 | "dslr": 1453, 121 | "total-perspective-vortex": 1460, 122 | "multilayer-perceptron": 1457, 123 | "ft_kalman": 2098, 124 | 125 | "ft_ls": 1479, 126 | "ft_select": 1469, 127 | "ft_script": 1466, 128 | "42sh": 1854, 129 | "lem-ipc": 1464, 130 | "corewar": 1475, 131 | "taskmaster": 1381, 132 | "ft_linux": 1415, 133 | "little-penguin-1": 1416, 134 | "drivers-and-interrupts": 1422, 135 | "process-and-memory": 1421, 136 | "userspace_digressions": 1456, 137 | "filesystem": 1423, 138 | "kfs-1": 1425, 139 | "kfs-2": 1424, 140 | "kfs-3": 1426, 141 | "kfs-4": 1431, 142 | "kfs-5": 1432, 143 | "kfs-6": 1438, 144 | "kfs-7": 1439, 145 | "kfs-8": 1440, 146 | "kfs-9": 1441, 147 | "kfs-x": 1442, 148 | 149 | "Part_Time I": 1650, 150 | "Open Project": 1635, 151 | "Internship II": 1644, 152 | "Inception-of-Things": 2064, 153 | 154 | "C Piscine Shell 00": 1255, 155 | "C Piscine Shell 01": 1256, 156 | "C Piscine C 00": 1257, 157 | "C Piscine C 01": 1258, 158 | "C Piscine C 02": 1259, 159 | "C Piscine C 03": 1260, 160 | "C Piscine C 04": 1261, 161 | "C Piscine C 05": 1262, 162 | "C Piscine C 06": 1263, 163 | "C Piscine C 07": 1270, 164 | "C Piscine C 08": 1264, 165 | "C Piscine C 09": 1265, 166 | "C Piscine C 10": 1266, 167 | "C Piscine C 11": 1267, 168 | "C Piscine C 12": 1268, 169 | "C Piscine C 13": 1271, 170 | "C Piscine Rush 00": 1308, 171 | "C Piscine Rush 01": 1310, 172 | "C Piscine Rush 02": 1309, 173 | "C Piscine BSQ": 1305 174 | } 175 | -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -o errexit # Exit on error 4 | npm run lint:check 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codam-coding-college/find-peers", 3 | "author": "Joppe Koers", 4 | "main": "./build/src/app.js", 5 | "scripts": { 6 | "start": "node ./build/src/app.js", 7 | "dev": "ts-node-dev --quiet --clear --rs --respawn src/app.ts", 8 | "build": "tsc", 9 | "lint:check": "eslint --cache --cache-location node_modules/.eslintcache src; prettier --check src", 10 | "lint:fix": "eslint --fix src; prettier --write src" 11 | }, 12 | "dependencies": { 13 | "@types/compression": "^1.7.2", 14 | "@types/ejs": "3.1.0", 15 | "@types/express": "^4.17.17", 16 | "@types/express-session": "1.17.4", 17 | "@types/node": "18.15.1", 18 | "@types/node-fetch": "2.1.0", 19 | "@types/passport": "1.0.7", 20 | "@types/request": "^2.48.8", 21 | "@types/session-file-store": "1.2.2", 22 | "@types/strip-bom": "3.0.0", 23 | "@types/strip-json-comments": "3.0.0", 24 | "@typescript-eslint/eslint-plugin": "^5.48.2", 25 | "@typescript-eslint/parser": "^5.48.2", 26 | "42-connector": "github:codam-coding-college/42-connector#3.1.0", 27 | "compression": "^1.7.4", 28 | "crypto": "^1.0.1", 29 | "dotenv": "^16.0.3", 30 | "ejs": "3.1.8", 31 | "eslint": "^8.32.0", 32 | "eslint-config-prettier": "^8.6.0", 33 | "eslint-plugin-prettier": "^4.2.1", 34 | "express": "4.17.3", 35 | "express-session": "1.17.2", 36 | "hot-shots": "^10.0.0", 37 | "node-fetch": "2.6.7", 38 | "passport": "0.6.0", 39 | "passport-oauth": "1.0.0", 40 | "path": "0.12.7", 41 | "request": "^2.88.2", 42 | "typescript": "4.9.5" 43 | }, 44 | "engines": { 45 | "node": ">=18.0.0" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /public/new.svg: -------------------------------------------------------------------------------- 1 | 2 | new 3 | 4 | 7 | 8 | -------------------------------------------------------------------------------- /public/placeholder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codam-coding-college/find-peers/54c8b4142826ba699fbfbd5ac3daee41c7236de5/public/placeholder.png -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line 2 | require('dotenv').config({ path: __dirname + '/../env/.env' }) 3 | 4 | import { syncCampuses, campusDBs } from './db' 5 | import { startWebserver } from './express' 6 | import { env } from './env' 7 | import util from 'util' 8 | 9 | // set depth of object expansion in terminal as printed by console.*() 10 | util.inspect.defaultOptions.depth = 10 11 | 12 | function msUntilNextPull(): number { 13 | let nextPull = env.pullTimeout 14 | for (const campus of Object.values(env.campuses)) { 15 | const lastPullAgo = Date.now() - campusDBs[campus.name].lastPull 16 | const msUntilNexPull = Math.max(0, env.pullTimeout - lastPullAgo) 17 | nextPull = Math.min(nextPull, msUntilNexPull) 18 | } 19 | return nextPull 20 | } 21 | 22 | ;(async () => { 23 | const port = parseInt(process.env['PORT'] || '8080') 24 | await startWebserver(port) 25 | 26 | while (true) { 27 | await syncCampuses() 28 | await new Promise(resolve => setTimeout(resolve, msUntilNextPull() + 1000)) 29 | } 30 | })() 31 | -------------------------------------------------------------------------------- /src/authentication.ts: -------------------------------------------------------------------------------- 1 | import passport from 'passport' 2 | // eslint-disable-next-line @typescript-eslint/no-var-requires 3 | const { OAuth2Strategy } = require('passport-oauth') 4 | import fetch from 'node-fetch' 5 | import fs from 'fs' 6 | import { env } from './env' 7 | import { UserProfile } from './types' 8 | import { Request, Response, NextFunction } from 'express' 9 | 10 | export function authenticate(req: Request, res: Response, next: NextFunction) { 11 | if (!req.user) { 12 | res.redirect(`/auth/${env.provider}`) 13 | } else { 14 | next() 15 | } 16 | } 17 | 18 | const usersDB: UserProfile[] = [] 19 | const emptyUsersDB: string = JSON.stringify(usersDB) 20 | if (!fs.existsSync(env.userDBpath) || fs.statSync(env.userDBpath).size < emptyUsersDB.length) { 21 | fs.writeFileSync(env.userDBpath, emptyUsersDB) 22 | } 23 | 24 | const users: UserProfile[] = JSON.parse(fs.readFileSync(env.userDBpath).toString()) 25 | 26 | passport.serializeUser((user, done) => { 27 | //@ts-ignore 28 | done(null, user.id) 29 | }) 30 | 31 | passport.deserializeUser((id, done) => { 32 | const user = users.find(user => user.id === id) 33 | done(null, user) 34 | }) 35 | 36 | async function getProfile(accessToken: string, refreshToken: string): Promise { 37 | try { 38 | const response = await fetch('https://api.intra.42.fr/v2/me', { 39 | headers: { 40 | Authorization: `Bearer ${accessToken}`, 41 | }, 42 | }) 43 | const json = await response.json() 44 | const profile: UserProfile = { 45 | id: json.id, 46 | login: json.login, 47 | first_name: json.first_name, 48 | displayname: json.displayname, 49 | campusID: json.campus.length > 0 ? json.campus[0].id : 42, // set user's campus to first one listed in API call 50 | campusName: json.campus.length > 0 ? json.campus[0].name : 'Paris', 51 | timeZone: json.campus.length > 0 ? json.campus[0].time_zone : 'Europe/Paris', 52 | accessToken, 53 | refreshToken, 54 | } 55 | for (const i in json.campus_users) { 56 | // get user's primary campus 57 | if (json.campus_users[i].is_primary) { 58 | for (const j in json.campus) { 59 | // get primary campus name and store it in UserProfile (overwriting the one assigned above, which might not be primary) 60 | if (json.campus[j].id === json.campus_users[i].campus_id) { 61 | profile.campusName = json.campus[j].name 62 | profile.timeZone = json.campus[j].time_zone 63 | profile.campusID = json.campus_users[i].campus_id 64 | break 65 | } 66 | } 67 | break 68 | } 69 | } 70 | return profile 71 | } catch (err) { 72 | return null 73 | } 74 | } 75 | const opt = { 76 | authorizationURL: env.authorizationURL, 77 | tokenURL: env.tokenURL, 78 | clientID: env.tokens.userAuth.UID, 79 | clientSecret: env.tokens.userAuth.secret, 80 | callbackURL: env.tokens.userAuth.callbackURL, 81 | // passReqToCallback: true 82 | } 83 | const client = new OAuth2Strategy(opt, async (accessToken: string, refreshToken: string, _profile: string, done: (err: string | null, user: UserProfile | null) => void) => { 84 | // fires when user clicked allow 85 | const newUser = await getProfile(accessToken, refreshToken) 86 | if (!newUser) { 87 | return done('cannot get user info', null) 88 | } 89 | const userIndex = users.findIndex(user => user.id === newUser.id) 90 | if (userIndex < 0) { 91 | users.push(newUser) 92 | } else { 93 | users[userIndex] = newUser 94 | } 95 | await fs.promises.writeFile(env.userDBpath, JSON.stringify(users)) 96 | done(null, newUser) 97 | }) 98 | passport.use(env.provider, client) 99 | 100 | export { passport } 101 | -------------------------------------------------------------------------------- /src/db.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import { API } from '42-connector' 3 | import { ApiProject, Project, ProjectSubscriber } from './types' 4 | import { env, Campus, ProjectStatus, CampusName } from './env' 5 | import { logCampus, log, msToHuman, nowISO } from './logger' 6 | import * as StatsD from './statsd' 7 | 8 | const Api: API = new API(env.tokens.sync.UID, env.tokens.sync.secret, { 9 | maxRequestPerSecond: env.tokens.sync.maxRequestPerSecond, 10 | logging: env.logLevel >= 3, 11 | }) 12 | 13 | export interface CampusDB { 14 | name: CampusName 15 | projects: Project[] 16 | lastPull: number 17 | } 18 | 19 | export const campusDBs: Record = {} as Record 20 | 21 | fs.mkdirSync(env.databaseRoot, { recursive: true }) 22 | function setupCampusDB(campus: Campus) { 23 | const campusDB: CampusDB = { 24 | name: campus.name, 25 | projects: [], 26 | lastPull: 0, 27 | } 28 | 29 | fs.mkdirSync(campus.databasePath, { recursive: true }) 30 | if (!fs.existsSync(campus.projectUsersPath)) { 31 | fs.writeFileSync(campus.projectUsersPath, '[]') 32 | } 33 | campusDB.projects = JSON.parse(fs.readFileSync(campus.projectUsersPath).toString()) 34 | if (!fs.existsSync(campus.lastPullPath)) { 35 | fs.writeFileSync(campus.lastPullPath, '0') 36 | } 37 | campusDB.lastPull = parseInt(fs.readFileSync(campus.lastPullPath).toString()) 38 | campusDBs[campus.name] = campusDB 39 | } 40 | 41 | for (const campus of Object.values(env.campuses)) { 42 | setupCampusDB(campus) 43 | } 44 | 45 | // Next time we use SQL 46 | function findProjectUserByLogin(login: string, projectName: string): ProjectSubscriber | undefined { 47 | for (const campus of Object.values(env.campuses)) { 48 | const projects = campusDBs[campus.name].projects as Project[] 49 | for (const project of projects) { 50 | if (project.name !== projectName) { 51 | continue 52 | } 53 | const user = project.users.find(x => x.login === login) 54 | if (user) { 55 | return user 56 | } 57 | } 58 | } 59 | return undefined 60 | } 61 | 62 | function getUpdate(status: ProjectStatus, existingUser?: ProjectSubscriber): { new: boolean; lastChangeD: Date } { 63 | if (!existingUser) { 64 | return { new: true, lastChangeD: new Date() } 65 | } 66 | 67 | if (status !== existingUser.status) { 68 | return { new: true, lastChangeD: new Date() } 69 | } 70 | 71 | const lastChangeD = new Date(existingUser.lastChangeD) 72 | const isNew = Date.now() - lastChangeD.getTime() < env.userNewStatusThresholdDays * 24 * 60 * 60 * 1000 73 | return { new: isNew, lastChangeD: lastChangeD } 74 | } 75 | 76 | // Intra's 'validated' key is sometimes wrong, therefore we use our own logic 77 | function getStatus(x: Readonly): ProjectStatus { 78 | let status: ProjectStatus = x['validated?'] ? 'finished' : x.status 79 | 80 | if (!env.projectStatuses.includes(x.status)) { 81 | console.error(`Invalid status: ${x.status} on user ${x.user}`) 82 | status = 'finished' 83 | } 84 | return status 85 | } 86 | 87 | function toProjectSubscriber(x: Readonly, projectName: string): ProjectSubscriber | undefined { 88 | try { 89 | const status = getStatus(x) 90 | const existing = findProjectUserByLogin(x.user.login, projectName) 91 | const valid: ProjectSubscriber = { 92 | login: x.user.login, 93 | status, 94 | staff: !!x.user['staff?'], 95 | image_url: x.user.image.versions.medium, 96 | ...getUpdate(status, existing), 97 | } 98 | return valid 99 | } catch (e) { 100 | console.error(e) 101 | return undefined 102 | } 103 | } 104 | 105 | export async function getProjectSubscribers(campus: Campus, projectID: number, projectName: string): Promise { 106 | const url = `/v2/projects/${projectID}/projects_users?filter[campus]=${campus.id}&page[size]=100` 107 | const onPage = () => StatsD.increment('dbfetch', StatsD.strToTag('campus', campus.name)) 108 | 109 | const { ok, json: users }: { ok: boolean; json?: ApiProject[] } = await Api.getPaged(url, onPage) 110 | if (!ok || !users) { 111 | throw new Error('Could not get project subscribers') 112 | } 113 | return users.map(u => toProjectSubscriber(u, projectName)).filter(x => !!x) as ProjectSubscriber[] 114 | } 115 | 116 | export async function writeAllProjectIds() { 117 | const a = (await Api.getPaged(`/v2/projects`)) as { 118 | ok: true 119 | status: 200 120 | json: ({ id: string; slug: string; name: string } & Record)[] 121 | } 122 | const path = 'env/allProjectIDs.json' 123 | const summary = a.json.map(x => ({ id: x.id, slug: x.slug, name: x.name })) 124 | fs.writeFileSync(path, JSON.stringify(summary, null, 4)) 125 | console.log('Project IDs written to', path) 126 | } 127 | // writeAllProjectIds() 128 | 129 | // @return number of users pulled 130 | export async function saveAllProjectSubscribers(campus: Campus): Promise { 131 | let usersPulled = 0 132 | const startPull = Date.now() 133 | const newProjects: Project[] = [] 134 | for (const [name, id] of Object.entries(env.projectIDs)) { 135 | let item: Project 136 | try { 137 | item = { 138 | name, 139 | users: await getProjectSubscribers(campus, id, name), 140 | } 141 | } catch (e) { 142 | return 0 143 | } 144 | usersPulled += item.users.length 145 | logCampus(2, campus.name, name, `total users: ${item.users.length}`) 146 | newProjects.push(item) 147 | } 148 | campusDBs[campus.name].projects = newProjects 149 | log(2, `Pull took ${msToHuman(Date.now() - startPull)}`) 150 | 151 | await fs.promises.writeFile(campus.projectUsersPath, JSON.stringify(newProjects)) 152 | await fs.promises.writeFile(campus.lastPullPath, String(Date.now())) 153 | campusDBs[campus.name].lastPull = parseInt((await fs.promises.readFile(campus.lastPullPath)).toString()) 154 | return usersPulled 155 | } 156 | 157 | // Sync all user statuses form all campuses if the env.pullTimeout for that campus has not been reached 158 | export async function syncCampuses(): Promise { 159 | const startPull = Date.now() 160 | 161 | log(1, 'starting pull') 162 | for (const campus of Object.values(campusDBs)) { 163 | const lastPullAgo = Date.now() - campus.lastPull 164 | logCampus(2, campus.name, '', `last pull was on ${nowISO(campus.lastPull)}, ${(lastPullAgo / 1000 / 60).toFixed(0)} minutes ago`) 165 | if (lastPullAgo < env.pullTimeout) { 166 | logCampus(2, campus.name, '', `not pulling, timeout of ${env.pullTimeout / 1000 / 60} minutes not reached`) 167 | continue 168 | } 169 | await saveAllProjectSubscribers(env.campuses[campus.name]) 170 | } 171 | log(1, `complete pull took ${msToHuman(Date.now() - startPull)}`) 172 | } 173 | -------------------------------------------------------------------------------- /src/env.ts: -------------------------------------------------------------------------------- 1 | import { log } from './logger' 2 | import path from 'path' 3 | import campusIDs from '../env/campusIDs.json' 4 | import projectIDs from '../env/projectIDs.json' 5 | import { assertEnvInt, assertEnvStr, mapObject } from './util' 6 | 7 | export type CampusID = (typeof campusIDs)[keyof typeof campusIDs] 8 | export type CampusName = keyof typeof campusIDs 9 | 10 | export interface Campus { 11 | name: CampusName 12 | id: number 13 | databasePath: string // path to the database subfolder for this campus 14 | projectUsersPath: string // users that are subscribed to a project 15 | lastPullPath: string // timestamp for when the server did a last pull 16 | } 17 | 18 | export interface Env { 19 | logLevel: 0 | 1 | 2 | 3 20 | pullTimeout: number 21 | projectIDs: typeof projectIDs 22 | campusIDs: typeof campusIDs 23 | databaseRoot: string 24 | campuses: Record 25 | projectStatuses: typeof projectStatuses 26 | sessionStorePath: string // session key data 27 | userDBpath: string // users associated with sessions 28 | scope: string[] 29 | authorizationURL: string 30 | tokenURL: string 31 | provider: string 32 | authPath: string 33 | tokens: { 34 | metricsSalt: string 35 | userAuth: { 36 | UID: string 37 | secret: string 38 | callbackURL: string 39 | } 40 | sync: { 41 | UID: string 42 | secret: string 43 | maxRequestPerSecond: number 44 | } 45 | } 46 | userNewStatusThresholdDays: number 47 | } 48 | 49 | const databaseRoot = 'database' 50 | const campuses: Record = mapObject(campusIDs, (name, id) => ({ 51 | name, 52 | id, 53 | databasePath: path.join(databaseRoot, name), 54 | projectUsersPath: path.join(databaseRoot, name, 'projectUsers.json'), 55 | lastPullPath: path.join(databaseRoot, name, 'lastpull.txt'), 56 | })) 57 | 58 | // known statuses, in the order we want them displayed on the website 59 | const projectStatuses = ['creating_group', 'searching_a_group', 'in_progress', 'waiting_for_correction', 'finished', 'parent'] as const 60 | export type ProjectStatus = (typeof projectStatuses)[number] 61 | 62 | export const env: Readonly = { 63 | logLevel: process.env['NODE_ENV'] === 'production' ? 3 : 1, // 0 being no logging 64 | pullTimeout: 24 * 60 * 60 * 1000, // how often to pull the project users statuses form the intra api (in Ms) 65 | projectIDs, 66 | campusIDs, 67 | databaseRoot, 68 | campuses, 69 | projectStatuses, 70 | sessionStorePath: path.join(databaseRoot, 'sessions'), 71 | authorizationURL: 'https://api.intra.42.fr/oauth/authorize', 72 | tokenURL: 'https://api.intra.42.fr/oauth/token', 73 | userDBpath: path.join(databaseRoot, 'users.json'), 74 | provider: '42', 75 | authPath: '/auth/42', 76 | scope: ['public'], 77 | tokens: { 78 | metricsSalt: assertEnvStr('METRICS_SALT'), 79 | userAuth: { 80 | UID: assertEnvStr('USERAUTH_UID'), 81 | secret: assertEnvStr('USERAUTH_SECRET'), 82 | callbackURL: assertEnvStr('USERAUTH_CALLBACK_URL'), 83 | }, 84 | sync: { 85 | UID: assertEnvStr('SYNC_UID'), 86 | secret: assertEnvStr('SYNC_SECRET'), 87 | maxRequestPerSecond: assertEnvInt('SYNC_MAX_REQUESTS_PER_SECOND'), 88 | }, 89 | }, 90 | userNewStatusThresholdDays: 7, 91 | } 92 | 93 | log(1, `Watching ${Object.keys(campusIDs).length} campuses`) 94 | log(1, `Watching ${Object.keys(projectIDs).length} projects`) 95 | -------------------------------------------------------------------------------- /src/express.ts: -------------------------------------------------------------------------------- 1 | import path from 'path' 2 | import express, { Response } from 'express' 3 | import { passport, authenticate } from './authentication' 4 | import { CampusName, env, ProjectStatus } from './env' 5 | import session from 'express-session' 6 | import { campusDBs, CampusDB } from './db' 7 | import { Project, ProjectSubscriber, UserProfile } from './types' 8 | import { log } from './logger' 9 | import { MetricsStorage } from './metrics' 10 | import compression from 'compression' 11 | import request from 'request' 12 | import { isLinguisticallySimilar } from './util' 13 | 14 | function errorPage(res: Response, error: string): void { 15 | const settings = { 16 | campuses: Object.values(env.campuses).sort((a, b) => (a.name < b.name ? -1 : 1)), 17 | error, 18 | } 19 | res.render('error.ejs', settings) 20 | } 21 | 22 | const cachingProxy = '/proxy' 23 | 24 | function filterUsers(users: ProjectSubscriber[], requestedStatus: string | undefined): ProjectSubscriber[] { 25 | const newUsers = users 26 | .filter(user => { 27 | if (user.staff) { 28 | return false 29 | } 30 | if (user.login.match(/^3b3/)) { 31 | // accounts who's login start with 3b3 are deactivated 32 | return false 33 | } 34 | if ((requestedStatus === 'finished' || user.status !== 'finished') && (!requestedStatus || user.status === requestedStatus)) { 35 | return true 36 | } 37 | return false 38 | }) 39 | .map(user => ({ ...user, image_url: `${cachingProxy}?q=${user.image_url}` })) 40 | .sort((a, b) => { 41 | if (a.status !== b.status) { 42 | const preferredOrder = env.projectStatuses 43 | const indexA = preferredOrder.findIndex(x => x === a.status) 44 | const indexB = preferredOrder.findIndex(x => x === b.status) 45 | return indexA < indexB ? -1 : 1 46 | } 47 | return a.login < b.login ? -1 : 1 48 | }) 49 | return newUsers 50 | } 51 | 52 | function filterProjects(projects: Project[], requestedStatus: string | undefined): Project[] { 53 | return projects.map(project => ({ 54 | name: project.name, 55 | users: filterUsers(project.users, requestedStatus), 56 | })) 57 | } 58 | 59 | const metrics = new MetricsStorage() 60 | 61 | export async function startWebserver(port: number) { 62 | const app = express() 63 | 64 | app.use( 65 | session({ 66 | secret: env.tokens.userAuth.secret.slice(5), 67 | resave: false, 68 | saveUninitialized: true, 69 | }) 70 | ) 71 | app.use(passport.initialize()) 72 | app.use(passport.session()) 73 | 74 | app.use(cachingProxy, (req, res) => { 75 | const url = req.query['q'] 76 | if (!url || typeof url !== 'string' || !url.startsWith('http')) { 77 | res.status(404).send('No URL provided') 78 | return 79 | } 80 | 81 | // inject cache header for images 82 | res.setHeader('Cache-Control', `public, max-age=${100 * 24 * 60 * 60}`) 83 | req.pipe(request(url)).pipe(res) 84 | }) 85 | 86 | app.use((req, res, next) => { 87 | try { 88 | compression()(req, res, next) 89 | return 90 | } catch (e) { 91 | console.error('Compression error', e) 92 | } 93 | next() 94 | }) 95 | 96 | app.get('/robots.txt', (_, res) => { 97 | res.type('text/plain') 98 | res.send('User-agent: *\nAllow: /') 99 | }) 100 | 101 | app.get(`/auth/${env.provider}/`, passport.authenticate(env.provider, { scope: env.scope })) 102 | app.get( 103 | `/auth/${env.provider}/callback`, 104 | passport.authenticate(env.provider, { 105 | successRedirect: '/', 106 | failureRedirect: `/auth/${env.provider}`, 107 | }) 108 | ) 109 | 110 | app.get('/', authenticate, (req, res) => { 111 | const user: UserProfile = req.user as UserProfile 112 | res.redirect(`/${user.campusName}`) 113 | }) 114 | 115 | app.get('/:campus', authenticate, (req, res) => { 116 | const user: UserProfile = req.user as UserProfile 117 | const requestedStatus: string | undefined = req.query['status']?.toString() 118 | 119 | const campus = req.params['campus'] as string 120 | const campusName: CampusName | undefined = Object.keys(campusDBs).find(k => isLinguisticallySimilar(k, campus)) as CampusName | undefined 121 | if (!campusName || !campusDBs[campusName]) { 122 | return errorPage(res, `Campus ${campus} is not supported by Find Peers (yet)`) 123 | } 124 | 125 | // saving anonymized metrics 126 | metrics.addVisitor(user) 127 | 128 | const campusDB: CampusDB = campusDBs[campusName] 129 | if (!campusDB.projects.length) { 130 | return errorPage(res, 'Empty database (please try again later)') 131 | } 132 | 133 | if (requestedStatus && !env.projectStatuses.includes(requestedStatus as ProjectStatus)) { 134 | return errorPage(res, `Unknown status ${req.query['status']}`) 135 | } 136 | 137 | const { uniqVisitorsTotal: v, uniqVisitorsCampus } = metrics.generateMetrics() 138 | const campuses = uniqVisitorsCampus.reduce((acc, visitors) => { 139 | acc += visitors.month > 0 ? 1 : 0 140 | return acc 141 | }, 0) 142 | const settings = { 143 | projects: filterProjects(campusDB.projects, requestedStatus), 144 | lastUpdate: new Date(campusDB.lastPull).toLocaleString('en-NL', { timeZone: user.timeZone }).slice(0, -3), 145 | hoursAgo: ((Date.now() - campusDB.lastPull) / 1000 / 60 / 60).toFixed(2), 146 | requestedStatus, 147 | projectStatuses: env.projectStatuses, 148 | campusName, 149 | campuses: Object.values(env.campuses).sort((a, b) => (a.name < b.name ? -1 : 1)), 150 | updateEveryHours: (env.pullTimeout / 1000 / 60 / 60).toFixed(0), 151 | usage: `${v.day} unique visitors today, ${v.month} this month, from ${campuses} different campuses`, 152 | userNewStatusThresholdDays: env.userNewStatusThresholdDays, 153 | } 154 | res.render('index.ejs', settings) 155 | }) 156 | 157 | app.get('/status/pull', (_, res) => { 158 | const obj = Object.values(campusDBs).map(campus => ({ 159 | name: campus.name, 160 | lastPull: new Date(campus.lastPull), 161 | hoursAgo: (Date.now() - campus.lastPull) / 1000 / 60 / 60, 162 | })) 163 | res.json(obj) 164 | }) 165 | 166 | app.get('/status/metrics', authenticate, (_, res) => { 167 | res.json(metrics.generateMetrics()) 168 | }) 169 | 170 | app.set('views', path.join(__dirname, '../views')) 171 | app.set('viewengine', 'ejs') 172 | app.use('/public', express.static('public/')) 173 | 174 | await app.listen(port) 175 | log(1, `${process.env['NODE_ENV'] ?? 'development'} app ready on http://localhost:${port}`) 176 | } 177 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import { env, Env } from './env' 2 | import { campusDBs } from './db' 3 | 4 | // eg. 24 5 | export function msToHuman(milliseconds: number): string { 6 | const hours = milliseconds / (1000 * 60 * 60) 7 | const h = Math.floor(hours) 8 | 9 | const minutes = (hours - h) * 60 10 | const m = Math.floor(minutes) 11 | 12 | const seconds = (minutes - m) * 60 13 | const s = Math.floor(seconds) 14 | 15 | return `${String(h).padStart(2, '0')}h ${String(m).padStart(2, '0')}m ${String(s).padStart(2, '0')}s` 16 | } 17 | 18 | function longestKeyLength(obj: Record): number { 19 | let longestCampusNameLength = 0 20 | for (const key in obj) { 21 | if (key.length > longestCampusNameLength) { 22 | longestCampusNameLength = key.length 23 | } 24 | } 25 | return longestCampusNameLength 26 | } 27 | 28 | export function nowISO(d: Date | number = new Date()): string { 29 | d = new Date(d) 30 | return `${d.toISOString().slice(0, -5)}Z` 31 | } 32 | 33 | export function logCampus(level: Env['logLevel'], campus: string, project: string, message: string) { 34 | if (level <= env.logLevel) { 35 | console.log(`${nowISO()} | ${campus.padEnd(longestKeyLength(campusDBs))} ${project.padEnd(longestKeyLength(env.projectIDs))} | ${message}`) 36 | } 37 | } 38 | 39 | export function log(level: Env['logLevel'], message: string) { 40 | if (level <= env.logLevel) { 41 | console.log(`${nowISO()} | ${message}`) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/metrics.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import { env } from './env' 3 | import crypto from 'crypto' 4 | import { UserProfile } from './types' 5 | import * as StatsD from './statsd' 6 | import { findLast, unique } from './util' 7 | 8 | interface Visitor { 9 | id: string 10 | campus: string 11 | date: Date 12 | } 13 | 14 | interface Metric { 15 | hour: number 16 | day: number 17 | month: number 18 | } 19 | 20 | interface Metrics { 21 | uniqVisitorsTotal: Metric 22 | uniqVisitorsCampus: ({ name: string } & Metric)[] 23 | nVisitors: number 24 | } 25 | 26 | export class MetricsStorage { 27 | constructor() { 28 | if (!fs.existsSync(this.dbPath)) { 29 | fs.writeFileSync(this.dbPath, '[]') 30 | } 31 | try { 32 | this.visitors = (JSON.parse(fs.readFileSync(this.dbPath, 'utf8')) as Visitor[]).map(x => ({ ...x, date: new Date(x.date) })) // in JSON Date is stored as a string, so now we convert it back to Date 33 | } catch (err) { 34 | console.error('Error while reading visitors database, resetting it...', err) 35 | this.visitors = [] 36 | } 37 | } 38 | 39 | public async addVisitor(user: UserProfile): Promise { 40 | // create a hash instead of storing the user id directly, for privacy 41 | const rawID = user.id.toString() + user.login + env.tokens.metricsSalt 42 | const id = crypto.createHash('sha256').update(rawID).digest('hex') 43 | 44 | // if the user has visited the page in the last n minutes, do not count it as a new visitor 45 | const lastVisit = findLast(this.visitors, x => x.id === id) 46 | if (lastVisit && Date.now() - lastVisit.date.getTime() < 1000 * 60 * 30) { 47 | return 48 | } 49 | 50 | this.visitors.push({ id, campus: user.campusName, date: new Date() }) 51 | StatsD.increment('visits', StatsD.strToTag('origin', user.campusName)) 52 | await fs.promises.writeFile(this.dbPath, JSON.stringify(this.visitors)) 53 | } 54 | 55 | uniqueVisitorsInLast(timeMs: number): Visitor[] { 56 | const now = Date.now() 57 | let visitors = this.visitors.filter(x => now - x.date.getTime() < timeMs) 58 | visitors = unique(visitors, (a, b) => a.id === b.id) 59 | visitors = visitors.map(x => ({ ...x, id: x.id.substring(5, -5) })) // cut a little of the id to keep it private 60 | return visitors 61 | } 62 | 63 | public generateMetrics(): Metrics { 64 | const hour = this.uniqueVisitorsInLast(3600 * 1000) 65 | const day = this.uniqueVisitorsInLast(24 * 3600 * 1000) 66 | const month = this.uniqueVisitorsInLast(30 * 24 * 3600 * 1000) 67 | 68 | const uniqVisitorsCampus = Object.values(env.campuses) 69 | .map(campus => ({ 70 | name: campus.name, 71 | hour: hour.filter(x => x.campus === campus.name).length, 72 | day: day.filter(x => x.campus === campus.name).length, 73 | month: month.filter(x => x.campus === campus.name).length, 74 | })) 75 | .sort((a, b) => b.day - a.day) 76 | 77 | return { 78 | uniqVisitorsTotal: { 79 | hour: hour.length, 80 | day: day.length, 81 | month: month.length, 82 | }, 83 | uniqVisitorsCampus, 84 | nVisitors: this.visitors.length, 85 | } 86 | } 87 | 88 | private readonly dbPath: string = `${env.databaseRoot}/visitors.json` 89 | private visitors: Visitor[] = [] 90 | } 91 | -------------------------------------------------------------------------------- /src/statsd.ts: -------------------------------------------------------------------------------- 1 | import { StatsD as StatsDObj } from 'hot-shots' 2 | 3 | const client = new StatsDObj({ 4 | port: 8125, 5 | host: 'datadog-agent', // TODO use env 6 | errorHandler: console.error, 7 | }) 8 | 9 | export function increment(stat: string, tag?: string): void { 10 | if (!isValidDataDogStr(stat)) { 11 | return console.error(`Invalid stat ${stat}`) 12 | } 13 | if (tag && !isValidDataDogStr(tag)) { 14 | return console.error(`Invalid tag ${tag} for stat ${stat}`) 15 | } 16 | client.increment(stat, tag ? [tag] : []) 17 | } 18 | 19 | export function strToTag(prefix: string, str: string): string { 20 | const normalized = str 21 | .toLowerCase() 22 | .normalize('NFD') 23 | .replace(/[\u0300-\u036f]/g, '') 24 | .replace(/[^a-z0-9]/g, '_') 25 | return `${prefix}:${normalized}` 26 | } 27 | 28 | function isValidDataDogStr(tag: string): boolean { 29 | return /^[a-z0-9_:]+$/.test(tag) 30 | } 31 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { ProjectStatus } from './env' 2 | 3 | export interface ApiProject { 4 | id: number 5 | occurrence: number 6 | final_mark: number 7 | status: ProjectStatus 8 | 'validated?': boolean 9 | current_team_id: number 10 | project: { 11 | id: number 12 | name: string 13 | slug: string 14 | parent_id?: unknown 15 | } 16 | cursus_ids: number[] 17 | marked_at: Date 18 | marked: boolean 19 | retriable_at: Date 20 | created_at: Date 21 | updated_at: Date 22 | user: { 23 | id: number 24 | email: string 25 | login: string 26 | first_name: string 27 | last_name: string 28 | usual_full_name: string 29 | 'usual_first_name?': unknown 30 | url: string 31 | phone: string 32 | displayname: string 33 | kind: string 34 | image_url: string 35 | image: { 36 | link: string 37 | versions: { 38 | large: string 39 | medium: string 40 | small: string 41 | micro: string 42 | } 43 | } 44 | new_image_url: string 45 | 'staff?': boolean 46 | correction_point: number 47 | pool_month: string 48 | pool_year: string 49 | 'location?': unknown 50 | wallet: number 51 | anonymize_date: Date 52 | data_erasure_date: Date 53 | created_at: Date 54 | updated_at: Date 55 | 'alumnized_at?': unknown 56 | 'alumni?': boolean 57 | 'active?': boolean 58 | } 59 | teams: { 60 | id: number 61 | name: string 62 | url: string 63 | final_mark: number 64 | project_id: number 65 | created_at: Date 66 | updated_at: Date 67 | status: string 68 | 'terminating_at?': unknown 69 | users: { 70 | id: number 71 | login: string 72 | url: string 73 | leader: boolean 74 | occurrence: number 75 | validated: boolean 76 | projects_user_id: number 77 | }[] 78 | 'locked?': boolean 79 | 'validated?': boolean 80 | 'closed?': boolean 81 | repo_url: string 82 | repo_uuid: string 83 | locked_at: Date 84 | closed_at: Date 85 | project_session_id: number 86 | project_gitlab_path: string 87 | }[] 88 | } 89 | 90 | export interface ProjectSubscriber { 91 | login: string 92 | status: ProjectStatus 93 | staff: boolean 94 | image_url: string 95 | lastChangeD: Date | string 96 | new: boolean 97 | } 98 | 99 | export interface Project { 100 | name: string 101 | users: ProjectSubscriber[] 102 | } 103 | 104 | export interface UserProfile { 105 | id: number 106 | login: string 107 | first_name: string 108 | displayname: string 109 | accessToken: string 110 | refreshToken: string 111 | campusID: number 112 | campusName: string 113 | timeZone: string 114 | } 115 | -------------------------------------------------------------------------------- /src/util.ts: -------------------------------------------------------------------------------- 1 | export function findLast(arr: T[], predicate: (x: T) => boolean): T | undefined { 2 | for (let i = arr.length - 1; i >= 0; i--) { 3 | if (predicate(arr[i] as T)) { 4 | return arr[i] 5 | } 6 | } 7 | return undefined 8 | } 9 | 10 | // get unique elements in array based on equalFn() 11 | export function unique(arr: T[], equalFn: (a: T, b: T) => boolean): T[] { 12 | return arr.filter((current, pos) => arr.findIndex(x => equalFn(x, current)) === pos) 13 | } 14 | 15 | // ignoring case, whitespace, -, _, non ascii chars 16 | export function isLinguisticallySimilar(a: string, b: string): boolean { 17 | a = a 18 | .toLowerCase() 19 | .replace(/\s|-|_/g, '') 20 | .normalize('NFKD') 21 | .replace(/[\u0300-\u036F]/g, '') 22 | b = b 23 | .toLowerCase() 24 | .replace(/\s|-|_/g, '') 25 | .normalize('NFKD') 26 | .replace(/[\u0300-\u036F]/g, '') 27 | return a === b 28 | } 29 | 30 | function assertEnv(env: string): string { 31 | const value = process.env[env] 32 | if (value === undefined) { 33 | throw new Error(`Environment variable "${env}" is not set`) 34 | } 35 | return value 36 | } 37 | 38 | export function assertEnvStr(env: string): string { 39 | const value = assertEnv(env) 40 | if (typeof value !== 'string' || value.length === 0) { 41 | throw new Error(`Environment variable "${value}" is not a non-empty string`) 42 | } 43 | return value 44 | } 45 | 46 | export function assertEnvInt(env: string): number { 47 | const value = assertEnv(env) 48 | const num = parseInt(value) 49 | if (isNaN(num)) { 50 | throw new Error(`Environment variable "${value}" is not a number`) 51 | } 52 | return num 53 | } 54 | 55 | export function mapObject(object: Record, mapFn: (key: Key, value: Value) => NewValue): Record { 56 | const newObj: Record = {} as Record 57 | 58 | for (const key in object) { 59 | newObj[key] = mapFn(key, object[key]) 60 | } 61 | return newObj 62 | } 63 | -------------------------------------------------------------------------------- /sync-production.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run this script to sync the local repo to production 4 | 5 | set -o xtrace # Print commands as they are executed 6 | set -o errexit # Exit on error 7 | 8 | # Makiig sure that we can acctually build 9 | docker build -t find-peers . 10 | 11 | SSH="ssh find-peers" 12 | 13 | # If there are local changes, this will fail. It should be like that 14 | $SSH 'cd /root/find-peers && git pull origin main' 15 | 16 | $SSH '(cd /root/find-peers && docker compose up --build -d)' 17 | 18 | echo Press Ctrl+C to stop following the logs 19 | $SSH 'docker logs -f find-peers' 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "NodeNext", 5 | "moduleResolution": "node", 6 | "esModuleInterop": true, 7 | "outDir": "./build", 8 | "rootDirs": ["./src", "./env"], 9 | // "declaration": true, 10 | // 11 | // code quality options 12 | "allowUnreachableCode": false, 13 | "allowUnusedLabels": false, 14 | "alwaysStrict": true, 15 | "allowJs": false, 16 | "forceConsistentCasingInFileNames": true, 17 | "noFallthroughCasesInSwitch": true, 18 | "noImplicitAny": true, 19 | "noPropertyAccessFromIndexSignature": true, 20 | "noUncheckedIndexedAccess": true, 21 | "noUnusedLocals": true, 22 | "noUnusedParameters": true, 23 | "removeComments": true, 24 | "resolveJsonModule": true, 25 | "strict": true, 26 | "allowSyntheticDefaultImports": true, 27 | "skipLibCheck": true, // skip checking types in node_modules 28 | // "sourceMap": true, // enable debugger 29 | }, 30 | "include": [ 31 | "src/" 32 | ], 33 | "ts-node": { 34 | "esm": true 35 | } 36 | } -------------------------------------------------------------------------------- /views/error.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Find peers 9 | 10 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |

Find Peers

43 |

<%= error %>

44 | 45 | 51 | 52 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /views/index.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Find peers 10 | 151 | 152 | 155 | 156 | 157 | 158 | 159 | 160 |
161 | Find Peers 162 |
163 |
164 | Built by Joppe Koers/@jkoers - Source
165 | This page is updated every <%= updateEveryHours %> hours, last update was on <%= lastUpdate %>: <%= hoursAgo %> hours ago
166 | indicates that the user's project status was changed in the last <%= userNewStatusThresholdDays %> days
167 | <%= usage %>
168 |
169 | 193 | 194 |
195 | <% projects.forEach(project=>{ %> 196 |
197 |
198 |
199 | <%= project.name %> 200 |
201 |
202 | <%= project.users.length %> users 203 |
204 |
205 |
206 | <% if (project.users.length == 0 && !requestedStatus) { %> 207 |

No users are subscribed to this project

208 | <% } else if (project.users.length == 0) { %> 209 |

No users with status <%= requestedStatus %>

210 | <% } else { %> 211 | <% project.users.forEach(user=>{ %> 212 | 213 | <% if (user.new) { %> 214 | 215 | <% } %> 216 | <%= user.login %> 217 |
218 | <%= user.login %> 219 |
220 |
221 | <%= user.status %> 222 |
223 | 224 | <% }) %> 225 | <% } %> 226 |
227 |
228 | <% }) %> 229 |
230 | 231 | 234 | 235 | 236 | 237 | --------------------------------------------------------------------------------