├── .deepsource.toml ├── .dockerignore ├── .editorconfig ├── .eslintrc ├── .github ├── FUNDING.yml └── workflows │ ├── ci.yml │ └── e2e.yml ├── .gitignore ├── .husky └── pre-commit ├── .mocharc.e2e.cjs ├── .mocharc.json ├── .nycrc ├── .spectral.adidas.yaml ├── .spectral.json ├── CONTRIBUTING.md ├── Dockerfile ├── INFRASTRUCTURE.md ├── README.md ├── SUPPORTERS.md ├── compose-stack.yml ├── config ├── create-dbs.sql ├── custom-environment-variables.cjs ├── default.cjs ├── development.cjs ├── production.cjs ├── redis │ ├── .env.redis │ ├── README.md │ ├── node.conf │ ├── node.sh │ ├── original.conf │ ├── standalone-1.conf │ └── standalone-2.conf ├── test.cjs └── whitelist-ips.txt ├── data └── .gitkeep ├── docker-compose.dev.yml ├── docker-compose.yml ├── docs ├── README.md ├── geoip.md ├── staging-env.md └── terms │ ├── Cookie Policy.md │ ├── Privacy Policy.md │ └── Terms of Use.md ├── elastic-apm-node.cjs ├── eslint.config.js ├── knexfile.js ├── migrations ├── create-tables.js └── create-tables.js.sql ├── package-lock.json ├── package.json ├── probes-stats ├── known-probes.json └── known.ts ├── public ├── demo │ ├── globals.js │ ├── index.html │ ├── measurements.vue.js │ └── probes.vue.js ├── favicon.ico └── v1 │ ├── components │ ├── examples.yaml │ ├── headers.yaml │ ├── parameters.yaml │ ├── responses.yaml │ └── schemas.yaml │ └── spec.yaml ├── seeds └── test │ └── index.js ├── src ├── adoption │ ├── adoption-token.ts │ ├── route │ │ └── adoption-code.ts │ ├── schema.ts │ ├── sender.ts │ └── types.ts ├── alternative-ip │ ├── route │ │ └── alternative-ip.ts │ ├── schema.ts │ └── types.ts ├── health │ ├── route │ │ └── get.ts │ └── term-listener.ts ├── index.ts ├── lib │ ├── alt-ips.ts │ ├── blocked-ip-ranges.ts │ ├── cache │ │ ├── cache-interface.ts │ │ ├── null-cache.ts │ │ └── redis-cache.ts │ ├── cloud-ip-ranges.ts │ ├── credits.ts │ ├── download-files.ts │ ├── flush-redis-cache.ts │ ├── geoip │ │ ├── altnames.ts │ │ ├── city-approximation.ts │ │ ├── client.ts │ │ ├── dc-cities.json │ │ ├── dc-cities.ts │ │ ├── fake-client.ts │ │ ├── overrides.ts │ │ ├── providers │ │ │ ├── fastly.ts │ │ │ ├── ip2location.ts │ │ │ ├── ipinfo.ts │ │ │ ├── ipmap.ts │ │ │ └── maxmind.ts │ │ ├── utils.ts │ │ └── whitelist.ts │ ├── get-probe-ip.ts │ ├── http │ │ ├── auth.ts │ │ ├── error-handler.ts │ │ ├── middleware │ │ │ ├── authenticate.ts │ │ │ ├── body-parser.ts │ │ │ ├── cors.ts │ │ │ ├── default-json.ts │ │ │ ├── docs-link.ts │ │ │ ├── error-handler.ts │ │ │ ├── is-admin.ts │ │ │ ├── is-system.ts │ │ │ └── validate.ts │ │ ├── server.ts │ │ └── spec.ts │ ├── location │ │ ├── continents.ts │ │ ├── countries.ts │ │ ├── location.ts │ │ ├── networks.ts │ │ ├── regions.ts │ │ ├── states-iso.ts │ │ ├── states.ts │ │ └── types.ts │ ├── logger.ts │ ├── malware │ │ ├── client.ts │ │ ├── domain.ts │ │ └── ip.ts │ ├── metrics.ts │ ├── override │ │ ├── admin-data.ts │ │ ├── adopted-probes.ts │ │ └── probe-override.ts │ ├── private-ip.ts │ ├── probe-error.ts │ ├── probe-validator.ts │ ├── rate-limiter │ │ ├── rate-limiter-get.ts │ │ └── rate-limiter-post.ts │ ├── redis │ │ ├── client.ts │ │ ├── measurement-client.ts │ │ ├── persistent-client.ts │ │ ├── scripts.ts │ │ ├── shared.ts │ │ └── subscription-client.ts │ ├── server.ts │ ├── sql │ │ └── client.ts │ └── ws │ │ ├── gateway.ts │ │ ├── helper │ │ ├── error-handler.ts │ │ ├── probe-ip-limit.ts │ │ ├── reconnect-probes.ts │ │ ├── subscribe-handler.ts │ │ └── throttle.ts │ │ ├── middleware │ │ └── probe-metadata.ts │ │ ├── server.ts │ │ ├── synced-probe-list.ts │ │ └── ws-error.ts ├── limits │ └── route │ │ └── get-limits.ts ├── measurement │ ├── handler │ │ ├── ack.ts │ │ ├── progress.ts │ │ ├── request.ts │ │ └── result.ts │ ├── route │ │ ├── create-measurement.ts │ │ └── get-measurement.ts │ ├── runner.ts │ ├── schema │ │ ├── command-schema.ts │ │ ├── global-schema.ts │ │ ├── location-schema.ts │ │ ├── probe-response-schema.ts │ │ └── utils.ts │ ├── store.ts │ └── types.ts ├── probe │ ├── builder.ts │ ├── handler │ │ ├── dns.ts │ │ ├── ip-version.ts │ │ ├── stats.ts │ │ └── status.ts │ ├── probes-location-filter.ts │ ├── route │ │ └── get-probes.ts │ ├── router.ts │ ├── schema │ │ └── probe-response-schema.ts │ └── types.ts └── types.d.ts ├── test-perf ├── artillery.yml └── index.ts ├── test ├── dist.js ├── e2e │ ├── cases │ │ ├── adopted-probes.test.ts │ │ ├── adoption-code.test.ts │ │ ├── dns.test.ts │ │ ├── health.test.ts │ │ ├── http.test.ts │ │ ├── limits.test.ts │ │ ├── location-overrides.test.ts │ │ ├── location.test.ts │ │ ├── mtr.test.ts │ │ ├── offline-probes.test.ts │ │ ├── ping.test.ts │ │ ├── probes-sync.test.ts │ │ ├── probes.test.ts │ │ └── traceroute.test.ts │ ├── docker.ts │ ├── setup.ts │ └── utils.ts ├── mocks │ ├── blocked-ip-ranges │ │ └── nock-apple-relay.csv │ ├── cities15000.txt │ ├── cloud-ip-ranges │ │ ├── nock-aws.json │ │ └── nock-gcp.json │ ├── malware │ │ ├── nock-domain.txt │ │ └── nock-ip.txt │ └── nock-geoip.json ├── plugins │ └── oas │ │ ├── index.d.ts │ │ └── index.js ├── setup.ts ├── tests │ ├── contract │ │ ├── newman-env.json │ │ ├── portman-cli.json │ │ └── portman-config.json │ ├── integration │ │ ├── adoption-code.test.ts │ │ ├── adoption-token.test.ts │ │ ├── alternative-ip.test.ts │ │ ├── health.test.ts │ │ ├── limits.test.ts │ │ ├── measurement │ │ │ ├── create-measurement.test.ts │ │ │ └── probe-communication.test.ts │ │ ├── middleware │ │ │ ├── authenticate.test.ts │ │ │ ├── compress.test.ts │ │ │ ├── cors.test.ts │ │ │ ├── etag.test.ts │ │ │ └── responsetime.test.ts │ │ ├── probes │ │ │ └── get-probes.test.ts │ │ └── ratelimit.test.ts │ └── unit │ │ ├── alt-ips.test.ts │ │ ├── auth.test.ts │ │ ├── blocked-ip-ranges.test.ts │ │ ├── credits.test.ts │ │ ├── geoip │ │ ├── city-approximation.test.ts │ │ └── client.test.ts │ │ ├── index.test.ts │ │ ├── ip-ranges.test.ts │ │ ├── malware.test.ts │ │ ├── measurement │ │ ├── runner.test.ts │ │ ├── schema │ │ │ ├── request-schema.test.ts │ │ │ └── response-schema.test.ts │ │ └── store.test.ts │ │ ├── middleware │ │ ├── error-handler.test.ts │ │ ├── is-system.test.ts │ │ └── validate.test.ts │ │ ├── override │ │ ├── admin-data.test.ts │ │ └── adopted-probes.test.ts │ │ ├── probe-validator.test.ts │ │ ├── probe │ │ └── router.test.ts │ │ ├── redis-cache.test.ts │ │ └── ws │ │ ├── error-handler.test.ts │ │ ├── probe-ip-limit.test.ts │ │ ├── reconnect-probes.test.ts │ │ ├── server.test.ts │ │ └── synced-probe-list.test.ts ├── types.ts └── utils │ ├── clock.ts │ ├── nock-geo-ip.ts │ ├── populate-static-files.ts │ └── server.ts ├── tsconfig.json ├── wallaby.e2e.js └── wallaby.js /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | test_patterns = ["test/**/*.test.ts"] 4 | 5 | exclude_patterns = ["public/**"] 6 | 7 | [[analyzers]] 8 | name = "shell" 9 | enabled = true 10 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .github/ 3 | .husky/ 4 | coverage/ 5 | dist/ 6 | node_modules/ 7 | probes-stats/ 8 | test/ 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | indent_style = tab 9 | 10 | [*.md] 11 | indent_style = space 12 | 13 | [*.{yaml,yml}] 14 | indent_style = space 15 | indent_size = 2 16 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@martin-kolarik/eslint-config/typescript", 3 | "ignorePatterns": [ 4 | "coverage/**", 5 | "data/**", 6 | "dist/**" 7 | ], 8 | "overrides": [ 9 | { 10 | "files": [ 11 | "src/**/*.ts" 12 | ], 13 | "extends": "@martin-kolarik/eslint-config/typescript-type-checking", 14 | "parserOptions": { 15 | "project": true 16 | } 17 | }, 18 | { 19 | "files": [ 20 | "**" 21 | ], 22 | "rules": { 23 | "no-duplicate-imports": "off", 24 | "no-extra-parens": "off" 25 | } 26 | }, 27 | { 28 | "files": [ 29 | "public/**" 30 | ], 31 | "parserOptions": { 32 | "sourceType": "script" 33 | }, 34 | "rules": { 35 | "no-undef": "off", 36 | "no-unused-vars": "off" 37 | } 38 | }, 39 | { 40 | "files": [ 41 | "test/**" 42 | ], 43 | "rules": { 44 | "@typescript-eslint/no-explicit-any": "off", 45 | "@typescript-eslint/no-non-null-assertion": "off", 46 | "no-restricted-properties": [ 47 | "error", 48 | { 49 | "object": "sinon", 50 | "property": "spy" 51 | }, 52 | { 53 | "object": "sinon", 54 | "property": "stub" 55 | }, 56 | { 57 | "object": "sinon", 58 | "property": "mock" 59 | }, 60 | { 61 | "object": "sinon", 62 | "property": "fake" 63 | }, 64 | { 65 | "object": "sinon", 66 | "property": "restore" 67 | }, 68 | { 69 | "object": "sinon", 70 | "property": "reset" 71 | }, 72 | { 73 | "object": "sinon", 74 | "property": "resetHistory" 75 | }, 76 | { 77 | "object": "sinon", 78 | "property": "resetBehavior" 79 | } 80 | ] 81 | } 82 | } 83 | ] 84 | } 85 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | github: jsdelivr 3 | open_collective: jsdelivr 4 | custom: ['https://www.jsdelivr.com/sponsors'] 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | pull_request: 7 | branches: [ "*" ] 8 | 9 | jobs: 10 | build: 11 | name: Run tests 12 | runs-on: ubuntu-latest 13 | 14 | env: 15 | NODE_ENV: test 16 | 17 | services: 18 | mariadb: 19 | image: mariadb:10.11.5 20 | ports: 21 | - 13306:3306 22 | env: 23 | MARIADB_DATABASE: dashboard-globalping-test 24 | MARIADB_USER: directus 25 | MARIADB_PASSWORD: password 26 | MARIADB_RANDOM_ROOT_PASSWORD: 1 27 | options: >- 28 | --health-cmd "mysqladmin ping" 29 | --health-interval 10s 30 | --health-timeout 5s 31 | --health-retries 5 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | - uses: actions/setup-node@v4 36 | with: 37 | node-version: 20.x 38 | - name: Set up Redis 39 | run: | 40 | cp config/redis/.env.redis ./ 41 | docker compose up -d 42 | - name: Build 43 | run: | 44 | npm ci 45 | npm run build 46 | - name: Test Unit, Integration, Contract 47 | run: | 48 | npm run lint 49 | npm run coverage 50 | npm run test:portman 51 | - name: Test Dist 52 | run: | 53 | rm -rf node_modules 54 | npm ci --omit=dev 55 | npm run test:dist 56 | -------------------------------------------------------------------------------- /.github/workflows/e2e.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | pull_request: 7 | branches: [ "*" ] 8 | 9 | jobs: 10 | build: 11 | name: Run e2e tests 12 | runs-on: ubicloud-standard-4 13 | 14 | env: 15 | NODE_ENV: test 16 | 17 | services: 18 | mariadb: 19 | image: mariadb:10.11.5 20 | ports: 21 | - 13306:3306 22 | env: 23 | MARIADB_DATABASE: dashboard-globalping-test 24 | MARIADB_USER: directus 25 | MARIADB_PASSWORD: password 26 | MARIADB_RANDOM_ROOT_PASSWORD: 1 27 | options: >- 28 | --health-cmd "mysqladmin ping" 29 | --health-interval 10s 30 | --health-timeout 5s 31 | --health-retries 5 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | - uses: ubicloud/setup-node@v4 36 | with: 37 | node-version: 20.x 38 | - name: Enable IPv6 for Docker containers 39 | run: | 40 | sudo cat /etc/docker/daemon.json | jq '. + { "ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "experimental": true, "ip6tables": true }' > daemon.json 41 | sudo mv daemon.json /etc/docker/ 42 | sudo systemctl restart docker 43 | docker restart $(docker ps -aq) 44 | - name: Set up Redis 45 | run: | 46 | cp config/redis/.env.redis ./ 47 | docker compose up -d 48 | - name: Build 49 | run: | 50 | npm ci 51 | npm run build 52 | - name: Test E2E 53 | run: | 54 | npm run test:e2e 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .vscode/ 3 | .nyc_output/ 4 | node_modules/ 5 | coverage/ 6 | dist/ 7 | tmp/ 8 | config/local* 9 | data/redis 10 | data/DOMAIN_BLACKLIST.json 11 | data/IP_BLACKLIST.json 12 | data/AWS_IP_RANGES.json 13 | data/GCP_IP_RANGES.json 14 | data/GEONAMES_CITIES.csv 15 | data/LAST_API_COMMIT_HASH.txt 16 | data/APPLE_RELAY_IP_RANGES.csv 17 | probes-stats/known-result.csv 18 | probes-stats/known-result.json 19 | probes-stats/all-result.csv 20 | probes-stats/all-result.json 21 | .eslintcache 22 | .env 23 | /.env.redis 24 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Add cygpath to PATH if missing to make this work with GitHub Desktop app 4 | # https://github.com/desktop/desktop/issues/12562 5 | if ! type cygpath > /dev/null 2>&1; then 6 | PATH="$PATH:$(type -ap git | grep 'cmd/git' | sed 's$cmd/git$usr/bin$')" 7 | fi 8 | 9 | node_modules/.bin/lint-staged --quiet 10 | -------------------------------------------------------------------------------- /.mocharc.e2e.cjs: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | 'exit': true, 5 | 'timeout': 40000, 6 | 'check-leaks': true, 7 | 'file': [ 8 | path.join(__dirname, 'test/e2e/setup.ts'), 9 | ], 10 | 'spec': [ 11 | path.join(__dirname, 'test/e2e/cases/**/*.test.ts'), 12 | ], 13 | 'node-option': [ 14 | 'enable-source-maps', 15 | 'experimental-specifier-resolution=node', 16 | 'loader=ts-node/esm', 17 | ], 18 | 'globals': [ 19 | '__extends', 20 | '__assign', 21 | '__rest', 22 | '__decorate', 23 | '__param', 24 | '__metadata', 25 | '__awaiter', 26 | '__generator', 27 | '__exportStar', 28 | '__createBinding', 29 | '__values', 30 | '__read', 31 | '__spread', 32 | '__spreadArrays', 33 | '__spreadArray', 34 | '__await', 35 | '__asyncGenerator', 36 | '__asyncDelegator', 37 | '__asyncValues', 38 | '__makeTemplateObject', 39 | '__importStar', 40 | '__importDefault', 41 | '__classPrivateFieldGet', 42 | '__classPrivateFieldSet', 43 | ], 44 | }; 45 | -------------------------------------------------------------------------------- /.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "exit": true, 3 | "timeout": 10000, 4 | "check-leaks": true, 5 | "file": [ 6 | "test/setup.ts" 7 | ], 8 | "spec": [ 9 | "test/tests/integration/**/*.test.ts", 10 | "test/tests/unit/**/*.test.ts" 11 | ], 12 | "node-option": [ 13 | "enable-source-maps", 14 | "experimental-specifier-resolution=node", 15 | "loader=ts-node/esm" 16 | ], 17 | "globals": [ 18 | "__extends", 19 | "__assign", 20 | "__rest", 21 | "__decorate", 22 | "__param", 23 | "__metadata", 24 | "__awaiter", 25 | "__generator", 26 | "__exportStar", 27 | "__createBinding", 28 | "__values", 29 | "__read", 30 | "__spread", 31 | "__spreadArrays", 32 | "__spreadArray", 33 | "__await", 34 | "__asyncGenerator", 35 | "__asyncDelegator", 36 | "__asyncValues", 37 | "__makeTemplateObject", 38 | "__importStar", 39 | "__importDefault", 40 | "__classPrivateFieldGet", 41 | "__classPrivateFieldSet" 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "check-coverage": false, 3 | "all": true, 4 | "exclude": [ 5 | "dist/", 6 | "test/", 7 | "public/", 8 | "**/*.d.ts" 9 | ], 10 | "reporter": [ 11 | "lcov", 12 | "text" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.spectral.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "spectral:oas", 4 | "./.spectral.adidas.yaml" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guide 2 | 3 | Hi! We're really excited that you're interested in contributing! Before submitting your contribution, please read through the following guide. 4 | 5 | ## General guidelines 6 | 7 | - Bug fixes and changes discussed in the existing issues are always welcome. 8 | - For new ideas, please open an issue to discuss them before sending a PR. 9 | - Make sure your PR passes `npm test` and has [appropriate commit messages](https://github.com/jsdelivr/globalping/commits/master). 10 | 11 | ## Project setup 12 | 13 | In order to run the Globalping API locally you will need Node.js 20 and Redis with [RedisJSON](https://oss.redis.com/redisjson/) module and MariaDB. All of them are included in docker-compose.dev.yml file. You will also need to run a development instance of the [Globalping Probe](https://github.com/jsdelivr/globalping-probe) at the same time when testing. 14 | 15 | The API uses 3000 port by default. This can be overridden by `PORT` environment variable. 16 | 17 | You can run the project by following these steps: 18 | 19 | 1. Clone this repository. 20 | 2. [Enable host networking in Docker Desktop](https://docs.docker.com/engine/network/drivers/host/#docker-desktop) if you haven't already. 21 | 3. `docker compose -f docker-compose.dev.yml up -d` - Run Redis and MariaDB 22 | 4. `npm install && npm run download:files` 23 | 5. Run `npm run start:dev` 24 | 25 | Once the API is live, you can spin up a probe instance by running as described at https://github.com/jsdelivr/globalping-probe/blob/master/CONTRIBUTING.md. 26 | 27 | ### Environment variables 28 | - `PORT=3000` environment variable can start the API on another port (default is 3000) 29 | - `FAKE_PROBE_IP=1` environment variable can be used to make debug easier. When defined, every Probe 30 | that connects to the API will get an IP address from the list of predefined "real" addresses. 31 | 32 | ### Testing 33 | 34 | A single command to run everything: `npm test` 35 | 36 | To run a specific linter or a test suite, please see the scripts section of [package.json](package.json). 37 | 38 | Most IDEs have plugins integrating the used linter (eslint), including support for automated fixes on save. 39 | 40 | ## Production config 41 | 42 | ### Environment variables 43 | 44 | - `ELASTIC_APM_SERVER_URL={value}` used in production to send APM metrics to elastic 45 | - `ELASTIC_APM_SECRET_TOKEN={value}` used in production to send APM metrics to elastic 46 | - `ELASTIC_SEARCH_URL={value}` used in production to send logs to elastic 47 | - `FAKE_PROBE_IP=1` used in development to use a random fake ip assigned by the API 48 | - `ADMIN_KEY={value}` used to access additional information over the API 49 | - `SYSTEM_API_KEY={value}` used for integration with the dashboard 50 | - `SERVER_SESSION_COOKIE_SECRET={value}` used to read the shared session cookie 51 | - `DB_CONNECTION_HOST`, `DB_CONNECTION_USER`, `DB_CONNECTION_PASSWORD`, and `DB_CONNECTION_DATABASE` database connection details 52 | - `REDIS_STANDALONE_PERSISTENT_URL`, `REDIS_STANDALONE_NON_PERSISTENT_URL`, `REDIS_CLUSTER_MEASUREMENTS_NODES_0`, `REDIS_CLUSTER_MEASUREMENTS_NODES_1`, `REDIS_CLUSTER_MEASUREMENTS_NODES_2`, and `REDIS_SHARED_OPTIONS_PASSWORD` - redis connection details 53 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20-bullseye-slim AS builder 2 | RUN apt-get update -y && apt-get install util-linux curl git -y 3 | 4 | ENV NODE_ENV=production 5 | 6 | COPY package.json package-lock.json /app/ 7 | WORKDIR /app 8 | RUN npm ci --include=dev 9 | COPY . /app 10 | RUN npm run build 11 | 12 | FROM node:20-bullseye-slim 13 | RUN apt-get update -y && apt-get install tini util-linux curl -y 14 | 15 | ENV NODE_ENV=production 16 | 17 | COPY package.json package-lock.json /app/ 18 | WORKDIR /app 19 | RUN npm ci 20 | COPY . /app 21 | COPY --from=builder /app/dist /app/dist 22 | COPY --from=builder /app/data /app/data 23 | 24 | ENV PORT=80 25 | EXPOSE 80 26 | ENTRYPOINT ["/usr/bin/tini", "--"] 27 | CMD [ "npm", "start" ] 28 | -------------------------------------------------------------------------------- /INFRASTRUCTURE.md: -------------------------------------------------------------------------------- 1 | # Globalping Infrastructure 2 | 3 | This file describes where and how the production Globalping infrastructure works. 4 | The purpose is to document everything for ourselves as well as allow everyone else to explore our services and contribute their own ideas for potential optimizations. 5 | 6 | ### API - api.globalping.io 7 | 8 | The main production API that all probes and users connect to. 9 | 10 | - Hosted on Hetzner in Falkenstein 11 | - 2xVMs with 8 threads and 16GB RAM per VM 12 | - 1 Load-balancer with health-checks and TLS termination and automated LetsEncrypt certs 13 | - Master branch is compiled into a Docker container automatically by Docker Hub. 14 | - Manually triggered deployments using Docker Swarm (Network host mode) 15 | - $60/month 16 | 17 | 18 | ### API - Redis 7.x 19 | 20 | Redis is used to cache GeoIP information from our 3 IP databases, to store all measurement results and to sync connected probes between multiple API instances. 21 | 22 | - Hosted with Hetzner in Falkenstein 23 | - Dedicated server with 12 threads and 64GB RAM 24 | - `maxmemory-policy allkeys-lru` Evict any key using approximated LRU. 25 | - RedisJSON 2.x module enabled 26 | - $46/month 27 | 28 | 29 | ### API - APM Service New Relic 30 | 31 | - We use New Relic in our API to monitor it's performance and stablity as well as to collect production logs 32 | - Addionally we use it to monitor the servers running the API and our self-hosted Redis database 33 | - EU region account 34 | - Free plan 35 | 36 | ### DNS for *.globalping.io 37 | 38 | - Hosted with Hetzner DNS 39 | - Free plan 40 | 41 | ### Probes - Seeded datacenter network 42 | 43 | We seeded the network with ~130 probes that we purchased from providers like Google Cloud, AWS, DigitalOcean, Vultr, OVH, Fly.io and Tencent. 44 | Help us by [running our probe on your servers](https://github.com/jsdelivr/globalping-probe#readme) with spare capacity or by [becoming a GitHub Sponsor](https://github.com/sponsors/jsdelivr). 45 | 46 | - ~$790/month 47 | -------------------------------------------------------------------------------- /SUPPORTERS.md: -------------------------------------------------------------------------------- 1 | | Supporter | Website | 2 | | :--------------------------------------------------------------------- | :---------------------------- | 3 | | RareCloud logo | [Website](https://rarecloud.io/) | -------------------------------------------------------------------------------- /compose-stack.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | globalping-api: 4 | image: jimaek/globalping-api:latest 5 | stop_grace_period: 30s 6 | stop_signal: SIGTERM 7 | networks: 8 | - hostnet 9 | deploy: 10 | mode: global 11 | endpoint_mode: dnsrr 12 | update_config: 13 | parallelism: 1 14 | delay: 5s 15 | failure_action: rollback 16 | monitor: 10s 17 | restart_policy: 18 | condition: any 19 | delay: 3s 20 | dns: 21 | - 8.8.8.8 22 | - 1.1.1.1 23 | env_file: 24 | - globalping.env 25 | healthcheck: 26 | test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] 27 | interval: 10s 28 | timeout: 8s 29 | retries: 2 30 | start_period: 3s 31 | networks: 32 | hostnet: 33 | external: true 34 | name: host 35 | -------------------------------------------------------------------------------- /config/create-dbs.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS `dashboard-globalping`; 2 | GRANT ALL PRIVILEGES ON `dashboard-globalping`.* to 'directus'@'%'; 3 | 4 | CREATE DATABASE IF NOT EXISTS `dashboard-globalping-test`; 5 | GRANT ALL PRIVILEGES ON `dashboard-globalping-test`.* to 'directus'@'%'; 6 | 7 | -- Directus issue https://github.com/directus/directus/discussions/11786 8 | ALTER DATABASE `dashboard-globalping` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; 9 | ALTER DATABASE `dashboard-globalping-test` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; 10 | -------------------------------------------------------------------------------- /config/custom-environment-variables.cjs: -------------------------------------------------------------------------------- 1 | const df = require('./default.cjs'); 2 | module.exports = require('config-mapper-env')(df); 3 | -------------------------------------------------------------------------------- /config/default.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | host: 'https://api.globalping.io', 4 | docsHost: 'https://globalping.io', 5 | port: 3000, 6 | processes: 2, 7 | cors: { 8 | trustedOrigins: [ 9 | 'https://globalping.io', 10 | 'https://staging.globalping.io', 11 | 'http://localhost:13000', 12 | ], 13 | }, 14 | session: { 15 | cookieName: 'dash_session_token', 16 | cookieSecret: '', 17 | }, 18 | }, 19 | redis: { 20 | standalonePersistent: { 21 | url: 'redis://localhost:7001', 22 | }, 23 | standaloneNonPersistent: { 24 | url: 'redis://localhost:7002', 25 | }, 26 | clusterMeasurements: { 27 | // listing three nodes here is enough, the rest will be discovered automatically 28 | nodes: { 29 | 0: 'redis://localhost:7101', 30 | 1: 'redis://localhost:7102', 31 | 2: 'redis://localhost:7103', 32 | }, 33 | options: {}, 34 | }, 35 | sharedOptions: { 36 | password: 'PASSWORD', 37 | socket: { 38 | tls: false, 39 | }, 40 | }, 41 | }, 42 | db: { 43 | type: 'mysql', 44 | connection: { 45 | host: 'localhost', 46 | user: 'directus', 47 | password: 'password', 48 | database: 'dashboard-globalping', 49 | port: 3306, 50 | }, 51 | }, 52 | dashboard: { 53 | directusUrl: 'https://dash-directus.globalping.io', 54 | }, 55 | admin: { 56 | key: '', 57 | }, 58 | systemApi: { 59 | key: '', 60 | }, 61 | geoip: { 62 | cache: { 63 | enabled: true, 64 | ttl: 3 * 24 * 60 * 60 * 1000, // 3 days 65 | }, 66 | }, 67 | maxmind: { 68 | accountId: '', 69 | licenseKey: '', 70 | }, 71 | ipinfo: { 72 | apiKey: '', 73 | }, 74 | ip2location: { 75 | apiKey: '', 76 | }, 77 | adoptedProbes: { 78 | syncInterval: 60000, 79 | }, 80 | adminData: { 81 | syncInterval: 60000, 82 | }, 83 | measurement: { 84 | maxInProgressTests: 5, 85 | // Timeout after which measurement will be marked as finished even if not all probes respond 86 | timeout: 30, // 30 seconds 87 | // measurement result TTL in redis 88 | resultTTL: 7 * 24 * 60 * 60, // 7 days 89 | rateLimit: { 90 | post: { 91 | anonymousLimit: 250, 92 | authenticatedLimit: 500, 93 | reset: 3600, 94 | }, 95 | getPerMeasurement: { 96 | limit: 5, 97 | reset: 2, 98 | }, 99 | }, 100 | limits: { 101 | anonymousTestsPerLocation: 200, 102 | anonymousTestsPerMeasurement: 500, 103 | authenticatedTestsPerLocation: 500, 104 | authenticatedTestsPerMeasurement: 500, 105 | }, 106 | globalDistribution: { 107 | AF: 5, 108 | AS: 15, 109 | EU: 30, 110 | OC: 10, 111 | NA: 30, 112 | SA: 10, 113 | }, 114 | }, 115 | reconnectProbesDelay: 2 * 60 * 1000, 116 | sigtermDelay: 15000, 117 | }; 118 | -------------------------------------------------------------------------------- /config/development.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | session: { 4 | cookieSecret: 'xxx', 5 | }, 6 | }, 7 | db: { 8 | connection: { 9 | port: 13306, 10 | }, 11 | }, 12 | dashboard: { 13 | directusUrl: 'http://localhost:18055', 14 | }, 15 | admin: { 16 | key: 'admin', 17 | }, 18 | systemApi: { 19 | key: 'system', 20 | }, 21 | adoptedProbes: { 22 | syncInterval: 5000, 23 | }, 24 | adminData: { 25 | syncInterval: 5000, 26 | }, 27 | reconnectProbesDelay: 0, 28 | }; 29 | -------------------------------------------------------------------------------- /config/production.cjs: -------------------------------------------------------------------------------- 1 | const physicalCpuCount = require('physical-cpu-count'); 2 | 3 | module.exports = { 4 | server: { 5 | processes: physicalCpuCount, 6 | }, 7 | }; 8 | -------------------------------------------------------------------------------- /config/redis/.env.redis: -------------------------------------------------------------------------------- 1 | # standalone: 2 | REDIS_ARGS="--requirepass PASSWORD" 3 | 4 | # cluster: 5 | REDIS_PASSWORD=PASSWORD 6 | REDIS_PUBLIC_IP=127.0.0.1 7 | -------------------------------------------------------------------------------- /config/redis/README.md: -------------------------------------------------------------------------------- 1 | ### Redis prod setup 2 | 3 | ``` 4 | echo never > /sys/kernel/mm/transparent_hugepage/enabled 5 | echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf 6 | echo 'vm.swappiness = 1' >> /etc/sysctl.conf 7 | 8 | fallocate -l 90G /swapfile 9 | chmod 600 /swapfile 10 | mkswap /swapfile 11 | swapon /swapfile 12 | echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab 13 | ``` 14 | 15 | ### Docker config 16 | 17 | Assuming you start in this directory: 18 | 19 | ``` 20 | cp .env.redis ../../ 21 | ``` 22 | 23 | Set the redis password and return to the project root. Then: 24 | 25 | ``` 26 | docker compose up -d 27 | ``` 28 | -------------------------------------------------------------------------------- /config/redis/node.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/dumb-init /bin/sh 2 | 3 | PORT=$1 4 | 5 | SRC_CONF_FILE="/home/runner/mounted/node.conf" 6 | DST_CONF_FILE="/node.conf" 7 | 8 | # copy the base config 9 | cp $SRC_CONF_FILE $DST_CONF_FILE 10 | 11 | # add node-specific values 12 | echo " 13 | port ${PORT} 14 | cluster-announce-ip $REDIS_PUBLIC_IP 15 | cluster-config-file node-cluster-config.conf 16 | requirepass $REDIS_PASSWORD 17 | masterauth $REDIS_PASSWORD" >> $DST_CONF_FILE 18 | 19 | # start the server 20 | cd /data 21 | exec redis-server $DST_CONF_FILE 22 | -------------------------------------------------------------------------------- /config/test.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | session: { 4 | cookieSecret: 'xxx', 5 | }, 6 | }, 7 | redis: { 8 | clusterMeasurements: { 9 | options: { 10 | nodeAddressMap (address) { 11 | if (process.env.TEST_MODE !== 'e2e') { 12 | return { 13 | host: address.substring(0, address.lastIndexOf(':')), 14 | port: address.substring(address.lastIndexOf(':') + 1), 15 | }; 16 | } 17 | 18 | return { 19 | host: 'host.docker.internal', 20 | port: address.substring(address.lastIndexOf(':') + 1), 21 | }; 22 | }, 23 | }, 24 | }, 25 | }, 26 | db: { 27 | connection: { 28 | port: 13306, 29 | database: 'dashboard-globalping-test', 30 | multipleStatements: true, 31 | }, 32 | }, 33 | admin: { 34 | key: 'admin', 35 | }, 36 | systemApi: { 37 | key: 'system', 38 | }, 39 | geoip: { 40 | cache: { 41 | enabled: false, 42 | }, 43 | }, 44 | measurement: { 45 | maxInProgressTests: 2, 46 | rateLimit: { 47 | getPerMeasurement: { 48 | limit: 1000, 49 | }, 50 | }, 51 | }, 52 | sigtermDelay: 0, 53 | }; 54 | -------------------------------------------------------------------------------- /config/whitelist-ips.txt: -------------------------------------------------------------------------------- 1 | 8.8.4.0/24 2 | 5.134.119.43 3 | 128.127.105.10 4 | 46.246.97.189 5 | 51.195.35.4 6 | 51.195.249.116 7 | 137.74.150.56 8 | 54.37.188.195 9 | 185.179.190.38 10 | 54.38.124.154 11 | 185.234.114.10 12 | 153.92.126.212 13 | 169.239.130.2 14 | 172.107.202.151 15 | 179.43.128.17 16 | 193.105.134.3 17 | 45.91.93.241 18 | 95.85.89.142 19 | 45.90.57.21 20 | 91.219.239.101 21 | 45.91.92.167 22 | 92.38.139.138 23 | 172.107.194.22 24 | 185.231.233.219 25 | 151.236.24.11 26 | 151.236.25.228 27 | 213.183.56.121 28 | 213.183.54.239 29 | 68.183.146.4 30 | 216.238.109.34 31 | 143.110.217.4 32 | 151.236.18.204 33 | 31.6.16.1 34 | 31.6.16.2 35 | 31.6.16.3 36 | 31.6.16.4 37 | 31.6.16.5 38 | -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsdelivr/globalping/9506024c91e633ef0fc1a36906522acd709223d0/data/.gitkeep -------------------------------------------------------------------------------- /docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | services: 2 | redis-standalone-1: 3 | image: redis/redis-stack-server:7.4.0-v1 4 | ports: 5 | - "7001:7001" 6 | volumes: 7 | - ./config/redis/standalone-1.conf:/redis-stack.conf 8 | - ./data/redis/7001:/data 9 | env_file: 10 | - ./config/redis/.env.redis 11 | redis-standalone-2: 12 | image: redis/redis-stack-server:7.4.0-v1 13 | ports: 14 | - "7002:7002" 15 | volumes: 16 | - ./config/redis/standalone-2.conf:/redis-stack.conf 17 | - ./data/redis/7002:/data 18 | env_file: 19 | - ./config/redis/.env.redis 20 | redis-node-01: 21 | image: redis/redis-stack-server:7.4.0-v1 22 | command: [ "bash", "/home/runner/mounted/node.sh", "7101" ] 23 | network_mode: "host" 24 | volumes: 25 | - ./config/redis:/home/runner/mounted 26 | - ./data/redis/7101:/data 27 | env_file: 28 | - ./config/redis/.env.redis 29 | redis-node-02: 30 | image: redis/redis-stack-server:7.4.0-v1 31 | command: [ "bash", "/home/runner/mounted/node.sh", "7102" ] 32 | network_mode: "host" 33 | volumes: 34 | - ./config/redis:/home/runner/mounted 35 | - ./data/redis/7102:/data 36 | env_file: 37 | - ./config/redis/.env.redis 38 | redis-node-03: 39 | image: redis/redis-stack-server:7.4.0-v1 40 | command: [ "bash", "/home/runner/mounted/node.sh", "7103" ] 41 | network_mode: "host" 42 | volumes: 43 | - ./config/redis:/home/runner/mounted 44 | - ./data/redis/7103:/data 45 | env_file: 46 | - ./config/redis/.env.redis 47 | redis-node-04: 48 | image: redis/redis-stack-server:7.4.0-v1 49 | command: [ "bash", "/home/runner/mounted/node.sh", "7104" ] 50 | network_mode: "host" 51 | volumes: 52 | - ./config/redis:/home/runner/mounted 53 | - ./data/redis/7104:/data 54 | env_file: 55 | - ./config/redis/.env.redis 56 | redis-cluster-creator: 57 | image: redis/redis-stack-server:7.4.0-v1 58 | command: bash -c 'redis-cli -a "$$REDIS_PASSWORD" --cluster create $$REDIS_PUBLIC_IP:7101 $$REDIS_PUBLIC_IP:7102 $$REDIS_PUBLIC_IP:7103 $$REDIS_PUBLIC_IP:7104 --cluster-replicas 0 --cluster-yes' 59 | network_mode: "host" 60 | depends_on: 61 | - redis-node-01 62 | - redis-node-02 63 | - redis-node-03 64 | - redis-node-04 65 | env_file: 66 | - ./config/redis/.env.redis 67 | 68 | mariadb: 69 | image: mariadb:10.11.5 70 | environment: 71 | - MARIADB_DATABASE=dashboard-globalping 72 | - MARIADB_USER=directus 73 | - MARIADB_PASSWORD=password 74 | - MARIADB_ROOT_PASSWORD=root 75 | ports: 76 | - "13306:3306" 77 | volumes: 78 | - ./config/create-dbs.sql:/docker-entrypoint-initdb.d/01-create-dbs.sql 79 | - ./migrations/create-tables.js.sql:/docker-entrypoint-initdb.d/02-create-tables.sql 80 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Globalping API 2 | 3 | The documentation has moved to https://www.jsdelivr.com/docs/api.globalping.io. 4 | To edit it, see the [OpenAPI document](../public/v1/spec.yaml). 5 | -------------------------------------------------------------------------------- /elastic-apm-node.cjs: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | let version; 4 | 5 | try { 6 | // We don't really increment the version so the commit hash is more useful. 7 | version = fs.readFileSync(path.join(path.resolve(), 'data/LAST_API_COMMIT_HASH.txt'), 'utf8').trim(); 8 | } catch { 9 | version = require('./package.json').version; 10 | } 11 | 12 | module.exports = { 13 | active: process.env.NODE_ENV === 'production', 14 | serviceName: 'globalping-api', 15 | serviceVersion: version, 16 | logLevel: 'fatal', 17 | centralConfig: false, 18 | captureExceptions: false, 19 | captureErrorLogStackTraces: 'always', 20 | ignoreUrls: [ '/favicon.ico', '/health', '/amp_preconnect_polyfill_404_or_other_error_expected._Do_not_worry_about_it' ], 21 | transactionSampleRate: 1, 22 | exitSpanMinDuration: '2ms', 23 | spanCompressionSameKindMaxDuration: '10ms', 24 | }; 25 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'eslint/config'; 2 | import typescript from '@martin-kolarik/eslint-config/typescript.js'; 3 | import typescriptTypeChecked from '@martin-kolarik/eslint-config/typescript-type-checked.js'; 4 | 5 | export default defineConfig([ 6 | typescript, 7 | { 8 | ignores: [ 9 | 'coverage/**/*', 10 | 'data/**/*', 11 | 'dist/**/*', 12 | '.geo-data-tests/**', 13 | ], 14 | }, 15 | { 16 | files: [ 'src/**/*.ts' ], 17 | extends: [ typescriptTypeChecked ], 18 | 19 | languageOptions: { 20 | sourceType: 'module', 21 | 22 | parserOptions: { 23 | project: true, 24 | }, 25 | }, 26 | }, 27 | { 28 | rules: { 29 | 'no-duplicate-imports': 'off', 30 | '@stylistic/no-extra-parens': 'off', 31 | }, 32 | }, 33 | { 34 | files: [ 'public/**' ], 35 | 36 | languageOptions: { 37 | sourceType: 'script', 38 | }, 39 | 40 | rules: { 41 | 'no-undef': 'off', 42 | 'no-unused-vars': 'off', 43 | }, 44 | }, 45 | { 46 | files: [ 'test/**' ], 47 | 48 | rules: { 49 | '@typescript-eslint/no-explicit-any': 'off', 50 | '@typescript-eslint/no-non-null-assertion': 'off', 51 | 52 | 'no-restricted-properties': [ 'error', 53 | { 54 | object: 'sinon', 55 | property: 'spy', 56 | }, 57 | { 58 | object: 'sinon', 59 | property: 'stub', 60 | }, 61 | { 62 | object: 'sinon', 63 | property: 'mock', 64 | }, 65 | { 66 | object: 'sinon', 67 | property: 'fake', 68 | }, 69 | { 70 | object: 'sinon', 71 | property: 'restore', 72 | }, 73 | { 74 | object: 'sinon', 75 | property: 'reset', 76 | }, 77 | { 78 | object: 'sinon', 79 | property: 'resetHistory', 80 | }, 81 | { 82 | object: 'sinon', 83 | property: 'resetBehavior', 84 | }], 85 | }, 86 | }, 87 | ]); 88 | -------------------------------------------------------------------------------- /knexfile.js: -------------------------------------------------------------------------------- 1 | import path from 'node:path'; 2 | import { fileURLToPath } from 'node:url'; 3 | import _ from 'lodash'; 4 | import config from 'config'; 5 | 6 | const __dirname = path.dirname(fileURLToPath(import.meta.url)); 7 | 8 | const dbConfig = config.get('db'); 9 | 10 | /** 11 | * @typedef {import('knex').Knex.Config} KnexConfig 12 | * @type {{ [key: string]: KnexConfig }} 13 | */ 14 | export default _.merge({}, ...[ 'development', 'production', 'staging', 'test' ].map((environment) => { 15 | return { 16 | [environment]: { 17 | client: dbConfig.type, 18 | connection: dbConfig.connection, 19 | pool: { 20 | min: 0, 21 | max: 10, 22 | propagateCreateError: false, 23 | }, 24 | acquireConnectionTimeout: 10000, 25 | seeds: { 26 | directory: path.join(__dirname, `./seeds/${environment}`), 27 | }, 28 | migrations: { 29 | directory: path.join(__dirname, `./migrations`), 30 | }, 31 | }, 32 | }; 33 | })); 34 | -------------------------------------------------------------------------------- /migrations/create-tables.js: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'path'; 3 | import { fileURLToPath } from 'url'; 4 | 5 | export const up = async (db) => { 6 | const __filename = fileURLToPath(import.meta.url); 7 | const query = fs.readFileSync(path.join(__filename + '.sql'), 'utf8'); 8 | await db.schema.raw(query); 9 | }; 10 | 11 | export const down = () => {}; 12 | -------------------------------------------------------------------------------- /probes-stats/known-probes.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ip": "exampleIp", 4 | "city": "exampleCity" 5 | } 6 | ] 7 | -------------------------------------------------------------------------------- /public/demo/globals.js: -------------------------------------------------------------------------------- 1 | // types 2 | const ALLOWED_QUERY_TYPES = [ 'ping', 'traceroute', 'dns', 'mtr', 'http' ]; 3 | 4 | // filters 5 | const ALLOWED_LOCATION_TYPES = [ 'continent', 'region', 'country', 'state', 'city', 'asn', 'network', 'tags', 'magic' ]; 6 | 7 | // ip versions 8 | const ALLOWED_IP_VERSIONS = [ 4, 6 ]; 9 | 10 | // traceroute 11 | const ALLOWED_TRACE_PROTOCOLS = [ 'TCP', 'UDP', 'ICMP' ]; 12 | 13 | // dns 14 | const ALLOWED_DNS_TYPES = [ 'A', 'AAAA', 'ANY', 'CNAME', 'DNSKEY', 'DS', 'HTTPS', 'MX', 'NS', 'NSEC', 'PTR', 'RRSIG', 'SOA', 'TXT', 'SRV', 'SVCB' ]; 15 | const ALLOWED_DNS_PROTOCOLS = [ 'UDP', 'TCP' ]; 16 | 17 | // mtr 18 | const ALLOWED_MTR_PROTOCOLS = [ 'TCP', 'UDP', 'ICMP' ]; 19 | 20 | // http 21 | const ALLOWED_HTTP_PROTOCOLS = [ 'HTTP', 'HTTPS', 'HTTP2' ]; 22 | const ALLOWED_HTTP_METHODS = [ 'GET', 'HEAD', 'OPTIONS' ]; 23 | -------------------------------------------------------------------------------- /public/demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 |
7 |
8 |
9 |
10 |
11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /public/demo/probes.vue.js: -------------------------------------------------------------------------------- 1 | const probes = () => ({ 2 | data () { 3 | return { 4 | probes: [], 5 | }; 6 | }, 7 | created () { 8 | this.fetchProbes(); 9 | }, 10 | methods: { 11 | getReadyColor (index) { 12 | const probe = this.probes[index]; 13 | 14 | if (!probe.status) { 15 | return 'green'; 16 | } 17 | 18 | return probe.status !== 'ready' ? 'orange' : 'green'; 19 | }, 20 | getNodeVersion (index) { 21 | const probe = this.probes[index]; 22 | return probe.nodeVersion ? `[${probe.nodeVersion}]` : ''; 23 | }, 24 | getReadyStatus (index) { 25 | const probe = this.probes[index]; 26 | return probe.status ? `[${probe.status.toUpperCase()}]` : ''; 27 | }, 28 | getHost (index) { 29 | const probe = this.probes[index]; 30 | return probe.host ? `[${probe.host}]` : ''; 31 | }, 32 | getIpAddresses (index) { 33 | const probe = this.probes[index]; 34 | return probe.ipAddress ? `[${[ probe.ipAddress, ...probe.altIpAddresses ].join(', ')}]` : ''; 35 | }, 36 | parsedLocation (index) { 37 | const probe = this.probes[index]; 38 | 39 | if (!probe) { 40 | return; 41 | } 42 | 43 | const city = probe.location.country === 'US' ? `${probe.location.city} (${probe.location.state})` : probe.location.city; 44 | 45 | return `${city}, ${probe.location.country}, ${probe.location.continent}, ${probe.location.asn}`; 46 | }, 47 | getTags (index) { 48 | const probe = this.probes[index]; 49 | return probe.tags.length ? `(${probe.tags.join(', ')})` : ''; 50 | }, 51 | getIsIPv4Supported (index) { 52 | const probe = this.probes[index]; 53 | return probe.isIPv4Supported === undefined ? '' : `IPv4: [${probe.isIPv4Supported}]`; 54 | }, 55 | getIsIPv6Supported (index) { 56 | const probe = this.probes[index]; 57 | return probe.isIPv6Supported === undefined ? '' : `IPv6: [${probe.isIPv6Supported}]`; 58 | }, 59 | async fetchProbes () { 60 | const adminKey = new URLSearchParams(window.location.search).get('adminkey'); 61 | const url = `/v1/probes?adminkey=${adminKey}`; 62 | const probes = await (await fetch(url)).json(); 63 | const sortNonReadyFirst = (probe1, probe2) => { 64 | if (probe1.status === 'ready' && probe2.status !== 'ready') { 65 | return 1; 66 | } else if (probe1.status !== 'ready' && probe2.status === 'ready') { 67 | return -1; 68 | } 69 | 70 | return 0; 71 | }; 72 | this.probes = probes.sort(sortNonReadyFirst); 73 | }, 74 | }, 75 | template: ` 76 |
77 |

78 | {{ probes.length }} probes available 79 |

80 | 87 |
88 | `, 89 | }); 90 | 91 | Vue.createApp(probes()).mount('#probes'); 92 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsdelivr/globalping/9506024c91e633ef0fc1a36906522acd709223d0/public/favicon.ico -------------------------------------------------------------------------------- /public/v1/components/headers.yaml: -------------------------------------------------------------------------------- 1 | components: 2 | headers: 3 | CreditsConsumed: 4 | description: The number of credits consumed by the request. Returned only when an attempt to use credits was made (requests with a valid token exceeding the hourly rate limit). 5 | required: false 6 | schema: 7 | type: integer 8 | CreditsRemaining: 9 | description: The number of credits remaining. Returned only when an attempt to use credits was made (requests with a valid token exceeding the hourly rate limit). 10 | required: false 11 | schema: 12 | type: integer 13 | MeasurementLocation: 14 | description: A link to the newly created measurement. 15 | required: true 16 | schema: 17 | type: string 18 | format: uri 19 | RateLimitLimit: 20 | description: The number of rate limit points available in a given time window. 21 | required: true 22 | schema: 23 | type: integer 24 | RateLimitConsumed: 25 | description: The number of rate limit points consumed by the request. 26 | required: true 27 | schema: 28 | type: integer 29 | RateLimitRemaining: 30 | description: The number of rate limit points remaining in the current time window. 31 | required: true 32 | schema: 33 | type: integer 34 | RateLimitReset: 35 | description: The number of seconds until the limit resets. 36 | required: true 37 | schema: 38 | type: integer 39 | RequestCost: 40 | description: The number of rate limit points or credits required to accept the request. 41 | required: false 42 | schema: 43 | type: integer 44 | RetryAfter: 45 | description: The number of seconds to wait before retrying this request. 46 | required: true 47 | schema: 48 | type: integer 49 | -------------------------------------------------------------------------------- /public/v1/components/parameters.yaml: -------------------------------------------------------------------------------- 1 | components: 2 | parameters: 3 | measurementId: 4 | description: The ID of the measurement you want to retrieve. 5 | in: path 6 | name: id 7 | required: true 8 | schema: 9 | type: string 10 | -------------------------------------------------------------------------------- /seeds/test/index.js: -------------------------------------------------------------------------------- 1 | export const seed = async () => { 2 | }; 3 | -------------------------------------------------------------------------------- /src/adoption/route/adoption-code.ts: -------------------------------------------------------------------------------- 1 | import type { Context } from 'koa'; 2 | import type Router from '@koa/router'; 3 | import createHttpError from 'http-errors'; 4 | import type { AdoptionCodeRequest } from '../types.js'; 5 | import { bodyParser } from '../../lib/http/middleware/body-parser.js'; 6 | import { validate } from '../../lib/http/middleware/validate.js'; 7 | import { schema } from '../schema.js'; 8 | import { codeSender } from '../sender.js'; 9 | import { AdoptedProbes } from '../../lib/override/adopted-probes.js'; 10 | 11 | const handle = async (ctx: Context): Promise => { 12 | if (!ctx['isSystem']) { 13 | throw createHttpError(403, 'Forbidden', { type: 'access_forbidden' }); 14 | } 15 | 16 | const request = ctx.request.body as AdoptionCodeRequest; 17 | const probe = await codeSender.sendCode(request); 18 | 19 | ctx.body = AdoptedProbes.formatProbeAsDProbe(probe); 20 | }; 21 | 22 | export const registerSendCodeRoute = (router: Router): void => { 23 | router.post('/adoption-code', '/adoption-code', bodyParser(), validate(schema), handle); 24 | }; 25 | -------------------------------------------------------------------------------- /src/adoption/schema.ts: -------------------------------------------------------------------------------- 1 | import Joi from 'joi'; 2 | import { globalIpOptions } from '../measurement/schema/utils.js'; 3 | 4 | export const schema = Joi.object({ 5 | ip: Joi.string().ip(globalIpOptions).required(), 6 | code: Joi.string().length(6).required(), 7 | }); 8 | -------------------------------------------------------------------------------- /src/adoption/sender.ts: -------------------------------------------------------------------------------- 1 | import createHttpError from 'http-errors'; 2 | import { getProbeByIp as serverGetProbeByIp, getWsServer, PROBES_NAMESPACE, type WsServer } from '../lib/ws/server.js'; 3 | import type { AdoptionCodeRequest } from './types.js'; 4 | import type { Probe } from '../probe/types.js'; 5 | 6 | export class CodeSender { 7 | constructor ( 8 | private readonly io: WsServer, 9 | private readonly getProbeByIp: typeof serverGetProbeByIp, 10 | ) {} 11 | 12 | async sendCode (request: AdoptionCodeRequest) { 13 | const probe = await this.getProbeByIp(request.ip); 14 | 15 | if (!probe) { 16 | throw createHttpError(422, 'No matching probes found.', { type: 'no_probes_found' }); 17 | } 18 | 19 | this.sendToProbe(probe, request.code); 20 | 21 | return probe; 22 | } 23 | 24 | private sendToProbe (probe: Probe, code: string) { 25 | this.io.of(PROBES_NAMESPACE).to(probe.client).emit('probe:adoption:code', { 26 | code, 27 | }); 28 | } 29 | } 30 | 31 | export const codeSender = new CodeSender(getWsServer(), serverGetProbeByIp); 32 | -------------------------------------------------------------------------------- /src/adoption/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Adoption code Objects 3 | */ 4 | 5 | export type AdoptionCodeRequest = { 6 | ip: string; 7 | code: string; 8 | }; 9 | -------------------------------------------------------------------------------- /src/alternative-ip/route/alternative-ip.ts: -------------------------------------------------------------------------------- 1 | import type { Context } from 'koa'; 2 | import type Router from '@koa/router'; 3 | import requestIp from 'request-ip'; 4 | import createHttpError from 'http-errors'; 5 | import type { AlternativeIpRequest } from '../types.js'; 6 | import { bodyParser } from '../../lib/http/middleware/body-parser.js'; 7 | import { validate } from '../../lib/http/middleware/validate.js'; 8 | import { schema } from '../schema.js'; 9 | import { getAltIpsClient } from '../../lib/alt-ips.js'; 10 | 11 | const handle = async (ctx: Context): Promise => { 12 | const request = ctx.request.body as AlternativeIpRequest; 13 | 14 | const ip = requestIp.getClientIp(ctx.request); 15 | 16 | if (!ip) { 17 | throw createHttpError(400, 'Unable to get requester ip.', { type: 'no_ip' }); 18 | } 19 | 20 | await getAltIpsClient().validateTokenFromHttp({ 21 | socketId: request.socketId, 22 | token: request.token, 23 | ip, 24 | }); 25 | 26 | ctx.body = { 27 | ip, 28 | }; 29 | }; 30 | 31 | export const registerAlternativeIpRoute = (router: Router): void => { 32 | router.post('/alternative-ip', '/alternative-ip', bodyParser(), validate(schema), handle); 33 | }; 34 | -------------------------------------------------------------------------------- /src/alternative-ip/schema.ts: -------------------------------------------------------------------------------- 1 | import Joi from 'joi'; 2 | 3 | export const schema = Joi.object({ 4 | socketId: Joi.string().length(20).required(), 5 | token: Joi.string().length(32).required(), 6 | }); 7 | -------------------------------------------------------------------------------- /src/alternative-ip/types.ts: -------------------------------------------------------------------------------- 1 | export type AlternativeIpRequest = { 2 | socketId: string; 3 | token: string; 4 | }; 5 | -------------------------------------------------------------------------------- /src/health/route/get.ts: -------------------------------------------------------------------------------- 1 | import type { DefaultContext, DefaultState, ParameterizedContext } from 'koa'; 2 | import type Router from '@koa/router'; 3 | 4 | import termListener from '../term-listener.js'; 5 | 6 | const handle = (ctx: ParameterizedContext): void => { 7 | const isTerminating = termListener.getIsTerminating(); 8 | ctx.body = isTerminating ? 'Received SIGTERM, shutting down' : 'Alive'; 9 | }; 10 | 11 | export const registerHealthRoute = (router: Router): void => { 12 | router.get('/health', '/health', handle); 13 | }; 14 | -------------------------------------------------------------------------------- /src/health/term-listener.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import process from 'node:process'; 3 | import { scopedLogger } from '../lib/logger.js'; 4 | 5 | const logger = scopedLogger('sigterm-listener'); 6 | 7 | class TermListener { 8 | private isTerminating: boolean; 9 | 10 | constructor () { 11 | this.isTerminating = false; 12 | const sigtermDelay = config.get('sigtermDelay'); 13 | sigtermDelay && this.attachListener(sigtermDelay); 14 | } 15 | 16 | public getIsTerminating () { 17 | return this.isTerminating; 18 | } 19 | 20 | private attachListener (delay: number) { 21 | process.on('SIGTERM', (signal) => { 22 | logger.info(`Process ${process.pid} received a ${signal} signal`); 23 | this.isTerminating = true; 24 | 25 | setTimeout(() => { 26 | logger.info('Exiting'); 27 | process.exit(0); 28 | }, delay); 29 | }); 30 | } 31 | } 32 | 33 | const termListener = new TermListener(); 34 | 35 | export default termListener; 36 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import process from 'node:process'; 3 | import cluster from 'node:cluster'; 4 | import { scopedLogger } from './lib/logger.js'; 5 | import { createServer } from './lib/server.js'; 6 | import { initRedisClient } from './lib/redis/client.js'; 7 | import { initPersistentRedisClient } from './lib/redis/persistent-client.js'; 8 | import { flushRedisCache } from './lib/flush-redis-cache.js'; 9 | 10 | const logger = scopedLogger('index'); 11 | const port = process.env['PORT'] ?? config.get('server.port'); 12 | const workerCount = config.get('server.processes'); 13 | 14 | const workerFn = async () => { 15 | const server = await createServer(); 16 | 17 | server.listen(port, () => { 18 | logger.info(`Application started at http://localhost:${port}`); 19 | }); 20 | }; 21 | 22 | if (cluster.isPrimary) { 23 | logger.info(`Master ${process.pid} is running with ${workerCount} workers.`); 24 | const redis = await initRedisClient(); 25 | const persistentRedis = await initPersistentRedisClient(); 26 | await flushRedisCache(); 27 | await redis.disconnect(); 28 | await persistentRedis.disconnect(); 29 | let syncAdoptionsPid: number | null = null; 30 | 31 | for (let i = 0; i < workerCount; i++) { 32 | if (!syncAdoptionsPid) { 33 | const worker = cluster.fork({ SHOULD_SYNC_ADOPTIONS: true }); 34 | logger.info(`Syncing adoptions on worker ${worker.process.pid!}.`); 35 | syncAdoptionsPid = worker.process.pid!; 36 | } else { 37 | cluster.fork(); 38 | } 39 | } 40 | 41 | cluster.on('exit', (worker, code, signal) => { 42 | logger.error(`Worker ${worker.process.pid!} died with code ${code} and signal ${signal}.`); 43 | 44 | if (process.env['TEST_DONT_RESTART_WORKERS']) { 45 | return; 46 | } 47 | 48 | if (worker.process.pid === syncAdoptionsPid) { 49 | const worker = cluster.fork({ SHOULD_SYNC_ADOPTIONS: true }); 50 | logger.info(`Syncing adoptions on worker ${worker.process.pid!}.`); 51 | syncAdoptionsPid = worker.process.pid!; 52 | } else { 53 | cluster.fork(); 54 | } 55 | }); 56 | } else { 57 | logger.info(`Worker ${process.pid} is running.`); 58 | 59 | workerFn().catch((error) => { 60 | logger.error('Failed to start cluster:', error); 61 | setTimeout(() => process.exit(1), 5000); 62 | }); 63 | } 64 | -------------------------------------------------------------------------------- /src/lib/blocked-ip-ranges.ts: -------------------------------------------------------------------------------- 1 | import { writeFile, readFile } from 'node:fs/promises'; 2 | import path from 'node:path'; 3 | import got from 'got'; 4 | import ipaddr from 'ipaddr.js'; 5 | 6 | type ParsedIpRange = [ipaddr.IPv4 | ipaddr.IPv6, number]; 7 | 8 | type Source = { 9 | url: string; 10 | file: string; 11 | }; 12 | 13 | export let blockedRangesIPv4 = new Set(); 14 | let blockedRangesIPv6 = new Set(); 15 | let ipsCache = new Map(); 16 | 17 | export const sources: Record<'appleRelay', Source> = { 18 | appleRelay: { 19 | url: 'https://mask-api.icloud.com/egress-ip-ranges.csv', 20 | file: 'data/APPLE_RELAY_IP_RANGES.csv', 21 | }, 22 | }; 23 | 24 | const query = async (url: string): Promise => { 25 | const result = await got(url, { 26 | timeout: { request: 10000 }, 27 | }).text(); 28 | 29 | return result; 30 | }; 31 | 32 | const populateAppleRelayList = async (newBlockedRangesIPv4: Set, newBlockedRangesIPv6: Set) => { 33 | const appleRelaySource = sources.appleRelay; 34 | const filePath = path.join(path.resolve(), appleRelaySource.file); 35 | const csv = await readFile(filePath, 'utf8'); 36 | 37 | csv.split('\n').forEach((line) => { 38 | const [ range ] = line.split(','); 39 | 40 | if (!range) { 41 | return; 42 | } 43 | 44 | const parsedRange = ipaddr.parseCIDR(range); 45 | 46 | if (parsedRange[0].kind() === 'ipv4') { 47 | newBlockedRangesIPv4.add(parsedRange); 48 | } else if (parsedRange[0].kind() === 'ipv6') { 49 | newBlockedRangesIPv6.add(parsedRange); 50 | } 51 | }); 52 | }; 53 | 54 | export const populateMemList = async (): Promise => { 55 | const newBlockedRangesIPv4 = new Set(); 56 | const newBlockedRangesIPv6 = new Set(); 57 | 58 | await Promise.all([ 59 | populateAppleRelayList(newBlockedRangesIPv4, newBlockedRangesIPv6), 60 | ]); 61 | 62 | blockedRangesIPv4 = newBlockedRangesIPv4; 63 | blockedRangesIPv6 = newBlockedRangesIPv6; 64 | ipsCache = new Map(); 65 | }; 66 | 67 | export const updateBlockedIpRangesFiles = async (): Promise => { 68 | await Promise.all(Object.values(sources).map(async (source) => { 69 | const response = await query(source.url); 70 | const filePath = path.join(path.resolve(), source.file); 71 | await writeFile(filePath, response, 'utf8'); 72 | })); 73 | }; 74 | 75 | export const isIpBlocked = (ip: string) => { 76 | const cached = ipsCache.get(ip); 77 | 78 | if (cached !== undefined) { 79 | return cached; 80 | } 81 | 82 | const parsedIp = ipaddr.process(ip); 83 | 84 | if (parsedIp.kind() === 'ipv4') { 85 | for (const ipRange of blockedRangesIPv4) { 86 | if (parsedIp.match(ipRange)) { 87 | ipsCache.set(ip, true); 88 | return true; 89 | } 90 | } 91 | } else if (parsedIp.kind() === 'ipv6') { 92 | for (const ipRange of blockedRangesIPv6) { 93 | if (parsedIp.match(ipRange)) { 94 | ipsCache.set(ip, true); 95 | return true; 96 | } 97 | } 98 | } 99 | 100 | ipsCache.set(ip, false); 101 | return false; 102 | }; 103 | -------------------------------------------------------------------------------- /src/lib/cache/cache-interface.ts: -------------------------------------------------------------------------------- 1 | export type CacheInterface = { 2 | set(key: string, value: T, ttl?: number): Promise; 3 | get(key: string): Promise; 4 | delete(key: string): Promise; 5 | }; 6 | -------------------------------------------------------------------------------- /src/lib/cache/null-cache.ts: -------------------------------------------------------------------------------- 1 | import type { CacheInterface } from './cache-interface.js'; 2 | 3 | export default class NullCache implements CacheInterface { 4 | async delete (): Promise { 5 | return null; 6 | } 7 | 8 | async get (): Promise { 9 | return null; 10 | } 11 | 12 | async set (): Promise {} 13 | } 14 | -------------------------------------------------------------------------------- /src/lib/cache/redis-cache.ts: -------------------------------------------------------------------------------- 1 | import { scopedLogger } from '../logger.js'; 2 | import type { RedisClient } from '../redis/client.js'; 3 | import type { CacheInterface } from './cache-interface.js'; 4 | 5 | const logger = scopedLogger('redis-cache'); 6 | 7 | export default class RedisCache implements CacheInterface { 8 | constructor (private readonly redis: RedisClient) {} 9 | 10 | async set (key: string, value: unknown, ttl: number = 0): Promise { 11 | try { 12 | await this.redis.set(this.buildCacheKey(key), JSON.stringify(value), { PX: ttl }); 13 | } catch (error) { 14 | logger.error('Failed to set redis cache value.', error, { key, ttl }); 15 | } 16 | } 17 | 18 | async get (key: string): Promise { 19 | try { 20 | const raw = await this.redis.get(this.buildCacheKey(key)); 21 | 22 | if (!raw) { 23 | return null; 24 | } 25 | 26 | return JSON.parse(raw) as T; 27 | } catch (error) { 28 | logger.error('Failed to get redis cache value.', error, { key }); 29 | return null; 30 | } 31 | } 32 | 33 | async delete (key: string): Promise { 34 | try { 35 | const raw = await this.redis.get(this.buildCacheKey(key)); 36 | 37 | if (!raw) { 38 | return null; 39 | } 40 | 41 | await this.redis.del(this.buildCacheKey(key)); 42 | 43 | return JSON.parse(raw) as T; 44 | } catch (error) { 45 | logger.error('Failed to del redis cache value.', error, { key }); 46 | return null; 47 | } 48 | } 49 | 50 | private buildCacheKey (key: string): string { 51 | return `gp:cache:${key}`; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/lib/credits.ts: -------------------------------------------------------------------------------- 1 | import type { Knex } from 'knex'; 2 | import { client } from './sql/client.js'; 3 | 4 | export const CREDITS_TABLE = 'gp_credits'; 5 | const ER_CONSTRAINT_FAILED_CODE = 4025; 6 | 7 | export class Credits { 8 | constructor (private readonly sql: Knex) {} 9 | 10 | async consume (userId: string, credits: number): Promise<{ isConsumed: boolean; remainingCredits: number }> { 11 | let numberOfUpdates = null; 12 | 13 | try { 14 | numberOfUpdates = await this.sql(CREDITS_TABLE).where({ user_id: userId }).update({ amount: this.sql.raw('amount - ?', [ credits ]) }); 15 | } catch (error) { 16 | if (error && (error as Error & { errno?: number }).errno === ER_CONSTRAINT_FAILED_CODE) { 17 | const remainingCredits = await this.getRemainingCredits(userId); 18 | return { isConsumed: false, remainingCredits }; 19 | } 20 | 21 | throw error; 22 | } 23 | 24 | if (numberOfUpdates === 0) { 25 | return { isConsumed: false, remainingCredits: 0 }; 26 | } 27 | 28 | const remainingCredits = await this.getRemainingCredits(userId); 29 | return { isConsumed: true, remainingCredits }; 30 | } 31 | 32 | async getRemainingCredits (userId: string): Promise { 33 | const result = await this.sql(CREDITS_TABLE).where({ user_id: userId }).first<{ amount: number } | undefined>('amount'); 34 | return result?.amount || 0; 35 | } 36 | } 37 | 38 | export const credits = new Credits(client); 39 | -------------------------------------------------------------------------------- /src/lib/download-files.ts: -------------------------------------------------------------------------------- 1 | import { scopedLogger } from './logger.js'; 2 | import { updateMalwareFiles } from './malware/client.js'; 3 | import { updateIpRangeFiles } from './cloud-ip-ranges.js'; 4 | import { updateBlockedIpRangesFiles } from './blocked-ip-ranges.js'; 5 | import { updateGeonamesCitiesFile } from './geoip/city-approximation.js'; 6 | 7 | const logger = scopedLogger('download-files'); 8 | 9 | logger.info('Updating malware blacklist JSON files.'); 10 | await updateMalwareFiles(); 11 | logger.info('Updating cloud ip ranges JSON files.'); 12 | await updateIpRangeFiles(); 13 | logger.info('Updating blocked ip ranges files.'); 14 | await updateBlockedIpRangesFiles(); 15 | logger.info('Updating geonames cities CSV file.'); 16 | await updateGeonamesCitiesFile(); 17 | logger.info('Update complete.'); 18 | -------------------------------------------------------------------------------- /src/lib/flush-redis-cache.ts: -------------------------------------------------------------------------------- 1 | import { readFile } from 'node:fs/promises'; 2 | import path from 'node:path'; 3 | import { scopedLogger } from './logger.js'; 4 | import { getRedisClient } from './redis/client.js'; 5 | import { getPersistentRedisClient } from './redis/persistent-client.js'; 6 | 7 | const logger = scopedLogger('flush-redis-cache'); 8 | 9 | export async function flushRedisCache () { 10 | if (process.env['NODE_ENV'] === 'production' && !process.env['HOSTNAME']) { 11 | throw new Error('HOSTNAME env variable is not specified'); 12 | } 13 | 14 | const redis = getRedisClient(); 15 | const persistentRedis = getPersistentRedisClient(); 16 | const hostname = process.env['HOSTNAME'] || process.env['NODE_ENV']; 17 | const filePath = path.join(path.resolve(), 'data/LAST_API_COMMIT_HASH.txt'); 18 | 19 | const lastCommitHashInRedis = await persistentRedis.get(`LAST_API_COMMIT_HASH_${hostname}`); 20 | const currentLastCommitHash = (await readFile(filePath, 'utf8').catch(() => '')).trim(); 21 | 22 | if (process.env['NODE_ENV'] === 'production' && !currentLastCommitHash) { 23 | throw new Error(`Current commit hash missing in ${filePath}`); 24 | } 25 | 26 | if (!currentLastCommitHash || lastCommitHashInRedis !== currentLastCommitHash) { 27 | logger.info('Latest commit hash changed. Clearing redis cache.'); 28 | await redis.flushDb(); 29 | await persistentRedis.set(`LAST_API_COMMIT_HASH_${hostname}`, currentLastCommitHash); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/lib/geoip/altnames.ts: -------------------------------------------------------------------------------- 1 | export const cities: Record = { 2 | 'Geneve': 'Geneva', 3 | 'Frankfurt am Main': 'Frankfurt', 4 | 'New York City': 'New York', 5 | 'Santiago de Queretaro': 'Queretaro', 6 | 'Nurnberg': 'Nuremberg', 7 | }; 8 | -------------------------------------------------------------------------------- /src/lib/geoip/dc-cities.ts: -------------------------------------------------------------------------------- 1 | 2 | import dcCitiesJson from './dc-cities.json' with { type: 'json' }; 3 | import { normalizeCityName } from './utils.js'; 4 | 5 | const dcCountries = new Map(dcCitiesJson.map(({ country, cities }) => { 6 | return [ country, new Set(cities.map(city => normalizeCityName(city))) ]; 7 | })); 8 | 9 | export const getIsDcCity = (city: string, country?: string) => { 10 | if (!country || !city) { 11 | return false; 12 | } 13 | 14 | const cities = dcCountries.get(country); 15 | 16 | if (!cities) { 17 | return false; 18 | } 19 | 20 | return cities.has(normalizeCityName(city)); 21 | }; 22 | -------------------------------------------------------------------------------- /src/lib/geoip/fake-client.ts: -------------------------------------------------------------------------------- 1 | import type { LocationInfo } from './client.js'; 2 | 3 | export const fakeLookup = (): LocationInfo => { 4 | return { 5 | continent: 'SA', 6 | country: 'AR', 7 | state: null, 8 | city: 'Buenos Aires', 9 | region: 'South America', 10 | normalizedCity: 'buenos aires', 11 | asn: 61003, 12 | latitude: -34.61, 13 | longitude: -58.38, 14 | network: 'InterBS S.R.L. (BAEHOST)', 15 | normalizedNetwork: 'interbs s.r.l. (baehost)', 16 | isProxy: false, 17 | isHosting: null, 18 | isAnycast: false, 19 | allowedCountries: [ 'AR' ], 20 | }; 21 | }; 22 | -------------------------------------------------------------------------------- /src/lib/geoip/overrides.ts: -------------------------------------------------------------------------------- 1 | export const isHostingOverrides = [ 2 | // https://baxetgroup.com/ 3 | { 4 | normalizedNetwork: /baxet group/, 5 | isHosting: true, 6 | }, 7 | // https://www.psychz.net/ 8 | { 9 | normalizedNetwork: /psychz networks/, 10 | isHosting: true, 11 | }, 12 | ]; 13 | -------------------------------------------------------------------------------- /src/lib/geoip/providers/fastly.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { getRegionByCountry } from '../../location/location.js'; 3 | import { getCity } from '../city-approximation.js'; 4 | import type { ProviderLocationInfo } from '../client.js'; 5 | import { 6 | normalizeCityName, 7 | normalizeCityNamePublic, 8 | normalizeNetworkName, 9 | } from '../utils.js'; 10 | 11 | type FastlyResponse = { 12 | 'as': { 13 | name: string; 14 | number: number; 15 | }; 16 | 'client': { 17 | proxy_desc: string; 18 | proxy_type: string; 19 | }; 20 | 'geo-digitalelement': { 21 | continent_code: string; 22 | country_code: string; 23 | city: string; 24 | region: string; 25 | latitude: number; 26 | longitude: number; 27 | network: string; 28 | }; 29 | }; 30 | 31 | export const fastlyLookup = async (addr: string): Promise => { 32 | const result = await got(`https://globalping-geoip.global.ssl.fastly.net/${addr}`, { 33 | timeout: { request: 5000 }, 34 | }).json(); 35 | 36 | const data = result['geo-digitalelement']; 37 | const originalCity = data.city.replace(/^(private|reserved)/, ''); 38 | const originalState = data.country_code === 'US' ? data.region : null; 39 | const { city, state } = await getCity({ city: originalCity, state: originalState }, data.country_code, Number(data.latitude), Number(data.longitude)); 40 | 41 | return { 42 | provider: 'fastly', 43 | continent: data.continent_code, 44 | region: getRegionByCountry(data.country_code), 45 | country: data.country_code, 46 | state, 47 | city: normalizeCityNamePublic(city), 48 | normalizedCity: normalizeCityName(city), 49 | asn: result.as.number, 50 | latitude: data.latitude, 51 | longitude: data.longitude, 52 | network: result.as.name, 53 | normalizedNetwork: normalizeNetworkName(result.as.name), 54 | isProxy: null, 55 | isHosting: null, 56 | isAnycast: null, 57 | }; 58 | }; 59 | -------------------------------------------------------------------------------- /src/lib/geoip/providers/ip2location.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import config from 'config'; 3 | import { 4 | getContinentByCountry, 5 | getRegionByCountry, 6 | getStateIsoByName, 7 | } from '../../location/location.js'; 8 | import type { ProviderLocationInfo } from '../client.js'; 9 | import { 10 | normalizeCityName, 11 | normalizeCityNamePublic, 12 | normalizeNetworkName, 13 | } from '../utils.js'; 14 | import { getCity } from '../city-approximation.js'; 15 | 16 | type Ip2LocationResponse = { 17 | ip?: string; 18 | country_code?: string; 19 | country_name?: string; 20 | region_name?: string; 21 | city_name?: string; 22 | zip_code?: string; 23 | time_zone?: string; 24 | asn?: string; 25 | latitude?: number; 26 | longitude?: number; 27 | as?: string; 28 | is_proxy?: boolean; 29 | usage_type?: string; 30 | }; 31 | 32 | // https://blog.ip2location.com/knowledge-base/what-is-usage-type/ 33 | const HOSTING_USAGE_TYPES = [ 'CDN', 'DCH' ]; 34 | 35 | export const ip2LocationLookup = async (addr: string): Promise => { 36 | const result = await got(`https://api.ip2location.io`, { 37 | searchParams: { 38 | key: config.get('ip2location.apiKey'), 39 | ip: addr, 40 | }, 41 | timeout: { request: 5000 }, 42 | }).json(); 43 | 44 | const originalCity = result.city_name || ''; 45 | const originalState = result.country_code === 'US' && result.region_name ? getStateIsoByName(result.region_name) : null; 46 | const { city, state } = await getCity({ city: originalCity, state: originalState }, result.country_code, result.latitude, result.longitude); 47 | 48 | return { 49 | provider: 'ip2location', 50 | continent: result.country_code ? getContinentByCountry(result.country_code) : '', 51 | region: result.country_code ? getRegionByCountry(result.country_code) : '', 52 | state, 53 | country: result.country_code ?? '', 54 | city: normalizeCityNamePublic(city), 55 | normalizedCity: normalizeCityName(city), 56 | asn: Number(result.asn ?? 0), 57 | latitude: result.latitude ?? 0, 58 | longitude: result.longitude ?? 0, 59 | network: result.as ?? '', 60 | normalizedNetwork: normalizeNetworkName(result.as ?? ''), 61 | isProxy: result.is_proxy ?? false, 62 | isHosting: result.usage_type ? HOSTING_USAGE_TYPES.includes(result.usage_type) : null, 63 | isAnycast: null, 64 | }; 65 | }; 66 | -------------------------------------------------------------------------------- /src/lib/geoip/providers/ipinfo.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import config from 'config'; 3 | import { getContinentByCountry, getRegionByCountry, getStateIsoByName } from '../../location/location.js'; 4 | import type { ProviderLocationInfo } from '../client.js'; 5 | import { 6 | normalizeCityName, 7 | normalizeCityNamePublic, 8 | normalizeNetworkName, 9 | } from '../utils.js'; 10 | import { getCity } from '../city-approximation.js'; 11 | 12 | type IpinfoResponse = { 13 | country: string | undefined; 14 | city: string | undefined; 15 | region: string | undefined; 16 | org: string | undefined; 17 | loc: string | undefined; 18 | privacy?: { 19 | hosting: boolean; 20 | }; 21 | anycast?: boolean; 22 | }; 23 | 24 | export const ipinfoLookup = async (addr: string): Promise => { 25 | const result = await got(`https://ipinfo.io/${addr}`, { 26 | username: config.get('ipinfo.apiKey'), 27 | timeout: { request: 5000 }, 28 | }).json(); 29 | 30 | const [ lat, lon ] = (result.loc ?? ',').split(','); 31 | const match = /^AS(\d+)/.exec(result.org ?? ''); 32 | const parsedAsn = match?.[1] ? Number(match[1]) : null; 33 | const network = (result.org ?? '').split(' ').slice(1).join(' '); 34 | 35 | const originalCity = result.city || ''; 36 | const originalState = result.country === 'US' && result.region ? getStateIsoByName(result.region) : null; 37 | const { city, state } = await getCity({ city: originalCity, state: originalState }, result.country, Number(lat), Number(lon)); 38 | 39 | return { 40 | provider: 'ipinfo', 41 | continent: result.country ? getContinentByCountry(result.country) : '', 42 | region: result.country ? getRegionByCountry(result.country) : '', 43 | state, 44 | country: result.country ?? '', 45 | city: normalizeCityNamePublic(city), 46 | normalizedCity: normalizeCityName(city), 47 | asn: Number(parsedAsn), 48 | latitude: Number(lat), 49 | longitude: Number(lon), 50 | network, 51 | normalizedNetwork: normalizeNetworkName(network), 52 | isProxy: null, 53 | isHosting: result.privacy?.hosting ?? null, 54 | isAnycast: result.anycast ?? null, 55 | }; 56 | }; 57 | -------------------------------------------------------------------------------- /src/lib/geoip/providers/ipmap.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { getContinentByCountry, getRegionByCountry } from '../../location/location.js'; 3 | import type { ProviderLocationInfo } from '../client.js'; 4 | import { 5 | normalizeCityName, 6 | normalizeCityNamePublic, 7 | } from '../utils.js'; 8 | import { getCity } from '../city-approximation.js'; 9 | 10 | export type IpmapResponse = { 11 | locations: { 12 | cityName?: string; 13 | stateAnsiCode?: string; 14 | countryCodeAlpha2?: string; 15 | latitude?: string; 16 | longitude?: string; 17 | }[]; 18 | }; 19 | 20 | export const ipmapLookup = async (addr: string): Promise => { 21 | const result = await got.get(`https://ipmap-api.ripe.net/v1/locate/${addr}`).json(); 22 | const location = result?.locations?.[0] || {}; 23 | 24 | const originalCity = location.cityName || ''; 25 | const originalState = location.countryCodeAlpha2 === 'US' && location.stateAnsiCode ? location.stateAnsiCode : null; 26 | const { city, state } = await getCity({ city: originalCity, state: originalState }, location.countryCodeAlpha2, Number(location.latitude), Number(location.longitude)); 27 | 28 | return { 29 | provider: 'ipmap', 30 | continent: location.countryCodeAlpha2 ? getContinentByCountry(location.countryCodeAlpha2) : '', 31 | region: location.countryCodeAlpha2 ? getRegionByCountry(location.countryCodeAlpha2) : '', 32 | state, 33 | country: location.countryCodeAlpha2 ?? '', 34 | city: normalizeCityNamePublic(city), 35 | normalizedCity: normalizeCityName(city), 36 | asn: 0, 37 | latitude: Number(location.latitude ?? 0), 38 | longitude: Number(location.longitude ?? 0), 39 | network: '', 40 | normalizedNetwork: '', 41 | isProxy: null, 42 | isHosting: null, 43 | isAnycast: null, 44 | }; 45 | }; 46 | -------------------------------------------------------------------------------- /src/lib/geoip/providers/maxmind.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import { type City, WebServiceClient } from '@maxmind/geoip2-node'; 3 | import type { WebServiceClientError } from '@maxmind/geoip2-node/dist/src/types.js'; 4 | import type { ProviderLocationInfo } from '../client.js'; 5 | import { 6 | normalizeCityName, 7 | normalizeCityNamePublic, 8 | normalizeNetworkName, 9 | } from '../utils.js'; 10 | import { getCity } from '../city-approximation.js'; 11 | import { getRegionByCountry } from '../../location/location.js'; 12 | import { scopedLogger } from '../../logger.js'; 13 | 14 | const logger = scopedLogger('geoip:maxmind'); 15 | const client = new WebServiceClient(config.get('maxmind.accountId'), config.get('maxmind.licenseKey')); 16 | 17 | export const isMaxmindError = (error: unknown): error is WebServiceClientError => error as WebServiceClientError['code'] !== undefined; 18 | 19 | const query = async (addr: string, retryCounter = 0): Promise => { 20 | try { 21 | const city = await client.city(addr); 22 | return city; 23 | } catch (error: unknown) { 24 | if (isMaxmindError(error)) { 25 | if ([ 'SERVER_ERROR', 'HTTP_STATUS_CODE_ERROR', 'INVALID_RESPONSE_BODY', 'FETCH_ERROR' ].includes(error.code) && retryCounter < 3) { 26 | return query(addr, retryCounter + 1); 27 | } 28 | 29 | if (error.code === 'ACCOUNT_ID_REQUIRED') { 30 | logger.error('Maxmind query error', new Error(error.error)); 31 | } 32 | } 33 | 34 | throw error; 35 | } 36 | }; 37 | 38 | export const maxmindLookup = async (addr: string): Promise => { 39 | const data = await query(addr); 40 | const originalCity = data.city?.names?.en || ''; 41 | const originalState = data.country?.isoCode === 'US' ? data.subdivisions?.map(s => s.isoCode)[0] ?? '' : null; 42 | const { city, state } = await getCity({ city: originalCity, state: originalState }, data.country?.isoCode, data.location?.latitude, data.location?.longitude); 43 | 44 | return { 45 | provider: 'maxmind', 46 | continent: data.continent?.code ?? '', 47 | region: data.country?.isoCode ? getRegionByCountry(data.country?.isoCode) : '', 48 | country: data.country?.isoCode ?? '', 49 | state, 50 | city: normalizeCityNamePublic(city), 51 | normalizedCity: normalizeCityName(city), 52 | asn: data.traits?.autonomousSystemNumber ?? 0, 53 | latitude: data.location?.latitude ?? 0, 54 | longitude: data.location?.longitude ?? 0, 55 | network: data.traits?.isp ?? '', 56 | normalizedNetwork: normalizeNetworkName(data.traits?.isp ?? ''), 57 | isProxy: null, 58 | isHosting: null, 59 | isAnycast: data.traits?.isAnycast ?? null, 60 | }; 61 | }; 62 | -------------------------------------------------------------------------------- /src/lib/geoip/utils.ts: -------------------------------------------------------------------------------- 1 | import anyAscii from 'any-ascii'; 2 | import { cities } from './altnames.js'; 3 | 4 | export const normalizeCityNamePublic = (name: string): string => { 5 | // We don't add city to the regex as there are valid names like 'Mexico City' or 'Kansas City' 6 | const asciiName = anyAscii(name).replace(/(?:\s+|^)the(?:\s+|$)/gi, ''); 7 | return cities[asciiName] ?? asciiName; 8 | }; 9 | 10 | export const normalizeCityName = (name: string): string => normalizeCityNamePublic(name).toLowerCase(); 11 | 12 | export const normalizeFromPublicName = (name: string): string => name.toLowerCase(); 13 | 14 | export const normalizeNetworkName = (name: string): string => name.toLowerCase(); 15 | 16 | export const normalizeCoordinate = (coordinate: number) => Math.round(coordinate * 100) / 100; 17 | -------------------------------------------------------------------------------- /src/lib/geoip/whitelist.ts: -------------------------------------------------------------------------------- 1 | import ipaddr from 'ipaddr.js'; 2 | import { readFile } from 'node:fs/promises'; 3 | import path from 'node:path'; 4 | 5 | const WHITELIST_FILE_PATH = 'config/whitelist-ips.txt'; 6 | 7 | let isInitialized = false; 8 | const whitelistIps = new Set(); 9 | let whitelistRanges: ReturnType[] = []; 10 | const whitelistRangesCache = new Set(); 11 | 12 | export const populateMemList = async () => { 13 | const listFilePath = path.join(path.resolve(), WHITELIST_FILE_PATH); 14 | const file = await readFile(listFilePath, 'utf8'); 15 | const whitelist = file.split(/\r?\n/).map(item => item.trim()).filter(item => item.length > 0); 16 | whitelistIps.clear(); 17 | whitelistRanges = []; 18 | whitelistRangesCache.clear(); 19 | 20 | for (const item of whitelist) { 21 | if (ipaddr.isValid(item)) { 22 | whitelistIps.add(item); 23 | } else if (ipaddr.isValidCIDR(item)) { 24 | const range = ipaddr.parseCIDR(item); 25 | whitelistRanges.push(range); 26 | } 27 | } 28 | 29 | isInitialized = true; 30 | }; 31 | 32 | export const isAddrWhitelisted = (addr: string) => { 33 | if (!isInitialized) { 34 | throw new Error('Whitelist ips are not initialized'); 35 | } 36 | 37 | if (whitelistIps.has(addr)) { 38 | return true; 39 | } 40 | 41 | if (isInWhitelistRanges(addr)) { 42 | return true; 43 | } 44 | 45 | return false; 46 | }; 47 | 48 | const isInWhitelistRanges = (addr: string) => { 49 | if (whitelistRangesCache.has(addr)) { 50 | return true; 51 | } 52 | 53 | const ip = ipaddr.parse(addr); 54 | const isInRanges = whitelistRanges.some(range => ip.kind() === range[0].kind() && ip.match(range)); 55 | 56 | if (isInRanges) { 57 | whitelistRangesCache.add(addr); 58 | return true; 59 | } 60 | 61 | return false; 62 | }; 63 | -------------------------------------------------------------------------------- /src/lib/get-probe-ip.ts: -------------------------------------------------------------------------------- 1 | import requestIp from 'request-ip'; 2 | import type { Socket } from 'socket.io'; 3 | 4 | const getProbeIp = (socket: Socket) => { 5 | if (process.env['TEST_MODE'] === 'unit') { 6 | return '1.2.3.4'; 7 | } 8 | 9 | if (process.env['TEST_MODE'] === 'e2e') { 10 | return '1.2.3.4'; // Uses a fake lookup. 11 | } 12 | 13 | // Use random ip assigned by the API 14 | if (process.env['FAKE_PROBE_IP']) { 15 | // Use fake ip provided by the probe if exists 16 | if (socket.handshake.query['fakeIp']) { 17 | return socket.handshake.query['fakeIp'] as string; 18 | } 19 | 20 | const samples = [ 21 | '131.255.7.26', // Buenos Aires 22 | '213.136.174.80', // Naples 23 | '95.155.94.127', // Krakow 24 | '18.200.0.1', // Dublin, AWS 25 | '34.140.0.10', // Brussels, GCP 26 | '65.49.22.66', 27 | '185.229.226.83', 28 | '94.214.253.78', 29 | '79.205.97.254', 30 | '2a04:4e42:200::485', // San Francisco 31 | ]; 32 | // Choosing ip based on the probe uuid to always return the same ip for the same probe. 33 | const lastGroup = (socket.handshake.query['uuid'] as string).split('-').pop() || '0'; 34 | const index = parseInt(lastGroup, 16) % samples.length; 35 | return samples[index]; 36 | } 37 | 38 | const clientIp = requestIp.getClientIp(socket.request); 39 | 40 | if (!clientIp) { 41 | return null; 42 | } 43 | 44 | const hasEmptyIpv6Prefix = clientIp.startsWith('::ffff:'); 45 | return hasEmptyIpv6Prefix ? clientIp.slice(7) : clientIp; 46 | }; 47 | 48 | export default getProbeIp; 49 | -------------------------------------------------------------------------------- /src/lib/http/error-handler.ts: -------------------------------------------------------------------------------- 1 | import { Context } from 'koa'; 2 | import { scopedLogger } from '../logger.js'; 3 | 4 | const logger = scopedLogger('error-handler-http'); 5 | 6 | export const errorHandler = (error: Error & { code?: string }, ctx: Context) => { 7 | const ignore = [ 'ECONNABORTED', 'ECONNRESET', 'EPIPE' ]; 8 | 9 | if (error?.code && ignore.includes(error.code)) { 10 | return; 11 | } 12 | 13 | logger.error('Koa server error:', error, { ctx }); 14 | }; 15 | -------------------------------------------------------------------------------- /src/lib/http/middleware/authenticate.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import { jwtVerify } from 'jose'; 3 | import apmAgent from 'elastic-apm-node'; 4 | 5 | import { auth } from '../auth.js'; 6 | import type { ExtendedMiddleware } from '../../../types.js'; 7 | 8 | const sessionConfig = config.get('server.session'); 9 | 10 | type SessionCookiePayload = { 11 | id?: string; 12 | role?: string; 13 | app_access?: number; 14 | admin_access?: number; 15 | github_username?: string; 16 | }; 17 | 18 | export const authenticate = (): ExtendedMiddleware => { 19 | const sessionKey = Buffer.from(sessionConfig.cookieSecret); 20 | 21 | return async (ctx, next) => { 22 | const authorization = ctx.headers.authorization; 23 | const sessionCookie = ctx.cookies.get(sessionConfig.cookieName); 24 | 25 | if (authorization) { 26 | const parts = authorization.split(' '); 27 | 28 | if (parts.length !== 2 || parts[0] !== 'Bearer') { 29 | ctx.status = 401; 30 | return; 31 | } 32 | 33 | const token = parts[1]!; 34 | const origin = ctx.get('Origin'); 35 | const result = await auth.validate(token, origin); 36 | 37 | if (!result) { 38 | ctx.status = 401; 39 | return; 40 | } 41 | 42 | ctx.state.user = { id: result.userId, username: result.username, scopes: result.scopes, authMode: 'token', hashedToken: result.hashedToken }; 43 | apmAgent.setUserContext({ id: result.userId || 'anonymous-token', username: result.username || 'anonymous-token' }); 44 | } else if (sessionCookie) { 45 | try { 46 | const result = await jwtVerify(sessionCookie, sessionKey); 47 | 48 | if (result.payload.id && result.payload.app_access) { 49 | ctx.state.user = { id: result.payload.id, username: result.payload.github_username || null, authMode: 'cookie' }; 50 | apmAgent.setUserContext({ id: result.payload.id, username: result.payload.github_username || `ID(${result.payload.id})` }); 51 | } 52 | } catch {} 53 | } 54 | 55 | return next(); 56 | }; 57 | }; 58 | 59 | export type AuthenticateOptions = { session: { cookieName: string; cookieSecret: string } }; 60 | export type AuthenticateState = { user?: { id: string | null; username: string | null; scopes?: string[]; hashedToken?: string; authMode: 'cookie' | 'token' } }; 61 | -------------------------------------------------------------------------------- /src/lib/http/middleware/body-parser.ts: -------------------------------------------------------------------------------- 1 | import createHttpError from 'http-errors'; 2 | import koaBodyParser from 'koa-bodyparser'; 3 | 4 | export const bodyParser = () => koaBodyParser({ 5 | enableTypes: [ 'json' ], 6 | jsonLimit: '100kb', 7 | onerror (error) { 8 | throw createHttpError(400, error.message); 9 | }, 10 | }); 11 | -------------------------------------------------------------------------------- /src/lib/http/middleware/cors.ts: -------------------------------------------------------------------------------- 1 | import type { Context, Next } from 'koa'; 2 | import config from 'config'; 3 | 4 | const corsConfig = config.get('server.cors'); 5 | const trustedOrigins = corsConfig.trustedOrigins || []; 6 | 7 | export const corsHandler = () => async (ctx: Context, next: Next) => { 8 | ctx.set('Access-Control-Allow-Origin', '*'); 9 | ctx.set('Access-Control-Allow-Headers', '*'); 10 | ctx.set('Access-Control-Expose-Headers', '*'); 11 | ctx.set('Access-Control-Max-Age', '600'); 12 | ctx.set('Cross-Origin-Resource-Policy', 'cross-origin'); 13 | ctx.set('Timing-Allow-Origin', '*'); 14 | ctx.set('Vary', 'Accept-Encoding'); 15 | 16 | await next(); 17 | }; 18 | 19 | export const corsAuthHandler = () => { 20 | const exposeHeaders = [ 21 | 'ETag', 22 | 'Link', 23 | 'Location', 24 | 'Retry-After', 25 | 'X-RateLimit-Limit', 26 | 'X-RateLimit-Consumed', 27 | 'X-RateLimit-Remaining', 28 | 'X-RateLimit-Reset', 29 | 'X-Credits-Consumed', 30 | 'X-Credits-Remaining', 31 | 'X-Request-Cost', 32 | 'X-Response-Time', 33 | 'Deprecation', 34 | 'Sunset', 35 | ].join(', '); 36 | 37 | return async (ctx: Context, next: Next) => { 38 | const origin = ctx.get('Origin'); 39 | 40 | // Allow credentials only if the request is coming from a trusted origin. 41 | if (trustedOrigins.includes(origin)) { 42 | ctx.set('Access-Control-Allow-Origin', ctx.get('Origin')); 43 | ctx.set('Access-Control-Allow-Credentials', 'true'); 44 | ctx.set('Vary', 'Accept-Encoding, Origin'); 45 | } 46 | 47 | ctx.set('Access-Control-Allow-Headers', 'Authorization, Content-Type'); 48 | ctx.set('Access-Control-Expose-Headers', exposeHeaders); 49 | 50 | await next(); 51 | }; 52 | }; 53 | 54 | export type CorsOptions = { trustedOrigins?: string[] }; 55 | -------------------------------------------------------------------------------- /src/lib/http/middleware/default-json.ts: -------------------------------------------------------------------------------- 1 | import type { ExtendedMiddleware } from '../../../types.js'; 2 | import createHttpError from 'http-errors'; 3 | import _ from 'lodash'; 4 | 5 | export const defaultJson = (): ExtendedMiddleware => async (ctx, next) => { 6 | await next(); 7 | 8 | if (ctx.status >= 400 && !ctx.body) { 9 | const error = createHttpError(ctx.status); 10 | 11 | // Fix a bit of koa's magic: setting the body below resets the status 12 | // to 200 if it wasn't explicitly set before. 13 | // eslint-disable-next-line no-self-assign 14 | ctx.status = ctx.status; 15 | 16 | ctx.body = { 17 | error: { 18 | type: _.snakeCase(error.message), 19 | message: `${error.message}.`, 20 | }, 21 | links: { 22 | documentation: ctx.getDocsLink(), 23 | }, 24 | }; 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /src/lib/http/middleware/docs-link.ts: -------------------------------------------------------------------------------- 1 | import type Koa from 'koa'; 2 | import type Router from '@koa/router'; 3 | 4 | export const docsLink = (options: DocsLinkOptions): DocsLinkMiddleware => async (ctx, next) => { 5 | ctx.getDocsLink = (routeName = ctx._matchedRouteName!, method = ctx.method === 'HEAD' ? 'GET' : ctx.method) => { 6 | return `${options.docsHost}/docs/api.globalping.io${getDocsPath(ctx.router, routeName, method)}`; 7 | }; 8 | 9 | await next(); 10 | }; 11 | 12 | const getDocsPath = (router: Router, routeName: string | undefined, method: string) => { 13 | if (!routeName || routeName === '/') { 14 | return ''; 15 | } 16 | 17 | const route = router.route(routeName); 18 | 19 | if (typeof route !== 'object' || !route.name) { 20 | throw new Error(`Unknown route ${routeName}.`); 21 | } 22 | 23 | return `#${method.toLowerCase()}-/v1${route.name.replace(/:(\w+)/g, '-$1-')}`; 24 | }; 25 | 26 | export type DocsLinkOptions = { docsHost: string }; 27 | export type DocsLinkContext = { getDocsLink(routeName?: string, method?: string): string }; 28 | export type DocsLinkMiddleware = Router.Middleware; 29 | -------------------------------------------------------------------------------- /src/lib/http/middleware/error-handler.ts: -------------------------------------------------------------------------------- 1 | import apmAgent from 'elastic-apm-node'; 2 | import createHttpError from 'http-errors'; 3 | import { scopedLogger } from '../../logger.js'; 4 | import type { ExtendedMiddleware } from '../../../types.js'; 5 | 6 | const logger = scopedLogger('error-handler-mw'); 7 | 8 | export const errorHandlerMw: ExtendedMiddleware = async (ctx, next) => { 9 | try { 10 | await next(); 11 | } catch (error: unknown) { 12 | apmAgent.addLabels({ 13 | gpErrorType: (error as { type?: string } | undefined)?.type || 'api_error', 14 | gpErrorMessage: (error as Error | undefined)?.message, 15 | }); 16 | 17 | if (createHttpError.isHttpError(error)) { 18 | ctx.status = error.status; 19 | 20 | ctx.body = { 21 | error: { 22 | type: error['type'] as string ?? 'api_error', 23 | message: error.expose ? error.message : `${createHttpError(error.status).message}.`, 24 | }, 25 | links: { 26 | documentation: ctx.getDocsLink(), 27 | }, 28 | }; 29 | 30 | return; 31 | } 32 | 33 | logger.error('Internal server error:', error); 34 | 35 | ctx.status = 500; 36 | 37 | ctx.body = { 38 | error: { 39 | type: 'api_error', 40 | message: 'Internal Server Error.', 41 | }, 42 | links: { 43 | documentation: ctx.getDocsLink(), 44 | }, 45 | }; 46 | } 47 | }; 48 | -------------------------------------------------------------------------------- /src/lib/http/middleware/is-admin.ts: -------------------------------------------------------------------------------- 1 | import type { UnknownNext } from '../../../types.js'; 2 | import type { Context } from 'koa'; 3 | import config from 'config'; 4 | 5 | export const isAdminMw = async (ctx: Context, next: UnknownNext) => { 6 | const adminKey = config.get('admin.key'); 7 | ctx['isAdmin'] = adminKey.length > 0 && ctx.query['adminkey'] === adminKey; 8 | return next(); 9 | }; 10 | -------------------------------------------------------------------------------- /src/lib/http/middleware/is-system.ts: -------------------------------------------------------------------------------- 1 | import type { UnknownNext } from '../../../types.js'; 2 | import type { Middleware } from 'koa'; 3 | import config from 'config'; 4 | 5 | const systemKey = config.get('systemApi.key'); 6 | 7 | export const isSystemMw: Middleware = async (ctx, next: UnknownNext) => { 8 | ctx['isSystem'] = ctx.headers['x-api-key'] === systemKey; 9 | return next(); 10 | }; 11 | -------------------------------------------------------------------------------- /src/lib/http/middleware/validate.ts: -------------------------------------------------------------------------------- 1 | import type { Schema } from 'joi'; 2 | import type { ExtendedMiddleware } from '../../../types.js'; 3 | 4 | export const validate = (schema: Schema): ExtendedMiddleware => async (ctx, next) => { 5 | const valid = schema.validate(ctx.request.body, { convert: true, context: ctx.state }); 6 | 7 | if (valid.error) { 8 | const fields = valid.error.details.map(field => [ field.path.join('.'), String(field?.message) ]); 9 | 10 | ctx.status = 400; 11 | 12 | ctx.body = { 13 | error: { 14 | type: 'validation_error', 15 | message: 'Parameter validation failed.', 16 | params: Object.fromEntries(fields) as never, 17 | }, 18 | links: { 19 | documentation: ctx.getDocsLink(), 20 | }, 21 | }; 22 | 23 | return; 24 | } 25 | 26 | ctx.request.body = valid.value as never; 27 | await next(); 28 | }; 29 | -------------------------------------------------------------------------------- /src/lib/http/spec.ts: -------------------------------------------------------------------------------- 1 | import type { Context } from 'koa'; 2 | import type Router from '@koa/router'; 3 | 4 | import _ from 'lodash'; 5 | import * as openApiCore from '@redocly/openapi-core'; 6 | 7 | const getYaml = async (): Promise => { 8 | const bundled = await openApiCore.bundle({ 9 | ref: 'public/v1/spec.yaml', 10 | config: await openApiCore.createConfig({}), 11 | }); 12 | 13 | return openApiCore.stringifyYaml(bundled.bundle.parsed, { 14 | lineWidth: -1, 15 | }); 16 | }; 17 | 18 | const getYamlMemoized = _.memoize(getYaml); 19 | 20 | const handle = async (ctx: Context): Promise => { 21 | ctx.body = await (ctx.app.env === 'production' ? getYamlMemoized : getYaml)(); 22 | }; 23 | 24 | export const registerSpecRoute = (router: Router): void => { 25 | router.get('/spec.yaml', handle); 26 | }; 27 | -------------------------------------------------------------------------------- /src/lib/location/continents.ts: -------------------------------------------------------------------------------- 1 | export const aliases = [ 2 | [ 'eu', 'europe' ], 3 | [ 'na', 'north america' ], 4 | [ 'sa', 'south america' ], 5 | [ 'as', 'asia' ], 6 | [ 'oc', 'oceania' ], 7 | [ 'af', 'africa' ], 8 | [ 'an', 'antarctica' ], 9 | ]; 10 | -------------------------------------------------------------------------------- /src/lib/location/networks.ts: -------------------------------------------------------------------------------- 1 | export const aliases = [ 2 | [ 'aws', 'amazon.com, inc.', 'amazon technologies inc.' ], 3 | [ 'vultr', 'choopa, llc' ], 4 | [ 'netia', 'internetia sp.z o.o.' ], 5 | ]; 6 | -------------------------------------------------------------------------------- /src/lib/location/regions.ts: -------------------------------------------------------------------------------- 1 | // Region names based on https://unstats.un.org/unsd/methodology/m49/ 2 | 3 | export const regions = { 4 | 'Northern Africa': [ 'DZ', 'EG', 'LY', 'MA', 'SD', 'TN', 'EH' ], 5 | 'Eastern Africa': [ 'BI', 'KM', 'DJ', 'ER', 'ET', 'KE', 'MG', 'MW', 'MU', 'MZ', 'RW', 'SC', 'SO', 'SS', 'TZ', 'UG', 'ZM', 'ZW', 'RE', 'TF', 'YT' ], 6 | 'Middle Africa': [ 'AO', 'CM', 'CF', 'TD', 'CG', 'CD', 'GQ', 'GA', 'ST' ], 7 | 'Southern Africa': [ 'BW', 'LS', 'NA', 'ZA', 'SZ' ], 8 | 'Western Africa': [ 'BJ', 'BF', 'CV', 'GM', 'GH', 'GN', 'GW', 'CI', 'LR', 'ML', 'MR', 'NE', 'NG', 'SN', 'SL', 'TG', 'SH' ], 9 | 10 | 'Caribbean': [ 'AG', 'BS', 'BB', 'CU', 'DM', 'DO', 'GD', 'HT', 'JM', 'KN', 'LC', 'VC', 'TT', 'GP', 'KY', 'MQ', 'MS', 'TC', 'AW', 'VG', 'VI', 'PR', 'AI', 'MF', 'BL', 'SX', 'CW', 'BQ' ], 11 | 'Central America': [ 'BZ', 'CR', 'SV', 'GT', 'HN', 'MX', 'NI', 'PA' ], 12 | 'South America': [ 'AR', 'BO', 'BR', 'CL', 'CO', 'EC', 'GY', 'PY', 'PE', 'SR', 'UY', 'VE', 'FK', 'GF', 'GS' ], 13 | 'Northern America': [ 'CA', 'US', 'BM', 'GL', 'PM' ], 14 | 15 | 'Central Asia': [ 'KZ', 'KG', 'TJ', 'TM', 'UZ' ], 16 | 'Eastern Asia': [ 'CN', 'JP', 'KP', 'KR', 'MN', 'HK', 'TW', 'MO' ], 17 | 'South-eastern Asia': [ 'BN', 'MM', 'KH', 'TL', 'ID', 'LA', 'MY', 'PH', 'SG', 'TH', 'VN' ], 18 | 'Southern Asia': [ 'AF', 'BD', 'BT', 'IN', 'IR', 'MV', 'NP', 'PK', 'LK', 'IO' ], 19 | 'Western Asia': [ 'BH', 'IQ', 'IL', 'JO', 'KW', 'LB', 'OM', 'QA', 'SA', 'SY', 'TR', 'AE', 'YE', 'AM', 'AZ', 'CY', 'GE', 'PS' ], 20 | 21 | 'Eastern Europe': [ 'RU', 'BY', 'BG', 'CZ', 'HU', 'MD', 'PL', 'RO', 'SK', 'UA', 'XK' ], 22 | 'Northern Europe': [ 'DK', 'EE', 'FI', 'IS', 'IE', 'LV', 'LT', 'NO', 'SE', 'GB', 'FO', 'GG', 'SJ', 'AX' ], 23 | 'Southern Europe': [ 'AL', 'AD', 'BA', 'HR', 'GR', 'IT', 'MK', 'MT', 'ME', 'PT', 'SM', 'RS', 'SI', 'ES', 'VA', 'GI' ], 24 | 'Western Europe': [ 'AT', 'BE', 'FR', 'DE', 'LI', 'LU', 'MC', 'NL', 'CH', 'JE', 'IM' ], 25 | 26 | 'Australia and New Zealand': [ 'AU', 'NZ', 'NF' ], 27 | 'Melanesia': [ 'FJ', 'PG', 'SB', 'VU', 'NC' ], 28 | 'Micronesia': [ 'KI', 'MH', 'FM', 'NR', 'PW', 'MP', 'GU' ], 29 | 'Polynesia': [ 'WS', 'TO', 'TV', 'CK', 'NU', 'PF', 'PN', 'TK', 'WF' ], 30 | }; 31 | 32 | export const aliases = [ 33 | [ 'northern africa', 'north africa' ], 34 | [ 'eastern africa', 'east africa' ], 35 | [ 'western africa', 'west africa' ], 36 | [ 'south america', 'southern america' ], 37 | [ 'eastern asia', 'east asia' ], 38 | [ 'south-eastern asia', 'south-east asia' ], 39 | [ 'southern asia', 'south asia' ], 40 | [ 'western asia', 'west asia' ], 41 | [ 'eastern europe', 'east europe' ], 42 | [ 'northern europe', 'north europe' ], 43 | [ 'southern europe', 'south europe' ], 44 | [ 'western europe', 'west europe' ], 45 | ]; 46 | 47 | export const regionNames = Object.keys(regions); 48 | -------------------------------------------------------------------------------- /src/lib/location/states-iso.ts: -------------------------------------------------------------------------------- 1 | 2 | export const statesIso: Record = { 3 | AL: 'US-AL', 4 | AK: 'US-AK', 5 | AS: 'US-AS', 6 | AZ: 'US-AZ', 7 | AR: 'US-AR', 8 | CA: 'US-CA', 9 | CO: 'US-CO', 10 | CT: 'US-CT', 11 | DE: 'US-DE', 12 | DC: 'US-DC', 13 | FL: 'US-FL', 14 | GA: 'US-GA', 15 | GU: 'US-GU', 16 | HI: 'US-HI', 17 | ID: 'US-ID', 18 | IL: 'US-IL', 19 | IN: 'US-IN', 20 | IA: 'US-IA', 21 | KS: 'US-KS', 22 | KY: 'US-KY', 23 | LA: 'US-LA', 24 | ME: 'US-ME', 25 | MH: 'US-MH', 26 | MD: 'US-MD', 27 | MA: 'US-MA', 28 | MI: 'US-MI', 29 | MN: 'US-MN', 30 | MS: 'US-MS', 31 | MO: 'US-MO', 32 | MT: 'US-MT', 33 | NE: 'US-NE', 34 | NV: 'US-NV', 35 | NH: 'US-NH', 36 | NJ: 'US-NJ', 37 | NM: 'US-NM', 38 | NY: 'US-NY', 39 | NC: 'US-NC', 40 | ND: 'US-ND', 41 | MP: 'US-MP', 42 | OH: 'US-OH', 43 | OK: 'US-OK', 44 | OR: 'US-OR', 45 | PW: 'US-PW', 46 | PA: 'US-PA', 47 | PR: 'US-PR', 48 | RI: 'US-RI', 49 | SC: 'US-SC', 50 | SD: 'US-SD', 51 | TN: 'US-TN', 52 | TX: 'US-TX', 53 | UT: 'US-UT', 54 | VT: 'US-VT', 55 | VI: 'US-VI', 56 | VA: 'US-VA', 57 | WA: 'US-WA', 58 | WV: 'US-WV', 59 | WI: 'US-WI', 60 | WY: 'US-WY', 61 | }; 62 | -------------------------------------------------------------------------------- /src/lib/location/states.ts: -------------------------------------------------------------------------------- 1 | export const states: Record = { 2 | 'Alabama': 'AL', 3 | 'Alaska': 'AK', 4 | 'American Samoa': 'AS', 5 | 'Arizona': 'AZ', 6 | 'Arkansas': 'AR', 7 | 'California': 'CA', 8 | 'Colorado': 'CO', 9 | 'Connecticut': 'CT', 10 | 'Delaware': 'DE', 11 | 'Washington, D.C.': 'DC', 12 | 'District of Columbia': 'DC', 13 | 'Florida': 'FL', 14 | 'Georgia': 'GA', 15 | 'Guam': 'GU', 16 | 'Hawaii': 'HI', 17 | 'Idaho': 'ID', 18 | 'Illinois': 'IL', 19 | 'Indiana': 'IN', 20 | 'Iowa': 'IA', 21 | 'Kansas': 'KS', 22 | 'Kentucky': 'KY', 23 | 'Louisiana': 'LA', 24 | 'Maine': 'ME', 25 | 'Marshall Islands': 'MH', 26 | 'Maryland': 'MD', 27 | 'Massachusetts': 'MA', 28 | 'Michigan': 'MI', 29 | 'Minnesota': 'MN', 30 | 'Mississippi': 'MS', 31 | 'Missouri': 'MO', 32 | 'Montana': 'MT', 33 | 'Nebraska': 'NE', 34 | 'Nevada': 'NV', 35 | 'New Hampshire': 'NH', 36 | 'New Jersey': 'NJ', 37 | 'New Mexico': 'NM', 38 | 'New York': 'NY', 39 | 'North Carolina': 'NC', 40 | 'North Dakota': 'ND', 41 | 'Northern Mariana Islands': 'MP', 42 | 'Ohio': 'OH', 43 | 'Oklahoma': 'OK', 44 | 'Oregon': 'OR', 45 | 'Palau': 'PW', 46 | 'Pennsylvania': 'PA', 47 | 'Puerto Rico': 'PR', 48 | 'Rhode Island': 'RI', 49 | 'South Carolina': 'SC', 50 | 'South Dakota': 'SD', 51 | 'Tennessee': 'TN', 52 | 'Texas': 'TX', 53 | 'Utah': 'UT', 54 | 'Vermont': 'VT', 55 | 'Virgin Islands': 'VI', 56 | 'Virginia': 'VA', 57 | 'Washington': 'WA', 58 | 'West Virginia': 'WV', 59 | 'Wisconsin': 'WI', 60 | 'Wyoming': 'WY', 61 | }; 62 | -------------------------------------------------------------------------------- /src/lib/location/types.ts: -------------------------------------------------------------------------------- 1 | export type Location = { 2 | continent?: string; 3 | region?: string; 4 | country?: string; 5 | state?: string; 6 | city?: string; 7 | asn?: number; 8 | magic?: string; 9 | tags?: string[]; 10 | }; 11 | -------------------------------------------------------------------------------- /src/lib/logger.ts: -------------------------------------------------------------------------------- 1 | import process from 'node:process'; 2 | import apmAgent from 'elastic-apm-node'; 3 | import ElasticWriter from 'h-logger2-elastic'; 4 | import { ConsoleWriter, Logger } from 'h-logger2'; 5 | import { Client as ElasticSearch } from '@elastic/elasticsearch'; 6 | import type { LogLevelValue } from 'h-logger2/src/types.js'; 7 | 8 | let esClient: ElasticSearch | undefined; 9 | 10 | // istanbul ignore next 11 | if (process.env['ELASTIC_SEARCH_URL']) { 12 | esClient = new ElasticSearch({ 13 | node: process.env['ELASTIC_SEARCH_URL'], 14 | }); 15 | } 16 | 17 | const loggerOptions = { 18 | inspectOptions: { breakLength: 120 }, 19 | }; 20 | 21 | const logger = new Logger( 22 | 'globalping-api', 23 | esClient ? [ 24 | new ConsoleWriter(Number(process.env['LOG_LEVEL']) as LogLevelValue || Logger.levels.info, loggerOptions), 25 | new ElasticWriter(Number(process.env['LOG_LEVEL']) as LogLevelValue || Logger.levels.info, { esClient, apmClient: apmAgent }), 26 | ] : [ 27 | new ConsoleWriter(Number(process.env['LOG_LEVEL']) as LogLevelValue || Logger.levels.trace, loggerOptions), 28 | ], 29 | ); 30 | 31 | export const scopedLogger = (scope: string, parent: Logger = logger): Logger => parent.scope(scope); 32 | 33 | export const scopedLoggerWithPrefix = (scope: string, prefix: string, parent: Logger = logger): Logger => new Proxy(parent.scope(scope), { 34 | get (target, prop: keyof Logger) { 35 | if (prop === 'log') { 36 | return (level: LogLevelValue, message: string, error?: never, context?: never) => { 37 | return target.log(level, `${prefix} ${message}`, error, context); 38 | }; 39 | } 40 | 41 | return target[prop]; 42 | }, 43 | }); 44 | -------------------------------------------------------------------------------- /src/lib/malware/client.ts: -------------------------------------------------------------------------------- 1 | import type { CustomHelpers, ErrorReport } from 'joi'; 2 | import { 3 | updateList as updateIpList, 4 | populateMemList as populateMemIpList, 5 | validate as validateIp, 6 | } from './ip.js'; 7 | 8 | import { 9 | updateList as updateDomainList, 10 | populateMemList as populateMemDomainList, 11 | validate as validateDomain, 12 | } from './domain.js'; 13 | 14 | export const updateMalwareFiles = async (): Promise => { 15 | await updateIpList(); 16 | await updateDomainList(); 17 | }; 18 | 19 | export const populateMemList = async (): Promise => { 20 | await populateMemIpList(); 21 | await populateMemDomainList(); 22 | }; 23 | 24 | export const validate = (target: string): boolean => { 25 | const ipCheck = validateIp(target); 26 | const domainCheck = validateDomain(target); 27 | 28 | return ipCheck && domainCheck; 29 | }; 30 | 31 | export const joiValidate = (value: string, helpers?: CustomHelpers): string | ErrorReport | Error => { 32 | const isValid = validate(value); 33 | 34 | if (!isValid) { 35 | if (helpers) { 36 | return helpers.error('any.blacklisted'); 37 | } 38 | 39 | throw new Error('any.blacklisted'); 40 | } 41 | 42 | return String(value); 43 | }; 44 | 45 | export const joiSchemaErrorMessages = { 46 | 'ip.blacklisted': `{{#label}} is a blacklisted address`, 47 | 'domain.blacklisted': `{{#label}} is a blacklisted domain`, 48 | 'any.blacklisted': `{{#label}} is a blacklisted address or domain`, 49 | }; 50 | -------------------------------------------------------------------------------- /src/lib/malware/domain.ts: -------------------------------------------------------------------------------- 1 | import { writeFile, readFile } from 'node:fs/promises'; 2 | import path from 'node:path'; 3 | import type { CustomHelpers, ErrorReport } from 'joi'; 4 | import got from 'got'; 5 | import validator from 'validator'; 6 | import { scopedLogger } from '../logger.js'; 7 | 8 | const logger = scopedLogger('malware-domain'); 9 | 10 | export const sourceList = [ 11 | 'https://phishing.army/download/phishing_army_blocklist.txt', 12 | 'https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt', 13 | 'https://urlhaus.abuse.ch/downloads/hostfile/', 14 | ]; 15 | 16 | export const domainListPath = path.join(path.resolve(), 'data/DOMAIN_BLACKLIST.json'); 17 | 18 | let domainListArray = new Set(); 19 | 20 | const isFulfilled = (input: PromiseSettledResult): input is PromiseFulfilledResult => input.status === 'fulfilled'; 21 | 22 | export const query = async (url: string): Promise => { 23 | const { body } = await got(url, { 24 | timeout: { request: 5000 }, 25 | }); 26 | 27 | const result = body.split(/r?\n?\s+/).filter(l => validator.isFQDN(l)); 28 | 29 | return result; 30 | }; 31 | 32 | export const updateList = async (): Promise => { 33 | const result = await Promise.allSettled(sourceList.map(source => query(source))); 34 | const list = [ ...new Set(result.flatMap((r) => { 35 | if (isFulfilled(r)) { 36 | return r.value; 37 | } 38 | 39 | logger.error('Error in domain updateList()', r.reason); 40 | return []; 41 | })) ].map(d => d.toLowerCase()); 42 | 43 | await writeFile(domainListPath, JSON.stringify(list), { encoding: 'utf8' }); 44 | }; 45 | 46 | export const populateMemList = async (): Promise => { 47 | const data = await readFile(domainListPath, 'utf8'); 48 | domainListArray = new Set(JSON.parse(data) as string[]); 49 | }; 50 | 51 | export const validate = (target: string): boolean => !domainListArray.has(target.toLowerCase()); 52 | 53 | export const joiValidate = (value: string, helpers?: CustomHelpers): string | ErrorReport | Error => { 54 | const isValid = validate(value); 55 | 56 | if (!isValid) { 57 | if (helpers) { 58 | return helpers.error('domain.blacklisted'); 59 | } 60 | 61 | throw new Error('domain.blacklisted'); 62 | } 63 | 64 | return String(value); 65 | }; 66 | 67 | -------------------------------------------------------------------------------- /src/lib/malware/ip.ts: -------------------------------------------------------------------------------- 1 | import { readFile, writeFile } from 'node:fs/promises'; 2 | import path from 'node:path'; 3 | import type { CustomHelpers, ErrorReport } from 'joi'; 4 | import got from 'got'; 5 | import validator from 'validator'; 6 | import ipaddr from 'ipaddr.js'; 7 | import { scopedLogger } from '../logger.js'; 8 | 9 | const logger = scopedLogger('malware-ip'); 10 | 11 | export const sourceList = [ 12 | 'https://osint.digitalside.it/Threat-Intel/lists/latestips.txt', 13 | 'https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset', 14 | 'https://raw.githubusercontent.com/stamparm/ipsum/master/levels/2.txt', 15 | 'https://www.spamhaus.org/drop/dropv6.txt', 16 | 'https://lists.blocklist.de/lists/all.txt', 17 | ]; 18 | 19 | export const ipListPath = path.join(path.resolve(), 'data/IP_BLACKLIST.json'); 20 | 21 | let ipList = new Set(); 22 | let ipListCIDR = new Set>(); 23 | 24 | const isFulfilled = (input: PromiseSettledResult): input is PromiseFulfilledResult => input.status === 'fulfilled'; 25 | 26 | export const query = async (url: string): Promise => { 27 | const { body } = await got(url, { 28 | timeout: { request: 5000 }, 29 | }); 30 | 31 | return body 32 | .split(/\r?\n/) 33 | .map(l => l.split(' ')[0] ?? '') 34 | .filter(l => validator.isIP(l.split('/')[0] ?? '')); 35 | }; 36 | 37 | export const populateMemList = async (): Promise => { 38 | const data = JSON.parse(await readFile(ipListPath, 'utf8')) as string[]; 39 | ipList = new Set(data.filter(address => ipaddr.isValid(address))); 40 | ipListCIDR = new Set(data.filter(address => ipaddr.isValidCIDR(address)).map(address => ipaddr.parseCIDR(address))); 41 | }; 42 | 43 | export const updateList = async (): Promise => { 44 | const result = await Promise.allSettled(sourceList.map(source => query(source))); 45 | const ipList = [ ...new Set(result.flatMap((r) => { 46 | if (isFulfilled(r)) { 47 | return r.value; 48 | } 49 | 50 | logger.error('Error in IP updateList()', r.reason); 51 | return []; 52 | })) ]; 53 | 54 | await writeFile(ipListPath, JSON.stringify(ipList), { encoding: 'utf8' }); 55 | }; 56 | 57 | function isContainedWithinSubnet (target: string, ipList: Set>): boolean { 58 | const targetAddr = ipaddr.parse(target); 59 | 60 | for (const listAddr of ipList) { 61 | if (targetAddr.kind() === listAddr[0].kind()) { 62 | if (targetAddr.match(listAddr)) { 63 | return true; 64 | } 65 | } 66 | } 67 | 68 | return false; 69 | } 70 | 71 | export const validate = (target: string): boolean => { 72 | if (!validator.isIP(target)) { 73 | return true; 74 | } 75 | 76 | return !ipList.has(target) && !isContainedWithinSubnet(target, ipListCIDR); 77 | }; 78 | 79 | export const joiValidate = (value: string, helpers?: CustomHelpers): string | ErrorReport | Error => { 80 | const isValid = validate(value); 81 | 82 | if (!isValid) { 83 | if (helpers) { 84 | return helpers.error('ip.blacklisted'); 85 | } 86 | 87 | throw new Error('ip.blacklisted'); 88 | } 89 | 90 | return String(value); 91 | }; 92 | -------------------------------------------------------------------------------- /src/lib/override/probe-override.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../../probe/types.js'; 2 | import type { AdoptedProbes } from './adopted-probes.js'; 3 | import type { AdminData } from './admin-data.js'; 4 | 5 | export class ProbeOverride { 6 | constructor ( 7 | private readonly adoptedProbes: AdoptedProbes, 8 | private readonly adminData: AdminData, 9 | ) {} 10 | 11 | async fetchDashboardData () { 12 | await Promise.all([ 13 | this.adoptedProbes.fetchDProbes(), 14 | this.adminData.syncDashboardData(), 15 | ]); 16 | } 17 | 18 | scheduleSync () { 19 | this.adoptedProbes.scheduleSync(); 20 | this.adminData.scheduleSync(); 21 | } 22 | 23 | getUpdatedLocation (probe: Probe) { 24 | const adminLocation = this.adminData.getUpdatedLocation(probe); 25 | const adoptedLocation = this.adoptedProbes.getUpdatedLocation(probe, adminLocation); 26 | return { ...probe.location, ...adminLocation, ...adoptedLocation }; 27 | } 28 | 29 | addAdminData (probes: Probe[]) { 30 | return this.adminData.getUpdatedProbes(probes); 31 | } 32 | 33 | addAdoptedData (probes: Probe[]) { 34 | return this.adoptedProbes.getUpdatedProbes(probes); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/lib/private-ip.ts: -------------------------------------------------------------------------------- 1 | import { isIP, BlockList } from 'net'; 2 | 3 | const privateBlockList = new BlockList(); 4 | 5 | // https://en.wikipedia.org/wiki/Reserved_IP_addresses 6 | // IPv4 7 | privateBlockList.addSubnet('0.0.0.0', 8, 'ipv4'); 8 | privateBlockList.addSubnet('10.0.0.0', 8, 'ipv4'); 9 | privateBlockList.addSubnet('100.64.0.0', 10, 'ipv4'); 10 | privateBlockList.addSubnet('127.0.0.0', 8, 'ipv4'); 11 | privateBlockList.addSubnet('169.254.0.0', 16, 'ipv4'); 12 | privateBlockList.addSubnet('172.16.0.0', 12, 'ipv4'); 13 | privateBlockList.addSubnet('192.0.0.0', 24, 'ipv4'); 14 | privateBlockList.addSubnet('192.0.2.0', 24, 'ipv4'); 15 | privateBlockList.addSubnet('192.88.99.0', 24, 'ipv4'); 16 | privateBlockList.addSubnet('192.168.0.0', 16, 'ipv4'); 17 | privateBlockList.addSubnet('198.18.0.0', 15, 'ipv4'); 18 | privateBlockList.addSubnet('198.51.100.0', 24, 'ipv4'); 19 | privateBlockList.addSubnet('203.0.113.0', 24, 'ipv4'); 20 | privateBlockList.addSubnet('224.0.0.0', 4, 'ipv4'); 21 | privateBlockList.addSubnet('240.0.0.0', 4, 'ipv4'); 22 | privateBlockList.addAddress('255.255.255.255', 'ipv4'); 23 | 24 | // https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml 25 | // IPv6 26 | privateBlockList.addSubnet('::', 128, 'ipv6'); 27 | privateBlockList.addSubnet('::1', 128, 'ipv6'); 28 | privateBlockList.addSubnet('64:ff9b:1::', 48, 'ipv6'); 29 | privateBlockList.addSubnet('100::', 64, 'ipv6'); 30 | privateBlockList.addSubnet('2001::', 32, 'ipv6'); 31 | privateBlockList.addSubnet('2001:10::', 28, 'ipv6'); 32 | privateBlockList.addSubnet('2001:20::', 28, 'ipv6'); 33 | privateBlockList.addSubnet('2001:db8::', 32, 'ipv6'); 34 | privateBlockList.addSubnet('2002::', 16, 'ipv6'); 35 | privateBlockList.addSubnet('fc00::', 7, 'ipv6'); 36 | privateBlockList.addSubnet('fe80::', 10, 'ipv6'); 37 | privateBlockList.addSubnet('ff00::', 8, 'ipv6'); 38 | 39 | export const isIpPrivate = (ip: string) => { 40 | const ipVersion = isIP(ip); 41 | 42 | if (ipVersion === 4) { 43 | return privateBlockList.check(ip, 'ipv4'); 44 | } 45 | 46 | if (ipVersion === 6) { 47 | return privateBlockList.check(ip, 'ipv6'); 48 | } 49 | 50 | // Not a valid IP 51 | return false; 52 | }; 53 | -------------------------------------------------------------------------------- /src/lib/probe-error.ts: -------------------------------------------------------------------------------- 1 | export class ProbeError extends Error {} 2 | -------------------------------------------------------------------------------- /src/lib/probe-validator.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import TTLCache from '@isaacs/ttlcache'; 3 | import { type RedisCluster, getMeasurementRedisClient } from './redis/measurement-client.js'; 4 | 5 | export class ProbeValidator { 6 | private readonly measurementIdToTests = new TTLCache>({ 7 | ttl: (config.get('measurement.timeout') + 30) * 1000, 8 | }); 9 | 10 | constructor (private readonly redis: RedisCluster) {} 11 | 12 | addValidIds (measurementId: string, testId: string, probeUuid: string): void { 13 | const measurement = this.measurementIdToTests.get(measurementId); 14 | 15 | if (!measurement) { 16 | this.measurementIdToTests.set(measurementId, new Map([ [ testId, probeUuid ] ])); 17 | } else { 18 | measurement.set(testId, probeUuid); 19 | } 20 | } 21 | 22 | async validateProbe (measurementId: string, testId: string, probeUuid: string): Promise { 23 | const measurement = this.measurementIdToTests.get(measurementId); 24 | let probeId = measurement && measurement.get(testId); 25 | 26 | if (!probeId) { 27 | probeId = await this.getProbeIdFromRedis(measurementId, testId); 28 | } 29 | 30 | if (!probeId) { 31 | throw new Error(`Probe ID not found for measurement ID: ${measurementId}, test ID: ${testId}`); 32 | } else if (probeId !== probeUuid) { 33 | throw new Error(`Probe ID is wrong for measurement ID: ${measurementId}, test ID: ${testId}. Expected: ${probeId}, actual: ${probeUuid}`); 34 | } 35 | } 36 | 37 | async getProbeIdFromRedis (measurementId: string, testId: string) { 38 | return this.redis.hGet('gp:test-to-probe', `${measurementId}_${testId}`); 39 | } 40 | } 41 | 42 | let probeValidator: ProbeValidator; 43 | 44 | export const getProbeValidator = () => { 45 | if (!probeValidator) { 46 | probeValidator = new ProbeValidator(getMeasurementRedisClient()); 47 | } 48 | 49 | return probeValidator; 50 | }; 51 | -------------------------------------------------------------------------------- /src/lib/rate-limiter/rate-limiter-get.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible'; 3 | import requestIp from 'request-ip'; 4 | import { getPersistentRedisClient } from '../redis/persistent-client.js'; 5 | import createHttpError from 'http-errors'; 6 | import type { ExtendedContext, UnknownNext } from '../../types.js'; 7 | 8 | const redisClient = getPersistentRedisClient(); 9 | 10 | export const rateLimiter = new RateLimiterRedis({ 11 | storeClient: redisClient, 12 | keyPrefix: 'rate:get', 13 | points: config.get('measurement.rateLimit.getPerMeasurement.limit'), 14 | duration: config.get('measurement.rateLimit.getPerMeasurement.reset'), 15 | blockDuration: 5, 16 | }); 17 | 18 | export const getMeasurementRateLimit = async (ctx: ExtendedContext, next: UnknownNext) => { 19 | if (ctx['isAdmin']) { 20 | return next(); 21 | } 22 | 23 | const ip = requestIp.getClientIp(ctx.req) ?? ''; 24 | const measurementId = ctx.params['id'] ?? ''; 25 | const id = `${ip}:${measurementId}`; 26 | 27 | try { 28 | await rateLimiter.consume(id); 29 | } catch (error) { 30 | if (error instanceof RateLimiterRes) { 31 | const retryAfter = Math.ceil(error.msBeforeNext / 1000); 32 | const units = retryAfter === 1 ? 'second' : 'seconds'; 33 | 34 | setRateLimitHeaders(ctx, error); 35 | throw createHttpError(429, `Too many requests. Please retry in ${retryAfter} ${units}.`, { type: 'too_many_requests' }); 36 | } 37 | 38 | throw createHttpError(500); 39 | } 40 | 41 | return next(); 42 | }; 43 | 44 | const setRateLimitHeaders = (ctx: ExtendedContext, error: RateLimiterRes) => { 45 | ctx.set('Retry-After', `${Math.ceil(error.msBeforeNext / 1000)}`); 46 | }; 47 | -------------------------------------------------------------------------------- /src/lib/redis/client.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import type { RedisClientOptions } from 'redis'; 3 | import { createRedisClientInternal, type RedisClient, type RedisClientInternal } from './shared.js'; 4 | import { scopedLogger } from '../logger.js'; 5 | 6 | export type { RedisClient } from './shared.js'; 7 | 8 | let redis: RedisClient; 9 | let redisConnectPromise: Promise; 10 | 11 | export const initRedisClient = async () => { 12 | if (redis) { 13 | await redisConnectPromise; 14 | return redis; 15 | } 16 | 17 | const { client, connectPromise } = createRedisClient(); 18 | 19 | redis = client; 20 | redisConnectPromise = connectPromise; 21 | 22 | await redisConnectPromise; 23 | return redis; 24 | }; 25 | 26 | const createRedisClient = (options?: RedisClientOptions): RedisClientInternal => { 27 | return createRedisClientInternal({ 28 | ...config.get('redis.sharedOptions'), 29 | ...config.get('redis.standaloneNonPersistent'), 30 | ...options, 31 | name: 'non-persistent', 32 | }, scopedLogger('redis-non-persistent')); 33 | }; 34 | 35 | export const getRedisClient = (): RedisClient => { 36 | if (!redis) { 37 | const { client, connectPromise } = createRedisClient(); 38 | redis = client; 39 | redisConnectPromise = connectPromise; 40 | } 41 | 42 | return redis; 43 | }; 44 | -------------------------------------------------------------------------------- /src/lib/redis/measurement-client.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import type { RedisClientOptions, RedisClusterOptions } from 'redis'; 3 | import { createRedisClusterInternal, type RedisCluster, type RedisClusterInternal } from './shared.js'; 4 | import { scopedLogger } from '../logger.js'; 5 | 6 | export type { RedisCluster } from './shared.js'; 7 | 8 | let redis: RedisCluster; 9 | let redisConnectPromise: Promise; 10 | 11 | export const initMeasurementRedisClient = async () => { 12 | if (redis) { 13 | await redisConnectPromise; 14 | return redis; 15 | } 16 | 17 | const { client, connectPromise } = createMeasurementRedisClient(); 18 | 19 | redis = client; 20 | redisConnectPromise = connectPromise; 21 | 22 | await redisConnectPromise; 23 | return redis; 24 | }; 25 | 26 | export const createMeasurementRedisClient = (options?: Partial): RedisClusterInternal => { 27 | return createRedisClusterInternal({ 28 | defaults: config.get('redis.sharedOptions'), 29 | rootNodes: Object.values(config.get<{ [index: string]: string }>('redis.clusterMeasurements.nodes')).map(url => ({ url })), 30 | ...config.get>('redis.clusterMeasurements.options'), 31 | ...options, 32 | }, scopedLogger('redis-measurement')); 33 | }; 34 | 35 | export const getMeasurementRedisClient = (): RedisCluster => { 36 | if (!redis) { 37 | const { client, connectPromise } = createMeasurementRedisClient(); 38 | redis = client; 39 | redisConnectPromise = connectPromise; 40 | } 41 | 42 | return redis; 43 | }; 44 | -------------------------------------------------------------------------------- /src/lib/redis/persistent-client.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import type { RedisClientOptions } from 'redis'; 3 | import { createRedisClientInternal, type RedisClient, type RedisClientInternal } from './shared.js'; 4 | import { scopedLogger } from '../logger.js'; 5 | 6 | export type { RedisClient } from './shared.js'; 7 | 8 | let redis: RedisClient; 9 | let redisConnectPromise: Promise; 10 | 11 | export const initPersistentRedisClient = async () => { 12 | if (redis) { 13 | await redisConnectPromise; 14 | return redis; 15 | } 16 | 17 | const { client, connectPromise } = createPersistentRedisClient(); 18 | 19 | redis = client; 20 | redisConnectPromise = connectPromise; 21 | 22 | await redisConnectPromise; 23 | return redis; 24 | }; 25 | 26 | export const createPersistentRedisClient = (options?: RedisClientOptions): RedisClientInternal => { 27 | return createRedisClientInternal({ 28 | ...config.get('redis.sharedOptions'), 29 | ...config.get('redis.standalonePersistent'), 30 | ...options, 31 | name: 'persistent', 32 | }, scopedLogger('redis-persistent')); 33 | }; 34 | 35 | export const getPersistentRedisClient = (): RedisClient => { 36 | if (!redis) { 37 | const { client, connectPromise } = createPersistentRedisClient(); 38 | redis = client; 39 | redisConnectPromise = connectPromise; 40 | } 41 | 42 | return redis; 43 | }; 44 | -------------------------------------------------------------------------------- /src/lib/redis/shared.ts: -------------------------------------------------------------------------------- 1 | import { 2 | createClient, 3 | createCluster, 4 | type RedisClientOptions, 5 | type RedisClientType, 6 | type RedisClusterOptions, 7 | type RedisClusterType, 8 | type RedisDefaultModules, 9 | type RedisFunctions, 10 | } from 'redis'; 11 | import _ from 'lodash'; 12 | import Bluebird from 'bluebird'; 13 | import { type RedisScripts, scripts } from './scripts.js'; 14 | import { type Logger } from 'h-logger2'; 15 | 16 | type ClusterExtensions = { 17 | mapMasters: typeof mapMasters; 18 | reduceMasters: typeof reduceMasters; 19 | }; 20 | 21 | export type RedisClient = RedisClientType; 22 | export type RedisCluster = RedisClusterType & ClusterExtensions; 23 | export type RedisClientInternal = { connectPromise: Promise; client: RedisClient }; 24 | export type RedisClusterInternal = { connectPromise: Promise; client: RedisCluster }; 25 | 26 | export const createRedisClientInternal = (options: RedisClientOptions, logger: Logger): RedisClientInternal => { 27 | const client = createClient({ 28 | ..._.cloneDeep(options), 29 | scripts, 30 | }); 31 | 32 | const connectPromise = client 33 | .on('error', (error: Error) => logger.error('Redis connection error:', error)) 34 | .on('ready', () => logger.info('Redis connection ready.')) 35 | .on('reconnecting', () => logger.info('Redis reconnecting.')) 36 | .connect().catch((error: Error) => logger.error('Redis connection error:', error)); 37 | 38 | return { client, connectPromise }; 39 | }; 40 | 41 | export const createRedisClusterInternal = (options: RedisClusterOptions, logger: Logger): RedisClusterInternal => { 42 | const cluster = createCluster({ 43 | ..._.cloneDeep(options), 44 | scripts, 45 | }); 46 | 47 | const client = Object.assign(cluster, { 48 | mapMasters, 49 | reduceMasters, 50 | }); 51 | 52 | const connectPromise = client 53 | .on('error', (error: Error) => logger.error('Redis connection error:', error)) 54 | .on('ready', () => logger.info('Redis connection ready.')) 55 | .on('reconnecting', () => logger.info('Redis reconnecting.')) 56 | .connect(); 57 | 58 | return { client, connectPromise }; 59 | }; 60 | 61 | function mapMasters (this: RedisCluster, mapper: (client: RedisClient) => Promise) { 62 | return Bluebird.map(this.masters, (node) => { 63 | return this.nodeClient(node); 64 | }).map(mapper); 65 | } 66 | 67 | function reduceMasters (this: RedisCluster, reducer: (accumulator: Result, client: RedisClient) => Promise, initialValue: Result) { 68 | return Bluebird.map(this.masters, (node) => { 69 | return this.nodeClient(node); 70 | }).reduce(reducer, initialValue); 71 | } 72 | -------------------------------------------------------------------------------- /src/lib/redis/subscription-client.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import type { RedisClientOptions } from 'redis'; 3 | import { createRedisClientInternal, type RedisClientInternal } from './shared.js'; 4 | import { scopedLogger } from '../logger.js'; 5 | 6 | export type { RedisClient } from './shared.js'; 7 | 8 | export const initSubscriptionRedisClient = async () => { 9 | const { connectPromise, client } = createSubscriptionRedisClient(); 10 | await connectPromise; 11 | return client; 12 | }; 13 | 14 | export const createSubscriptionRedisClient = (options?: RedisClientOptions): RedisClientInternal => { 15 | return createRedisClientInternal({ 16 | ...config.get('redis.sharedOptions'), 17 | ...config.get('redis.standaloneNonPersistent'), 18 | ...options, 19 | name: 'subscription', 20 | }, scopedLogger('redis-subscription')); 21 | }; 22 | -------------------------------------------------------------------------------- /src/lib/server.ts: -------------------------------------------------------------------------------- 1 | import type { Server } from 'node:http'; 2 | import { initRedisClient } from './redis/client.js'; 3 | import { probeOverride, probeIpLimit, initWsServer } from './ws/server.js'; 4 | import { getMetricsAgent } from './metrics.js'; 5 | import { populateMemList as populateMemMalwareList } from './malware/client.js'; 6 | import { populateMemList as populateMemCloudIpRangesList } from './cloud-ip-ranges.js'; 7 | import { populateMemList as populateMemBlockedIpRangesList } from './blocked-ip-ranges.js'; 8 | import { populateMemList as populateIpWhiteList } from './geoip/whitelist.js'; 9 | import { populateCitiesList } from './geoip/city-approximation.js'; 10 | import { reconnectProbes } from './ws/helper/reconnect-probes.js'; 11 | import { initPersistentRedisClient } from './redis/persistent-client.js'; 12 | import { initMeasurementRedisClient } from './redis/measurement-client.js'; 13 | import { initSubscriptionRedisClient } from './redis/subscription-client.js'; 14 | import { auth } from './http/auth.js'; 15 | import { adoptionToken } from '../adoption/adoption-token.js'; 16 | 17 | export const createServer = async (): Promise => { 18 | await initRedisClient(); 19 | await initPersistentRedisClient(); 20 | await initMeasurementRedisClient(); 21 | await initSubscriptionRedisClient(); 22 | 23 | // Populate memory malware list 24 | await populateMemMalwareList(); 25 | // Populate memory cloud regions list 26 | await populateMemCloudIpRangesList(); 27 | // Populate memory blocked ip ranges list 28 | await populateMemBlockedIpRangesList(); 29 | // Populate ip whitelist 30 | await populateIpWhiteList(); 31 | // Populate cities info 32 | await populateCitiesList(); 33 | // Populate Dashboard override data before using it during initWsServer() 34 | await probeOverride.fetchDashboardData(); 35 | probeOverride.scheduleSync(); 36 | 37 | adoptionToken.scheduleSync(); 38 | 39 | await initWsServer(); 40 | 41 | await auth.syncTokens(); 42 | auth.scheduleSync(); 43 | 44 | probeIpLimit.scheduleSync(); 45 | 46 | reconnectProbes(); 47 | 48 | const { getWsServer } = await import('./ws/server.js'); 49 | const { getHttpServer } = await import('./http/server.js'); 50 | 51 | const httpServer = getHttpServer(); 52 | const wsServer = getWsServer(); 53 | 54 | wsServer.attach(httpServer); 55 | 56 | await import('./ws/gateway.js'); 57 | 58 | const metricsAgent = getMetricsAgent(); 59 | metricsAgent.run(); 60 | 61 | return httpServer; 62 | }; 63 | -------------------------------------------------------------------------------- /src/lib/sql/client.ts: -------------------------------------------------------------------------------- 1 | import knex, { Knex } from 'knex'; 2 | import knexfile from '../../../knexfile.js'; 3 | 4 | const env = process.env['NODE_ENV'] || 'development'; 5 | 6 | export const client: Knex = knex(knexfile[env] || {}); 7 | -------------------------------------------------------------------------------- /src/lib/ws/gateway.ts: -------------------------------------------------------------------------------- 1 | import { getMetricsAgent } from '../metrics.js'; 2 | import { listenMeasurementRequest } from '../../measurement/handler/request.js'; 3 | import { handleMeasurementAck } from '../../measurement/handler/ack.js'; 4 | import { handleMeasurementResult } from '../../measurement/handler/result.js'; 5 | import { handleMeasurementProgress } from '../../measurement/handler/progress.js'; 6 | import { handleStatusUpdate } from '../../probe/handler/status.js'; 7 | import { handleDnsUpdate } from '../../probe/handler/dns.js'; 8 | import { handleStatsReport } from '../../probe/handler/stats.js'; 9 | import { scopedLogger } from '../logger.js'; 10 | import { probeOverride, getWsServer, PROBES_NAMESPACE, ServerSocket } from './server.js'; 11 | import { probeMetadata } from './middleware/probe-metadata.js'; 12 | import { errorHandler } from './helper/error-handler.js'; 13 | import { subscribeWithHandler } from './helper/subscribe-handler.js'; 14 | import { handleIsIPv4SupportedUpdate, handleIsIPv6SupportedUpdate } from '../../probe/handler/ip-version.js'; 15 | import { getAltIpsClient } from '../alt-ips.js'; 16 | import { adoptionToken } from '../../adoption/adoption-token.js'; 17 | 18 | const io = getWsServer(); 19 | const logger = scopedLogger('gateway'); 20 | const metricsAgent = getMetricsAgent(); 21 | 22 | io 23 | .of(PROBES_NAMESPACE) 24 | .use(probeMetadata) 25 | .on('connect', errorHandler(async (socket: ServerSocket) => { 26 | const probe = socket.data.probe; 27 | const location = probeOverride.getUpdatedLocation(probe); 28 | 29 | adoptionToken.validate(socket).catch(err => logger.warn('Error during adoption token validation:', err)); 30 | socket.emit('api:connect:alt-ips-token', { token: await getAltIpsClient().generateToken(socket), socketId: socket.id, ip: probe.ipAddress }); 31 | socket.emit('api:connect:location', location); 32 | logger.info(`WS client connected.`, { client: { id: socket.id, ip: probe.ipAddress }, location: { city: location.city, country: location.country, network: location.network } }); 33 | 34 | // Handlers 35 | subscribeWithHandler(socket, 'probe:status:update', handleStatusUpdate(probe)); 36 | subscribeWithHandler(socket, 'probe:isIPv6Supported:update', handleIsIPv6SupportedUpdate(probe)); 37 | subscribeWithHandler(socket, 'probe:isIPv4Supported:update', handleIsIPv4SupportedUpdate(probe)); 38 | subscribeWithHandler(socket, 'probe:dns:update', handleDnsUpdate(probe)); 39 | subscribeWithHandler(socket, 'probe:stats:report', handleStatsReport(probe)); 40 | socket.onAnyOutgoing(listenMeasurementRequest(probe)); 41 | subscribeWithHandler(socket, 'probe:measurement:ack', handleMeasurementAck(probe)); 42 | subscribeWithHandler(socket, 'probe:measurement:progress', handleMeasurementProgress(probe)); 43 | subscribeWithHandler(socket, 'probe:measurement:result', handleMeasurementResult(probe)); 44 | 45 | socket.on('disconnect', (reason) => { 46 | logger.debug(`Probe disconnected. (reason: ${reason}) [${socket.id}][${probe.ipAddress}]`); 47 | 48 | if (reason === 'server namespace disconnect') { 49 | return; // Probe was disconnected by the .disconnect() call from the API, no need to record that 50 | } 51 | 52 | metricsAgent.recordDisconnect(reason); 53 | }); 54 | })); 55 | -------------------------------------------------------------------------------- /src/lib/ws/helper/error-handler.ts: -------------------------------------------------------------------------------- 1 | import type { Socket } from 'socket.io'; 2 | import type { ServerSocket } from '../server.js'; 3 | 4 | import getProbeIp from '../../get-probe-ip.js'; 5 | import { scopedLogger } from '../../logger.js'; 6 | 7 | const logger = scopedLogger('ws:error'); 8 | 9 | type NextConnectArgument = ( 10 | socket: Socket, 11 | ) => Promise; 12 | 13 | type NextMwArgument = ( 14 | socket: Socket, 15 | next: () => void 16 | ) => Promise; 17 | 18 | type NextArgument = NextConnectArgument | NextMwArgument; 19 | 20 | const isError = (error: unknown): error is Error => Boolean(error as Error['message']); 21 | 22 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 23 | export const errorHandler = (next: NextArgument) => (socket: ServerSocket, mwNext?: (error?: any) => void) => { 24 | next(socket, mwNext!).catch((error) => { 25 | const clientIp = getProbeIp(socket) ?? ''; 26 | const reason = isError(error) ? error.message : 'unknown'; 27 | 28 | logger.info(`Disconnecting client for (${reason})`, { client: { id: socket.id, ip: clientIp } }); 29 | logger.debug('Details:', error); 30 | 31 | if (mwNext) { 32 | mwNext(error); 33 | } 34 | 35 | socket.disconnect(); 36 | }); 37 | }; 38 | -------------------------------------------------------------------------------- /src/lib/ws/helper/probe-ip-limit.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import config from 'config'; 3 | import type { fetchProbes as serverFetchProbes, fetchRawSockets as serverFetchRawSockets, getProbeByIp as serverGetProbeByIp } from '../server.js'; 4 | import { scopedLogger } from '../../logger.js'; 5 | import { ProbeError } from '../../probe-error.js'; 6 | 7 | const numberOfProcesses = config.get('server.processes'); 8 | 9 | const logger = scopedLogger('ws:limit'); 10 | 11 | export class ProbeIpLimit { 12 | private timer: NodeJS.Timeout | undefined; 13 | 14 | constructor ( 15 | private readonly fetchProbes: typeof serverFetchProbes, 16 | private readonly fetchRawSockets: typeof serverFetchRawSockets, 17 | private readonly getProbeByIp: typeof serverGetProbeByIp, 18 | ) {} 19 | 20 | scheduleSync () { 21 | clearTimeout(this.timer); 22 | 23 | this.timer = setTimeout(() => { 24 | this.syncIpLimit() 25 | .finally(() => this.scheduleSync()) 26 | .catch(error => logger.error('Error in ProbeIpLimit.syncIpLimit()', error)); 27 | }, 60_000 * 2 * Math.random() * numberOfProcesses).unref(); 28 | } 29 | 30 | async syncIpLimit () { 31 | if (process.env['FAKE_PROBE_IP']) { 32 | return; 33 | } 34 | 35 | const probes = await this.fetchProbes(); 36 | // Sorting probes by "client" (socket id), so all workers will treat the same probe as "first". 37 | const sortedProbes = _.sortBy(probes, [ 'client' ]); 38 | 39 | const ipToSocketId = new Map(); 40 | const socketIdsToDisconnect = new Set(); 41 | 42 | for (const probe of sortedProbes) { 43 | for (const ip of [ probe.ipAddress, ...probe.altIpAddresses ]) { 44 | const prevSocketId = ipToSocketId.get(ip); 45 | 46 | if (prevSocketId && prevSocketId !== probe.client) { 47 | logger.warn(`Probe ip duplication occurred (${ip}). Socket id to preserve: ${prevSocketId}, socket id to disconnect: ${probe.client}`); 48 | socketIdsToDisconnect.add(probe.client); 49 | } else { 50 | ipToSocketId.set(ip, probe.client); 51 | } 52 | } 53 | } 54 | 55 | if (socketIdsToDisconnect.size > 0) { 56 | const sockets = await this.fetchRawSockets(); 57 | sockets 58 | .filter(socket => socketIdsToDisconnect.has(socket.id)) 59 | .forEach(socket => socket.disconnect()); 60 | } 61 | } 62 | 63 | async verifyIpLimit (ip: string, socketId: string): Promise { 64 | if (process.env['FAKE_PROBE_IP'] || process.env['TEST_MODE'] === 'unit') { 65 | return; 66 | } 67 | 68 | const previousProbe = await this.getProbeByIp(ip, { allowStale: false }); 69 | 70 | if (previousProbe && previousProbe.client !== socketId) { 71 | logger.warn(`WS client ${socketId} has reached the concurrent IP limit.`, { message: previousProbe.ipAddress }); 72 | throw new ProbeError('ip limit'); 73 | } 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /src/lib/ws/helper/reconnect-probes.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import { scopedLogger } from '../../logger.js'; 3 | import { fetchRawSockets } from '../server.js'; 4 | 5 | const logger = scopedLogger('reconnect-probes'); 6 | const reconnectProbesDelay = config.get('reconnectProbesDelay'); 7 | 8 | const TIME_UNTIL_VM_BECOMES_HEALTHY = 8000; 9 | 10 | const disconnectProbes = async () => { 11 | const sockets = await fetchRawSockets(); 12 | 13 | for (const socket of sockets) { 14 | setTimeout(() => socket.disconnect(), Math.random() * reconnectProbesDelay); 15 | } 16 | }; 17 | 18 | export const reconnectProbes = () => { 19 | if (!reconnectProbesDelay) { 20 | return; 21 | } 22 | 23 | setTimeout(() => { 24 | disconnectProbes().catch(error => logger.error('Error in disconnectProbes()', error)); 25 | }, TIME_UNTIL_VM_BECOMES_HEALTHY); 26 | }; 27 | -------------------------------------------------------------------------------- /src/lib/ws/helper/subscribe-handler.ts: -------------------------------------------------------------------------------- 1 | import Joi from 'joi'; 2 | import { scopedLogger } from '../../logger.js'; 3 | import type { ServerSocket } from '../server.js'; 4 | 5 | const logger = scopedLogger('ws:handler:error'); 6 | const isError = (error: unknown): error is Error => Boolean(error as Error['message']); 7 | 8 | type HandlerMethod = (...args: never[]) => Promise | void; 9 | 10 | export const subscribeWithHandler = (socket: ServerSocket, event: string, method: HandlerMethod) => { 11 | socket.on(event, async (...args) => { 12 | try { 13 | await method(...args as never[]); 14 | } catch (error: unknown) { 15 | const probe = socket.data.probe; 16 | const clientIp = probe.ipAddress; 17 | const metadata: Record = { 18 | client: { id: socket.id, ip: clientIp }, 19 | message: 'unknown', 20 | args, 21 | }; 22 | 23 | if (isError(error)) { 24 | metadata['message'] = error.message; 25 | } 26 | 27 | if (Joi.isError(error)) { 28 | metadata['details'] = error; 29 | } 30 | 31 | logger.warn(`Event "${event}" failed to handle`, metadata); 32 | } 33 | }); 34 | }; 35 | -------------------------------------------------------------------------------- /src/lib/ws/helper/throttle.ts: -------------------------------------------------------------------------------- 1 | import { LRUCache } from 'lru-cache'; 2 | 3 | export const throttle = >(func: () => Promise, time: number, maxStale?: number) => { 4 | type FetchOptions = LRUCache.FetchOptions; 5 | 6 | const cache = new LRUCache({ 7 | max: 1, 8 | ttl: time, 9 | fetchMethod: func, 10 | }); 11 | 12 | return (options?: FetchOptions) => { 13 | const allowStale = maxStale ? cache.getRemainingTTL('') > -maxStale : false; 14 | const newOptions: FetchOptions = Object.assign({ allowStale }, options); 15 | 16 | return cache.fetch('', newOptions) as Promise; 17 | }; 18 | }; 19 | -------------------------------------------------------------------------------- /src/lib/ws/middleware/probe-metadata.ts: -------------------------------------------------------------------------------- 1 | import type { ServerSocket } from '../server.js'; 2 | import { WsError } from '../ws-error.js'; 3 | import { buildProbe } from '../../../probe/builder.js'; 4 | import { ProbeError } from '../../probe-error.js'; 5 | import { errorHandler } from '../helper/error-handler.js'; 6 | import { scopedLogger } from '../../logger.js'; 7 | import getProbeIp from '../../get-probe-ip.js'; 8 | 9 | const logger = scopedLogger('probe-metadata'); 10 | 11 | export const probeMetadata = errorHandler(async (socket: ServerSocket, next: (error?: Error) => void) => { 12 | const clientIp = getProbeIp(socket); 13 | 14 | try { 15 | parseHandshakeQuery(socket); 16 | socket.data.probe = await buildProbe(socket); 17 | next(); 18 | } catch (error: unknown) { 19 | let message = 'failed to collect probe metadata'; 20 | 21 | if (error instanceof ProbeError) { 22 | logger.warn(message, error); 23 | message = error.message; 24 | } else { 25 | logger.error(message, error); 26 | } 27 | 28 | throw new WsError(message, { 29 | ipAddress: clientIp ?? '', 30 | }); 31 | } 32 | }); 33 | 34 | 35 | const parseHandshakeQuery = (socket: ServerSocket) => { 36 | for (const [ key, value ] of Object.entries(socket.handshake.query)) { 37 | if (value === 'undefined') { socket.handshake.query[key] = undefined; } 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /src/lib/ws/ws-error.ts: -------------------------------------------------------------------------------- 1 | type Data = { 2 | ipAddress: string; 3 | }; 4 | 5 | export class WsError extends Error { 6 | data: Data; 7 | constructor (message: string, data: Data) { 8 | super(message); 9 | this.data = data; 10 | } 11 | } 12 | 13 | export default WsError; 14 | -------------------------------------------------------------------------------- /src/limits/route/get-limits.ts: -------------------------------------------------------------------------------- 1 | import type Router from '@koa/router'; 2 | import { getRateLimitState } from '../../lib/rate-limiter/rate-limiter-post.js'; 3 | import type { ExtendedContext } from '../../types.js'; 4 | import { credits } from '../../lib/credits.js'; 5 | import { authenticate } from '../../lib/http/middleware/authenticate.js'; 6 | import { corsAuthHandler } from '../../lib/http/middleware/cors.js'; 7 | 8 | const handle = async (ctx: ExtendedContext): Promise => { 9 | const [ rateLimitState, remainingCredits ] = await Promise.all([ 10 | getRateLimitState(ctx), 11 | ctx.state.user?.id && credits.getRemainingCredits(ctx.state.user.id), 12 | ]); 13 | 14 | ctx.body = { 15 | rateLimit: { 16 | measurements: { 17 | create: rateLimitState, 18 | }, 19 | }, 20 | ...(ctx.state.user?.id && { 21 | credits: { 22 | remaining: remainingCredits, 23 | }, 24 | }), 25 | }; 26 | }; 27 | 28 | export const registerLimitsRoute = (router: Router): void => { 29 | router.get('/limits', '/limits', corsAuthHandler(), authenticate(), handle); 30 | }; 31 | -------------------------------------------------------------------------------- /src/measurement/handler/ack.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../../probe/types.js'; 2 | 3 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 4 | export const handleMeasurementAck = (_probe: Probe) => async (_data: null, ack: () => void): Promise => { 5 | ack(); 6 | }; 7 | -------------------------------------------------------------------------------- /src/measurement/handler/progress.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../../probe/types.js'; 2 | import type { MeasurementProgressMessage } from '../types.js'; 3 | import { getMeasurementRunner } from '../runner.js'; 4 | import { getProbeValidator } from '../../lib/probe-validator.js'; 5 | import { progressSchema } from '../schema/probe-response-schema.js'; 6 | 7 | const runner = getMeasurementRunner(); 8 | 9 | export const handleMeasurementProgress = (probe: Probe) => async (data: MeasurementProgressMessage): Promise => { 10 | const validation = progressSchema.validate(data); 11 | 12 | if (validation.error) { 13 | throw validation.error; 14 | } 15 | 16 | await getProbeValidator().validateProbe(validation.value.measurementId, validation.value.testId, probe.uuid); 17 | await runner.recordProgress(validation.value); 18 | }; 19 | -------------------------------------------------------------------------------- /src/measurement/handler/request.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../../probe/types.js'; 2 | import { getProbeValidator } from '../../lib/probe-validator.js'; 3 | import { MeasurementRequestMessage } from '../types.js'; 4 | 5 | export const listenMeasurementRequest = (probe: Probe) => (event: string, data: unknown) => { 6 | if (event !== 'probe:measurement:request') { 7 | return; 8 | } 9 | 10 | const message = data as MeasurementRequestMessage; 11 | getProbeValidator().addValidIds(message.measurementId, message.testId, probe.uuid); 12 | }; 13 | -------------------------------------------------------------------------------- /src/measurement/handler/result.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../../probe/types.js'; 2 | import type { MeasurementResultMessage } from '../types.js'; 3 | import { getMeasurementRunner } from '../runner.js'; 4 | import { getProbeValidator } from '../../lib/probe-validator.js'; 5 | import { resultSchema } from '../schema/probe-response-schema.js'; 6 | 7 | const runner = getMeasurementRunner(); 8 | 9 | export const handleMeasurementResult = (probe: Probe) => async (data: MeasurementResultMessage): Promise => { 10 | await getProbeValidator().validateProbe(data.measurementId, data.testId, probe.uuid); 11 | 12 | const validation = resultSchema.validate(data); 13 | 14 | if (validation.error) { 15 | (data.measurementId && data.testId) && await runner.recordResult({ 16 | measurementId: data.measurementId, 17 | testId: data.testId, 18 | result: { 19 | status: 'failed', 20 | rawOutput: 'The probe reported an invalid result.', 21 | }, 22 | }); 23 | 24 | throw validation.error; 25 | } 26 | 27 | await runner.recordResult(validation.value); 28 | }; 29 | -------------------------------------------------------------------------------- /src/measurement/route/create-measurement.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import type Router from '@koa/router'; 3 | import { getMeasurementRunner } from '../runner.js'; 4 | import { bodyParser } from '../../lib/http/middleware/body-parser.js'; 5 | import { corsAuthHandler } from '../../lib/http/middleware/cors.js'; 6 | import { validate } from '../../lib/http/middleware/validate.js'; 7 | import { authenticate } from '../../lib/http/middleware/authenticate.js'; 8 | import { schema } from '../schema/global-schema.js'; 9 | import type { ExtendedContext } from '../../types.js'; 10 | 11 | const hostConfig = config.get('server.host'); 12 | const runner = getMeasurementRunner(); 13 | 14 | const handle = async (ctx: ExtendedContext): Promise => { 15 | const { measurementId, probesCount } = await runner.run(ctx); 16 | 17 | ctx.status = 202; 18 | ctx.set('Location', `${hostConfig}/v1/measurements/${measurementId}`); 19 | 20 | ctx.body = { 21 | id: measurementId, 22 | probesCount, 23 | }; 24 | }; 25 | 26 | export const registerCreateMeasurementRoute = (router: Router): void => { 27 | router 28 | .post('/measurements', '/measurements', corsAuthHandler(), authenticate(), bodyParser(), validate(schema), handle) 29 | .options('/measurements', '/measurements', corsAuthHandler()); 30 | }; 31 | -------------------------------------------------------------------------------- /src/measurement/route/get-measurement.ts: -------------------------------------------------------------------------------- 1 | import type { DefaultContext, DefaultState, ParameterizedContext } from 'koa'; 2 | import type Router from '@koa/router'; 3 | import apmAgent from 'elastic-apm-node'; 4 | import createHttpError from 'http-errors'; 5 | import { getMeasurementStore } from '../store.js'; 6 | import { getMeasurementRateLimit } from '../../lib/rate-limiter/rate-limiter-get.js'; 7 | 8 | const store = getMeasurementStore(); 9 | 10 | const handle = async (ctx: ParameterizedContext): Promise => { 11 | const { id } = ctx.params; 12 | 13 | if (!id) { 14 | ctx.status = 400; 15 | return; 16 | } 17 | 18 | const result = await store.getMeasurementString(id); 19 | apmAgent.addLabels({ gpMeasurementId: id }); 20 | 21 | if (!result) { 22 | throw createHttpError(404, `Couldn't find the requested measurement.`, { type: 'not_found' }); 23 | } 24 | 25 | ctx.type = 'application/json'; 26 | ctx.body = result; 27 | }; 28 | 29 | export const registerGetMeasurementRoute = (router: Router): void => { 30 | router.get('/measurements/:id', '/measurements/:id([a-zA-Z0-9]+)', getMeasurementRateLimit, handle); 31 | }; 32 | -------------------------------------------------------------------------------- /src/measurement/schema/global-schema.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import Joi from 'joi'; 3 | import { 4 | targetSchema, 5 | measurementSchema, 6 | } from './command-schema.js'; 7 | import { schema as locationSchema } from './location-schema.js'; 8 | import { GLOBAL_DEFAULTS } from './utils.js'; 9 | 10 | const authenticatedTestsPerMeasurement = config.get('measurement.limits.authenticatedTestsPerMeasurement'); 11 | const anonymousTestsPerMeasurement = config.get('measurement.limits.anonymousTestsPerMeasurement'); 12 | 13 | export const schema = Joi.object({ 14 | type: Joi.string().valid('ping', 'traceroute', 'dns', 'mtr', 'http').insensitive().required(), 15 | target: targetSchema, 16 | measurementOptions: measurementSchema, 17 | locations: locationSchema, 18 | limit: Joi.number().min(1).when('$userId', { 19 | is: Joi.exist(), 20 | then: Joi.number().max(authenticatedTestsPerMeasurement), 21 | otherwise: Joi.number().max(anonymousTestsPerMeasurement), 22 | }).default(GLOBAL_DEFAULTS.limit), 23 | inProgressUpdates: Joi.bool().default(GLOBAL_DEFAULTS.inProgressUpdates), 24 | }); 25 | -------------------------------------------------------------------------------- /src/probe/handler/dns.ts: -------------------------------------------------------------------------------- 1 | import { dnsSchema } from '../schema/probe-response-schema.js'; 2 | import type { Probe } from '../types.js'; 3 | 4 | export const handleDnsUpdate = (probe: Probe) => (list: string[]): void => { 5 | const validation = dnsSchema.validate(list); 6 | 7 | if (validation.error) { 8 | throw validation.error; 9 | } 10 | 11 | probe.resolvers = validation.value; 12 | }; 13 | -------------------------------------------------------------------------------- /src/probe/handler/ip-version.ts: -------------------------------------------------------------------------------- 1 | import type { Probe } from '../types.js'; 2 | import { ipVersionSchema } from '../schema/probe-response-schema.js'; 3 | 4 | export const handleIsIPv4SupportedUpdate = (probe: Probe) => (isIPv4Supported: boolean): void => { 5 | const validation = ipVersionSchema.validate(isIPv4Supported); 6 | 7 | if (validation.error) { 8 | throw validation.error; 9 | } 10 | 11 | probe.isIPv4Supported = validation.value; 12 | }; 13 | 14 | export const handleIsIPv6SupportedUpdate = (probe: Probe) => (isIPv6Supported: boolean): void => { 15 | const validation = ipVersionSchema.validate(isIPv6Supported); 16 | 17 | if (validation.error) { 18 | throw validation.error; 19 | } 20 | 21 | probe.isIPv6Supported = validation.value; 22 | }; 23 | -------------------------------------------------------------------------------- /src/probe/handler/stats.ts: -------------------------------------------------------------------------------- 1 | import type { Probe, ProbeStats } from '../types.js'; 2 | import { statsSchema } from '../schema/probe-response-schema.js'; 3 | 4 | export const handleStatsReport = (probe: Probe) => (report: ProbeStats): void => { 5 | const validation = statsSchema.validate(report); 6 | 7 | if (validation.error) { 8 | throw validation.error; 9 | } 10 | 11 | probe.stats = validation.value; 12 | }; 13 | -------------------------------------------------------------------------------- /src/probe/handler/status.ts: -------------------------------------------------------------------------------- 1 | import { statusSchema } from '../schema/probe-response-schema.js'; 2 | import type { Probe } from '../types.js'; 3 | 4 | export const handleStatusUpdate = (probe: Probe) => (status: Probe['status']) => { 5 | const validation = statusSchema.validate(status); 6 | 7 | if (validation.error) { 8 | throw validation.error; 9 | } 10 | 11 | probe.status = validation.value; 12 | }; 13 | -------------------------------------------------------------------------------- /src/probe/route/get-probes.ts: -------------------------------------------------------------------------------- 1 | import type { DefaultContext, DefaultState, ParameterizedContext } from 'koa'; 2 | import type Router from '@koa/router'; 3 | import type { Probe } from '../types.js'; 4 | import { fetchProbes } from '../../lib/ws/server.js'; 5 | 6 | const handle = async (ctx: ParameterizedContext): Promise => { 7 | const { isAdmin } = ctx; 8 | let probes = await fetchProbes(); 9 | 10 | if (!isAdmin) { 11 | probes = probes.filter(probe => probe.status === 'ready'); 12 | } 13 | 14 | ctx.body = probes.map((probe: Probe) => ({ 15 | status: isAdmin ? probe.status : undefined, 16 | version: probe.version, 17 | isIPv4Supported: isAdmin ? probe.isIPv4Supported : undefined, 18 | isIPv6Supported: isAdmin ? probe.isIPv6Supported : undefined, 19 | nodeVersion: isAdmin ? probe.nodeVersion : undefined, 20 | uuid: isAdmin ? probe.uuid : undefined, 21 | ipAddress: isAdmin ? probe.ipAddress : undefined, 22 | altIpAddresses: isAdmin ? probe.altIpAddresses : undefined, 23 | location: { 24 | continent: probe.location.continent, 25 | region: probe.location.region, 26 | country: probe.location.country, 27 | state: probe.location.state, 28 | city: probe.location.city, 29 | asn: probe.location.asn, 30 | latitude: probe.location.latitude, 31 | longitude: probe.location.longitude, 32 | network: probe.location.network, 33 | }, 34 | tags: probe.tags.map(({ value }) => value), 35 | isHardware: isAdmin ? probe.isHardware : undefined, 36 | hardwareDevice: isAdmin ? probe.hardwareDevice : undefined, 37 | hardwareDeviceFirmware: isAdmin ? probe.hardwareDeviceFirmware : undefined, 38 | resolvers: probe.resolvers, 39 | host: isAdmin ? probe.host : undefined, 40 | stats: isAdmin ? probe.stats : undefined, 41 | hostInfo: isAdmin ? probe.hostInfo : undefined, 42 | })); 43 | }; 44 | 45 | export const registerGetProbesRoute = (router: Router): void => { 46 | router.get('/probes', '/probes', handle); 47 | }; 48 | -------------------------------------------------------------------------------- /src/probe/schema/probe-response-schema.ts: -------------------------------------------------------------------------------- 1 | import Joi from 'joi'; 2 | import { Probe, ProbeStats } from '../types.js'; 3 | 4 | export const statusSchema = Joi.string().valid('initializing', 'ready', 'unbuffer-missing', 'ping-test-failed', 'sigterm').required(); 5 | 6 | export const ipVersionSchema = Joi.boolean().required(); 7 | 8 | export const dnsSchema = Joi.array().max(1024).items(Joi.string().max(1024)).required(); 9 | 10 | export const statsSchema = Joi.object({ 11 | cpu: Joi.object({ 12 | load: Joi.array().max(1024).items(Joi.object({ 13 | usage: Joi.number().required(), 14 | })).required(), 15 | }).required(), 16 | jobs: Joi.object({ 17 | count: Joi.number().required(), 18 | }).required(), 19 | }).required(); 20 | -------------------------------------------------------------------------------- /src/probe/types.ts: -------------------------------------------------------------------------------- 1 | export type ProbeLocation = { 2 | continent: string; 3 | region: string; 4 | country: string; 5 | city: string; 6 | normalizedCity: string; 7 | asn: number; 8 | latitude: number; 9 | longitude: number; 10 | state: string | null; 11 | network: string; 12 | normalizedNetwork: string; 13 | allowedCountries: string[]; 14 | }; 15 | 16 | export type ProbeStats = { 17 | cpu: { 18 | load: Array<{ 19 | usage: number; 20 | }>; 21 | }; 22 | jobs: { 23 | count: number; 24 | }; 25 | }; 26 | 27 | export type HostInfo = { 28 | totalMemory: number; 29 | totalDiskSize: number; 30 | availableDiskSpace: number; 31 | }; 32 | 33 | export type Tag = { 34 | type: 'system' | 'admin' | 'user'; 35 | value: string; 36 | }; 37 | 38 | export type Probe = { 39 | status: 'initializing' | 'ready' | 'unbuffer-missing' | 'ping-test-failed' | 'sigterm'; 40 | isIPv4Supported: boolean; 41 | isIPv6Supported: boolean; 42 | client: string; 43 | version: string; 44 | nodeVersion: string; 45 | uuid: string; 46 | isHardware: boolean; 47 | hardwareDevice: string | null; 48 | hardwareDeviceFirmware: string | null; 49 | ipAddress: string; 50 | altIpAddresses: string[]; 51 | host: string; 52 | location: ProbeLocation; 53 | index: string[][]; 54 | resolvers: string[]; 55 | tags: Tag[]; 56 | stats: ProbeStats; 57 | hostInfo: HostInfo; 58 | owner?: { id: string }; 59 | adoptionToken: string | null; 60 | }; 61 | 62 | type Modify = Omit & Fields; 63 | 64 | export type OfflineProbe = Modify; 93 | -------------------------------------------------------------------------------- /src/types.d.ts: -------------------------------------------------------------------------------- 1 | import Koa, { ParameterizedContext } from 'koa'; 2 | import type Router from '@koa/router'; 3 | import type { DocsLinkContext } from './lib/http/middleware/docs-link.js'; 4 | import type { AuthenticateState } from './lib/http/middleware/authenticate.js'; 5 | 6 | export type CustomState = Koa.DefaultState & AuthenticateState; 7 | export type CustomContext = Koa.DefaultContext & Router.RouterParamContext & DocsLinkContext; 8 | 9 | export type UnknownNext = () => Promise; 10 | export type ExtendedContext = Router.RouterContext; 11 | export type ExtendedMiddleware = (context: ParameterizedContext, next: UnknownNext) => Promise; 12 | -------------------------------------------------------------------------------- /test-perf/artillery.yml: -------------------------------------------------------------------------------- 1 | config: 2 | target: "{{ $processEnvironment.HOST }}" 3 | http: 4 | timeout: 240 5 | 6 | phases: 7 | - name: "POST measurement" 8 | duration: "{{ $processEnvironment.DURATION }}" 9 | arrivalRate: "{{ $processEnvironment.RPS }}" 10 | 11 | scenarios: 12 | - name: "Ping 100 probes" 13 | flow: 14 | - post: 15 | url: "/v1/measurements?adminkey=admin" 16 | json: 17 | target: "google.com" 18 | type: "mtr" 19 | limit: "{{ $processEnvironment.LIMIT }}" 20 | locations: [] 21 | -------------------------------------------------------------------------------- /test/dist.js: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import cluster from 'node:cluster'; 3 | import { after, before, describe, it } from 'node:test'; 4 | import config from 'config'; 5 | 6 | if (!cluster.isPrimary) { 7 | import('../dist/src/index.js'); 8 | } else { 9 | describe('dist build', () => { 10 | before(async () => { 11 | await import('../dist/src/index.js'); 12 | }); 13 | 14 | after(() => { 15 | cluster.removeAllListeners('exit'); 16 | 17 | Object.values(cluster.workers).forEach((worker) => { 18 | worker.kill(); 19 | }); 20 | 21 | setTimeout(() => { 22 | process.exit(process.exitCode); 23 | }, 1000).unref(); 24 | }); 25 | 26 | it('loads and doesn\'t crash', async () => { 27 | await new Promise((resolve, reject) => { 28 | setTimeout(resolve, 10000).unref(); 29 | cluster.removeAllListeners('exit'); 30 | 31 | cluster.on('exit', ({ code, signal }) => { 32 | reject(new assert.AssertionError({ message: `Exited with code ${code}, signal ${signal}.` })); 33 | }); 34 | }); 35 | 36 | const response = await fetch(`http://localhost:${config.get('server.port')}/favicon.ico`); 37 | assert.equal(response.status, 200); 38 | }); 39 | }); 40 | } 41 | -------------------------------------------------------------------------------- /test/e2e/cases/adoption-code.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { client } from '../../../src/lib/sql/client.js'; 4 | 5 | describe('/adoption-code endpoint', () => { 6 | after(async () => { 7 | await client('gp_probes').delete(); 8 | }); 9 | 10 | it('should send code to the probe', async () => { 11 | const response = await got.post('http://localhost:80/v1/adoption-code', { 12 | json: { 13 | ip: '1.2.3.4', 14 | code: '123456', 15 | }, 16 | headers: { 17 | 'X-Api-Key': 'system', 18 | }, 19 | responseType: 'json', 20 | }); 21 | 22 | expect(response.statusCode).to.equal(200); 23 | 24 | expect(response.body).to.deep.include({ 25 | city: 'Buenos Aires', 26 | country: 'AR', 27 | hardwareDevice: null, 28 | hardwareDeviceFirmware: null, 29 | state: null, 30 | status: 'ready', 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /test/e2e/cases/health.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | 4 | describe('/probes endpoint', () => { 5 | it('should return an array of probes', async () => { 6 | const response = await got('http://localhost:80/health'); 7 | 8 | expect(response.statusCode).to.equal(200); 9 | expect(response.body).to.equal('Alive'); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /test/e2e/cases/limits.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { getPersistentRedisClient } from '../../../src/lib/redis/persistent-client.js'; 4 | 5 | describe('/limits endpoint', () => { 6 | const redis = getPersistentRedisClient(); 7 | 8 | before(async () => { 9 | const keys = await redis.keys('rate:post:anon:*'); 10 | await redis.del(keys); 11 | }); 12 | 13 | it('should return a default limits object', async () => { 14 | const response = await got('http://localhost:80/v1/limits', { 15 | responseType: 'json', 16 | }); 17 | 18 | expect(response.statusCode).to.equal(200); 19 | 20 | expect(response.body).to.deep.equal({ 21 | rateLimit: { 22 | measurements: { 23 | create: { 24 | type: 'ip', 25 | limit: 250, 26 | remaining: 250, 27 | reset: 0, 28 | }, 29 | }, 30 | }, 31 | }); 32 | }); 33 | 34 | it('should return an active limits object', async () => { 35 | await got.post('http://localhost:80/v1/measurements', { 36 | json: { 37 | target: 'www.jsdelivr.com', 38 | type: 'ping', 39 | }, 40 | }); 41 | 42 | const response = await got('http://localhost:80/v1/limits', { 43 | responseType: 'json', 44 | }); 45 | 46 | expect(response.statusCode).to.equal(200); 47 | expect(response.body.rateLimit.measurements.create.reset).to.be.a('number'); 48 | 49 | expect(response.body.rateLimit.measurements.create).to.deep.include({ 50 | type: 'ip', 51 | limit: 250, 52 | remaining: 249, 53 | }); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /test/e2e/cases/location-overrides.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { client } from '../../../src/lib/sql/client.js'; 4 | import { waitProbeInCity } from '../utils.js'; 5 | 6 | const LOCATION_OVERRIDES_TABLE = 'gp_location_overrides'; 7 | 8 | describe('location overrides', () => { 9 | before(async function () { 10 | this.timeout(80000); 11 | 12 | await client(LOCATION_OVERRIDES_TABLE).insert({ 13 | user_created: '89da69bd-a236-4ab7-9c5d-b5f52ce09959', 14 | date_created: new Date(), 15 | user_updated: null, 16 | date_updated: null, 17 | ip_range: '1.2.3.4/24', 18 | country: 'US', 19 | state: 'FL', 20 | city: 'Miami', 21 | latitude: 25.77, 22 | longitude: -80.19, 23 | }); 24 | 25 | await waitProbeInCity('Miami'); 26 | }); 27 | 28 | after(async function () { 29 | this.timeout(80000); 30 | await client(LOCATION_OVERRIDES_TABLE).where({ city: 'Miami' }).delete(); 31 | await waitProbeInCity('Buenos Aires'); 32 | }); 33 | 34 | it('should return probe list with updated location', async () => { 35 | const probes = await got('http://localhost:80/v1/probes').json(); 36 | 37 | expect(probes[0].location).to.include({ 38 | continent: 'NA', 39 | region: 'Northern America', 40 | country: 'US', 41 | state: 'FL', 42 | city: 'Miami', 43 | latitude: 25.77, 44 | longitude: -80.19, 45 | }); 46 | }); 47 | 48 | it('should create measurement by its new location', async () => { 49 | const response = await got.post('http://localhost:80/v1/measurements', { 50 | json: { 51 | target: 'www.jsdelivr.com', 52 | type: 'ping', 53 | locations: [{ 54 | city: 'Miami', 55 | }], 56 | }, 57 | }); 58 | 59 | expect(response.statusCode).to.equal(202); 60 | }); 61 | 62 | it('should not create measurement by its old location', async () => { 63 | const response = await got.post('http://localhost:80/v1/measurements', { 64 | json: { 65 | target: 'www.jsdelivr.com', 66 | type: 'ping', 67 | locations: [{ 68 | city: 'Buenos Aires', 69 | }], 70 | }, 71 | throwHttpErrors: false, 72 | }); 73 | 74 | expect(response.statusCode).to.equal(422); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /test/e2e/cases/location.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | 4 | describe('locations filter', () => { 5 | it('should create measurement without location', async () => { 6 | const response = await got.post('http://localhost:80/v1/measurements', { 7 | json: { 8 | target: 'www.jsdelivr.com', 9 | type: 'ping', 10 | }, 11 | }); 12 | 13 | expect(response.statusCode).to.equal(202); 14 | }); 15 | 16 | it('should create measurement by valid "city" value', async () => { 17 | const response = await got.post('http://localhost:80/v1/measurements', { 18 | json: { 19 | target: 'www.jsdelivr.com', 20 | type: 'ping', 21 | locations: [{ 22 | city: 'Buenos Aires', 23 | }], 24 | }, 25 | }); 26 | 27 | expect(response.statusCode).to.equal(202); 28 | }); 29 | 30 | it('should not create measurement by invalid "city" value', async () => { 31 | const response = await got.post('http://localhost:80/v1/measurements', { 32 | json: { 33 | target: 'www.jsdelivr.com', 34 | type: 'ping', 35 | locations: [{ 36 | city: 'Ouagadougou', 37 | }], 38 | }, 39 | throwHttpErrors: false, 40 | }); 41 | 42 | expect(response.statusCode).to.equal(422); 43 | }); 44 | 45 | it('should create measurement by valid "magic" value', async () => { 46 | const response = await got.post('http://localhost:80/v1/measurements', { 47 | json: { 48 | target: 'www.jsdelivr.com', 49 | type: 'ping', 50 | locations: [{ 51 | magic: 'Buenos Aires', 52 | }], 53 | }, 54 | }); 55 | 56 | expect(response.statusCode).to.equal(202); 57 | }); 58 | 59 | it('should not create measurement by invalid "magic" value', async () => { 60 | const response = await got.post('http://localhost:80/v1/measurements', { 61 | json: { 62 | target: 'www.jsdelivr.com', 63 | type: 'ping', 64 | locations: [{ 65 | magic: 'Ouagadougou', 66 | }], 67 | }, 68 | throwHttpErrors: false, 69 | }); 70 | 71 | expect(response.statusCode).to.equal(422); 72 | }); 73 | 74 | it('should create measurement by id of another measurement', async () => { 75 | const { id } = await got.post('http://localhost:80/v1/measurements', { 76 | json: { 77 | target: 'www.jsdelivr.com', 78 | type: 'ping', 79 | }, 80 | }).json(); 81 | 82 | const response = await got.post('http://localhost:80/v1/measurements', { 83 | json: { 84 | target: 'www.jsdelivr.com', 85 | type: 'ping', 86 | locations: id, 87 | }, 88 | }); 89 | 90 | expect(response.statusCode).to.equal(202); 91 | }); 92 | 93 | it('should not create measurement by wrong id', async () => { 94 | const response = await got.post('http://localhost:80/v1/measurements', { 95 | json: { 96 | target: 'www.jsdelivr.com', 97 | type: 'ping', 98 | locations: 'wrongIdValue', 99 | }, 100 | throwHttpErrors: false, 101 | }); 102 | 103 | expect(response.statusCode).to.equal(422); 104 | }); 105 | }); 106 | -------------------------------------------------------------------------------- /test/e2e/cases/mtr.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { waitMeasurementFinish } from '../utils.js'; 4 | 5 | describe('mtr measurement', () => { 6 | it('should finish successfully', async () => { 7 | const { id } = await got.post('http://localhost:80/v1/measurements', { 8 | json: { 9 | target: 'www.jsdelivr.com', 10 | type: 'mtr', 11 | }, 12 | }).json(); 13 | 14 | const response = await waitMeasurementFinish(id); 15 | 16 | expect(response.body.status).to.equal('finished'); 17 | expect(response.body.results[0].result.status).to.equal('finished'); 18 | expect(response).to.matchApiSchema(); 19 | }); 20 | 21 | it('should finish successfully in case of IPv6 domain target', async () => { 22 | const { id } = await got.post('http://localhost:80/v1/measurements', { 23 | json: { 24 | target: 'www.jsdelivr.com', 25 | type: 'mtr', 26 | measurementOptions: { 27 | ipVersion: 6, 28 | }, 29 | }, 30 | }).json(); 31 | 32 | const response = await waitMeasurementFinish(id); 33 | 34 | expect(response.body.status).to.equal('finished'); 35 | expect(response.body.results[0].result.status).to.equal('finished'); 36 | expect(response).to.matchApiSchema(); 37 | }); 38 | 39 | it('should finish successfully in case of IPv6 address target', async () => { 40 | const { id } = await got.post('http://localhost:80/v1/measurements', { 41 | json: { 42 | target: '2606:4700:3037::ac43:d071', 43 | type: 'mtr', 44 | }, 45 | }).json(); 46 | 47 | const response = await waitMeasurementFinish(id); 48 | 49 | expect(response.body.status).to.equal('finished'); 50 | expect(response.body.results[0].result.status).to.equal('finished'); 51 | expect(response).to.matchApiSchema(); 52 | }); 53 | 54 | it('should return 400 for blacklisted target', async () => { 55 | const response = await got.post('http://localhost:80/v1/measurements', { 56 | json: { 57 | target: 'dpd.96594345154.xyz', 58 | type: 'mtr', 59 | }, 60 | throwHttpErrors: false, 61 | }); 62 | 63 | expect(response.statusCode).to.equal(400); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /test/e2e/cases/offline-probes.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | 4 | import { waitMeasurementFinish, waitProbeToConnect, waitProbeToDisconnect } from '../utils.js'; 5 | import { docker } from '../docker.js'; 6 | 7 | describe('api', () => { 8 | beforeEach(async function () { 9 | this.timeout(60000); 10 | await docker.startProbeContainer(); 11 | await waitProbeToConnect(); 12 | }); 13 | 14 | after(async function () { 15 | this.timeout(60000); 16 | await docker.startProbeContainer(); 17 | await waitProbeToConnect(); 18 | }); 19 | 20 | it('should create measurement with "offline" result if requested probe is not connected', async () => { 21 | const { id: locationId } = await got.post('http://localhost:80/v1/measurements', { 22 | json: { 23 | target: 'www.jsdelivr.com', 24 | type: 'ping', 25 | }, 26 | }).json(); 27 | 28 | await docker.stopProbeContainer(); 29 | await waitProbeToDisconnect(); 30 | 31 | const { id } = await got.post('http://localhost:80/v1/measurements', { 32 | json: { 33 | target: 'www.jsdelivr.com', 34 | type: 'ping', 35 | locations: locationId, 36 | }, 37 | }).json(); 38 | 39 | const response = await waitMeasurementFinish(id); 40 | 41 | expect(response.body.status).to.equal('finished'); 42 | expect(response.body.results[0].result.status).to.equal('offline'); 43 | expect(response).to.matchApiSchema(); 44 | }); 45 | 46 | it('should create measurement with "failed" result if probe failed to send result', async () => { 47 | const { id } = await got.post('http://localhost:80/v1/measurements', { 48 | json: { 49 | target: 'www.jsdelivr.com', 50 | type: 'ping', 51 | }, 52 | }).json(); 53 | 54 | await docker.stopProbeContainer(); 55 | 56 | const response = await waitMeasurementFinish(id); 57 | 58 | expect(response.body.status).to.equal('finished'); 59 | expect(response.body.results[0].result.status).to.equal('failed'); 60 | expect(response.body.results[0].result.rawOutput).to.equal('\n\nThe measurement timed out.'); 61 | expect(response).to.matchApiSchema(); 62 | }).timeout(40000); 63 | }); 64 | -------------------------------------------------------------------------------- /test/e2e/cases/ping.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { waitMeasurementFinish } from '../utils.js'; 4 | 5 | describe('ping measurement', () => { 6 | it('should finish successfully', async () => { 7 | const { id } = await got.post('http://localhost:80/v1/measurements', { 8 | json: { 9 | target: 'www.jsdelivr.com', 10 | type: 'ping', 11 | }, 12 | }).json(); 13 | 14 | const response = await waitMeasurementFinish(id); 15 | 16 | expect(response.body.status).to.equal('finished'); 17 | expect(response.body.results[0].result.status).to.equal('finished'); 18 | expect(response).to.matchApiSchema(); 19 | }); 20 | 21 | it('should finish successfully in case of IPv6 domain target', async () => { 22 | const { id } = await got.post('http://localhost:80/v1/measurements', { 23 | json: { 24 | target: 'www.jsdelivr.com', 25 | type: 'ping', 26 | measurementOptions: { 27 | ipVersion: 6, 28 | }, 29 | }, 30 | }).json(); 31 | 32 | const response = await waitMeasurementFinish(id); 33 | 34 | expect(response.body.status).to.equal('finished'); 35 | expect(response.body.results[0].result.status).to.equal('finished'); 36 | expect(response).to.matchApiSchema(); 37 | }); 38 | 39 | it('should finish successfully in case of IPv6 address target', async () => { 40 | const { id } = await got.post('http://localhost:80/v1/measurements', { 41 | json: { 42 | target: '2606:4700:3037::ac43:d071', 43 | type: 'ping', 44 | }, 45 | }).json(); 46 | 47 | const response = await waitMeasurementFinish(id); 48 | 49 | expect(response.body.status).to.equal('finished'); 50 | expect(response.body.results[0].result.status).to.equal('finished'); 51 | expect(response).to.matchApiSchema(); 52 | }); 53 | 54 | it('should return 400 for blacklisted target', async () => { 55 | const response = await got.post('http://localhost:80/v1/measurements', { 56 | json: { 57 | target: 'dpd.96594345154.xyz', 58 | type: 'ping', 59 | }, 60 | throwHttpErrors: false, 61 | }); 62 | 63 | expect(response.statusCode).to.equal(400); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /test/e2e/cases/probes-sync.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import { client } from '../../../src/lib/sql/client.js'; 3 | import { waitRowInTable } from '../utils.js'; 4 | 5 | const ADOPTIONS_TABLE = 'gp_probes'; 6 | 7 | describe('probes sync', () => { 8 | it('should insert new probe row to sql table', async () => { 9 | await client(ADOPTIONS_TABLE).delete(); 10 | const row = await waitRowInTable(ADOPTIONS_TABLE); 11 | expect(row).to.include({ 12 | date_updated: null, 13 | userId: null, 14 | ip: '1.2.3.4', 15 | altIps: '[]', 16 | tags: '[]', 17 | systemTags: '[]', 18 | hardwareDevice: null, 19 | hardwareDeviceFirmware: null, 20 | country: 'AR', 21 | city: 'Buenos Aires', 22 | state: null, 23 | latitude: -34.61, 24 | longitude: -58.38, 25 | asn: 61003, 26 | network: 'InterBS S.R.L. (BAEHOST)', 27 | allowedCountries: '["AR"]', 28 | customLocation: null, 29 | }); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /test/e2e/cases/probes.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | 4 | describe('/probes endpoint', () => { 5 | it('should return an array of probes', async () => { 6 | const response = await got('http://localhost:80/v1/probes', { responseType: 'json' }); 7 | 8 | expect(response.statusCode).to.equal(200); 9 | expect(response.body.length).to.equal(1); 10 | expect(response).to.matchApiSchema(); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /test/e2e/cases/traceroute.test.ts: -------------------------------------------------------------------------------- 1 | import got from 'got'; 2 | import { expect } from 'chai'; 3 | import { waitMeasurementFinish } from '../utils.js'; 4 | 5 | describe('traceroute measurement', () => { 6 | it('should finish successfully', async () => { 7 | const { id } = await got.post('http://localhost:80/v1/measurements', { 8 | json: { 9 | target: 'www.jsdelivr.com', 10 | type: 'traceroute', 11 | }, 12 | }).json(); 13 | 14 | const response = await waitMeasurementFinish(id); 15 | 16 | expect(response.body.status).to.equal('finished'); 17 | expect(response.body.results[0].result.status).to.equal('finished'); 18 | expect(response).to.matchApiSchema(); 19 | }); 20 | 21 | it('should finish successfully in case of IPv6 domain target', async () => { 22 | const { id } = await got.post('http://localhost:80/v1/measurements', { 23 | json: { 24 | target: 'www.jsdelivr.com', 25 | type: 'traceroute', 26 | measurementOptions: { 27 | ipVersion: 6, 28 | }, 29 | }, 30 | }).json(); 31 | 32 | const response = await waitMeasurementFinish(id); 33 | 34 | expect(response.body.status).to.equal('finished'); 35 | expect(response.body.results[0].result.status).to.equal('finished'); 36 | expect(response).to.matchApiSchema(); 37 | }); 38 | 39 | it('should finish successfully in case of IPv6 address target', async () => { 40 | const { id } = await got.post('http://localhost:80/v1/measurements', { 41 | json: { 42 | target: '2606:4700:3037::ac43:d071', 43 | type: 'traceroute', 44 | }, 45 | }).json(); 46 | 47 | const response = await waitMeasurementFinish(id); 48 | 49 | expect(response.body.status).to.equal('finished'); 50 | expect(response.body.results[0].result.status).to.equal('finished'); 51 | expect(response).to.matchApiSchema(); 52 | }); 53 | 54 | it('should return 400 for blacklisted target', async () => { 55 | const response = await got.post('http://localhost:80/v1/measurements', { 56 | json: { 57 | target: 'dpd.96594345154.xyz', 58 | type: 'traceroute', 59 | }, 60 | throwHttpErrors: false, 61 | }); 62 | 63 | expect(response.statusCode).to.equal(400); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /test/e2e/setup.ts: -------------------------------------------------------------------------------- 1 | import Bluebird from 'bluebird'; 2 | import type { Knex } from 'knex'; 3 | import path from 'node:path'; 4 | import { fileURLToPath } from 'node:url'; 5 | import * as chai from 'chai'; 6 | 7 | import { waitProbeToConnect } from './utils.js'; 8 | import chaiOas from '../plugins/oas/index.js'; 9 | import { docker } from './docker.js'; 10 | import { client as sql } from '../../src/lib/sql/client.js'; 11 | import { initRedisClient } from '../../src/lib/redis/client.js'; 12 | import { initPersistentRedisClient } from '../../src/lib/redis/persistent-client.js'; 13 | import { initMeasurementRedisClient } from '../../src/lib/redis/measurement-client.js'; 14 | 15 | before(async () => { 16 | chai.use(await chaiOas({ specPath: path.join(fileURLToPath(new URL('.', import.meta.url)), '../../public/v1/spec.yaml') })); 17 | 18 | await docker.removeProbeContainer(); 19 | await docker.removeApiContainer(); 20 | 21 | await flushRedis(); 22 | 23 | await dropAllTables(sql); 24 | await sql.migrate.latest(); 25 | await sql.seed.run(); 26 | 27 | await docker.createApiContainer(); 28 | await docker.createProbeContainer(); 29 | 30 | await waitProbeToConnect(); 31 | }); 32 | 33 | after(async () => { 34 | await docker.removeProbeContainer(); 35 | await docker.removeApiContainer(); 36 | }); 37 | 38 | const dropAllTables = async (sql: Knex) => { 39 | const allTables = (await sql('information_schema.tables') 40 | .whereRaw(`table_schema = database()`) 41 | .select(`table_name as table`) 42 | ).map(({ table }: { table: string }) => table); 43 | await Bluebird.map(allTables, table => sql.schema.raw(`drop table \`${table}\``)); 44 | }; 45 | 46 | const flushRedis = async () => { 47 | const [ client1, client2, cluster1 ] = await Promise.all([ 48 | initRedisClient(), 49 | initPersistentRedisClient(), 50 | initMeasurementRedisClient(), 51 | ]); 52 | 53 | await Bluebird.all([ 54 | client1.flushDb(), 55 | client2.flushDb(), 56 | cluster1.mapMasters(client => client.flushDb()), 57 | ]); 58 | }; 59 | -------------------------------------------------------------------------------- /test/e2e/utils.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import got, { type RequestError } from 'got'; 3 | import _ from 'lodash'; 4 | import { setTimeout } from 'timers/promises'; 5 | import { client } from '../../src/lib/sql/client.js'; 6 | import { scopedLogger } from '../../src/lib/logger.js'; 7 | 8 | const logger = scopedLogger('e2e-utils'); 9 | 10 | const processes = config.get('server.processes'); 11 | logger.info(`There are ${processes} workers running.`); 12 | 13 | export const waitProbeToDisconnect = async () => { 14 | let responses; 15 | 16 | for (;;) { 17 | try { 18 | // Probe list sync across workers takes a few seconds. So we should retry until all workers return the same result. Multiplying by 2 for safety. 19 | responses = await Promise.all(_.times(processes * 2, (() => got('http://localhost:80/v1/probes', { responseType: 'json' })))); 20 | } catch (err) { 21 | logger.info((err as RequestError).code); 22 | await setTimeout(100); 23 | continue; 24 | } 25 | 26 | if (responses.every(response => response.body.length === 0)) { 27 | return; 28 | } 29 | 30 | await setTimeout(100); 31 | } 32 | }; 33 | 34 | export const waitProbeToConnect = async () => { 35 | let responses; 36 | 37 | for (;;) { 38 | try { 39 | responses = await Promise.all(_.times(processes * 2, (() => got('http://localhost:80/v1/probes', { responseType: 'json' })))); 40 | } catch (err) { 41 | logger.info((err as RequestError).code); 42 | await setTimeout(100); 43 | continue; 44 | } 45 | 46 | if (responses.every(response => response.body.length > 0)) { 47 | return; 48 | } 49 | 50 | await setTimeout(100); 51 | } 52 | }; 53 | 54 | export const waitProbeInCity = async (city: string) => { 55 | let responses; 56 | 57 | for (;;) { 58 | try { 59 | responses = await Promise.all(_.times(processes * 2, (() => got('http://localhost:80/v1/probes', { responseType: 'json' })))); 60 | } catch (err) { 61 | logger.info((err as RequestError).code); 62 | throw err; 63 | } 64 | 65 | if (responses.every(response => response.body.length > 0 && response.body[0].location.city === city)) { 66 | return; 67 | } 68 | 69 | await setTimeout(100); 70 | } 71 | }; 72 | 73 | export const waitMeasurementFinish = async (id: string) => { 74 | for (;;) { 75 | const response = await got(`http://localhost:80/v1/measurements/${id}`, { responseType: 'json' }); 76 | 77 | if (response.body.status !== 'in-progress') { 78 | return response; 79 | } 80 | 81 | await setTimeout(100); 82 | } 83 | }; 84 | 85 | export const waitRowInTable = async (table: string) => { 86 | for (;;) { 87 | const row = await client(table).first(); 88 | 89 | if (row) { 90 | return row; 91 | } 92 | 93 | await setTimeout(100); 94 | } 95 | }; 96 | -------------------------------------------------------------------------------- /test/mocks/blocked-ip-ranges/nock-apple-relay.csv: -------------------------------------------------------------------------------- 1 | 172.224.226.0/27,GB,GB-EN,London, 2 | 172.224.226.32/31,GB,GB-SC,Aberdeen, 3 | 172.224.226.34/31,GB,GB-EN,Oxford, 4 | 172.224.226.36/31,GB,GB-EN,Luton, 5 | 172.224.226.38/31,GB,GB-NI,Belfast, 6 | 172.224.226.40/31,GB,GB-SC,Dundee, 7 | 172.224.226.42/31,GB,GB-EN,Brighton, 8 | 172.224.226.44/31,GB,GB-EN,Leicester, 9 | 172.224.226.46/31,GB,GB-EN,Liverpool, 10 | 172.224.226.48/31,GB,GB-SC,Edinburgh, 11 | 172.224.226.50/31,GB,GB-EN,Cambridge, 12 | 172.224.226.52/31,GB,GB-EN,Bristol, 13 | 172.224.226.54/31,GB,GB-EN,Maidstone, 14 | 172.224.226.56/31,GB,GB-EN,Broomfield, 15 | 172.224.226.58/31,GB,GB-EN,Egg Buckland, 16 | 2a02:26f7:b000:4000::/64,US,US-AK,Anchorage, 17 | 2a02:26f7:b001:4000::/64,US,US-AK,Anchorage, 18 | 2a02:26f7:b002:4000::/64,US,US-AK,Anchorage, 19 | 2a02:26f7:b003:4000::/64,US,US-AK,Anchorage, 20 | 2a02:26f7:b004:4000::/64,US,US-AK,Anchorage, 21 | -------------------------------------------------------------------------------- /test/mocks/cloud-ip-ranges/nock-aws.json: -------------------------------------------------------------------------------- 1 | { 2 | "syncToken": "1670585588", 3 | "createDate": "2022-12-09-11-33-08", 4 | "prefixes": [ 5 | { 6 | "ip_prefix": "3.2.34.0/26", 7 | "region": "af-south-1", 8 | "service": "AMAZON", 9 | "network_border_group": "af-south-1" 10 | }, 11 | { 12 | "ip_prefix": "3.5.140.0/22", 13 | "region": "ap-northeast-2", 14 | "service": "AMAZON", 15 | "network_border_group": "ap-northeast-2" 16 | } 17 | ], 18 | "ipv6_prefixes": [ 19 | { 20 | "ipv6_prefix": "2600:1ff2:4000::/40", 21 | "region": "us-west-2", 22 | "service": "AMAZON", 23 | "network_border_group": "us-west-2" 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/mocks/cloud-ip-ranges/nock-gcp.json: -------------------------------------------------------------------------------- 1 | { 2 | "syncToken": "1670493908499", 3 | "creationTime": "2022-12-08T02:05:08.499208", 4 | "prefixes": [ 5 | { 6 | "ipv4Prefix": "34.104.116.0/24", 7 | "service": "Google Cloud", 8 | "scope": "europe-central2" 9 | }, 10 | { 11 | "ipv4Prefix": "34.137.0.0/16", 12 | "service": "Google Cloud", 13 | "scope": "asia-east1" 14 | }, 15 | { 16 | "ipv6Prefix": "2600:1900:4180::/44", 17 | "service": "Google Cloud", 18 | "scope": "us-west4" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /test/mocks/malware/nock-ip.txt: -------------------------------------------------------------------------------- 1 | # The list contains uniques IPs collected over # 2 | # the last 7 days by OSINT.digitalside.it # 3 | # The list is released without any warranty to the end users. # 4 | # INFO: info@digitalside.it # 5 | # PGP key ID: 30B31BDA # 6 | # PGP fingerprint: 0B4C F801 E8FF E9A3 A602 D2C7 9C36 93B2 30B3 1BDA # 7 | # Every link reported should be considered harmefull and should # 8 | # result in an unwanted malware download. Use this file carefully # 9 | # Generated at: 2022-04-07 09:16:55 # 10 | ####################################################################### 11 | 100.0.41.228 12 | 100.12.102.168 13 | 100.12.115.24 14 | 100.12.181.52 15 | 2803:5380:ffff::/48 ; SBL262056 16 | 100.2.231.58 17 | 100.33.107.62 18 | 100.33.225.244 19 | 100.33.75.218 20 | 101.108.10.224 21 | 101.108.103.38 22 | 101.108.106.55 23 | 101.108.107.2 24 | 101.108.107.242 25 | 101.108.108.101 26 | 101.108.110.138 27 | 101.108.1.121 28 | 101.108.11.251 29 | 101.108.12.5 30 | 101.108.128.101 31 | 101.108.128.148 32 | 101.108.128.187 33 | 101.108.128.254 34 | 101.108.128.96 35 | 101.108.129.152 36 | 101.108.130.109 37 | 101.108.133.211 38 | 101.108.134.213 39 | 101.108.134.4 40 | 101.108.134.97 41 | 101.108.135.203 42 | 101.108.135.75 43 | 101.108.144.244 44 | 101.108.147.137 45 | 101.108.148.250 46 | 101.108.149.109 47 | 101.108.15.183 48 | 101.108.177.135 49 | 101.108.181.187 50 | 101.108.185.79 51 | 101.108.243.132 52 | 101.108.244.86 53 | 101.108.246.79 54 | 101.108.247.105 55 | 101.108.247.54 56 | 101.108.251.137 57 | 101.108.252.211 58 | 101.108.3.89 59 | -------------------------------------------------------------------------------- /test/plugins/oas/index.d.ts: -------------------------------------------------------------------------------- 1 | declare global { 2 | namespace Chai { 3 | interface Assertion { 4 | matchApiSchema (): Assertion; 5 | } 6 | } 7 | } 8 | 9 | interface OasOptions { 10 | specPath: string; 11 | } 12 | 13 | declare function chaiOas (options: OasOptions): typeof chaiOasPlugin; 14 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 15 | declare function chaiOasPlugin (chai: any, utils: any): void; 16 | 17 | export default chaiOas; 18 | -------------------------------------------------------------------------------- /test/tests/contract/newman-env.json: -------------------------------------------------------------------------------- 1 | { 2 | "values": [] 3 | } 4 | -------------------------------------------------------------------------------- /test/tests/contract/portman-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "local": "tmp/spec.yaml", 3 | "baseUrl": "http://localhost:3000", 4 | "portmanConfigFile": "test/tests/contract/portman-config.json" 5 | } 6 | -------------------------------------------------------------------------------- /test/tests/contract/portman-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "tests": { 4 | "contractTests": [ 5 | { 6 | "openApiOperation": "*::/*", 7 | "excludeForOperations": [ 8 | "createMeasurement" 9 | ], 10 | "statusSuccess": { 11 | "enabled": true 12 | }, 13 | "contentType": { 14 | "enabled": true 15 | }, 16 | "jsonBody": { 17 | "enabled": true 18 | }, 19 | "schemaValidation": { 20 | "enabled": true 21 | }, 22 | "headersPresent": { 23 | "enabled": true 24 | } 25 | }, 26 | { 27 | "openApiOperation": "createMeasurement", 28 | "contentType": { 29 | "enabled": true 30 | }, 31 | "jsonBody": { 32 | "enabled": true 33 | }, 34 | "schemaValidation": { 35 | "enabled": true 36 | }, 37 | "headersPresent": { 38 | "enabled": true 39 | } 40 | } 41 | ] 42 | }, 43 | "overwrites": [ 44 | { 45 | "openApiOperationId": "getMeasurement", 46 | "overwriteRequestPathVariables": [ 47 | { 48 | "key": "id", 49 | "value": "measurementid", 50 | "insert": false 51 | } 52 | ] 53 | } 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /test/tests/integration/adoption-token.test.ts: -------------------------------------------------------------------------------- 1 | import nock from 'nock'; 2 | import * as sinon from 'sinon'; 3 | import { setTimeout } from 'node:timers/promises'; 4 | import { getTestServer, addFakeProbe, deleteFakeProbes } from '../../utils/server.js'; 5 | import nockGeoIpProviders from '../../utils/nock-geo-ip.js'; 6 | import { expect } from 'chai'; 7 | import { client } from '../../../src/lib/sql/client.js'; 8 | import { adoptionToken } from '../../../src/adoption/adoption-token.js'; 9 | import { randomUUID } from 'crypto'; 10 | 11 | describe('Adoption token', () => { 12 | const sandbox = sinon.createSandbox(); 13 | 14 | const adoptionStatusStub = sandbox.stub(); 15 | 16 | before(async () => { 17 | await getTestServer(); 18 | await client('directus_users').insert({ id: 'userIdValue', adoption_token: 'adoptionTokenValue', default_prefix: 'defaultPrefixValue' }); 19 | await adoptionToken.syncTokens(); 20 | }); 21 | 22 | afterEach(async () => { 23 | sandbox.resetHistory(); 24 | await deleteFakeProbes(); 25 | await client('gp_probes').delete(); 26 | await client('directus_notifications').delete(); 27 | }); 28 | 29 | after(async () => { 30 | nock.cleanAll(); 31 | await deleteFakeProbes(); 32 | await client('directus_users').delete(); 33 | }); 34 | 35 | it('should adopt probe by token', async () => { 36 | nockGeoIpProviders(); 37 | 38 | nock('https://dash-directus.globalping.io').put('/adoption-code/adopt-by-token', (body) => { 39 | expect(body).to.deep.equal({ 40 | probe: { 41 | userId: null, 42 | ip: '1.2.3.4', 43 | name: null, 44 | altIps: [], 45 | uuid: '1-1-1-1-1', 46 | tags: [], 47 | systemTags: [ 'datacenter-network' ], 48 | status: 'initializing', 49 | isIPv4Supported: false, 50 | isIPv6Supported: false, 51 | version: '0.14.0', 52 | nodeVersion: 'v18.17.0', 53 | hardwareDevice: null, 54 | hardwareDeviceFirmware: null, 55 | city: 'Dallas', 56 | state: 'TX', 57 | country: 'US', 58 | latitude: 32.78, 59 | longitude: -96.81, 60 | asn: 20004, 61 | network: 'The Constant Company LLC', 62 | adoptionToken: 'adoptionTokenValue', 63 | customLocation: null, 64 | allowedCountries: [ 'US' ], 65 | }, 66 | user: { id: 'userIdValue' }, 67 | }); 68 | 69 | return true; 70 | }).reply(200); 71 | 72 | await addFakeProbe({ 'api:connect:adoption': adoptionStatusStub }, { query: { adoptionToken: 'adoptionTokenValue' } }); 73 | 74 | await setTimeout(100); 75 | expect(adoptionStatusStub.callCount).to.equal(1); 76 | expect(adoptionStatusStub.args[0]).to.deep.equal([{ message: 'Probe successfully adopted by token.' }]); 77 | }); 78 | 79 | it('should do nothing if it is the same user', async () => { 80 | await client('gp_probes').insert({ 81 | id: randomUUID(), 82 | uuid: '1-1-1-1-1', 83 | ip: '1.2.3.4', 84 | userId: 'userIdValue', 85 | lastSyncDate: new Date(), 86 | status: 'offline', 87 | }); 88 | 89 | nockGeoIpProviders(); 90 | 91 | await addFakeProbe({ 'api:connect:adoption': adoptionStatusStub }, { query: { adoptionToken: 'adoptionTokenValue' } }); 92 | 93 | await setTimeout(100); 94 | expect(adoptionStatusStub.callCount).to.equal(0); 95 | }); 96 | }); 97 | -------------------------------------------------------------------------------- /test/tests/integration/health.test.ts: -------------------------------------------------------------------------------- 1 | import type { Server } from 'node:http'; 2 | import process from 'node:process'; 3 | import { expect } from 'chai'; 4 | import * as sinon from 'sinon'; 5 | import request, { type Agent } from 'supertest'; 6 | import { getTestServer } from '../../utils/server.js'; 7 | 8 | after(() => { 9 | process.removeAllListeners('SIGTERM'); 10 | process.removeAllListeners('SIGINT'); 11 | }); 12 | 13 | describe('Get health', () => { 14 | let app: Server; 15 | let requestAgent: Agent; 16 | const sandbox = sinon.createSandbox(); 17 | 18 | before(async () => { 19 | app = await getTestServer(); 20 | requestAgent = request(app); 21 | }); 22 | 23 | afterEach(() => { 24 | sandbox.restore(); 25 | }); 26 | 27 | describe('health endpoint', () => { 28 | it('should respond with "Alive" message by default', async () => { 29 | await requestAgent.get('/health') 30 | .send() 31 | .expect(200) 32 | .expect((response) => { 33 | expect(response.text).to.equal('Alive'); 34 | }); 35 | }); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /test/tests/integration/middleware/compress.test.ts: -------------------------------------------------------------------------------- 1 | import request, { type Response } from 'supertest'; 2 | import { expect } from 'chai'; 3 | import nock from 'nock'; 4 | import type { Socket } from 'socket.io-client'; 5 | import { getTestServer, addFakeProbes, deleteFakeProbes, waitForProbesUpdate } from '../../../utils/server.js'; 6 | import geoIpMocks from '../../../mocks/nock-geoip.json' with { type: 'json' }; 7 | 8 | describe('compression', () => { 9 | let requestAgent: any; 10 | let probes: Socket[] = []; 11 | 12 | describe('headers', () => { 13 | before(async () => { 14 | const app = await getTestServer(); 15 | requestAgent = request(app); 16 | }); 17 | 18 | after(async () => { 19 | nock.cleanAll(); 20 | await deleteFakeProbes(); 21 | }); 22 | 23 | it('should include compression headers', async () => { 24 | nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).times(3).reply(200, geoIpMocks.ipmap.default); 25 | nock('https://api.ip2location.io').get(/.*/).times(3).reply(200, geoIpMocks.ip2location.default); 26 | nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).times(3).reply(200, geoIpMocks.fastly.default); 27 | nock('https://ipinfo.io').get(/.*/).times(3).reply(200, geoIpMocks.ipinfo.default); 28 | nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).times(3).reply(200, geoIpMocks.maxmind.default); 29 | probes = await addFakeProbes(3); 30 | 31 | for (const probe of probes) { 32 | probe.emit('probe:status:update', 'ready'); 33 | } 34 | 35 | await waitForProbesUpdate(); 36 | 37 | const response = await requestAgent 38 | .get('/v1/probes') 39 | .set('accept-encoding', '*') 40 | .send() as Response; 41 | 42 | expect(response.headers['transfer-encoding']).to.equal('chunked'); 43 | expect(response.headers['content-length']).to.not.exist; 44 | }); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /test/tests/integration/middleware/cors.test.ts: -------------------------------------------------------------------------------- 1 | import type { Server } from 'node:http'; 2 | import request, { type Response } from 'supertest'; 3 | import { expect } from 'chai'; 4 | 5 | import { getTestServer } from '../../../utils/server.js'; 6 | 7 | describe('cors', () => { 8 | let app: Server; 9 | let requestAgent: any; 10 | 11 | before(async () => { 12 | app = await getTestServer(); 13 | requestAgent = request(app); 14 | }); 15 | 16 | describe('Access-Control-Allow-Origin header', () => { 17 | it('should include the header with value of *', async () => { 18 | const response = await requestAgent.get('/v1/').set('Origin', 'elocast.com').send() as Response; 19 | 20 | expect(response.headers['access-control-allow-origin']).to.equal('*'); 21 | }); 22 | 23 | it('should include the header at root', async () => { 24 | const response = await requestAgent.get('/').send() as Response; 25 | 26 | expect(response.headers['access-control-allow-origin']).to.equal('*'); 27 | }); 28 | 29 | describe('POST /v1/measurements', () => { 30 | it('should include the explicit origin if it is trusted', async () => { 31 | const response = await requestAgent.options('/v1/measurements').set('Origin', 'https://globalping.io').send() as Response; 32 | 33 | expect(response.headers['access-control-allow-origin']).to.equal('https://globalping.io'); 34 | expect(response.headers['vary']).to.include('Origin'); 35 | }); 36 | 37 | it('should include the wildcard if the origin is not trusted', async () => { 38 | const response = await requestAgent.options('/v1/measurements').send() as Response; 39 | 40 | expect(response.headers['access-control-allow-origin']).to.equal('*'); 41 | }); 42 | }); 43 | }); 44 | 45 | describe('Access-Control-Allow-Headers header', () => { 46 | it('should include the header with value of *', async () => { 47 | const response = await requestAgent.get('/v1/').set('Origin', 'elocast.com').send() as Response; 48 | 49 | expect(response.headers['access-control-allow-headers']).to.equal('*'); 50 | }); 51 | 52 | it('should include the header with value of Authorization, Content-Type', async () => { 53 | const response = await requestAgent.options('/v1/measurements').send() as Response; 54 | 55 | expect(response.headers['access-control-allow-headers']).to.equal('Authorization, Content-Type'); 56 | }); 57 | }); 58 | }); 59 | -------------------------------------------------------------------------------- /test/tests/integration/middleware/etag.test.ts: -------------------------------------------------------------------------------- 1 | import type { Server } from 'node:http'; 2 | import request, { type Response } from 'supertest'; 3 | import { expect } from 'chai'; 4 | 5 | import { getTestServer } from '../../../utils/server.js'; 6 | 7 | describe('etag', () => { 8 | let app: Server; 9 | let requestAgent: any; 10 | 11 | before(async () => { 12 | app = await getTestServer(); 13 | requestAgent = request(app); 14 | }); 15 | 16 | describe('ETag header', () => { 17 | it('should include the header', async () => { 18 | const response = await requestAgent.get('/v1/probes').send() as Response; 19 | 20 | expect(response.headers['etag']).to.exist; 21 | }); 22 | }); 23 | 24 | describe('conditional get', () => { 25 | it('should redirect to cache', async () => { 26 | const response = await requestAgent 27 | .get('/v1/probes') 28 | .set('if-none-match', 'W/"2-l9Fw4VUO7kr8CvBlt4zaMCqXZ0w"') 29 | .send() as Response; 30 | 31 | expect(response.status).to.equal(304); 32 | }); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /test/tests/integration/middleware/responsetime.test.ts: -------------------------------------------------------------------------------- 1 | import type { Server } from 'node:http'; 2 | import request, { type Response } from 'supertest'; 3 | import { expect } from 'chai'; 4 | 5 | import { getTestServer } from '../../../utils/server.js'; 6 | 7 | describe('response time', () => { 8 | let app: Server; 9 | let requestAgent: any; 10 | 11 | before(async () => { 12 | app = await getTestServer(); 13 | requestAgent = request(app); 14 | }); 15 | 16 | describe('X-Response-Time header', () => { 17 | describe('should include the header', (): void => { 18 | it('should succeed', async () => { 19 | const response = await requestAgent.get('/v1/').send() as Response; 20 | 21 | expect(response.headers['x-response-time']).to.exist; 22 | }); 23 | }); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /test/tests/unit/blocked-ip-ranges.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import { isIpBlocked, blockedRangesIPv4, populateMemList } from '../../../src/lib/blocked-ip-ranges.js'; 3 | import ipaddr from 'ipaddr.js'; 4 | 5 | describe('blocked-ip-ranges', () => { 6 | after(async () => { 7 | await populateMemList(); 8 | }); 9 | 10 | describe('validate', () => { 11 | it('should check IPv4 ranges', () => { 12 | expect(isIpBlocked('172.224.226.1')).to.equal(true); 13 | expect(isIpBlocked('172.228.226.1')).to.equal(false); 14 | }); 15 | 16 | it('should check IPv6 ranges', () => { 17 | expect(isIpBlocked('2a02:26f7:b000:4000::0001')).to.equal(true); 18 | expect(isIpBlocked('2a02:26f7:1337:4000::0001')).to.equal(false); 19 | }); 20 | 21 | it('should cache results', () => { 22 | expect(isIpBlocked('172.224.226.1')).to.equal(true); 23 | expect(isIpBlocked('172.228.226.1')).to.equal(false); 24 | blockedRangesIPv4.add(ipaddr.parseCIDR('172.228.226.1/27')); 25 | expect(isIpBlocked('172.224.226.1')).to.equal(true); 26 | expect(isIpBlocked('172.228.226.1')).to.equal(false); 27 | }); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /test/tests/unit/credits.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import type { Knex } from 'knex'; 3 | import * as sinon from 'sinon'; 4 | import { Credits } from '../../../src/lib/credits.js'; 5 | 6 | describe('Credits', () => { 7 | const sandbox = sinon.createSandbox(); 8 | const updateStub = sandbox.stub(); 9 | const firstStub = sandbox.stub(); 10 | const whereStub = sandbox.stub().returns({ 11 | update: updateStub, 12 | first: firstStub, 13 | }); 14 | const sqlStub = sandbox.stub().returns({ 15 | where: whereStub, 16 | }) as sinon.SinonStub & { raw: any }; 17 | sqlStub.raw = sandbox.stub(); 18 | 19 | beforeEach(() => { 20 | sandbox.resetHistory(); 21 | }); 22 | 23 | it('should return true if row was updated', async () => { 24 | updateStub.resolves(1); 25 | firstStub.resolves({ amount: 5 }); 26 | const credits = new Credits(sqlStub as unknown as Knex); 27 | const result = await credits.consume('userId', 10); 28 | expect(result).to.deep.equal({ isConsumed: true, remainingCredits: 5 }); 29 | }); 30 | 31 | it('should return true if row was updated to 0', async () => { 32 | updateStub.resolves(1); 33 | firstStub.resolves({ amount: 0 }); 34 | const credits = new Credits(sqlStub as unknown as Knex); 35 | const result = await credits.consume('userId', 10); 36 | expect(result).to.deep.equal({ isConsumed: true, remainingCredits: 0 }); 37 | }); 38 | 39 | it(`should return false if row wasn't updated`, async () => { 40 | updateStub.resolves(0); 41 | const credits = new Credits(sqlStub as unknown as Knex); 42 | const result = await credits.consume('userId', 10); 43 | expect(firstStub.callCount).to.equal(0); 44 | expect(result).to.deep.equal({ isConsumed: false, remainingCredits: 0 }); 45 | }); 46 | 47 | it(`should return false if update throws ER_CONSTRAINT_FAILED_CODE`, async () => { 48 | const error: Error & { errno?: number } = new Error('constraint'); 49 | error.errno = 4025; 50 | updateStub.rejects(error); 51 | firstStub.resolves({ amount: 5 }); 52 | const credits = new Credits(sqlStub as unknown as Knex); 53 | const result = await credits.consume('userId', 10); 54 | expect(result).to.deep.equal({ isConsumed: false, remainingCredits: 5 }); 55 | }); 56 | 57 | it(`should throw if update throws other error`, async () => { 58 | const error = new Error('other error'); 59 | updateStub.rejects(error); 60 | const credits = new Credits(sqlStub as unknown as Knex); 61 | const result = await credits.consume('userId', 10).catch(err => err); 62 | expect(result).to.equal(error); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /test/tests/unit/index.test.ts: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import { EventEmitter } from 'node:events'; 3 | import { expect } from 'chai'; 4 | import * as td from 'testdouble'; 5 | import * as sinon from 'sinon'; 6 | import { getRedisClient, type RedisClient } from '../../../src/lib/redis/client.js'; 7 | import { getPersistentRedisClient } from '../../../src/lib/redis/persistent-client.js'; 8 | 9 | describe('index file', () => { 10 | const sandbox = sinon.createSandbox(); 11 | const cluster: any = new EventEmitter(); 12 | cluster.isPrimary = true; 13 | cluster.fork = sandbox.stub().returns({ process: { pid: 0 } }); 14 | let redis: RedisClient; 15 | let persistentRedis: RedisClient; 16 | 17 | const readFile = sandbox.stub().resolves('commitHash'); 18 | 19 | before(async () => { 20 | redis = getRedisClient(); 21 | persistentRedis = getPersistentRedisClient(); 22 | }); 23 | 24 | beforeEach(async () => { 25 | sandbox.resetHistory(); 26 | await td.replaceEsm('node:cluster', null, cluster); 27 | await td.replaceEsm('node:fs/promises', { readFile, writeFile: sandbox.stub() }); 28 | }); 29 | 30 | after(() => { 31 | td.reset(); 32 | redis.del('testfield'); 33 | persistentRedis.del('testfield'); 34 | }); 35 | 36 | it('master should restart a worker if it dies', async () => { 37 | cluster.fork.onFirstCall().returns({ process: { pid: 1 } }); 38 | cluster.fork.onSecondCall().returns({ process: { pid: 2 } }); 39 | await import('../../../src/index.js'); 40 | 41 | cluster.emit('exit', { process: { pid: 1 } }); 42 | cluster.emit('exit', { process: { pid: 2 } }); 43 | 44 | expect(cluster.fork.callCount).to.equal(4); 45 | expect(cluster.fork.args[0]).to.deep.equal([{ SHOULD_SYNC_ADOPTIONS: true }]); 46 | expect(cluster.fork.args[1]).to.deep.equal([]); 47 | expect(cluster.fork.args[2]).to.deep.equal([{ SHOULD_SYNC_ADOPTIONS: true }]); 48 | expect(cluster.fork.args[3]).to.deep.equal([]); 49 | }); 50 | 51 | it('master should fork the configured number of processes', async () => { 52 | await import('../../../src/index.js'); 53 | 54 | expect(cluster.fork.callCount).to.equal(config.get('server.processes')); 55 | }); 56 | 57 | it('master should flush non-persistent redis if commits hashes do not match', async () => { 58 | redis.set('testfield', 'testvalue'); 59 | persistentRedis.set('testfield', 'testvalue'); 60 | readFile.resolves('oldCommitHash'); 61 | 62 | persistentRedis.set('LAST_API_COMMIT_HASH_test', 'commitHash'); 63 | 64 | await import('../../../src/index.js'); 65 | 66 | const value = await redis.get('testfield'); 67 | const persistentValue = await persistentRedis.get('testfield'); 68 | expect(value).to.equal(null); 69 | expect(persistentValue).to.equal('testvalue'); 70 | }); 71 | 72 | it('master should not flush non-persistent redis if commits hashes match', async () => { 73 | redis.set('testfield', 'testvalue'); 74 | persistentRedis.set('testfield', 'testvalue'); 75 | readFile.resolves('commitHash'); 76 | 77 | persistentRedis.set('LAST_API_COMMIT_HASH_test', 'commitHash'); 78 | 79 | await import('../../../src/index.js'); 80 | 81 | const value = await redis.get('testfield'); 82 | const persistentValue = await persistentRedis.get('testfield'); 83 | expect(value).to.equal('testvalue'); 84 | expect(persistentValue).to.equal('testvalue'); 85 | }); 86 | }); 87 | -------------------------------------------------------------------------------- /test/tests/unit/ip-ranges.test.ts: -------------------------------------------------------------------------------- 1 | import { writeFile, readFile } from 'node:fs/promises'; 2 | import path from 'node:path'; 3 | import nock from 'nock'; 4 | import { expect } from 'chai'; 5 | import { updateIpRangeFiles, sources, populateMemList, getRegion } from '../../../src/lib/cloud-ip-ranges.js'; 6 | 7 | const mockDataPath = path.join(path.resolve(), 'test/mocks/cloud-ip-ranges'); 8 | const gcpMockRanges = await readFile(path.join(mockDataPath, 'nock-gcp.json'), 'utf8'); 9 | const awsMockRanges = await readFile(path.join(mockDataPath, 'nock-aws.json'), 'utf8'); 10 | const gcpUrl = new URL(sources.gcp.url); 11 | const awsUrl = new URL(sources.aws.url); 12 | 13 | describe('cloud ip ranges', () => { 14 | before(() => { 15 | nock(gcpUrl.origin).get(gcpUrl.pathname).reply(200, gcpMockRanges); 16 | nock(awsUrl.origin).get(awsUrl.pathname).reply(200, awsMockRanges); 17 | }); 18 | 19 | after(() => { 20 | nock.cleanAll(); 21 | }); 22 | 23 | describe('updateList', () => { 24 | const gcpFilePath = path.join(path.resolve(), sources.gcp.file); 25 | 26 | it('should override blacklist file', async () => { 27 | // Reset the file 28 | await writeFile(gcpFilePath, '', { encoding: 'utf8' }); 29 | 30 | const preFile = await readFile(gcpFilePath, 'utf8').catch(() => null); 31 | await updateIpRangeFiles(); 32 | const postFile = await readFile(gcpFilePath, 'utf8'); 33 | 34 | expect(preFile).to.not.equal(postFile); 35 | }); 36 | }); 37 | 38 | describe('validate', () => { 39 | before(async () => { 40 | await populateMemList(); 41 | }); 42 | 43 | it('should return null', () => { 44 | const region = getRegion('100.0.41.228'); 45 | expect(region).to.equal(null); 46 | }); 47 | 48 | it('should return gcp region', () => { 49 | const region = getRegion('34.104.116.1'); 50 | expect(region).to.equal('gcp-europe-central2'); 51 | }); 52 | 53 | it('should return aws region', () => { 54 | const region = getRegion('3.2.34.1'); 55 | expect(region).to.equal('aws-af-south-1'); 56 | }); 57 | 58 | it('should return region for gcp IPv6 ips', () => { 59 | const region = getRegion('2600:1900:4180::0001'); 60 | expect(region).to.equal('gcp-us-west4'); 61 | }); 62 | 63 | it('should return region for aws IPv6 ips', () => { 64 | const region = getRegion('2600:1ff2:4000::0001'); 65 | expect(region).to.equal('aws-us-west-2'); 66 | }); 67 | }); 68 | }); 69 | -------------------------------------------------------------------------------- /test/tests/unit/middleware/error-handler.test.ts: -------------------------------------------------------------------------------- 1 | import createHttpError from 'http-errors'; 2 | import { expect } from 'chai'; 3 | import { errorHandlerMw } from '../../../../src/lib/http/middleware/error-handler.js'; 4 | 5 | describe('Error handler middleware', () => { 6 | const documentation = 'link://'; 7 | const getDocsLink = () => documentation; 8 | 9 | it('should handle http errors', async () => { 10 | const ctx: any = { getDocsLink }; 11 | 12 | await errorHandlerMw(ctx, () => { 13 | throw createHttpError(400, 'bad request'); 14 | }); 15 | 16 | expect(ctx.status).to.equal(400); 17 | expect(ctx.body).to.deep.equal({ error: { message: 'bad request', type: 'api_error' }, links: { documentation } }); 18 | }); 19 | 20 | it('should handle http errors with expose=false', async () => { 21 | const ctx: any = { getDocsLink }; 22 | await errorHandlerMw(ctx, () => { 23 | throw createHttpError(400, 'custom error message', { expose: false }); 24 | }); 25 | 26 | expect(ctx.status).to.equal(400); 27 | expect(ctx.body).to.deep.equal({ error: { message: 'Bad Request.', type: 'api_error' }, links: { documentation } }); 28 | }); 29 | 30 | it('should handle custom errors', async () => { 31 | const ctx: any = { getDocsLink }; 32 | await errorHandlerMw(ctx, () => { 33 | throw new Error('custom error message'); 34 | }); 35 | 36 | expect(ctx.status).to.equal(500); 37 | expect(ctx.body).to.deep.equal({ error: { message: 'Internal Server Error.', type: 'api_error' }, links: { documentation } }); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /test/tests/unit/middleware/is-system.test.ts: -------------------------------------------------------------------------------- 1 | import type { Context } from 'koa'; 2 | import * as sinon from 'sinon'; 3 | import { expect } from 'chai'; 4 | import { isSystemMw } from '../../../../src/lib/http/middleware/is-system.js'; 5 | 6 | const sandbox = sinon.createSandbox(); 7 | const next = sandbox.stub(); 8 | 9 | beforeEach(() => { 10 | sandbox.resetHistory(); 11 | }); 12 | 13 | describe('rate limit middleware', () => { 14 | it('should set to "false" for requests without system key', async () => { 15 | const ctx = { headers: {} } as unknown as Context; 16 | await isSystemMw(ctx, next); 17 | expect(ctx['isSystem']).to.equal(false); 18 | }); 19 | 20 | it('should set to "false" for requests with invalid system key', async () => { 21 | const ctx = { headers: { 'x-api-key': 'wrongkey' } } as unknown as Context; 22 | await isSystemMw(ctx, next); 23 | expect(ctx['isSystem']).to.equal(false); 24 | }); 25 | 26 | it('should set to "true" for requests with valid system key', async () => { 27 | const ctx = { headers: { 'x-api-key': 'system' } } as unknown as Context; 28 | await isSystemMw(ctx, next); 29 | expect(ctx['isSystem']).to.equal(true); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /test/tests/unit/middleware/validate.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import Joi from 'joi'; 3 | import * as sinon from 'sinon'; 4 | import { validate } from '../../../../src/lib/http/middleware/validate.js'; 5 | 6 | describe('Validate middleware', () => { 7 | const sandbox = sinon.createSandbox(); 8 | 9 | const documentation = 'link://'; 10 | const getDocsLink = () => documentation; 11 | 12 | const schema = Joi.object({ 13 | hello: Joi.string().valid('world!').required(), 14 | }); 15 | const nextMock = sandbox.stub(); 16 | 17 | beforeEach(() => { 18 | nextMock.reset(); 19 | }); 20 | 21 | it('should call next', async () => { 22 | const ctx: any = { request: { body: { hello: 'world!' } }, getDocsLink }; 23 | const next = sandbox.stub(); 24 | 25 | await validate(schema)(ctx, next); 26 | 27 | expect(next.calledOnce).to.be.true; 28 | expect(ctx.status).to.not.exist; 29 | }); 30 | 31 | it('should return validation error', async () => { 32 | const ctx: any = { request: { body: { hello: 'no one' } }, getDocsLink }; 33 | 34 | await validate(schema)(ctx, nextMock); 35 | 36 | expect(nextMock.notCalled).to.be.true; 37 | 38 | expect(ctx.status).to.equal(400); 39 | 40 | expect(ctx.body).to.deep.equal({ 41 | error: { 42 | message: 'Parameter validation failed.', 43 | type: 'validation_error', 44 | params: { hello: '"hello" must be [world!]' }, 45 | }, 46 | links: { 47 | documentation, 48 | }, 49 | }); 50 | }); 51 | 52 | it('should normalise incorrect input case', async () => { 53 | const ctx: any = { request: { body: { input: 'text' } }, getDocsLink }; 54 | 55 | const schema = Joi.object({ 56 | input: Joi.string().valid('TEXT').insensitive().required(), 57 | }); 58 | 59 | await validate(schema)(ctx, nextMock); 60 | 61 | expect(ctx.request.body.input).to.equal('TEXT'); 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /test/tests/unit/probe-validator.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import * as sinon from 'sinon'; 3 | import { ProbeValidator } from '../../../src/lib/probe-validator.js'; 4 | import { RedisCluster } from '../../../src/lib/redis/shared.js'; 5 | 6 | describe('ProbeValidator', () => { 7 | const sandbox = sinon.createSandbox(); 8 | const redis = { hGet: sandbox.stub() }; 9 | const probeValidator = new ProbeValidator(redis as unknown as RedisCluster); 10 | 11 | beforeEach(() => { 12 | redis.hGet.resolves(undefined); 13 | }); 14 | 15 | it('should pass through valid probe id', async () => { 16 | probeValidator.addValidIds('measurement-id', 'test-id-0', 'probe-uuid-0'); 17 | probeValidator.addValidIds('measurement-id', 'test-id-1', 'probe-uuid-1'); 18 | await probeValidator.validateProbe('measurement-id', 'test-id-0', 'probe-uuid-0'); 19 | await probeValidator.validateProbe('measurement-id', 'test-id-1', 'probe-uuid-1'); 20 | }); 21 | 22 | it('should throw for invalid probe id', async () => { 23 | probeValidator.addValidIds('measurement-id', 'test-id', 'probe-uuid'); 24 | const error = await probeValidator.validateProbe('measurement-id', 'test-id', 'invalid-probe-uuid').catch(err => err); 25 | expect(error.message).to.equal('Probe ID is wrong for measurement ID: measurement-id, test ID: test-id. Expected: probe-uuid, actual: invalid-probe-uuid'); 26 | }); 27 | 28 | it('should throw for missing key', async () => { 29 | const error = await probeValidator.validateProbe('missing-measurement-id', 'test-id', 'probe-uuid').catch(err => err); 30 | expect(error.message).to.equal('Probe ID not found for measurement ID: missing-measurement-id, test ID: test-id'); 31 | }); 32 | 33 | it('should search key in redis if not found locally', async () => { 34 | redis.hGet.resolves('probe-uuid'); 35 | await probeValidator.validateProbe('only-redis-measurement-id', 'test-id', 'probe-uuid'); 36 | }); 37 | 38 | it('should throw if redis probe id is different', async () => { 39 | redis.hGet.resolves('different-probe-uuid'); 40 | const error = await probeValidator.validateProbe('only-redis-measurement-id', 'test-id', 'probe-uuid').catch(err => err); 41 | expect(error.message).to.equal('Probe ID is wrong for measurement ID: only-redis-measurement-id, test ID: test-id. Expected: different-probe-uuid, actual: probe-uuid'); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /test/tests/unit/ws/error-handler.test.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'chai'; 2 | import type { Socket } from 'socket.io'; 3 | import * as sinon from 'sinon'; 4 | 5 | import { WsError } from '../../../../src/lib/ws/ws-error.js'; 6 | import { errorHandler } from '../../../../src/lib/ws/helper/error-handler.js'; 7 | 8 | class MockSocket { 9 | public isConnected = true; 10 | 11 | public store: Array<{ type: string; event: string; payload: any }> = []; 12 | 13 | public request: any = {}; 14 | 15 | public handshake: any = { 16 | query: {}, 17 | }; 18 | 19 | constructor (public id: string) {} 20 | 21 | emit (event: string, payload: string | Record) { 22 | this.store.push({ type: 'emit', event, payload }); 23 | } 24 | 25 | disconnect () { 26 | this.isConnected = false; 27 | } 28 | } 29 | 30 | type BundledMockSocket = Socket & MockSocket; 31 | 32 | describe('ws error', () => { 33 | const sandbox = sinon.createSandbox(); 34 | 35 | afterEach(() => { 36 | sandbox.restore(); 37 | }); 38 | 39 | describe('ws error handler', () => { 40 | it('should catch Error and execute cb', async () => { 41 | const socket = new MockSocket('abc') as BundledMockSocket; 42 | let cbError: Error | null = null; 43 | 44 | const testMethod = async (socket: Socket): Promise => { 45 | // Prevent unused variable err 46 | socket.emit('connect', ''); 47 | throw new Error('abc'); 48 | }; 49 | 50 | const testCb = (error: Error) => { 51 | cbError = error; 52 | }; 53 | 54 | errorHandler(testMethod)(socket as Socket, testCb); 55 | 56 | expect(socket.isConnected).to.equal(true); 57 | await clock.nextAsync(); 58 | 59 | expect(socket.isConnected).to.equal(false); 60 | expect(cbError).to.not.be.null; 61 | expect(cbError).to.be.instanceof(Error); 62 | expect(cbError!.toString()).to.equal('Error: abc'); 63 | }); 64 | 65 | it('should catch WsError and execute cb', async () => { 66 | const socket = new MockSocket('abc') as BundledMockSocket; 67 | let cbError: Error | null = null; 68 | 69 | const testMethod = async (socket: Socket): Promise => { 70 | // Prevent unused variable err 71 | socket.emit('connect', ''); 72 | throw new WsError('vpn detected', { ipAddress: '' }); 73 | }; 74 | 75 | const testCb = (error: Error) => { 76 | cbError = error; 77 | }; 78 | 79 | errorHandler(testMethod)(socket as Socket, testCb); 80 | 81 | expect(socket.isConnected).to.equal(true); 82 | await clock.nextAsync(); 83 | 84 | expect(socket.isConnected).to.equal(false); 85 | expect(cbError).to.not.be.null; 86 | expect(cbError).to.be.instanceof(WsError); 87 | expect(cbError!.toString()).to.equal('Error: vpn detected'); 88 | }); 89 | }); 90 | }); 91 | -------------------------------------------------------------------------------- /test/tests/unit/ws/reconnect-probes.test.ts: -------------------------------------------------------------------------------- 1 | import * as sinon from 'sinon'; 2 | import * as td from 'testdouble'; 3 | import { expect } from 'chai'; 4 | 5 | const sandbox = sinon.createSandbox(); 6 | const disconnect = sandbox.stub(); 7 | const fetchRawSockets = sandbox.stub().resolves([{ disconnect }, { disconnect }]); 8 | 9 | describe('reconnectProbes', () => { 10 | let reconnectProbes: () => void; 11 | 12 | before(async () => { 13 | await td.replaceEsm('../../../../src/lib/ws/server.ts', { 14 | fetchRawSockets, 15 | }); 16 | 17 | ({ reconnectProbes } = await import('../../../../src/lib/ws/helper/reconnect-probes.js')); 18 | }); 19 | 20 | afterEach(() => { 21 | sandbox.restore(); 22 | }); 23 | 24 | after(() => { 25 | td.reset(); 26 | }); 27 | 28 | it('should disconnect every probe in configured time', async () => { 29 | reconnectProbes(); 30 | 31 | expect(fetchRawSockets.callCount).to.equal(0); 32 | expect(disconnect.callCount).to.equal(0); 33 | 34 | await clock.tickAsync(8000 + 2 * 60_000 + 1000); 35 | 36 | expect(fetchRawSockets.callCount).to.equal(1); 37 | expect(disconnect.callCount).to.equal(2); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /test/tests/unit/ws/server.test.ts: -------------------------------------------------------------------------------- 1 | import * as sinon from 'sinon'; 2 | import * as td from 'testdouble'; 3 | import { expect } from 'chai'; 4 | 5 | describe('ws server', () => { 6 | let initWsServer: () => any, getWsServer: () => any; 7 | 8 | const sandbox = sinon.createSandbox(); 9 | const redisClient = { 10 | duplicate: () => redisClient, 11 | connect: sandbox.stub(), 12 | xAdd: sandbox.stub().resolves(null), 13 | xRange: sandbox.stub().resolves([]), 14 | pExpire: sandbox.stub().resolves(null), 15 | json: { 16 | get: sandbox.stub().resolves(null), 17 | set: sandbox.stub().resolves(null), 18 | }, 19 | }; 20 | const disconnect = sandbox.stub(); 21 | const fetchSocketsSocketIo = sandbox.stub(); 22 | const getRedisClient = sandbox.stub().returns(redisClient); 23 | const io = { 24 | adapter: sandbox.stub(), 25 | of: sandbox.stub().returns({ 26 | on: sandbox.stub(), 27 | serverSideEmit: sandbox.stub(), 28 | local: { 29 | fetchSockets: fetchSocketsSocketIo, 30 | }, 31 | }), 32 | }; 33 | 34 | before(async () => { 35 | await td.replaceEsm('socket.io', { Server: sandbox.stub().returns(io) }); 36 | await td.replaceEsm('../../../../src/lib/redis/client.ts', { getRedisClient }); 37 | }); 38 | 39 | beforeEach(async () => { 40 | ({ initWsServer, getWsServer } = await import('../../../../src/lib/ws/server.js')); 41 | fetchSocketsSocketIo.reset(); 42 | 43 | fetchSocketsSocketIo.resolves([ 44 | { data: { probe: { ipAddress: '1.2.3.4', altIpAddresses: [] } }, disconnect }, 45 | { data: { probe: { ipAddress: '1.2.3.4', altIpAddresses: [] } }, disconnect }, 46 | ]); 47 | }); 48 | 49 | it('getWsServer should return the same instance every time', async () => { 50 | await initWsServer(); 51 | const wsServer1 = getWsServer(); 52 | const wsServer2 = getWsServer(); 53 | const wsServer3 = getWsServer(); 54 | expect(wsServer1).to.equal(wsServer2); 55 | expect(wsServer1).to.equal(wsServer3); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /test/types.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-var */ 2 | import type { ExtendedFakeTimers } from './utils/clock.js'; 3 | 4 | export type DeepPartial = T extends Record ? { 5 | [P in keyof T]?: DeepPartial; 6 | } : T; 7 | 8 | declare global { 9 | var clock: ExtendedFakeTimers; 10 | } 11 | -------------------------------------------------------------------------------- /test/utils/clock.ts: -------------------------------------------------------------------------------- 1 | import type { SinonFakeTimers } from 'sinon'; 2 | 3 | export type ExtendedFakeTimers = SinonFakeTimers & { 4 | pause(): ExtendedFakeTimers; 5 | unpause(): ExtendedFakeTimers; 6 | }; 7 | 8 | export const extendSinonClock = (clock: SinonFakeTimers): ExtendedFakeTimers => { 9 | const pause = () => { 10 | // @ts-expect-error need to use the original clearInterval here 11 | clock._clearInterval(clock.attachedInterval); 12 | 13 | return clock; 14 | }; 15 | 16 | const unpause = () => { 17 | pause(); 18 | 19 | // @ts-expect-error need to use the original delta here 20 | const advanceTimeDelta = clock.advanceTimeDelta; 21 | 22 | // @ts-expect-error need to use the original setInterval here 23 | clock._setInterval(() => { 24 | clock.tick(advanceTimeDelta); 25 | }, advanceTimeDelta); 26 | 27 | return clock; 28 | }; 29 | 30 | return Object.assign(clock, { 31 | pause, 32 | unpause, 33 | }) as ExtendedFakeTimers; 34 | }; 35 | -------------------------------------------------------------------------------- /test/utils/nock-geo-ip.ts: -------------------------------------------------------------------------------- 1 | import nock from 'nock'; 2 | import geoIpMocks from '../mocks/nock-geoip.json' with { type: 'json' }; 3 | 4 | type ProviderToMockname = { 5 | ipmap?: keyof(typeof geoIpMocks.ipmap); 6 | ip2location?: keyof(typeof geoIpMocks.ip2location); 7 | maxmind?: keyof(typeof geoIpMocks.maxmind); 8 | ipinfo?: keyof(typeof geoIpMocks.ipinfo); 9 | fastly?: keyof(typeof geoIpMocks.fastly); 10 | }; 11 | 12 | const nockGeoIpProviders = (providersToMockname: ProviderToMockname = {}) => { 13 | const mockNames: Required = { 14 | ipmap: 'default', 15 | ip2location: 'default', 16 | maxmind: 'default', 17 | ipinfo: 'default', 18 | fastly: 'default', 19 | ...providersToMockname, 20 | }; 21 | 22 | nock('https://ipmap-api.ripe.net/v1/locate/').get(/.*/).reply(200, geoIpMocks.ipmap[mockNames.ipmap]); 23 | nock('https://api.ip2location.io').get(/.*/).reply(200, geoIpMocks.ip2location[mockNames.ip2location]); 24 | nock('https://geoip.maxmind.com/geoip/v2.1/city/').get(/.*/).reply(200, geoIpMocks.maxmind[mockNames.maxmind]); 25 | nock('https://ipinfo.io').get(/.*/).reply(200, geoIpMocks.ipinfo[mockNames.ipinfo]); 26 | nock('https://globalping-geoip.global.ssl.fastly.net').get(/.*/).reply(200, geoIpMocks.fastly[mockNames.fastly]); 27 | }; 28 | 29 | export default nockGeoIpProviders; 30 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/strictest/tsconfig", 3 | "compilerOptions": { 4 | "outDir": "./dist", 5 | "lib": [ 6 | "ES2023" 7 | ], 8 | "module": "NodeNext", 9 | "target": "ES2022", 10 | "moduleResolution": "NodeNext", 11 | "resolveJsonModule": true, 12 | "sourceMap": true 13 | }, 14 | "include": [ 15 | "src/**/*", 16 | "test/**/*.ts" 17 | ], 18 | "exclude": [ 19 | "dist", 20 | "public", 21 | "node_modules" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /wallaby.e2e.js: -------------------------------------------------------------------------------- 1 | export default function w (wallaby) { 2 | return { 3 | testFramework: 'mocha', 4 | files: [ 5 | 'config/*', 6 | 'public/v1/*', 7 | 'public/**/*.yaml', 8 | 'test/plugins/**/*', 9 | 'test/e2e/setup.ts', 10 | 'test/e2e/utils.ts', 11 | 'test/e2e/docker.ts', 12 | 'src/**/*.ts', 13 | 'migrations/*', 14 | 'seeds/**/*', 15 | 'knexfile.js', 16 | 'package.json', 17 | ], 18 | tests: [ 19 | 'test/e2e/**/*.test.ts', 20 | ], 21 | 22 | setup (w) { 23 | const path = require('path'); 24 | w.testFramework.addFile(path.resolve(process.cwd(), 'test/e2e/setup.js')); 25 | w.testFramework.timeout(20000); 26 | }, 27 | 28 | env: { 29 | type: 'node', 30 | params: { 31 | env: 'NODE_ENV=test', 32 | }, 33 | }, 34 | compilers: { 35 | '**/*.ts?(x)': wallaby.compilers.typeScript({ 36 | module: 'ESNext', 37 | }), 38 | }, 39 | workers: { restart: true, initial: 1, regular: 1 }, 40 | runMode: 'onsave', 41 | }; 42 | } 43 | -------------------------------------------------------------------------------- /wallaby.js: -------------------------------------------------------------------------------- 1 | import * as path from 'node:path'; 2 | import * as url from 'node:url'; 3 | 4 | export default function w (wallaby) { 5 | const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); 6 | 7 | return { 8 | testFramework: 'mocha', 9 | files: [ 10 | 'public/v1/*', 11 | 'src/**/*.ts', 12 | 'src/**/*.cjs', 13 | 'src/**/*.json', 14 | 'config/*', 15 | 'public/**/*.yaml', 16 | 'seeds/**/*', 17 | 'migrations/**/*', 18 | 'test/utils/**/*.ts', 19 | 'test/mocks/**/*', 20 | 'test/plugins/**/*', 21 | 'test/setup.ts', 22 | 'test/types.ts', 23 | 'package.json', 24 | 'knexfile.js', 25 | 'data/GCP_IP_RANGES.json', 26 | 'data/AWS_IP_RANGES.json', 27 | 'data/DOMAIN_BLACKLIST.json', 28 | 'data/IP_BLACKLIST.json', 29 | 'data/GEONAMES_CITIES.csv', 30 | 'data/LAST_API_COMMIT_HASH.txt', 31 | ], 32 | tests: [ 33 | 'test/tests/integration/**/*.test.ts', 34 | 'test/tests/unit/**/*.test.ts', 35 | ], 36 | setup (w) { 37 | const path = require('path'); 38 | w.testFramework.files.unshift(path.resolve(process.cwd(), 'test/setup.js')); 39 | const mocha = w.testFramework; 40 | mocha.timeout(10000); 41 | }, 42 | env: { 43 | type: 'node', 44 | params: { 45 | runner: '--experimental-specifier-resolution=node --loader ' 46 | + url.pathToFileURL(path.join(__dirname, 'node_modules/testdouble/lib/index.mjs')), 47 | env: 'NODE_ENV=test;TEST_MODE=unit', 48 | }, 49 | }, 50 | compilers: { 51 | '**/*.ts?(x)': wallaby.compilers.typeScript({ 52 | module: 'ESNext', 53 | }), 54 | }, 55 | preprocessors: { 56 | '**/*.ts': file => file.content.replace(/\.ts/g, '.js'), 57 | }, 58 | workers: { restart: true, initial: 1, regular: 1 }, 59 | runMode: 'onsave', 60 | }; 61 | } 62 | --------------------------------------------------------------------------------