├── .github ├── ISSUE_TEMPLATE │ └── bug-report.md └── workflows │ ├── codeql-analysis.yml │ └── e2e.yml ├── .gitignore ├── .gitpod.Dockerfile ├── .gitpod.yml ├── .prettierignore ├── README.md ├── package-lock.json ├── package.json ├── packages ├── clickhouse │ ├── package-lock.json │ ├── package.json │ └── src │ │ ├── events.js │ │ ├── index.js │ │ ├── setup.js │ │ ├── stats │ │ ├── browser │ │ │ └── index.js │ │ ├── fragments.js │ │ ├── index.js │ │ ├── os │ │ │ └── index.js │ │ ├── screen │ │ │ └── index.js │ │ ├── top-pages │ │ │ └── index.js │ │ ├── top-referrers │ │ │ └── index.js │ │ ├── total-pageviews │ │ │ └── index.js │ │ ├── unique-visitors │ │ │ └── index.js │ │ ├── visitors │ │ │ └── index.js │ │ └── world-map │ │ │ ├── code-name-map.json │ │ │ └── index.js │ │ └── tests.js ├── events-script │ ├── .gitignore │ ├── README.md │ ├── package-lock.json │ ├── package.json │ └── src │ │ └── ya.js └── faunadb │ ├── package-lock.json │ ├── package.json │ └── src │ ├── domain-db │ ├── admin.js │ ├── index.js │ └── settings.js │ ├── index.js │ └── root │ ├── index.js │ └── users.js └── services ├── admin-api ├── .env ├── Dockerfile ├── README.md ├── cloudbuild.yaml ├── package-lock.json ├── package.json └── src │ ├── index.js │ ├── middlewares │ ├── index.js │ └── magic.js │ ├── routes │ ├── index.js │ ├── tests │ │ ├── db-reset.js │ │ ├── domain-db.js │ │ ├── index.js │ │ └── user.js │ ├── user.js │ └── website.js │ └── utils │ └── cookies.js ├── db-analytics ├── .gitignore ├── README.md ├── dev │ ├── Dockerfile │ ├── db │ │ └── .keep │ └── sql │ │ └── 1-init.sql └── package.json ├── events-api ├── .env ├── Dockerfile ├── README.md ├── cloudbuild.yaml ├── geo-db │ ├── Dockerfile │ ├── GeoIP.conf │ └── README.md ├── package-lock.json ├── package.json ├── src │ └── index.js ├── test-event.json └── ya-load-testing │ ├── .editorconfig │ ├── .gitignore │ ├── README.md │ ├── bin │ ├── run │ └── run.cmd │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── api.ts │ ├── index.ts │ └── load-testing.ts │ ├── test │ ├── index.test.ts │ ├── mocha.opts │ └── tsconfig.json │ └── tsconfig.json ├── query-api ├── .env ├── Dockerfile ├── README.md ├── cloudbuild.yaml ├── package-lock.json ├── package.json └── src │ └── index.js └── website ├── .gitignore ├── README.md ├── cypress.json ├── cypress ├── fixtures │ └── example.json ├── integration │ ├── auth.spec.ts │ └── onboarding.spec.ts ├── plugins │ ├── db.ts │ └── index.ts ├── support │ ├── commands.d.ts │ ├── commands.ts │ └── index.ts └── tsconfig.json ├── package-lock.json ├── package.json ├── postcss.config.cjs ├── src ├── api │ ├── onboarding.ts │ ├── settings.ts │ └── stats.ts ├── app.html ├── auth │ └── magic.ts ├── components │ ├── card.svelte │ ├── date-range.svelte │ ├── footer.svelte │ ├── header │ │ ├── index.svelte │ │ ├── nav-item.svelte │ │ ├── nav-mobile-menu-button.svelte │ │ ├── nav-mobile.svelte │ │ └── nav.svelte │ ├── landing-page │ │ ├── feature-card.svelte │ │ ├── features.svelte │ │ ├── frontend-integrations.svelte │ │ ├── hero.svelte │ │ ├── pricing.svelte │ │ ├── section.svelte │ │ ├── stats-card.svelte │ │ └── stats.svelte │ ├── main-content.svelte │ ├── onboarding │ │ ├── step-1.svelte │ │ ├── step-2.svelte │ │ ├── step-3.svelte │ │ ├── step-header.svelte │ │ └── step-wrapper.svelte │ ├── stats │ │ ├── devices.svelte │ │ ├── elements │ │ │ ├── comparison-numbers.svelte │ │ │ └── number.svelte │ │ ├── loading.svelte │ │ ├── top-pages.svelte │ │ ├── top-referrers.svelte │ │ ├── total-pageviews.svelte │ │ ├── unique-visitors.svelte │ │ ├── visitors.svelte │ │ └── world-map.svelte │ ├── table │ │ ├── cell.svelte │ │ ├── index.svelte │ │ └── row.svelte │ ├── timezone-select.svelte │ └── website-select.svelte ├── config.ts ├── hooks.ts ├── routes │ ├── [site] │ │ ├── index.svelte │ │ └── settings.svelte │ ├── __error.svelte │ ├── __layout.svelte │ ├── auth.svelte │ ├── dashboard.svelte │ ├── index.svelte │ ├── onboarding.svelte │ └── websites.svelte ├── service-worker.ts ├── stores │ ├── date-range.ts │ ├── mobile-menu.ts │ ├── onboarding.ts │ ├── stats-filters-query-string.ts │ └── stats-filters-query.ts └── tailwind.css ├── static ├── countries-110m.json ├── favicon.png ├── logo-192.png ├── logo-3d.png ├── logo-512.png ├── logo-color.svg ├── manifest.json ├── robots.txt ├── svg │ ├── clipboard-copy.svg │ ├── cog.svg │ ├── document-report.svg │ ├── frontend-stacks │ │ ├── ghost.svg │ │ ├── react.svg │ │ ├── svelte.svg │ │ └── wordpress.svg │ ├── mail.svg │ └── template.svg └── ya.js ├── svelte.config.js ├── tailwind.config.cjs ├── tsconfig.json └── vercel.json /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots - Optional** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information) - Optional:** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information) - Optional:** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context - Optional** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | # The branches below must be a subset of the branches above 8 | branches: [master] 9 | schedule: 10 | - cron: '0 6 * * 0' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | # Override automatic language detection by changing the below list 21 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 22 | language: ['javascript'] 23 | # Learn more... 24 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 25 | 26 | steps: 27 | - name: Checkout repository 28 | uses: actions/checkout@v2 29 | with: 30 | # We must fetch at least the immediate parents so that if this is 31 | # a pull request then we can checkout the head. 32 | fetch-depth: 2 33 | 34 | # If this run was triggered by a pull request event, then checkout 35 | # the head of the pull request instead of the merge commit. 36 | - run: git checkout HEAD^2 37 | if: ${{ github.event_name == 'pull_request' }} 38 | 39 | # Initializes the CodeQL tools for scanning. 40 | - name: Initialize CodeQL 41 | uses: github/codeql-action/init@v1 42 | with: 43 | languages: ${{ matrix.language }} 44 | 45 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 46 | # If this step fails, then you should remove it and run the build manually (see below) 47 | - name: Autobuild 48 | uses: github/codeql-action/autobuild@v1 49 | 50 | # ℹ️ Command-line programs to run using the OS shell. 51 | # 📚 https://git.io/JvXDl 52 | 53 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 54 | # and modify them (or add more) to build your code if your project 55 | # uses a compiled language 56 | 57 | #- run: | 58 | # make bootstrap 59 | # make release 60 | 61 | - name: Perform CodeQL Analysis 62 | uses: github/codeql-action/analyze@v1 63 | -------------------------------------------------------------------------------- /.github/workflows/e2e.yml: -------------------------------------------------------------------------------- 1 | name: "e2e" 2 | on: [push] 3 | jobs: 4 | e2e: 5 | name: Test end-to-end 6 | runs-on: ubuntu-latest 7 | services: 8 | clickhouse: 9 | image: yandex/clickhouse-server 10 | ports: 11 | - 8123:8123 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-node@v2 15 | with: 16 | node-version: "14" 17 | - name: Install dependencies 18 | run: npm install 19 | - name: Start all services 20 | run: npm run dev & 21 | env: 22 | FAUNADB_ADMIN_SECRET: ${{ secrets.FAUNADB_ADMIN_SECRET }} 23 | FAUNADB_SERVER_SECRET: ${{ secrets.FAUNADB_SERVER_SECRET }} 24 | MAGIC_SECRET_KEY: ${{ secrets.MAGIC_SECRET_KEY }} 25 | NODE_ENV: development 26 | - name: Run Cypress 27 | uses: cypress-io/github-action@v2 28 | with: 29 | install: false 30 | record: true 31 | wait-on: "http://localhost:3000" 32 | wait-on-timeout: 120 33 | working-directory: services/website 34 | env: 35 | CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | NODE_ENV: development 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/local-packages -------------------------------------------------------------------------------- /.gitpod.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitpod/workspace-full-vnc 2 | 3 | # Install custom tools, runtimes, etc. 4 | # For example "bastet", a command-line tetris clone: 5 | # RUN brew install bastet 6 | # 7 | # More information: https://www.gitpod.io/docs/config-docker/ 8 | 9 | RUN brew install gh 10 | 11 | RUN sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 \ 12 | && echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list \ 13 | && sudo apt-get update \ 14 | && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y clickhouse-server clickhouse-client \ 15 | && sudo rm -rf /var/lib/apt/lists/* 16 | 17 | # Install Cypress dependencies 18 | RUN sudo apt-get update \ 19 | && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ 20 | libgtk2.0-0 \ 21 | libgtk-3-0 \ 22 | libnotify-dev \ 23 | libgconf-2-4 \ 24 | libnss3 \ 25 | libxss1 \ 26 | libasound2 \ 27 | libxtst6 \ 28 | xauth \ 29 | xvfb \ 30 | && sudo rm -rf /var/lib/apt/lists/* 31 | 32 | # `gitpod-workspace-full-vnc` may get updated with a newer version of Node 33 | # Let's make sure we stick to what works for us. 34 | RUN curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | PROFILE=/dev/null bash \ 35 | && bash -c ". .nvm/nvm.sh \ 36 | && nvm install 14 \ 37 | && nvm alias default 14" 38 | 39 | RUN echo "nvm use default &>/dev/null" >> ~/.bashrc.d/51-nvm-fix 40 | 41 | # Install Firefox 42 | RUN sudo apt-get update -q \ 43 | && sudo apt-get install -yq \ 44 | firefox \ 45 | && sudo rm -rf /var/lib/apt/lists/* 46 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | image: 2 | file: .gitpod.Dockerfile 3 | 4 | tasks: 5 | - command: | 6 | cd services/website 7 | gp await-port 3000 8 | CYPRESS_BASE_URL=`gp url 3000` npm run cy:run 9 | cd ../events-api/ya-load-testing 10 | ./bin/run -d dev.com 11 | env: 12 | CYPRESS_CACHE_FOLDER: /workspace/.cache/cypress 13 | - init: npm install 14 | command: | 15 | gp await-port 8123 16 | export HMR_HOST=`gp url 3000` 17 | FAUNADB_ADMIN_SECRET=$YA_FAUNADB_ADMIN_SECRET FAUNADB_SERVER_SECRET=$YA_FAUNADB_SERVER_SECRET MAGIC_SECRET_KEY=$YA_MAGIC_SECRET_KEY npm run dev 18 | env: 19 | CYPRESS_CACHE_FOLDER: /workspace/.cache/cypress 20 | - command: cd services/db-analytics/dev/db && clickhouse-server 21 | - command: | 22 | cd services/events-api/ya-load-testing 23 | gp await-port 8082 24 | curl -X POST -H "content-type: application/json" -d '{"url": "your-analytics.org"}' http://localhost:8082/tests/domain-db 25 | ./bin/run -d your-analytics.org 26 | 27 | ports: 28 | - port: 3000 29 | onOpen: ignore 30 | visibility: public 31 | - port: 5900 32 | onOpen: ignore 33 | - port: 6080 34 | onOpen: ignore 35 | - port: 8080-8082 36 | onOpen: ignore 37 | visibility: public 38 | - port: 8123 39 | onOpen: ignore 40 | - port: 9000 41 | onOpen: ignore 42 | - port: 9004 43 | onOpen: ignore 44 | - port: 10000 45 | onOpen: ignore 46 | 47 | github: 48 | prebuilds: 49 | addCheck: false 50 | addBadge: true 51 | branches: true 52 | 53 | vscode: 54 | extensions: 55 | - svelte.svelte-vscode 56 | - bradlc.vscode-tailwindcss 57 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | packages/events-script/public/ya.js 2 | 3 | services/website/__sapper__/ 4 | services/website/static/ya.js -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/mikenikles/your-analytics) 2 | 3 | # your-analytics 4 | 5 | ## Architecture 6 | 7 | Please refer to [#1](../../issues/1). 8 | 9 | ## Development 10 | 11 | Follow the steps outlined in the chapters below, then click the Gitpod link above. 12 | This will open your development environment, install dependencies and start all necessary services. 13 | The web application is available on port 3000. Use the "Open Ports" tab next to the terminal in Gitpod 14 | to open a new browser tab on port 3000. 15 | 16 | ### FaunaDB 17 | 18 | 1. Create a `your-analytics-dev` database at https://fauna.com. 19 | 1. In your Gitpod.io account, set the following environment variables: 20 | - `YA_FAUNADB_ADMIN_SECRET`: [Follow instructions here](./services/admin-api/README.md) 21 | - `YA_FAUNADB_SERVER_SECRET`: [Follow instructions here](./services/admin-api/README.md) 22 | 23 | ### Magic Link 24 | 25 | 1. Create an account at https://magic.link. 26 | 1. In your Gitpod.io account, set the following environment variable: 27 | - `YA_MAGIC_SECRET_KEY`: Your Magic **TEST**\_SECRET_KEY value 28 | 29 | ## Tests 30 | 31 | All test results are publicly available on the [Cypress Dashboard](https://dashboard.cypress.io/projects/gynhxr/runs). 32 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/monorepo", 3 | "version": "1.0.0", 4 | "description": "An analytis platform to collect events, such as from website visits, user behavior in native apps or IoT-enabled devices.", 5 | "main": "index.js", 6 | "scripts": { 7 | "dev": "run-p dev:*", 8 | "dev:admin-api": "npm run dev --prefix services/admin-api", 9 | "dev:events-api": "npm run dev --prefix services/events-api", 10 | "dev:query-api": "npm run dev --prefix services/query-api", 11 | "dev:website": "npm run dev --prefix services/website", 12 | "install:admin-api": "npm i --prefix services/admin-api", 13 | "install:events-api": "npm i --prefix services/events-api", 14 | "install:events-api-load-testing": "npm i --prefix services/events-api/ya-load-testing", 15 | "install:query-api": "npm i --prefix services/query-api", 16 | "install:website": "npm i --prefix services/website", 17 | "postinstall": "run-p install:*", 18 | "test": "echo \"Error: no test specified\" && exit 1" 19 | }, 20 | "husky": { 21 | "hooks": { 22 | "pre-commit": "pretty-quick --staged" 23 | } 24 | }, 25 | "repository": { 26 | "type": "git", 27 | "url": "git+https://github.com/your-analytics-org/your-analytics.git" 28 | }, 29 | "author": "", 30 | "license": "ISC", 31 | "bugs": { 32 | "url": "https://github.com/your-analytics-org/your-analytics/issues" 33 | }, 34 | "homepage": "https://github.com/your-analytics-org/your-analytics#readme", 35 | "devDependencies": { 36 | "husky": "^4.2.5", 37 | "npm-run-all": "^4.1.5", 38 | "prettier": "^2.0.5", 39 | "pretty-quick": "^2.0.1" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /packages/clickhouse/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/clickhouse", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@apla/clickhouse": { 8 | "version": "1.6.4", 9 | "resolved": "https://registry.npmjs.org/@apla/clickhouse/-/clickhouse-1.6.4.tgz", 10 | "integrity": "sha512-zOLL/lOaXrk+l8XS8eDGZfqOy1yKwYQirQtHtyrLxAHlJMNRKBosjK2gVUyCZ94fAIpE8lLUZRxFDid2Vmqxnw==" 11 | }, 12 | "date-fns": { 13 | "version": "2.16.1", 14 | "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.16.1.tgz", 15 | "integrity": "sha512-sAJVKx/FqrLYHAQeN7VpJrPhagZc9R4ImZIWYRFZaaohR3KzmuK88touwsSwSVT8Qcbd4zoDsnGfX4GFB4imyQ==" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/clickhouse/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/clickhouse", 3 | "version": "1.0.0", 4 | "description": "Contains ClickHouse models and common code.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@apla/clickhouse": "^1.6.4", 13 | "date-fns": "^2.16.1" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /packages/clickhouse/src/events.js: -------------------------------------------------------------------------------- 1 | const { convertUrlToDbName } = require("./setup"); 2 | 3 | const insertBuffers = {}; 4 | let flushIntervalId; 5 | 6 | const recordEvent = (ch) => (event) => { 7 | if (!insertBuffers[event.domain]) { 8 | insertBuffers[event.domain] = []; 9 | } 10 | insertBuffers[event.domain].push(event); 11 | 12 | if (!flushIntervalId) { 13 | const flushBuffer = () => { 14 | Object.entries(insertBuffers).forEach(([domain, events]) => { 15 | if (events.length > 0) { 16 | console.log(`Flushing ${events.length} events for domain ${domain}`); 17 | } 18 | const dbName = convertUrlToDbName(domain); 19 | const writableStream = ch.query( 20 | `INSERT INTO ${dbName}.events`, 21 | { 22 | format: "JSONEachRow", 23 | queryOptions: { 24 | input_format_import_nested_json: 1, 25 | }, 26 | }, 27 | (err) => { 28 | if (err) { 29 | console.error(err); 30 | } 31 | } 32 | ); 33 | 34 | while (events.length > 0) { 35 | writableStream.write(events.shift()); 36 | } 37 | writableStream.end(); 38 | }); 39 | }; 40 | // TODO: Clean this up. Flush also when there are 10k events. 41 | // TODO: Consider intervals for individual domains and randomizing the delay 42 | // to spread DB writes over a few seconds. 43 | flushIntervalId = setInterval(flushBuffer, 5000); 44 | } 45 | }; 46 | 47 | module.exports = { 48 | recordEvent, 49 | }; 50 | -------------------------------------------------------------------------------- /packages/clickhouse/src/index.js: -------------------------------------------------------------------------------- 1 | const ClickHouse = require("@apla/clickhouse"); 2 | const { recordEvent } = require("./events"); 3 | const { addNewWebsite, convertUrlToDbName } = require("./setup"); 4 | const stats = require("./stats"); 5 | const { deleteWebsite } = require("./tests"); 6 | 7 | const ch = new ClickHouse({ 8 | host: process.env.CH_HOST, 9 | port: process.env.CH_PORT, 10 | user: process.env.CH_USER, 11 | password: process.env.CH_PASSWORD, 12 | // Uncomment the protocol if we give ClickHouse Cloud a try again in the future. 13 | // See https://github.com/mikenikles/your-analytics/issues/299 14 | // protocol: process.env.NODE_ENV === "production" ? "https:" : "http:", 15 | }); 16 | 17 | module.exports = { 18 | addNewWebsite: addNewWebsite(ch), 19 | convertUrlToDbName, 20 | recordEvent: recordEvent(ch), 21 | stats: stats(ch), 22 | deleteWebsite: deleteWebsite(ch), 23 | }; 24 | -------------------------------------------------------------------------------- /packages/clickhouse/src/setup.js: -------------------------------------------------------------------------------- 1 | const createDb = (ch, dbName) => 2 | ch.querying(`CREATE DATABASE IF NOT EXISTS ${dbName}`); 3 | 4 | const createEventsTable = (ch, dbName) => 5 | ch.querying(`CREATE TABLE IF NOT EXISTS ${dbName}.events ( 6 | browser_major LowCardinality(String), 7 | browser_name LowCardinality(String), 8 | browser_version LowCardinality(String), 9 | device_model LowCardinality(String), 10 | device_type LowCardinality(String), 11 | device_vendor LowCardinality(String), 12 | domain String, 13 | geo_city String, 14 | geo_country LowCardinality(FixedString(2)), 15 | geo_lat Float64, 16 | geo_long Float64, 17 | hostname String, 18 | name String, 19 | os_name LowCardinality(String), 20 | os_version LowCardinality(String), 21 | path String, 22 | referrer String, 23 | screen_size UInt16, 24 | session_id UInt64, 25 | timestamp DateTime, 26 | user_id UInt64 27 | ) ENGINE = MergeTree() 28 | PARTITION BY toYYYYMM(timestamp) 29 | ORDER BY (name, user_id, timestamp) 30 | SETTINGS index_granularity = 8192;`); 31 | 32 | const convertUrlToDbName = (url) => url.replace(/\./g, "__").replace(/-/g, "_"); 33 | 34 | const addNewWebsite = (ch) => async (url) => { 35 | const dbName = convertUrlToDbName(url); 36 | await createDb(ch, dbName); 37 | await createEventsTable(ch, dbName); 38 | }; 39 | 40 | module.exports = { 41 | addNewWebsite, 42 | convertUrlToDbName, 43 | }; 44 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/browser/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchBrowser = (ch) => async (dateRange, domain, websiteSettings) => { 4 | const { chDbName, timezone } = websiteSettings; 5 | const sql = `SELECT COALESCE(NULLIF(browser_name, ''), 'Unknown') AS browser, COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 6 | dateRange, 7 | timezone 8 | )} AND domain = '${domain}' GROUP BY browser ORDER BY total DESC`; 9 | const stream = ch.query(sql); 10 | 11 | return new Promise((resolve, reject) => { 12 | const result = {}; 13 | stream.on("error", (error) => reject(error)); 14 | 15 | stream.on("data", (row) => { 16 | // row: [browser, total] 17 | result[row[0]] = row[1] * 1; 18 | }); 19 | 20 | stream.on("end", () => { 21 | resolve(result); 22 | }); 23 | }); 24 | }; 25 | 26 | module.exports = { 27 | fetchBrowser, 28 | }; 29 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/fragments.js: -------------------------------------------------------------------------------- 1 | const { formatISO } = require("date-fns"); 2 | 3 | const formatISODate = (date) => formatISO(date, { representation: "date" }); 4 | 5 | const getDateRange = (dateRange, timezone) => 6 | `toDateTime(timestamp, '${timezone}') >= toStartOfDay(toDate('${formatISODate( 7 | dateRange.from 8 | )}', '${timezone}'), '${timezone}') AND toDateTime(timestamp, '${timezone}') < toStartOfDay(addDays(toDate('${formatISODate( 9 | dateRange.to 10 | )}', '${timezone}'), 1), '${timezone}')`; 11 | 12 | module.exports = { 13 | getDateRange, 14 | formatISODate, 15 | }; 16 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/index.js: -------------------------------------------------------------------------------- 1 | const { fetchBrowser } = require("./browser"); 2 | const { fetchOs } = require("./os"); 3 | const { fetchScreen } = require("./screen"); 4 | const { fetchTopPages } = require("./top-pages"); 5 | const { fetchTopReferrers } = require("./top-referrers"); 6 | const { fetchTotalPageviews } = require("./total-pageviews"); 7 | const { fetchUniqueVisitors } = require("./unique-visitors"); 8 | const { fetchVisitors } = require("./visitors"); 9 | const { fetchWorldMap } = require("./world-map"); 10 | 11 | module.exports = (ch) => ({ 12 | fetchBrowser: fetchBrowser(ch), 13 | fetchOs: fetchOs(ch), 14 | fetchScreen: fetchScreen(ch), 15 | fetchTopPages: fetchTopPages(ch), 16 | fetchTopReferrers: fetchTopReferrers(ch), 17 | fetchTotalPageviews: fetchTotalPageviews(ch), 18 | fetchUniqueVisitors: fetchUniqueVisitors(ch), 19 | fetchVisitors: fetchVisitors(ch), 20 | fetchWorldMap: fetchWorldMap(ch), 21 | }); 22 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/os/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchOs = (ch) => async (dateRange, domain, websiteSettings) => { 4 | const { chDbName, timezone } = websiteSettings; 5 | const sql = `SELECT COALESCE(NULLIF(os_name, ''), 'Unknown') AS os, COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 6 | dateRange, 7 | timezone 8 | )} AND domain = '${domain}' GROUP BY os ORDER BY total DESC`; 9 | const stream = ch.query(sql); 10 | 11 | return new Promise((resolve, reject) => { 12 | const result = {}; 13 | stream.on("error", (error) => reject(error)); 14 | 15 | stream.on("data", (row) => { 16 | // row: [os, total] 17 | result[row[0]] = row[1] * 1; 18 | }); 19 | 20 | stream.on("end", () => { 21 | resolve(result); 22 | }); 23 | }); 24 | }; 25 | 26 | module.exports = { 27 | fetchOs, 28 | }; 29 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/screen/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchScreen = (ch) => async (dateRange, domain, websiteSettings) => { 4 | const { chDbName, timezone } = websiteSettings; 5 | const sql = `SELECT screen_size, COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 6 | dateRange, 7 | timezone 8 | )} AND domain = '${domain}' GROUP BY screen_size ORDER BY total DESC`; 9 | const stream = ch.query(sql); 10 | return new Promise((resolve, reject) => { 11 | const result = { 12 | mobile: 0, 13 | tablet: 0, 14 | laptop: 0, 15 | desktop: 0, 16 | }; 17 | stream.on("error", (error) => reject(error)); 18 | 19 | stream.on("data", (row) => { 20 | // row: [screen_size, total] 21 | 22 | // TODO: Can this be done in SQL? 23 | if (row[0] <= 640) { 24 | result.mobile += row[1] * 1; 25 | } else if (row[0] <= 768) { 26 | result.tablet += row[1] * 1; 27 | } else if (row[0] <= 1024) { 28 | result.laptop += row[1] * 1; 29 | } else { 30 | result.desktop += row[1] * 1; 31 | } 32 | }); 33 | 34 | stream.on("end", () => { 35 | const sorted = []; 36 | for (let screenCategory in result) { 37 | sorted.push([screenCategory, result[screenCategory]]); 38 | } 39 | sorted.sort((a, b) => b[1] - a[1]); 40 | resolve(sorted); 41 | }); 42 | }); 43 | }; 44 | 45 | module.exports = { 46 | fetchScreen, 47 | }; 48 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/top-pages/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchTopPages = (ch) => async (dateRange, domain, websiteSettings) => { 4 | const { chDbName, timezone } = websiteSettings; 5 | const sql = `SELECT path, COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 6 | dateRange, 7 | timezone 8 | )} AND domain = '${domain}' GROUP BY path ORDER BY total DESC`; 9 | const stream = ch.query(sql); 10 | 11 | return new Promise((resolve, reject) => { 12 | const result = {}; 13 | stream.on("error", (error) => reject(error)); 14 | 15 | stream.on("data", (row) => { 16 | // row: [path, total] 17 | result[row[0]] = row[1] * 1; 18 | }); 19 | 20 | stream.on("end", () => { 21 | resolve(result); 22 | }); 23 | }); 24 | }; 25 | 26 | module.exports = { 27 | fetchTopPages, 28 | }; 29 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/top-referrers/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchTopReferrers = (ch) => async ( 4 | dateRange, 5 | domain, 6 | websiteSettings 7 | ) => { 8 | const { chDbName, timezone } = websiteSettings; 9 | const sql = `SELECT extract(referrer, '^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n?]+)') AS referrer_domain, COUNT(*) AS total FROM ${chDbName}.events WHERE referrer != '' AND ${getDateRange( 10 | dateRange, 11 | timezone 12 | )} AND domain = '${domain}' GROUP BY referrer_domain ORDER BY total DESC`; 13 | const stream = ch.query(sql); 14 | 15 | return new Promise((resolve, reject) => { 16 | const result = {}; 17 | stream.on("error", (error) => reject(error)); 18 | 19 | stream.on("data", (row) => { 20 | // row: [referrer_domain, total] 21 | result[row[0]] = row[1] * 1; 22 | }); 23 | 24 | stream.on("end", () => { 25 | resolve(result); 26 | }); 27 | }); 28 | }; 29 | 30 | module.exports = { 31 | fetchTopReferrers, 32 | }; 33 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/total-pageviews/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchTotalPageviews = (ch) => async ( 4 | dateRange, 5 | domain, 6 | websiteSettings 7 | ) => { 8 | const { chDbName, timezone } = websiteSettings; 9 | const sql = `SELECT COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 10 | dateRange, 11 | timezone 12 | )} AND domain = '${domain}'`; 13 | 14 | const stream = ch.query(sql); 15 | 16 | return new Promise((resolve, reject) => { 17 | let result = 0; 18 | stream.on("error", (error) => reject(error)); 19 | 20 | stream.on("data", (row) => { 21 | // row: [total] 22 | result = row[0] * 1; 23 | }); 24 | 25 | stream.on("end", () => { 26 | resolve(result); 27 | }); 28 | }); 29 | }; 30 | 31 | module.exports = { 32 | fetchTotalPageviews, 33 | }; 34 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/unique-visitors/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | 3 | const fetchUniqueVisitors = (ch) => async ( 4 | dateRange, 5 | domain, 6 | websiteSettings 7 | ) => { 8 | const { chDbName, timezone } = websiteSettings; 9 | const sql = `SELECT COUNT(DISTINCT user_id) AS total FROM ${chDbName}.events WHERE ${getDateRange( 10 | dateRange, 11 | timezone 12 | )} AND domain = '${domain}'`; 13 | 14 | const stream = ch.query(sql); 15 | 16 | return new Promise((resolve, reject) => { 17 | let result = 0; 18 | stream.on("error", (error) => reject(error)); 19 | 20 | stream.on("data", (row) => { 21 | // row: [total] 22 | result = row[0] * 1; 23 | }); 24 | 25 | stream.on("end", () => { 26 | resolve(result); 27 | }); 28 | }); 29 | }; 30 | 31 | module.exports = { 32 | fetchUniqueVisitors, 33 | }; 34 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/visitors/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange, formatISODate } = require("../fragments"); 2 | 3 | const ONE_DAY = 1000 * 60 * 60 * 24; 4 | const ONE_MONTH = ONE_DAY * 31; 5 | const ONE_YEAR = ONE_MONTH * 12; 6 | const MONTHS = { 7 | 1: "January", 8 | 2: "February", 9 | 3: "March", 10 | 4: "April", 11 | 5: "May", 12 | 6: "June", 13 | 7: "July", 14 | 8: "August", 15 | 9: "September", 16 | 10: "October", 17 | 11: "November", 18 | 12: "December", 19 | }; 20 | 21 | /** 22 | * Each `dateRange.to` value represents 23:59:59 on a given day. 23 | * The `+ 1` in the calculations below are there to represent a full day. 24 | * 25 | * Note: Use a `format` that can be used in an `ORDER BY` query to naturally 26 | * sort the results, e.g. "2020 08". Use `getLabel` to turn that into a label 27 | * that is displayed in the UI. 28 | */ 29 | const dateRangeOptions = [ 30 | { 31 | test: ({ from, to }) => to - from + 1 === ONE_DAY, 32 | format: "%H", 33 | period: "Hour", 34 | getLabel: (valueString) => { 35 | const value = valueString * 1; 36 | return value < 12 37 | ? `${value === 0 ? 12 : value}am` 38 | : `${value === 12 ? 12 : value - 12}pm`; 39 | }, 40 | }, 41 | { 42 | test: ({ from, to }) => 43 | to - from + 1 > ONE_DAY && to - from + 1 <= ONE_MONTH, 44 | format: "%F", 45 | period: "Day", 46 | getLabel: (value) => value, 47 | }, 48 | { 49 | test: ({ from, to }) => 50 | to - from + 1 > ONE_MONTH && to - from + 1 <= ONE_YEAR, 51 | format: "%Y %m", 52 | period: "Month", 53 | getLabel: (value) => 54 | `${MONTHS[value.substring(5) * 1]} ${value.substring(0, 4)}`, // `* 1` to convert "01" to 1 55 | }, 56 | { 57 | test: ({ from, to }) => to - from + 1 > ONE_YEAR, 58 | format: "%Y", 59 | period: "Year", 60 | getLabel: (value) => value, 61 | }, 62 | { 63 | test: () => true, 64 | format: "%F", 65 | getLabel: (value) => value, 66 | }, 67 | ]; 68 | 69 | const fetchVisitors = (ch) => async (dateRange, domain, websiteSettings) => { 70 | const { chDbName, timezone } = websiteSettings; 71 | const { format, getLabel, period } = dateRangeOptions.find( 72 | (dateRangeOption) => 73 | dateRangeOption.test({ 74 | from: dateRange.from.getTime(), 75 | to: dateRange.to.getTime(), 76 | }) 77 | ); 78 | 79 | const sql = ` 80 | SELECT t, SUM(total) AS total FROM ( 81 | SELECT 82 | arrayJoin( 83 | arrayMap( x -> formatDateTime(add${period}s(toDateTime('${formatISODate( 84 | dateRange.from 85 | )} 00:00:00', '${timezone}'), x), '${format}', '${timezone}'), 86 | range(toUInt64( 87 | dateDiff('${period.toLowerCase()}', 88 | toDateTime('${formatISODate( 89 | dateRange.from 90 | )} 00:00:00', '${timezone}'), 91 | toDateTime(toStartOfDay(addDays(toDate('${formatISODate( 92 | dateRange.to 93 | )}', '${timezone}'), 1), '${timezone}')))))) 94 | ) as t, 95 | 0 as total 96 | 97 | UNION ALL 98 | 99 | SELECT 100 | formatDateTime(timestamp, '${format}', '${timezone}') as t, 101 | COUNT(DISTINCT user_id) as total 102 | FROM ${chDbName}.events 103 | WHERE ${getDateRange(dateRange, timezone)} 104 | AND domain = '${domain}' 105 | GROUP BY t 106 | ) 107 | GROUP BY t ORDER BY t 108 | `; 109 | const stream = ch.query(sql); 110 | 111 | return new Promise((resolve, reject) => { 112 | const result = {}; 113 | stream.on("error", (error) => reject(error)); 114 | 115 | stream.on("data", (row) => { 116 | // row: [label, total] 117 | result[getLabel(row[0])] = row[1] * 1; 118 | }); 119 | 120 | stream.on("end", () => { 121 | resolve(result); 122 | }); 123 | }); 124 | }; 125 | 126 | module.exports = { 127 | fetchVisitors, 128 | }; 129 | -------------------------------------------------------------------------------- /packages/clickhouse/src/stats/world-map/index.js: -------------------------------------------------------------------------------- 1 | const { getDateRange } = require("../fragments"); 2 | const countryCodeToNameMap = require("./code-name-map.json"); 3 | 4 | const getCountryNameByCode = (code) => 5 | countryCodeToNameMap[code] ? countryCodeToNameMap[code] : code; 6 | 7 | const fetchWorldMap = (ch) => async (dateRange, domain, websiteSettings) => { 8 | const { chDbName, timezone } = websiteSettings; 9 | const sql = `SELECT geo_country, COUNT(*) AS total FROM ${chDbName}.events WHERE ${getDateRange( 10 | dateRange, 11 | timezone 12 | )} AND domain = '${domain}' GROUP BY geo_country ORDER BY total DESC`; 13 | const stream = ch.query(sql); 14 | 15 | return new Promise((resolve, reject) => { 16 | const result = {}; 17 | stream.on("error", (error) => reject(error)); 18 | 19 | stream.on("data", (row) => { 20 | // row: [geo_country, total] 21 | const countryName = countryCodeToNameMap[row[0]] 22 | ? countryCodeToNameMap[row[0]] 23 | : row[0]; 24 | result[countryName] = row[1] * 1; 25 | }); 26 | 27 | stream.on("end", () => { 28 | resolve(result); 29 | }); 30 | }); 31 | }; 32 | 33 | module.exports = { 34 | fetchWorldMap, 35 | }; 36 | -------------------------------------------------------------------------------- /packages/clickhouse/src/tests.js: -------------------------------------------------------------------------------- 1 | const { convertUrlToDbName } = require("./setup"); 2 | 3 | const deleteDb = (ch, dbName) => 4 | ch.querying(`DROP DATABASE IF EXISTS ${dbName}`); 5 | 6 | const deleteWebsite = (ch) => async (url) => { 7 | const dbName = convertUrlToDbName(url); 8 | await deleteDb(ch, dbName); 9 | }; 10 | 11 | module.exports = { 12 | deleteWebsite, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/events-script/.gitignore: -------------------------------------------------------------------------------- 1 | public 2 | -------------------------------------------------------------------------------- /packages/events-script/README.md: -------------------------------------------------------------------------------- 1 | # events-script 2 | 3 | The JavaScript events script file loaded on websites that track events. 4 | -------------------------------------------------------------------------------- /packages/events-script/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/events-script", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "commander": { 8 | "version": "2.13.0", 9 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz", 10 | "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA==", 11 | "dev": true 12 | }, 13 | "source-map": { 14 | "version": "0.6.1", 15 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", 16 | "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", 17 | "dev": true 18 | }, 19 | "uglify-es": { 20 | "version": "3.3.9", 21 | "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz", 22 | "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==", 23 | "dev": true, 24 | "requires": { 25 | "commander": "~2.13.0", 26 | "source-map": "~0.6.1" 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/events-script/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/events-script", 3 | "version": "1.0.0", 4 | "description": "The JavaScript events script file loaded on websites that track events.", 5 | "main": "src/ya.js", 6 | "scripts": { 7 | "build": "npm run clean && mkdir ./public && uglifyjs --compress --mangle -o ./public/ya.js src/ya.js", 8 | "clean": "rm -fr ./public", 9 | "deploy": "npm run build && cp ./public/ya.js ../../services/website/static/", 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "author": "", 13 | "license": "ISC", 14 | "devDependencies": { 15 | "uglify-es": "^3.3.9" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /packages/events-script/src/ya.js: -------------------------------------------------------------------------------- 1 | (() => { 2 | try { 3 | const sendEvent = () => { 4 | if (navigator.doNotTrack || navigator.msDoNotTrack || window.doNotTrack) { 5 | return; 6 | } 7 | fetch("https://events-api.your-analytics.org", { 8 | method: "POST", 9 | body: JSON.stringify({ 10 | name: "pageview", 11 | domain: document 12 | .querySelector('[src*="your-analytics.org"]') 13 | .getAttribute("data-domain"), 14 | url: 15 | location.protocol + 16 | "//" + 17 | location.hostname + 18 | location.pathname + 19 | location.search, 20 | // session_id: 9, 21 | referrer: document.referrer, 22 | screen_size: window.innerWidth, 23 | }), 24 | credentials: "omit", 25 | cache: "no-store", 26 | mode: "no-cors", 27 | headers: { 28 | "Content-Type": "text/plain", 29 | }, 30 | }); 31 | }; 32 | 33 | const originalHistory = window.history; 34 | if (originalHistory.pushState) { 35 | const originalPushState = originalHistory["pushState"]; 36 | originalHistory.pushState = function () { 37 | originalPushState.apply(this, arguments); 38 | sendEvent(); 39 | }; 40 | window.addEventListener("popstate", () => { 41 | sendEvent(); 42 | }); 43 | } 44 | sendEvent(); 45 | } catch (error) { 46 | new Image().src = `https://events-api.your-analytics.org/error?m=${encodeURIComponent( 47 | error.message 48 | )}`; 49 | } 50 | })(); 51 | -------------------------------------------------------------------------------- /packages/faunadb/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/faunadb", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "base64-js": { 8 | "version": "1.3.1", 9 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", 10 | "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" 11 | }, 12 | "btoa-lite": { 13 | "version": "1.0.0", 14 | "resolved": "https://registry.npmjs.org/btoa-lite/-/btoa-lite-1.0.0.tgz", 15 | "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=" 16 | }, 17 | "cross-fetch": { 18 | "version": "3.0.5", 19 | "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.5.tgz", 20 | "integrity": "sha512-FFLcLtraisj5eteosnX1gf01qYDCOc4fDy0+euOt8Kn9YBY2NtXL/pCoYPavw24NIQkQqm5ZOLsGD5Zzj0gyew==", 21 | "requires": { 22 | "node-fetch": "2.6.0" 23 | } 24 | }, 25 | "dotenv": { 26 | "version": "8.2.0", 27 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", 28 | "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" 29 | }, 30 | "faunadb": { 31 | "version": "3.0.1", 32 | "resolved": "https://registry.npmjs.org/faunadb/-/faunadb-3.0.1.tgz", 33 | "integrity": "sha512-WlfPjC0V9xHs4NTunOWmYZtJfbJ45Z1VAIKKka6+mRrmijWOFQzJVDY9CqS6X9kvepM36EjmtNkIvV0OJ1wTEA==", 34 | "requires": { 35 | "base64-js": "^1.2.0", 36 | "btoa-lite": "^1.0.0", 37 | "cross-fetch": "^3.0.4", 38 | "dotenv": "^8.2.0", 39 | "fn-annotate": "^1.1.3", 40 | "object-assign": "^4.1.0", 41 | "url-parse": "^1.4.7", 42 | "util-deprecate": "^1.0.2" 43 | } 44 | }, 45 | "fn-annotate": { 46 | "version": "1.2.0", 47 | "resolved": "https://registry.npmjs.org/fn-annotate/-/fn-annotate-1.2.0.tgz", 48 | "integrity": "sha1-KNoAARfephhC/mHzU/Qc9Mk6en4=" 49 | }, 50 | "node-fetch": { 51 | "version": "2.6.0", 52 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", 53 | "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" 54 | }, 55 | "object-assign": { 56 | "version": "4.1.1", 57 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 58 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" 59 | }, 60 | "querystringify": { 61 | "version": "2.2.0", 62 | "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", 63 | "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" 64 | }, 65 | "requires-port": { 66 | "version": "1.0.0", 67 | "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", 68 | "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" 69 | }, 70 | "url-parse": { 71 | "version": "1.4.7", 72 | "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", 73 | "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", 74 | "requires": { 75 | "querystringify": "^2.1.1", 76 | "requires-port": "^1.0.0" 77 | } 78 | }, 79 | "util-deprecate": { 80 | "version": "1.0.2", 81 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 82 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /packages/faunadb/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/faunadb", 3 | "version": "1.0.0", 4 | "description": "Contains FaunaDB models and common code.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "faunadb": "^3.0.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/faunadb/src/domain-db/admin.js: -------------------------------------------------------------------------------- 1 | const faunadb = require("faunadb"); 2 | 3 | const q = faunadb.query; 4 | 5 | const addNewWebsite = (adminClient) => (url) => 6 | adminClient.query( 7 | q.CreateDatabase({ 8 | name: url, 9 | }) 10 | ); 11 | 12 | const createCollection = (websiteClient) => (name) => 13 | websiteClient.query( 14 | q.CreateCollection({ 15 | name, 16 | }) 17 | ); 18 | 19 | const createWebsiteServerKey = (adminClient) => (url) => 20 | adminClient.query( 21 | q.CreateKey({ 22 | database: q.Database(url), 23 | role: "server", 24 | }) 25 | ); 26 | 27 | const deleteWebsite = (adminClient) => (url) => 28 | adminClient.query(q.Delete(q.Database(url))); 29 | 30 | module.exports = { 31 | addNewWebsite, 32 | createCollection, 33 | createWebsiteServerKey, 34 | deleteWebsite, 35 | }; 36 | -------------------------------------------------------------------------------- /packages/faunadb/src/domain-db/index.js: -------------------------------------------------------------------------------- 1 | const faunadb = require("faunadb"); 2 | 3 | const { 4 | addNewWebsite, 5 | createCollection, 6 | createWebsiteServerKey, 7 | deleteWebsite, 8 | } = require("./admin"); 9 | 10 | const { 11 | getSettings, 12 | getSettingsPublic, 13 | getVisibility, 14 | insertSettings, 15 | setVisibility, 16 | } = require("./settings"); 17 | 18 | module.exports = (adminClient) => ({ 19 | admin: { 20 | addNewWebsite: addNewWebsite(adminClient), 21 | createCollection: (websiteServerKeySecret) => 22 | createCollection( 23 | new faunadb.Client({ 24 | secret: websiteServerKeySecret, 25 | }) 26 | ), 27 | createWebsiteServerKey: createWebsiteServerKey(adminClient), 28 | deleteWebsite: deleteWebsite(adminClient), 29 | }, 30 | settings: { 31 | getSettings: (websiteServerKeySecret) => 32 | getSettings( 33 | new faunadb.Client({ 34 | secret: websiteServerKeySecret, 35 | }) 36 | ), 37 | getSettingsPublic: getSettingsPublic(adminClient), 38 | getVisibility: getVisibility(adminClient), 39 | insertSettings: (websiteServerKeySecret) => 40 | insertSettings( 41 | new faunadb.Client({ 42 | secret: websiteServerKeySecret, 43 | }) 44 | ), 45 | setVisibility: (websiteServerKeySecret) => 46 | setVisibility( 47 | new faunadb.Client({ 48 | secret: websiteServerKeySecret, 49 | }) 50 | ), 51 | }, 52 | }); 53 | -------------------------------------------------------------------------------- /packages/faunadb/src/domain-db/settings.js: -------------------------------------------------------------------------------- 1 | const faunadb = require("faunadb"); 2 | 3 | const q = faunadb.query; 4 | 5 | const insertSettings = (websiteClient) => (data) => 6 | websiteClient.query( 7 | q.Insert(q.Ref(q.Collection("settings"), "1"), 1, "create", { 8 | data, 9 | }) 10 | ); 11 | 12 | const getSettings = (websiteClient) => () => 13 | websiteClient.query(q.Get(q.Ref(q.Collection("settings"), "1"))); 14 | 15 | const getSettingsPublic = (adminClient) => (url) => 16 | adminClient.query( 17 | q.Get(q.Ref(q.Collection("settings", q.Database(url)), "1")) 18 | ); 19 | 20 | const getVisibility = (adminClient) => (url) => 21 | adminClient.query( 22 | q.Get(q.Ref(q.Collection("settings", q.Database(url)), "1")) 23 | ); 24 | 25 | const setVisibility = (websiteClient) => (visibility) => 26 | websiteClient.query( 27 | q.Update(q.Ref(q.Collection("settings"), "1"), { 28 | data: { 29 | visibility, 30 | }, 31 | }) 32 | ); 33 | 34 | module.exports = { 35 | insertSettings, 36 | getSettings, 37 | getSettingsPublic, 38 | getVisibility, 39 | setVisibility, 40 | }; 41 | -------------------------------------------------------------------------------- /packages/faunadb/src/index.js: -------------------------------------------------------------------------------- 1 | const faunadb = require("faunadb"); 2 | 3 | const domainDb = require("./domain-db"); 4 | const root = require("./root"); 5 | 6 | const adminClient = new faunadb.Client({ 7 | secret: process.env.FAUNADB_ADMIN_SECRET, 8 | }); 9 | 10 | const serverClient = new faunadb.Client({ 11 | secret: process.env.FAUNADB_SERVER_SECRET, 12 | }); 13 | 14 | module.exports = { 15 | rootDb: root(serverClient), 16 | domainDb: domainDb(adminClient), 17 | }; 18 | -------------------------------------------------------------------------------- /packages/faunadb/src/root/index.js: -------------------------------------------------------------------------------- 1 | const { 2 | createUser, 3 | updateUser, 4 | deleteUser, 5 | findUser, 6 | findUserWithAllData, 7 | setLastLoginAt, 8 | addNewWebsiteServerKey, 9 | getDomainServerKeySecret, 10 | } = require("./users"); 11 | 12 | module.exports = (serverClient) => ({ 13 | users: { 14 | create: createUser(serverClient), 15 | update: updateUser(serverClient), 16 | delete: deleteUser(serverClient), 17 | find: findUser(serverClient), 18 | findUserWithAllData: findUserWithAllData(serverClient), 19 | setLastLoginAt: setLastLoginAt(serverClient), 20 | addNewWebsiteServerKey: addNewWebsiteServerKey(serverClient), 21 | getDomainServerKeySecret: getDomainServerKeySecret(serverClient), 22 | }, 23 | }); 24 | -------------------------------------------------------------------------------- /packages/faunadb/src/root/users.js: -------------------------------------------------------------------------------- 1 | const crypto = require("crypto"); 2 | const faunadb = require("faunadb"); 3 | 4 | const q = faunadb.query; 5 | 6 | const prepareUserForPublic = async (user) => { 7 | const { firstName, email, issuer, sites } = user; 8 | return { 9 | sites: Object.keys(sites).reduce((result, site) => { 10 | result[site] = {}; 11 | // Extend the empty object with site values that are meant to be public 12 | return result; 13 | }, {}), 14 | issuer, 15 | firstName, 16 | email, 17 | emailHash: crypto.createHash("md5").update(email).digest("hex"), 18 | }; 19 | }; 20 | 21 | const createUser = (serverClient) => (user) => 22 | serverClient.query( 23 | q.Create(q.Collection("users"), { 24 | data: { ...user, sites: {} }, 25 | }) 26 | ); 27 | 28 | const updateUser = (serverClient) => (issuer, updates) => 29 | serverClient.query( 30 | q.Update( 31 | q.Select( 32 | ["data", 0], 33 | q.Paginate(q.Match(q.Index("user_by_issuer"), issuer)) 34 | ), 35 | { 36 | data: { 37 | // Whitelist properties that are save to be updated 38 | firstName: updates.firstName, 39 | }, 40 | } 41 | ) 42 | ); 43 | 44 | const deleteUser = (serverClient) => (issuer) => 45 | serverClient.query( 46 | q.Delete(q.Select("ref", q.Get(q.Match(q.Index("user_by_issuer"), issuer)))) 47 | ); 48 | 49 | const findUserWithAllData = (serverClient) => async (issuer) => { 50 | const response = await serverClient.query( 51 | q.Paginate(q.Match(q.Index("user_by_issuer"), issuer)) 52 | ); 53 | 54 | if (response.data.length > 1) { 55 | const error = new Error(`More than one user found for issuer: ${issuer}`); 56 | console.error(error); 57 | throw error; 58 | } 59 | 60 | if (response.data.length === 0) { 61 | return null; 62 | } 63 | 64 | const user = await serverClient.query(q.Get(response.data[0])); 65 | if (!user) { 66 | return null; 67 | } 68 | 69 | return user.data; 70 | }; 71 | 72 | const findUser = (serverClient) => async (issuer) => { 73 | const user = await findUserWithAllData(serverClient)(issuer); 74 | if (!user) { 75 | return null; 76 | } 77 | 78 | return prepareUserForPublic(user); 79 | }; 80 | 81 | const setLastLoginAt = (serverClient) => (issuer, lastLoginAt) => 82 | serverClient.query( 83 | q.Update( 84 | q.Select( 85 | ["data", 0], 86 | q.Paginate(q.Match(q.Index("user_by_issuer"), issuer)) 87 | ), 88 | { 89 | data: { 90 | lastLoginAt, 91 | }, 92 | } 93 | ) 94 | ); 95 | 96 | const addNewWebsiteServerKey = (serverClient) => (issuer, data) => 97 | serverClient.query( 98 | q.Update( 99 | q.Select( 100 | ["data", 0], 101 | q.Paginate(q.Match(q.Index("user_by_issuer"), issuer)) 102 | ), 103 | { 104 | data, 105 | } 106 | ) 107 | ); 108 | 109 | const getDomainServerKeySecret = (serverClient) => async (issuer, domain) => { 110 | const response = await serverClient.query( 111 | q.Paginate(q.Match(q.Index("user_by_issuer"), issuer)) 112 | ); 113 | 114 | if (response.data.length > 1) { 115 | const error = new Error(`More than one user found for issuer: ${issuer}`); 116 | console.error(error); 117 | throw error; 118 | } 119 | 120 | if (response.data.length === 0) { 121 | return null; 122 | } 123 | 124 | const user = await serverClient.query(q.Get(response.data[0])); 125 | if (!user) { 126 | return null; 127 | } 128 | 129 | if (!user.data.sites) { 130 | return null; 131 | } 132 | 133 | if (!user.data.sites[domain]) { 134 | return null; 135 | } 136 | 137 | return user.data.sites[domain].serverKeySecret; 138 | }; 139 | 140 | module.exports = { 141 | createUser, 142 | updateUser, 143 | deleteUser, 144 | findUser, 145 | findUserWithAllData, 146 | setLastLoginAt, 147 | addNewWebsiteServerKey, 148 | getDomainServerKeySecret, 149 | }; 150 | -------------------------------------------------------------------------------- /services/admin-api/.env: -------------------------------------------------------------------------------- 1 | COOKIE_SECRET=cookie-dev-secret 2 | JWT_SECRET=jwt-dev-secret 3 | NODE_ENV=development -------------------------------------------------------------------------------- /services/admin-api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-slim 2 | ARG SERVICE_PATH=services/admin-api 3 | WORKDIR /app 4 | COPY ./local-packages /app/packages/ 5 | RUN for D in /app/packages/*; do npm ci --prod --prefix $D; done 6 | COPY ./package*.json /app/${SERVICE_PATH}/ 7 | RUN npm ci --prod --prefix ${SERVICE_PATH} 8 | 9 | FROM node:12-slim 10 | ARG SERVICE_PATH=services/admin-api 11 | WORKDIR /app 12 | COPY --from=0 /app . 13 | COPY . ./${SERVICE_PATH}/ 14 | RUN rm -fr ./${SERVICE_PATH}/local-packages 15 | CMD ["node", "./services/admin-api/src/index.js"] -------------------------------------------------------------------------------- /services/admin-api/README.md: -------------------------------------------------------------------------------- 1 | # admin-api 2 | 3 | Provides functionality used by the website, e.g. account creation, auth, etc. 4 | 5 | ## Fauna DB 6 | 7 | ### Main database (`your-analytics`) 8 | 9 | **Note**: Use the FaunaDB web shell to run the following commands. 10 | 11 | #### Server keys 12 | 13 | 1. Create an admin key 14 | ``` 15 | CreateKey({ 16 | role: "admin" 17 | }) 18 | ``` 19 | **Important**: Copy the `secret` to a secure location. It is only visible once. 20 | 1. Create a server key 21 | ``` 22 | CreateKey({ 23 | role: "server" 24 | }) 25 | ``` 26 | **Important**: Copy the `secret` to a secure location. It is only visible once. 27 | 28 | #### Collections 29 | 30 | 1. Create a `users` collection 31 | ``` 32 | CreateCollection({name: "users"}) 33 | ``` 34 | 35 | #### Indices 36 | 37 | 1. Create a `user_by_issuer` index 38 | ``` 39 | CreateIndex({ 40 | name: "user_by_issuer", 41 | source: Collection("users"), 42 | terms: [{ field: ["data", "issuer"]}], 43 | unique: true 44 | }) 45 | ``` 46 | -------------------------------------------------------------------------------- /services/admin-api/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: "bash" 3 | args: ["mkdir", "local-packages"] 4 | dir: "services/admin-api" 5 | - name: "bash" 6 | args: 7 | [ 8 | "cp", 9 | "-r", 10 | "../../packages/clickhouse", 11 | "../../packages/faunadb", 12 | "local-packages/", 13 | ] 14 | dir: "services/admin-api" 15 | - name: "gcr.io/cloud-builders/docker" 16 | args: ["build", "-t", "gcr.io/$PROJECT_ID/admin-api:$COMMIT_SHA", "."] 17 | dir: "services/admin-api" 18 | - name: "bash" 19 | args: ["rm", "-fr", "local-packages"] 20 | dir: "services/admin-api" 21 | - name: "gcr.io/cloud-builders/docker" 22 | args: ["push", "gcr.io/$PROJECT_ID/admin-api:$COMMIT_SHA"] 23 | dir: "services/admin-api" 24 | - name: "gcr.io/cloud-builders/gcloud" 25 | args: 26 | - "beta" 27 | - "run" 28 | - "deploy" 29 | - "admin-api" 30 | - "--image" 31 | - "gcr.io/$PROJECT_ID/admin-api:$COMMIT_SHA" 32 | - "--region" 33 | - "us-central1" 34 | - "--platform" 35 | - "managed" 36 | - "--vpc-connector" 37 | - "cloud-run-vpc-connector" 38 | - "--allow-unauthenticated" 39 | - "--set-env-vars" 40 | - "CH_HOST=$_CH_HOST,CH_PORT=$_CH_PORT,CH_USER=$_CH_USER,CH_PASSWORD=$_CH_PASSWORD,FAUNADB_ADMIN_SECRET=$_FAUNADB_ADMIN_SECRET,FAUNADB_SERVER_SECRET=$_FAUNADB_SERVER_SECRET,MAGIC_SECRET_KEY=$_MAGIC_SECRET_KEY,COOKIE_SECRET=$_COOKIE_SECRET,JWT_SECRET=$_JWT_SECRET,NODE_ENV=production" 41 | dir: "services/admin-api" 42 | images: 43 | - "gcr.io/$PROJECT_ID/admin-api:$COMMIT_SHA" 44 | -------------------------------------------------------------------------------- /services/admin-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/admin-api", 3 | "version": "1.0.0", 4 | "description": "Provides functionality used by the website, e.g. account creation, auth, etc.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "dev": "nodemon -r dotenv/config .", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@magic-sdk/admin": "^1.2.2", 14 | "@your-analytics/clickhouse": "file:../../packages/clickhouse", 15 | "@your-analytics/faunadb": "file:../../packages/faunadb", 16 | "cookie-parser": "^1.4.5", 17 | "cors": "^2.8.5", 18 | "express": "^4.17.1", 19 | "helmet": "^4.1.1", 20 | "jsonwebtoken": "^8.5.1", 21 | "passport": "^0.4.1", 22 | "passport-cookie": "^1.0.8", 23 | "passport-magic": "^1.0.0" 24 | }, 25 | "devDependencies": { 26 | "dotenv": "^8.2.0", 27 | "nodemon": "^2.0.4" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /services/admin-api/src/index.js: -------------------------------------------------------------------------------- 1 | const cookieParser = require("cookie-parser"); 2 | const cors = require("cors"); 3 | const express = require("express"); 4 | const helmet = require("helmet"); 5 | const passport = require("passport"); 6 | const passportCookie = require("passport-cookie"); 7 | const jwt = require("jsonwebtoken"); 8 | const { magic } = require("./middlewares"); 9 | const { user, website } = require("./routes"); 10 | const tests = require("./routes/tests"); 11 | 12 | const app = express(); 13 | const port = process.env.PORT || 8082; 14 | 15 | process.env.NODE_ENV === "development" && 16 | app.use( 17 | cors({ 18 | credentials: true, 19 | origin: /\.gitpod.io/, 20 | }) 21 | ); 22 | 23 | app.use(helmet()); 24 | app.use(express.json()); 25 | app.use(cookieParser(process.env.COOKIE_SECRET)); 26 | passport.use(magic); 27 | passport.use( 28 | new passportCookie.Strategy( 29 | { 30 | cookieName: "jwt", 31 | signed: true, 32 | }, 33 | (token, done) => { 34 | const payload = jwt.verify(token, process.env.JWT_SECRET); 35 | return done(null, payload.user); 36 | } 37 | ) 38 | ); 39 | app.use(passport.initialize()); 40 | 41 | const authenticateMagic = passport.authenticate("magic", { session: false }); 42 | const authenticateJwtCookieCombo = passport.authenticate("cookie", { 43 | session: false, 44 | }); 45 | 46 | app.get("/", async (req, res) => { 47 | res.status(200).end(); 48 | }); 49 | 50 | app.use("/user", user(authenticateMagic, authenticateJwtCookieCombo)); 51 | app.use("/website", website(authenticateJwtCookieCombo)); 52 | 53 | if (process.env.NODE_ENV === "development") { 54 | console.log("[TESTS] Adding a /tests API endpoint."); 55 | app.use("/tests", tests); 56 | } 57 | 58 | app.listen(port, () => { 59 | console.log(`admin-api started at http://localhost:${port}`); 60 | }); 61 | -------------------------------------------------------------------------------- /services/admin-api/src/middlewares/index.js: -------------------------------------------------------------------------------- 1 | const magic = require("./magic"); 2 | 3 | module.exports = { 4 | magic, 5 | }; 6 | -------------------------------------------------------------------------------- /services/admin-api/src/middlewares/magic.js: -------------------------------------------------------------------------------- 1 | const { Magic } = require("@magic-sdk/admin"); 2 | const { rootDb } = require("@your-analytics/faunadb"); 3 | const MagicStrategy = require("passport-magic").Strategy; 4 | const passport = require("passport"); 5 | 6 | const magic = new Magic(process.env.MAGIC_SECRET_KEY); 7 | 8 | const signup = async (user, userMetadata, done) => { 9 | let newUser = { 10 | issuer: user.issuer, 11 | email: userMetadata.email, 12 | lastLoginAt: user.claim.iat, 13 | }; 14 | await rootDb.users.create(newUser); 15 | return done(null, newUser); 16 | }; 17 | 18 | const login = async (user, done) => { 19 | // Replay attack protection (https://go.magic.link/replay-attack) 20 | if (user.claim.iat <= user.lastLoginAt) { 21 | return done(null, false, { 22 | message: `Replay attack detected for user ${user.issuer}}.`, 23 | }); 24 | } 25 | await rootDb.users.setLastLoginAt(user.issuer, user.claim.iat); 26 | return done(null, user); 27 | }; 28 | 29 | const strategy = new MagicStrategy(async function (user, done) { 30 | const userMetadata = await magic.users.getMetadataByIssuer(user.issuer); 31 | 32 | const dbUser = await rootDb.users.find(user.issuer); 33 | return dbUser ? login(user, done) : signup(user, userMetadata, done); 34 | }); 35 | 36 | module.exports = strategy; 37 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/index.js: -------------------------------------------------------------------------------- 1 | const user = require("./user"); 2 | const website = require("./website"); 3 | 4 | module.exports = { 5 | user, 6 | website, 7 | }; 8 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/tests/db-reset.js: -------------------------------------------------------------------------------- 1 | const { deleteWebsite } = require("@your-analytics/clickhouse"); 2 | const { domainDb, rootDb } = require("@your-analytics/faunadb"); 3 | const express = require("express"); 4 | 5 | const router = express.Router(); 6 | 7 | const ignoreError = (promise) => promise.catch((e) => undefined); 8 | 9 | router.post("/", async (req, res) => { 10 | console.log("[admin-api]: TESTS - Resetting database..."); 11 | 12 | const testUserIssuer = "hello+ya-automated-tests@mikenikles.com"; 13 | const testUser = await rootDb.users.find(testUserIssuer); 14 | 15 | if (testUser) { 16 | const sites = Object.keys(testUser.sites || {}); 17 | const steps = []; 18 | for (const site of sites) { 19 | steps.push(domainDb.admin.deleteWebsite(site)); 20 | steps.push(deleteWebsite(site)); 21 | } 22 | steps.push(rootDb.users.delete(testUserIssuer)); 23 | 24 | await Promise.all(steps.map(ignoreError)); 25 | } 26 | 27 | return res.status(200).end(); 28 | }); 29 | 30 | module.exports = router; 31 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/tests/domain-db.js: -------------------------------------------------------------------------------- 1 | const { addNewWebsite } = require("@your-analytics/clickhouse"); 2 | const express = require("express"); 3 | 4 | const router = express.Router(); 5 | 6 | const isValidWebsite = (domain) => 7 | !!domain.match(/^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$/); 8 | 9 | router.post("/", async (req, res) => { 10 | console.log("[admin-api]: TESTS - Adding new website: ", req.body.url); 11 | try { 12 | try { 13 | if (!isValidWebsite(req.body.url)) { 14 | throw new Error(`Invalid website provided: ${req.body.url}`); 15 | } 16 | await addNewWebsite(req.body.url); 17 | } catch (error) { 18 | console.error(error); 19 | return res.status(400).end(); 20 | } 21 | 22 | return res.status(201).end(); 23 | } catch (error) { 24 | console.error(error); 25 | return res.status(500).end(); 26 | } 27 | }); 28 | 29 | module.exports = router; 30 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/tests/index.js: -------------------------------------------------------------------------------- 1 | const express = require("express"); 2 | const dbReset = require("./db-reset"); 3 | const domainDb = require("./domain-db"); 4 | const user = require("./user"); 5 | 6 | const router = express.Router(); 7 | 8 | router.use("/user", user); 9 | router.use("/domain-db", domainDb); 10 | router.use("/db-reset", dbReset); 11 | 12 | module.exports = router; 13 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/tests/user.js: -------------------------------------------------------------------------------- 1 | const { rootDb } = require("@your-analytics/faunadb"); 2 | const express = require("express"); 3 | const jwt = require("jsonwebtoken"); 4 | 5 | const router = express.Router(); 6 | 7 | const cookieTestConfig = { 8 | path: "/", 9 | maxAge: 1000 * 60 * 60 * 24 * 7, 10 | httpOnly: true, 11 | sameSite: true, 12 | signed: true, 13 | secure: false, 14 | }; 15 | 16 | /** 17 | * This endpoint allows the Cypress `loginWithApi` custom command to 18 | * obtain a signed cookie with the user's JWT token. 19 | * 20 | * It create a test account if none exists or returns the existing 21 | * user information in case an account already existed. 22 | */ 23 | router.post("/login-with-api", async (req, res) => { 24 | let testUser = await rootDb.users.find(req.body.email); 25 | if (testUser) { 26 | await rootDb.users.setLastLoginAt( 27 | testUser.issuer, 28 | Math.floor(Date.now() / 1000) 29 | ); 30 | } else { 31 | await rootDb.users.create({ 32 | issuer: req.body.email, 33 | email: req.body.email, 34 | lastLoginAt: Math.floor(Date.now() / 1000), 35 | }); 36 | testUser = await rootDb.users.find(req.body.email); 37 | } 38 | 39 | jwt.sign({ user: testUser }, process.env.JWT_SECRET, (error, token) => { 40 | if (error) { 41 | return res.status(401).end("Could not log user in."); 42 | } 43 | 44 | res.cookie("jwt", token, cookieTestConfig); 45 | return res.status(200).json(testUser).end(); 46 | }); 47 | }); 48 | 49 | module.exports = router; 50 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/user.js: -------------------------------------------------------------------------------- 1 | const { rootDb } = require("@your-analytics/faunadb"); 2 | const { Magic } = require("@magic-sdk/admin"); 3 | const express = require("express"); 4 | const { cookieConfig, signJwtAndSetCookie } = require("../utils/cookies"); 5 | 6 | const magic = new Magic(process.env.MAGIC_SECRET_KEY); 7 | const router = express.Router(); 8 | 9 | module.exports = (authenticateMagic, authenticateJwtCookieCombo) => { 10 | router.get("/", authenticateJwtCookieCombo, async (req, res) => { 11 | const dbUser = await rootDb.users.find(req.user.issuer); 12 | if (dbUser) { 13 | return res.status(200).json(dbUser).end(); 14 | } 15 | return res.status(404).end(); 16 | }); 17 | 18 | router.put("/", authenticateJwtCookieCombo, async (req, res) => { 19 | try { 20 | await rootDb.users.update(req.user.issuer, req.body); 21 | const dbUser = await rootDb.users.find(req.user.issuer); 22 | 23 | try { 24 | await signJwtAndSetCookie(res, dbUser); 25 | return res.status(200).json(req.user).end(); 26 | } catch (error) { 27 | return res.status(401).end("Something went wrong."); 28 | } 29 | } catch (error) { 30 | console.error(error); 31 | return res.status(500).end(); 32 | } 33 | }); 34 | 35 | router.post("/login", authenticateMagic, async (req, res) => { 36 | if (req.user) { 37 | const dbUser = await rootDb.users.find(req.user.issuer); 38 | try { 39 | await signJwtAndSetCookie(res, dbUser); 40 | return res.status(200).json(req.user).end(); 41 | } catch (error) { 42 | return res.status(401).end("Could not log user in."); 43 | } 44 | } else { 45 | return res.status(401).end("Could not log user in."); 46 | } 47 | }); 48 | 49 | router.post("/logout", authenticateJwtCookieCombo, async (req, res) => { 50 | await magic.users.logoutByIssuer(req.user.issuer); 51 | req.logout(); 52 | res.clearCookie("jwt", cookieConfig); 53 | return res.status(200).end(); 54 | }); 55 | 56 | return router; 57 | }; 58 | -------------------------------------------------------------------------------- /services/admin-api/src/routes/website.js: -------------------------------------------------------------------------------- 1 | const { 2 | addNewWebsite, 3 | convertUrlToDbName, 4 | } = require("@your-analytics/clickhouse"); 5 | const { domainDb, rootDb } = require("@your-analytics/faunadb"); 6 | const express = require("express"); 7 | const { signJwtAndSetCookie } = require("../utils/cookies"); 8 | 9 | const router = express.Router(); 10 | 11 | const isValidWebsite = (domain) => 12 | !!domain.match(/^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$/); 13 | 14 | module.exports = (authenticate) => { 15 | router.get("/", authenticate, async (req, res) => { 16 | try { 17 | const user = await rootDb.users.find(req.user.issuer); 18 | return res.status(200).json(user.sites); 19 | } catch (error) { 20 | console.error(error); 21 | return res.status(500).end(); 22 | } 23 | }); 24 | 25 | router.post("/", authenticate, async (req, res) => { 26 | try { 27 | try { 28 | if (!isValidWebsite(req.body.url)) { 29 | throw new Error(`Invalid website provided: ${req.body.url}`); 30 | } 31 | await domainDb.admin.addNewWebsite(req.body.url); 32 | await addNewWebsite(req.body.url); 33 | } catch (error) { 34 | console.error(error); 35 | return res.status(400).end(); 36 | } 37 | const { 38 | secret: websiteServerKeySecret, 39 | } = await domainDb.admin.createWebsiteServerKey(req.body.url); 40 | await rootDb.users.addNewWebsiteServerKey(req.user.issuer, { 41 | sites: { 42 | [req.body.url]: { 43 | serverKeySecret: websiteServerKeySecret, 44 | }, 45 | }, 46 | }); 47 | await domainDb.admin.createCollection(websiteServerKeySecret)("settings"); 48 | await domainDb.settings.insertSettings(websiteServerKeySecret)({ 49 | chDbName: convertUrlToDbName(req.body.url), 50 | timezone: req.body.timezone, 51 | visibility: "private", 52 | }); 53 | 54 | try { 55 | const newUser = Object.assign({}, req.user); 56 | newUser.sites[req.body.url] = {}; 57 | await signJwtAndSetCookie(res, newUser); 58 | return res.status(201).end(); 59 | } catch (error) { 60 | return res.status(401).end("Something went wrong."); 61 | } 62 | } catch (error) { 63 | console.error(error); 64 | return res.status(500).end(); 65 | } 66 | }); 67 | 68 | router.get("/:domain/settings", authenticate, async (req, res) => { 69 | try { 70 | const domain = req.params.domain; 71 | const user = await rootDb.users.find(req.user.issuer); 72 | if (!user.sites[domain]) { 73 | console.error( 74 | new Error( 75 | `User ${req.user.issuer} tried to access domain ${domain} but is not authorized.` 76 | ) 77 | ); 78 | res.status(401).end(); 79 | return; 80 | } 81 | 82 | const domainServerKeySecret = await rootDb.users.getDomainServerKeySecret( 83 | user.issuer, 84 | domain 85 | ); 86 | const settings = await domainDb.settings.getSettings( 87 | domainServerKeySecret 88 | )(); 89 | res.status(200).send(JSON.stringify(settings.data)); 90 | } catch (error) { 91 | console.error(error); 92 | return res.status(500).end(); 93 | } 94 | }); 95 | 96 | router.get("/:domain/settings/visibility", async (req, res) => { 97 | try { 98 | const { data } = await domainDb.settings.getVisibility(req.params.domain); 99 | return res.status(200).json({ visibility: data.visibility }); 100 | } catch (error) { 101 | console.error(error); 102 | // To prevent bad actors from scanning the database for URLs to see whether 103 | // they are configured in our database or not. Returning a 500 error would indicate 104 | // the req.query.url value does not exist in our database; returning visibility: private 105 | // gives the impression the URL is configured, even if it may not be. 106 | return res.status(200).json({ visibility: "private" }); 107 | } 108 | }); 109 | 110 | router.put("/:domain/settings/visibility", authenticate, async (req, res) => { 111 | try { 112 | const domain = req.params.domain; 113 | const visibility = req.body.visibility; 114 | 115 | const user = await rootDb.users.findUserWithAllData(req.user.issuer); 116 | if (!user.sites[domain]) { 117 | console.error( 118 | new Error( 119 | `User ${req.user.issuer} tried to access domain ${domain} but is not authorized.` 120 | ) 121 | ); 122 | res.status(401).end(); 123 | return; 124 | } 125 | await domainDb.settings.setVisibility(user.sites[domain].serverKeySecret)( 126 | visibility 127 | ); 128 | return res.status(200).end(); 129 | } catch (error) { 130 | console.error(error); 131 | return res.status(500).end(); 132 | } 133 | }); 134 | 135 | return router; 136 | }; 137 | -------------------------------------------------------------------------------- /services/admin-api/src/utils/cookies.js: -------------------------------------------------------------------------------- 1 | const jwt = require("jsonwebtoken"); 2 | 3 | const cookieConfig = { 4 | path: "/", 5 | maxAge: 1000 * 60 * 60 * 24 * 7, 6 | httpOnly: true, 7 | sameSite: true, 8 | signed: true, 9 | secure: true, 10 | }; 11 | 12 | const signJwtAndSetCookie = (res, user) => 13 | new Promise((resolve, reject) => { 14 | jwt.sign({ user }, process.env.JWT_SECRET, (error, token) => { 15 | if (error) { 16 | console.error(error); 17 | reject(); 18 | } 19 | res.cookie("jwt", token, cookieConfig); 20 | resolve(); 21 | }); 22 | }); 23 | 24 | module.exports = { 25 | cookieConfig, 26 | signJwtAndSetCookie, 27 | }; 28 | -------------------------------------------------------------------------------- /services/db-analytics/.gitignore: -------------------------------------------------------------------------------- 1 | dbvolume 2 | dev/db/** 3 | !dev/db/.keep 4 | -------------------------------------------------------------------------------- /services/db-analytics/README.md: -------------------------------------------------------------------------------- 1 | # db-analytics 2 | 3 | ## Host 4 | 5 | A VM on GCE as part of the POC. 6 | 7 | **TODO**: Migrate to a Zookeeper, multi-node, HA environment for production. 8 | 9 | ## Local development 10 | 11 | There are a number of `dev:docker:*` NPM scripts to assist with local development. Mainly, 12 | the following are important: 13 | 14 | - `npm run dev:docker:build` 15 | - `npm run dev:docker:start` 16 | - `npm run dev:docker:clean` 17 | 18 | The database is seeded with all SQL scripts located in `dev/sql`. 19 | 20 | ## Transfer prod data to local DB 21 | 22 | 1. Replace `[DB_NAME]` in all commands below with the correct DB name depending on which domain events you want to export 23 | 1. On prod: `clickhouse-client --password --query="SELECT * FROM [DB_NAME].events FORMAT Native" > events.native` 24 | 1. Download `events.native` 25 | 1. Upload `events.native` to GCP shell 26 | 1. In GCP shell 27 | 1. Start DB 28 | 1. Run: `cat ~/events.native | curl 'http://localhost:8123/?query=INSERT%20INTO%20[DB_NAME].events%20FORMAT%20Native' --data-binary @-` 29 | 30 | ## Ubuntu 18.04 installation instructions 31 | 32 | https://www.digitalocean.com/community/tutorials/how-to-install-and-use-clickhouse-on-ubuntu-18-04 33 | 34 | ### Installation script 35 | 36 | `1-install-clickhouse.sh` 37 | 38 | ```bash 39 | #!/bin/bash 40 | sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 41 | 42 | echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list 43 | 44 | sudo apt-get update 45 | 46 | sudo apt-get install -y clickhouse-server clickhouse-client 47 | ``` 48 | 49 | ### DB & tables creation script 50 | 51 | Each domain has its own database, with at least an `events` table. This is created at runtime 52 | when a new domain is configured. 53 | -------------------------------------------------------------------------------- /services/db-analytics/dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM yandex/clickhouse-server:20.5 2 | 3 | COPY sql/*.sql /docker-entrypoint-initdb.d -------------------------------------------------------------------------------- /services/db-analytics/dev/db/.keep: -------------------------------------------------------------------------------- 1 | Keep this directory. This is where we start the ClickHouse server. -------------------------------------------------------------------------------- /services/db-analytics/dev/sql/1-init.sql: -------------------------------------------------------------------------------- 1 | -- This is a placeholder. 2 | -- Should there be any seed data required, 3 | -- please add it to this script or create 4 | -- other *.sql scripts in this directory. 5 | SHOW DATABASES; -------------------------------------------------------------------------------- /services/db-analytics/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/db-analytics", 3 | "version": "1.0.0", 4 | "description": "The database service that stores events, sessions, etc.", 5 | "main": "index.js", 6 | "scripts": { 7 | "dev:docker:build": "cd dev && docker build -t ya-clickhouse-dev .", 8 | "dev:docker:clean": "npm run dev:docker:stopAndRm && npm run dev:docker:removeimage", 9 | "dev:docker:cli": "docker run -it --rm --link ch-your-analytics:clickhouse-server yandex/clickhouse-client --host clickhouse-server", 10 | "dev:docker:removeimage": "docker image rm ya-clickhouse-dev", 11 | "dev:docker:start": "docker run -d -p 8123:8123 --volume $PWD/dbvolume:/var/lib/clickhouse --name ch-your-analytics --ulimit nofile=262144:262144 ya-clickhouse-dev:latest", 12 | "dev:docker:stopAndRm": "docker rm $(docker stop $(docker ps -a -q --filter ancestor=ya-clickhouse-dev --format=\"{{.ID}}\"))", 13 | "test": "echo \"Error: no test specified\" && exit 1" 14 | }, 15 | "author": "", 16 | "license": "ISC" 17 | } 18 | -------------------------------------------------------------------------------- /services/events-api/.env: -------------------------------------------------------------------------------- 1 | NODE_ENV=development -------------------------------------------------------------------------------- /services/events-api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mhart/alpine-node:12 2 | ARG SERVICE_PATH=services/events-api 3 | WORKDIR /app 4 | COPY ./local-packages /app/packages/ 5 | RUN for D in /app/packages/*; do npm ci --prod --prefix $D; done 6 | COPY ./package*.json /app/${SERVICE_PATH}/ 7 | RUN npm ci --prod --prefix ${SERVICE_PATH} 8 | 9 | FROM mhart/alpine-node:slim-12 10 | ARG SERVICE_PATH=services/events-api 11 | WORKDIR /app 12 | COPY --from=0 /app . 13 | COPY . ./${SERVICE_PATH}/ 14 | RUN rm -fr ./${SERVICE_PATH}/local-packages 15 | CMD ["node", "./services/events-api/src/index.js"] -------------------------------------------------------------------------------- /services/events-api/README.md: -------------------------------------------------------------------------------- 1 | # events-api 2 | 3 | Processes incoming events. In the initial phase, these events are sent from the `ya.js` file embedded in websites. 4 | 5 | ## Send a test event 6 | 7 | ``` 8 | curl -v -X POST -H "Content-Type: text/plain" -H "user-agent:Mozilla/5.0 (X11; CrOS x86_64 13099.48.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.64 Safari/537.36" -d @./test-event.json "http://localhost:8080" 9 | ``` 10 | 11 | ## Serverless VPC Access 12 | 13 | https://cloud.google.com/vpc/docs/configure-serverless-vpc-access#gcloud 14 | 15 | ```bash 16 | gcloud services enable vpcaccess.googleapis.com 17 | gcloud compute networks vpc-access connectors create [CONNECTOR_NAME] \ 18 | --network [VPC_NETWORK] \ 19 | --region [REGION] \ 20 | --range [IP_RANGE] 21 | ``` 22 | -------------------------------------------------------------------------------- /services/events-api/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: "bash" 3 | args: ["mkdir", "local-packages"] 4 | dir: "services/events-api" 5 | - name: "bash" 6 | args: ["cp", "-r", "../../packages/clickhouse", "local-packages/"] 7 | dir: "services/events-api" 8 | - name: "ubuntu" 9 | entrypoint: bash 10 | args: 11 | - "-c" 12 | - | 13 | sed -i "s|YOUR_ACCOUNT_ID_HERE|$_MAXMIND_ACCOUNT_ID |g" ./GeoIP.conf 14 | sed -i "s|YOUR_LICENSE_KEY_HERE|$_MAXMIND_LICENSE_KEY |g" ./GeoIP.conf 15 | dir: "services/events-api/geo-db" 16 | - name: "gcr.io/your-analytics/maxmind-geoipupdater:latest" 17 | dir: "services/events-api/geo-db" 18 | - name: "bash" 19 | args: ["test", "-f", "GeoLite2-City.mmdb"] 20 | dir: "services/events-api/geo-db" 21 | - name: "gcr.io/cloud-builders/docker" 22 | args: ["build", "-t", "gcr.io/$PROJECT_ID/events-api:$COMMIT_SHA", "."] 23 | dir: "services/events-api" 24 | - name: "bash" 25 | args: ["rm", "-fr", "local-packages"] 26 | dir: "services/events-api" 27 | - name: "gcr.io/cloud-builders/docker" 28 | args: ["push", "gcr.io/$PROJECT_ID/events-api:$COMMIT_SHA"] 29 | dir: "services/events-api" 30 | - name: "gcr.io/cloud-builders/gcloud" 31 | args: 32 | - "beta" 33 | - "run" 34 | - "deploy" 35 | - "events-api" 36 | - "--image" 37 | - "gcr.io/$PROJECT_ID/events-api:$COMMIT_SHA" 38 | - "--region" 39 | - "us-central1" 40 | - "--platform" 41 | - "managed" 42 | - "--vpc-connector" 43 | - "cloud-run-vpc-connector" 44 | - "--allow-unauthenticated" 45 | - "--set-env-vars" 46 | - "CH_HOST=$_CH_HOST,CH_PORT=$_CH_PORT,CH_USER=$_CH_USER,CH_PASSWORD=$_CH_PASSWORD,NODE_ENV=production" 47 | dir: "services/events-api" 48 | images: 49 | - "gcr.io/$PROJECT_ID/events-api:$COMMIT_SHA" 50 | -------------------------------------------------------------------------------- /services/events-api/geo-db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | 3 | RUN apt-get update -q \ 4 | && apt-get install -yq software-properties-common \ 5 | && add-apt-repository ppa:maxmind/ppa \ 6 | && apt-get install -yq \ 7 | geoipupdate \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | CMD geoipupdate --config-file ./GeoIP.conf -------------------------------------------------------------------------------- /services/events-api/geo-db/GeoIP.conf: -------------------------------------------------------------------------------- 1 | # GeoIP.conf file for `geoipupdate` program, for versions >= 3.1.1. 2 | # Used to update GeoIP databases from https://www.maxmind.com. 3 | # For more information about this config file, visit the docs at 4 | # https://dev.maxmind.com/geoip/geoipupdate/. 5 | 6 | AccountID YOUR_ACCOUNT_ID_HERE 7 | LicenseKey YOUR_LICENSE_KEY_HERE 8 | EditionIDs GeoLite2-City 9 | 10 | DatabaseDirectory ./ -------------------------------------------------------------------------------- /services/events-api/geo-db/README.md: -------------------------------------------------------------------------------- 1 | # GEO IP updater Cloud Build custom builder 2 | 3 | This custom Cloud Build builder updates the GeoLite2-City database. 4 | 5 | ## Push a new image to the GCP Container Registry 6 | 7 | In a terminal, navigate to this directory and run the following command: 8 | 9 | ```bash 10 | gcloud builds submit --tag gcr.io/your-analytics/maxmind-geoipupdater:latest 11 | ``` 12 | 13 | ## Usage in a `cloudbuild.yaml` config file 14 | 15 | First, create the following `GeoIP.conf` file: 16 | 17 | ``` 18 | AccountID YOUR_ACCOUNT_ID_HERE 19 | LicenseKey YOUR_LICENSE_KEY_HERE 20 | EditionIDs GeoLite2-City 21 | 22 | DatabaseDirectory ./ 23 | ``` 24 | 25 | Next, update the `cloudbuild.yaml` with the steps below. 26 | Set the `dir` option to the directory that contains your `GeoIP.conf` file. 27 | 28 | ```yaml 29 | - name: "ubuntu" 30 | entrypoint: bash 31 | args: 32 | - "-c" 33 | - | 34 | sed -i "s|YOUR_ACCOUNT_ID_HERE|$_MAXMIND_ACCOUNT_ID |g" ./GeoIP.conf 35 | sed -i "s|YOUR_LICENSE_KEY_HERE|$_MAXMIND_LICENSE_KEY |g" ./GeoIP.conf 36 | dir: "services/events-api/geo-db" 37 | - name: "gcr.io/your-analytics/maxmind-geoipupdater:latest" 38 | dir: "services/events-api/geo-db" 39 | - name: "bash" 40 | args: ["test", "-f", "GeoLite2-City.mmdb"] 41 | dir: "services/events-api/geo-db" 42 | ``` 43 | 44 | Set the following substitutions for your build (either through the Cloud Build UI or the CLI): 45 | 46 | - `_MAXMIND_ACCOUNT_ID` 47 | - `_MAXMIND_LICENSE_KEY` 48 | -------------------------------------------------------------------------------- /services/events-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/events-api", 3 | "version": "1.0.0", 4 | "description": "Processes incoming events.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "dev": "nodemon -r dotenv/config .", 8 | "start": "node .", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "author": "", 12 | "license": "ISC", 13 | "devDependencies": { 14 | "dotenv": "^8.2.0", 15 | "nodemon": "^2.0.4" 16 | }, 17 | "dependencies": { 18 | "@your-analytics/clickhouse": "file:../../packages/clickhouse", 19 | "@maxmind/geoip2-node": "^1.4.0", 20 | "express": "^4.17.1", 21 | "ua-parser-js": "^0.7.21", 22 | "url-parse": "^1.4.7" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /services/events-api/src/index.js: -------------------------------------------------------------------------------- 1 | const express = require("express"); 2 | const userAgentParser = require("ua-parser-js"); 3 | const urlParse = require("url-parse"); 4 | const Geo2IpReader = require("@maxmind/geoip2-node").Reader; 5 | const { recordEvent } = require("@your-analytics/clickhouse"); 6 | 7 | const GEO_DB_PATH_PREFIX = 8 | process.env.NODE_ENV === "development" ? "." : "./services/events-api"; 9 | 10 | /** 11 | * @see https://github.com/darkskyapp/string-hash/blob/master/index.js 12 | */ 13 | const hash = (str) => { 14 | let result = 5381; 15 | let i = str.length; 16 | 17 | while (i) { 18 | result = (result * 33) ^ str.charCodeAt(--i); 19 | } 20 | 21 | return result >>> 0; 22 | }; 23 | 24 | const logDetailsIfUserAgentCannotBeParsed = ( 25 | userAgentHeader, 26 | { browser, device, os } 27 | ) => { 28 | if (!(browser.major && browser.name && browser.version)) { 29 | console.log(`Cannot determine browser given UA: ${userAgentHeader}`); 30 | } 31 | 32 | if (!(device.model && device.type && device.vendor)) { 33 | console.log(`Cannot determine device given UA: ${userAgentHeader}`); 34 | } 35 | 36 | if (!(os.name && os.version)) { 37 | console.log(`Cannot determine os given UA: ${userAgentHeader}`); 38 | } 39 | }; 40 | 41 | const app = express(); 42 | const port = process.env.PORT || 8080; 43 | let readGeoFromIp; 44 | 45 | app.use( 46 | express.json({ 47 | type: "text/plain", 48 | }) 49 | ); 50 | 51 | app.post("/", async (req, res) => { 52 | try { 53 | const { domain, name, referrer, screen_size, url } = req.body; 54 | 55 | const userAgentHeader = req.get("user-agent"); 56 | const xForwardedFor = req.get("x-forwarded-for"); 57 | const userAgent = userAgentParser(userAgentHeader); 58 | const urlParsed = urlParse(url, true); 59 | const userId = hash(userAgentHeader + xForwardedFor); 60 | 61 | logDetailsIfUserAgentCannotBeParsed(userAgentHeader, userAgent); 62 | 63 | const event = { 64 | browser_major: userAgent.browser.major, 65 | browser_name: userAgent.browser.name, 66 | browser_version: userAgent.browser.version, 67 | device_model: userAgent.device.model, 68 | device_type: userAgent.device.type, 69 | device_vendor: userAgent.device.vendor, 70 | domain, 71 | hostname: urlParsed.hostname, 72 | name, 73 | os_name: userAgent.os.name, 74 | os_version: userAgent.os.version, 75 | path: urlParsed.pathname, 76 | referrer, 77 | screen_size, 78 | session_id: 0, 79 | timestamp: 80 | process.env.NODE_ENV === "development" 81 | ? new Date(req.body.timestamp) || new Date() 82 | : new Date(), 83 | user_id: userId, 84 | }; 85 | 86 | if (xForwardedFor) { 87 | try { 88 | if (!readGeoFromIp) { 89 | readGeoFromIp = await Geo2IpReader.open( 90 | // See this service's Dockerfile. 91 | // We start this service from the monorepo root, hence the path prefix in production. 92 | `${GEO_DB_PATH_PREFIX}/geo-db/GeoLite2-City.mmdb` 93 | ); 94 | } 95 | const geoFromIpResponse = await readGeoFromIp.city(xForwardedFor); 96 | 97 | event.geo_city = geoFromIpResponse.city.names.en; 98 | event.geo_country = geoFromIpResponse.country.isoCode; 99 | event.geo_lat = geoFromIpResponse.location.latitude; 100 | event.geo_long = geoFromIpResponse.location.longitude; 101 | } catch (geoFromIpError) { 102 | console.log( 103 | `Could not determine GEO from IP: %s. Error: %s`, 104 | xForwardedFor, 105 | geoFromIpError 106 | ); 107 | } 108 | } 109 | 110 | recordEvent(event); 111 | } catch (error) { 112 | console.error(error); 113 | } finally { 114 | res.status(201).end(); 115 | } 116 | }); 117 | 118 | app.get("/error", (req, res) => { 119 | try { 120 | console.error(new Error(req.query.m)); 121 | } catch (error) { 122 | console.error(error); 123 | } finally { 124 | res.status(200).end(); 125 | } 126 | }); 127 | 128 | app.listen(port, () => { 129 | console.log(`events-api started at http://localhost:${port}`); 130 | }); 131 | -------------------------------------------------------------------------------- /services/events-api/test-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "domain": "example.com", 3 | "name": "pageview", 4 | "referrer": "", 5 | "screen_size": "1240x900", 6 | "url": "http://www.example.com/test-path" 7 | } 8 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.md] 11 | trim_trailing_whitespace = false 12 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/.gitignore: -------------------------------------------------------------------------------- 1 | *-debug.log 2 | *-error.log 3 | /.nyc_output 4 | /dist 5 | /lib 6 | /tmp 7 | /yarn.lock 8 | node_modules 9 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/README.md: -------------------------------------------------------------------------------- 1 | # ya-load-testing 2 | 3 | A load testing CLI for your-analytics.org. 4 | 5 | [![oclif](https://img.shields.io/badge/cli-oclif-brightgreen.svg)](https://oclif.io) 6 | [![Version](https://img.shields.io/npm/v/ya-load-testing.svg)](https://npmjs.org/package/ya-load-testing) 7 | [![Downloads/week](https://img.shields.io/npm/dw/ya-load-testing.svg)](https://npmjs.org/package/ya-load-testing) 8 | [![License](https://img.shields.io/npm/l/ya-load-testing.svg)](https://github.com/mikenikles/your-analytics/blob/master/package.json) 9 | 10 | 11 | 12 | - [Usage](#usage) 13 | - [Commands](#commands) 14 | 15 | 16 | # Usage 17 | 18 | 19 | 20 | ```sh-session 21 | $ npm install -g ya-load-testing 22 | $ ya-load-testing COMMAND 23 | running command... 24 | $ ya-load-testing (-v|--version|version) 25 | ya-load-testing/1.0.0 linux-x64 node-v12.18.4 26 | $ ya-load-testing --help [COMMAND] 27 | USAGE 28 | $ ya-load-testing COMMAND 29 | ... 30 | ``` 31 | 32 | 33 | 34 | # Commands 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/bin/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ":" //# comment; exec /usr/bin/env node --expose-gc "$0" "$@" 3 | 4 | const fs = require('fs') 5 | const path = require('path') 6 | const project = path.join(__dirname, '../tsconfig.json') 7 | const dev = fs.existsSync(project) 8 | 9 | if (dev) { 10 | require('ts-node').register({project}) 11 | } 12 | 13 | require(`../${dev ? 'src' : 'lib'}`).run() 14 | .catch(require('@oclif/errors/handle')) 15 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/bin/run.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | node "%~dp0\run" %* 4 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ya-load-testing", 3 | "description": "A load testing CLI for your-analytics.org.", 4 | "version": "1.0.0", 5 | "author": "Mike Nikles @mikenikles", 6 | "bin": { 7 | "ya-load-testing": "./bin/run" 8 | }, 9 | "bugs": "https://github.com/mikenikles/your-analytics/issues", 10 | "dependencies": { 11 | "@oclif/command": "^1.8.0", 12 | "@oclif/config": "^1.17.0", 13 | "@oclif/plugin-help": "^3.2.0", 14 | "date-fns": "^2.16.1", 15 | "tslib": "^1.14.1" 16 | }, 17 | "devDependencies": { 18 | "@oclif/dev-cli": "^1.22.2", 19 | "@oclif/test": "^1.2.7", 20 | "@types/chai": "^4.2.13", 21 | "@types/mocha": "^5.2.7", 22 | "@types/node": "^10.17.39", 23 | "chai": "^4.2.0", 24 | "got": "^11.7.0", 25 | "mocha": "^5.2.0", 26 | "nock": "^13.0.4", 27 | "nyc": "^14.1.1", 28 | "ts-node": "^8.10.2", 29 | "typescript": "^3.9.7" 30 | }, 31 | "engines": { 32 | "node": ">=8.0.0" 33 | }, 34 | "files": [ 35 | "/bin", 36 | "/lib" 37 | ], 38 | "homepage": "https://github.com/mikenikles/your-analytics", 39 | "keywords": [ 40 | "oclif" 41 | ], 42 | "license": "MIT", 43 | "main": "lib/index.js", 44 | "oclif": { 45 | "bin": "ya-load-testing" 46 | }, 47 | "repository": "mikenikles/your-analytics", 48 | "scripts": { 49 | "prepack": "rm -rf lib && tsc -b && oclif-dev readme", 50 | "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", 51 | "version": "oclif-dev readme && git add README.md" 52 | }, 53 | "types": "lib/index.d.ts" 54 | } 55 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/src/api.ts: -------------------------------------------------------------------------------- 1 | import got from "got"; 2 | 3 | export interface IEvent { 4 | name: string; 5 | domain: string; 6 | url: string; 7 | referrer: string; 8 | screen_size: number; 9 | timestamp: Date; 10 | } 11 | 12 | const defaultUserAgent = 13 | "Mozilla/5.0 (Linux; Android 7.1.1; SM-T555 Build/NMF26X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/83.0.4103.96 Safari/537.36"; 14 | 15 | export const sendEvent = ( 16 | event: IEvent, 17 | apiEndpoint = "http://localhost:8080" 18 | ) => 19 | got.post(apiEndpoint, { 20 | body: JSON.stringify(event), 21 | headers: { 22 | "Content-Type": "text/plain", 23 | "User-Agent": defaultUserAgent, 24 | }, 25 | }); 26 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/src/index.ts: -------------------------------------------------------------------------------- 1 | import { Command, flags } from "@oclif/command"; 2 | import { 3 | formatDistanceToNowStrict, 4 | formatISO, 5 | parseISO, 6 | startOfMonth, 7 | sub, 8 | } from "date-fns"; 9 | import { performLoadTesting } from "./load-testing"; 10 | 11 | class YaLoadTesting extends Command { 12 | static description = "describe the command here"; 13 | 14 | static flags = { 15 | version: flags.version({ char: "v" }), 16 | help: flags.help({ char: "h" }), 17 | from: flags.string({ 18 | char: "f", 19 | default: () => 20 | formatISO( 21 | startOfMonth( 22 | sub(new Date(), { 23 | months: 3, 24 | }) 25 | ), 26 | { representation: "date" } 27 | ), 28 | description: "The first day for which to generate events.", 29 | }), 30 | to: flags.string({ 31 | char: "t", 32 | default: () => formatISO(new Date(), { representation: "date" }), 33 | description: "The last day (inclusive) for which to generate events.", 34 | }), 35 | min: flags.string({ 36 | default: "0", 37 | description: "The minimum number of events to create for a given period.", 38 | }), 39 | max: flags.string({ 40 | default: "50", 41 | description: "The maximum number of events to create for a given period.", 42 | }), 43 | period: flags.string({ 44 | char: "p", 45 | default: "hour", 46 | description: "The period for which the min/max flags apply.", 47 | options: ["hour", "day", "month"], 48 | }), 49 | domain: flags.string({ 50 | char: "d", 51 | default: "local-testing.com", 52 | description: "The domain name for which to create events.", 53 | }), 54 | }; 55 | 56 | async run() { 57 | const start = Date.now(); 58 | const { flags } = this.parse(YaLoadTesting); 59 | 60 | this.log(`Running load testing with flags: %o`, flags); 61 | await performLoadTesting({ 62 | from: parseISO(flags.from), 63 | to: parseISO(flags.to), 64 | // @ts-ignore 65 | min: flags.min * 1, 66 | // @ts-ignore 67 | max: flags.max * 1, 68 | period: flags.period, 69 | domain: flags.domain, 70 | }); 71 | this.log("Load testing completed in %s.", formatDistanceToNowStrict(start)); 72 | } 73 | } 74 | 75 | export = YaLoadTesting; 76 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/src/load-testing.ts: -------------------------------------------------------------------------------- 1 | import { 2 | add, 3 | startOfDay, 4 | startOfHour, 5 | startOfMonth, 6 | endOfDay, 7 | endOfHour, 8 | endOfMonth, 9 | } from "date-fns"; 10 | 11 | import { IEvent, sendEvent } from "./api"; 12 | 13 | interface IFlags { 14 | from: Date; 15 | to: Date; 16 | min: number; 17 | max: number; 18 | period: string; 19 | domain: string; 20 | } 21 | 22 | const startAndEndPeriodCalculations: { 23 | [key: string]: { 24 | getStart: (date: Date) => number; 25 | getEnd: (date: Date) => number; 26 | }; 27 | } = { 28 | hour: { 29 | getStart: (date: Date) => startOfHour(date).getTime(), 30 | getEnd: (date: Date) => endOfHour(date).getTime(), 31 | }, 32 | day: { 33 | getStart: (date: Date) => startOfDay(date).getTime(), 34 | getEnd: (date: Date) => endOfDay(date).getTime(), 35 | }, 36 | month: { 37 | getStart: (date: Date) => startOfMonth(date).getTime(), 38 | getEnd: (date: Date) => endOfMonth(date).getTime(), 39 | }, 40 | }; 41 | 42 | const createEvent = (timestamp: Date, domain: string) => ({ 43 | name: "pageview", 44 | domain, 45 | url: `http://${domain}/tests`, 46 | referrer: "http://www.test-referrer.com", 47 | screen_size: 800, 48 | timestamp, 49 | }); 50 | 51 | const createEventsForPeriod = async ( 52 | current: Date, 53 | { max, min, period, domain }: IFlags 54 | ) => { 55 | const randomEventsCount = Math.round(Math.random() * (max - min) + min); 56 | console.log("Creating %s event(s) for %s", randomEventsCount, current); 57 | 58 | let start = startAndEndPeriodCalculations[period].getStart(current); 59 | let end = startAndEndPeriodCalculations[period].getEnd(current); 60 | let requestPromises: any[] | null = []; 61 | for (let i = 0; i < randomEventsCount; i++) { 62 | let event: IEvent | null = createEvent( 63 | new Date(Math.random() * (end - start) + start), 64 | domain 65 | ); 66 | requestPromises.push(sendEvent(event)); 67 | 68 | if (requestPromises.length === 150 || i === randomEventsCount - 1) { 69 | await Promise.all(requestPromises); 70 | requestPromises = null; 71 | requestPromises = []; 72 | } 73 | event = null; 74 | } 75 | }; 76 | 77 | export const performLoadTesting = async (flags: IFlags) => { 78 | let current = startOfDay(flags.from); 79 | const end = endOfDay(flags.to); 80 | while (current.getTime() <= end.getTime()) { 81 | await createEventsForPeriod(current, flags); 82 | current = add(current, { 83 | [`${flags.period}s`]: 1, 84 | }); 85 | global.gc(); 86 | // console.log(process.memoryUsage()) 87 | } 88 | }; 89 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/test/index.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from "@oclif/test"; 2 | import { formatISO, startOfMonth, sub } from "date-fns"; 3 | import { inspect } from "util"; 4 | 5 | import cmd = require("../src"); 6 | 7 | describe("ya-load-testing", () => { 8 | test 9 | .nock("http://localhost:8080", (api) => 10 | api 11 | .post("/", { 12 | name: "pageview", 13 | domain: "local-testing.com", 14 | url: "http://local-testing.com/tests", 15 | referrer: "http://www.test-referrer.com", 16 | screen_size: 800, 17 | timestamp: new Date(2020, 5, 28, 12, 13, 14).toISOString(), 18 | }) 19 | .reply(201) 20 | ) 21 | .stdout() 22 | .do(() => cmd.run([])) 23 | .it("runs ya-load-testing", (ctx) => { 24 | const defaultOptions = { 25 | from: "2020-07-01", 26 | to: "2020-10-12", 27 | min: "0", 28 | max: "50", 29 | period: "hour", 30 | }; 31 | expect(ctx.stdout).to.contain( 32 | `Running load testing with flags: ${inspect(defaultOptions)}` 33 | ); 34 | expect(ctx.stdout).to.contain("Load testing completed."); 35 | }); 36 | 37 | // test 38 | // .stdout() 39 | // .do(() => cmd.run(["--name", "jeff"])) 40 | // .it("runs hello --name jeff", (ctx) => { 41 | // expect(ctx.stdout).to.contain("hello jeff"); 42 | // }); 43 | }); 44 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/test/mocha.opts: -------------------------------------------------------------------------------- 1 | --require ts-node/register 2 | --watch-extensions ts 3 | --recursive 4 | --reporter spec 5 | --timeout 5000 6 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig", 3 | "compilerOptions": { 4 | "noEmit": true 5 | }, 6 | "references": [{ "path": ".." }] 7 | } 8 | -------------------------------------------------------------------------------- /services/events-api/ya-load-testing/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "importHelpers": true, 5 | "module": "commonjs", 6 | "outDir": "lib", 7 | "rootDir": "src", 8 | "strict": true, 9 | "target": "es2017" 10 | }, 11 | "include": ["src/**/*"] 12 | } 13 | -------------------------------------------------------------------------------- /services/query-api/.env: -------------------------------------------------------------------------------- 1 | CH_HOST=localhost 2 | CH_PORT=8123 3 | CH_USER= 4 | CH_PASSWORD= 5 | COOKIE_SECRET=cookie-dev-secret 6 | JWT_SECRET=jwt-dev-secret 7 | NODE_ENV=development -------------------------------------------------------------------------------- /services/query-api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-slim 2 | ARG SERVICE_PATH=services/query-api 3 | WORKDIR /app 4 | COPY ./local-packages /app/packages/ 5 | RUN for D in /app/packages/*; do npm ci --prod --prefix $D; done 6 | COPY ./package*.json /app/${SERVICE_PATH}/ 7 | RUN npm ci --prod --prefix ${SERVICE_PATH} 8 | 9 | FROM node:12-slim 10 | ARG SERVICE_PATH=services/query-api 11 | WORKDIR /app 12 | COPY --from=0 /app . 13 | COPY . ./${SERVICE_PATH}/ 14 | RUN rm -fr ./${SERVICE_PATH}/local-packages 15 | CMD ["node", "./services/query-api/src/index.js"] -------------------------------------------------------------------------------- /services/query-api/README.md: -------------------------------------------------------------------------------- 1 | # query-api 2 | 3 | Provides a query interface to the analytics database. 4 | 5 | ## Endpoints 6 | 7 | See the `src/index.js` file for all available API endpoints. 8 | 9 | ## Country code to name mapping (`src/clickhouse/world-map/code-name-map.json`) 10 | 11 | Navigate to https://www.iban.com/country-codes and run the following code in the dev console: 12 | 13 | ```js 14 | copy( 15 | Array.from($("#myTable tbody tr")).reduce((result, tr) => { 16 | result[tr.children[1].innerHTML] = tr.children[0].innerHTML 17 | .replace(" (the)", "") 18 | .replace("Russian Federation", "Russia") 19 | .replace(" of Great Britain and Northern Ireland", "") 20 | .replace("Korea (the Democratic People's Republic of)", "North Korea") 21 | .replace("Korea (the Republic of)", "South Korea") 22 | .replace("Virgin Islands (British)", "British Virgin Is.") 23 | .replace("Virgin Islands (U.S.)", "U.S. Virgin Is.") 24 | .replace("Taiwan (Province of China)", "Taiwan") 25 | .replace("Venezuela (Bolivarian Republic of)", "Venezuela") 26 | .replace("Dominican Republic", "Dominican Rep.") 27 | .replace("Bosnia and Herzegovina", "Bosnia and Herz.") 28 | .replace("Tanzania, United Republic of", "Tanzania") 29 | .replace("Lao People's Democratic Republic", "Laos"); 30 | return result; 31 | }, {}) 32 | ); 33 | ``` 34 | 35 | ## Serverless VPC Access 36 | 37 | https://cloud.google.com/vpc/docs/configure-serverless-vpc-access#gcloud 38 | 39 | ```bash 40 | gcloud services enable vpcaccess.googleapis.com 41 | gcloud compute networks vpc-access connectors create [CONNECTOR_NAME] \ 42 | --network [VPC_NETWORK] \ 43 | --region [REGION] \ 44 | --range [IP_RANGE] 45 | ``` 46 | -------------------------------------------------------------------------------- /services/query-api/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: "bash" 3 | args: ["mkdir", "local-packages"] 4 | dir: "services/query-api" 5 | - name: "bash" 6 | args: 7 | [ 8 | "cp", 9 | "-r", 10 | "../../packages/clickhouse", 11 | "../../packages/faunadb", 12 | "local-packages/", 13 | ] 14 | dir: "services/query-api" 15 | - name: "gcr.io/cloud-builders/docker" 16 | args: ["build", "-t", "gcr.io/$PROJECT_ID/query-api:$COMMIT_SHA", "."] 17 | dir: "services/query-api" 18 | - name: "bash" 19 | args: ["rm", "-fr", "local-packages"] 20 | dir: "services/query-api" 21 | - name: "gcr.io/cloud-builders/docker" 22 | args: ["push", "gcr.io/$PROJECT_ID/query-api:$COMMIT_SHA"] 23 | dir: "services/query-api" 24 | - name: "gcr.io/cloud-builders/gcloud" 25 | args: 26 | - "beta" 27 | - "run" 28 | - "deploy" 29 | - "query-api" 30 | - "--image" 31 | - "gcr.io/$PROJECT_ID/query-api:$COMMIT_SHA" 32 | - "--region" 33 | - "us-central1" 34 | - "--platform" 35 | - "managed" 36 | - "--vpc-connector" 37 | - "cloud-run-vpc-connector" 38 | - "--allow-unauthenticated" 39 | - "--set-env-vars" 40 | - "FAUNADB_ADMIN_SECRET=$_FAUNADB_ADMIN_SECRET,FAUNADB_SERVER_SECRET=$_FAUNADB_SERVER_SECRET,CH_HOST=$_CH_HOST,CH_PORT=$_CH_PORT,CH_USER=$_CH_USER,CH_PASSWORD=$_CH_PASSWORD,COOKIE_SECRET=$_COOKIE_SECRET,JWT_SECRET=$_JWT_SECRET,NODE_ENV=production" 41 | dir: "services/query-api" 42 | images: 43 | - "gcr.io/$PROJECT_ID/query-api:$COMMIT_SHA" 44 | -------------------------------------------------------------------------------- /services/query-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/query-api", 3 | "version": "1.0.0", 4 | "description": "Provides a query interface to the analytics database.", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "dev": "nodemon -r dotenv/config .", 8 | "start": "node .", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "author": "", 12 | "license": "ISC", 13 | "dependencies": { 14 | "@your-analytics/clickhouse": "file:../../packages/clickhouse", 15 | "@your-analytics/faunadb": "file:../../packages/faunadb", 16 | "cookie-parser": "^1.4.5", 17 | "cors": "^2.8.5", 18 | "date-fns": "^2.16.1", 19 | "express": "^4.17.1", 20 | "helmet": "^4.1.1", 21 | "jsonwebtoken": "^8.5.1", 22 | "rate-limiter-flexible": "^2.1.10" 23 | }, 24 | "devDependencies": { 25 | "dotenv": "^8.2.0", 26 | "nodemon": "^2.0.4" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /services/query-api/src/index.js: -------------------------------------------------------------------------------- 1 | const { rootDb, domainDb } = require("@your-analytics/faunadb"); 2 | const cookieParser = require("cookie-parser"); 3 | const cors = require("cors"); 4 | const { 5 | endOfDay, 6 | parseISO, 7 | startOfDay, 8 | startOfYear, 9 | sub, 10 | } = require("date-fns"); 11 | const express = require("express"); 12 | const helmet = require("helmet"); 13 | const jwt = require("jsonwebtoken"); 14 | const { RateLimiterMemory } = require("rate-limiter-flexible"); 15 | const { stats } = require("@your-analytics/clickhouse"); 16 | 17 | const rateLimiter = new RateLimiterMemory({ 18 | points: 20, // # of requests 19 | duration: 1, // per x seconds by IP 20 | }); 21 | 22 | const rateLimiterMiddleware = (req, res, next) => { 23 | rateLimiter 24 | .consume(req.ip) 25 | .then(() => { 26 | next(); 27 | }) 28 | .catch(() => { 29 | res.status(429).send("Too Many Requests"); 30 | }); 31 | }; 32 | 33 | const app = express(); 34 | const port = process.env.PORT || 8081; 35 | 36 | process.env.NODE_ENV === "development" && 37 | app.use( 38 | cors({ 39 | origin: /\.gitpod.io/, 40 | }) 41 | ); 42 | 43 | app.use(helmet()); 44 | app.use(rateLimiterMiddleware); 45 | app.use(cookieParser(process.env.COOKIE_SECRET)); 46 | 47 | const datePresets = { 48 | // Also update `services/website/src/components/date-range.svelte` 49 | today: { 50 | calculateFromDate: () => new Date(), 51 | calculateToDate: () => new Date(), 52 | }, 53 | "7days": { 54 | calculateFromDate: () => 55 | sub(new Date(), { 56 | days: 7, 57 | }), 58 | calculateToDate: () => new Date(), 59 | }, 60 | "30days": { 61 | calculateFromDate: () => 62 | sub(new Date(), { 63 | days: 30, 64 | }), 65 | calculateToDate: () => new Date(), 66 | }, 67 | thisyear: { 68 | calculateFromDate: () => startOfYear(new Date()), 69 | calculateToDate: () => new Date(), 70 | }, 71 | custom: { 72 | calculateFromDate: (from) => parseISO(from), 73 | calculateToDate: (to) => parseISO(to), 74 | }, 75 | }; 76 | const determineDateRange = (preset, from, to) => ({ 77 | from: startOfDay(datePresets[preset].calculateFromDate(from)), 78 | to: endOfDay(datePresets[preset].calculateToDate(to)), 79 | }); 80 | 81 | const createStatsEndpoint = (path, fetcher) => { 82 | app.get(`/:domain/${path}`, async (req, res) => { 83 | try { 84 | const domain = req.params.domain; 85 | const { data: visibilityData } = await domainDb.settings.getVisibility( 86 | domain 87 | ); 88 | 89 | let websiteSettings = {}; 90 | if (visibilityData.visibility === "private") { 91 | const jwtCookie = req.signedCookies["jwt"]; 92 | 93 | let reqUser; 94 | try { 95 | const { user } = jwt.verify(jwtCookie, process.env.JWT_SECRET); 96 | reqUser = user; 97 | } catch (jwtError) { 98 | res.status(401).end(); 99 | return; 100 | } 101 | 102 | const user = await rootDb.users.find(reqUser.issuer); 103 | if (!user.sites[domain]) { 104 | console.error( 105 | new Error( 106 | `User ${user.issuer} tried to access domain ${domain} but is not authorized.` 107 | ) 108 | ); 109 | res.status(401).end(); 110 | return; 111 | } 112 | 113 | const domainServerKeySecret = await rootDb.users.getDomainServerKeySecret( 114 | user.issuer, 115 | domain 116 | ); 117 | const { data } = await domainDb.settings.getSettings( 118 | domainServerKeySecret 119 | )(); 120 | websiteSettings = data; 121 | } else { 122 | const { data } = await domainDb.settings.getSettingsPublic(domain); 123 | websiteSettings = data; 124 | } 125 | 126 | const dateRange = determineDateRange( 127 | req.query.preset, 128 | req.query.from, 129 | req.query.to 130 | ); 131 | const data = await fetcher(dateRange, domain, websiteSettings); 132 | res.json({ data }); 133 | } catch (error) { 134 | console.error(error); 135 | res.status(500).end(); 136 | } 137 | }); 138 | }; 139 | 140 | createStatsEndpoint("browser", stats.fetchBrowser); 141 | createStatsEndpoint("os", stats.fetchOs); 142 | createStatsEndpoint("screen", stats.fetchScreen); 143 | createStatsEndpoint("top-pages", stats.fetchTopPages); 144 | createStatsEndpoint("top-referrers", stats.fetchTopReferrers); 145 | createStatsEndpoint("total-pageviews", stats.fetchTotalPageviews); 146 | createStatsEndpoint("unique-visitors", stats.fetchUniqueVisitors); 147 | createStatsEndpoint("visitors", stats.fetchVisitors); 148 | createStatsEndpoint("world-map", stats.fetchWorldMap); 149 | 150 | app.listen(port, () => { 151 | console.log(`query-api started at http://localhost:${port}`); 152 | }); 153 | -------------------------------------------------------------------------------- /services/website/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.svelte-kit 3 | /.vercel_build_output 4 | /node_modules/ 5 | yarn-error.log 6 | /cypress/screenshots/ 7 | /cypress/videos/ 8 | /static/tailwind.css -------------------------------------------------------------------------------- /services/website/README.md: -------------------------------------------------------------------------------- 1 | # website 2 | 3 | The website for www.your-analytics.org. 4 | 5 | For anonymous website visitors, a marketing landing page is displayed. 6 | For logged-in website visitors, their analytics dashboards are displayed. 7 | -------------------------------------------------------------------------------- /services/website/cypress.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://localhost:3000", 3 | "video": false, 4 | "projectId": "gynhxr", 5 | "retries": { 6 | "runMode": 1, 7 | "openMode": 0 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /services/website/cypress/fixtures/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Using fixtures to represent data", 3 | "email": "hello@cypress.io", 4 | "body": "Fixtures are a great way to mock data for responses to routes" 5 | } 6 | -------------------------------------------------------------------------------- /services/website/cypress/integration/auth.spec.ts: -------------------------------------------------------------------------------- 1 | describe("/auth", () => { 2 | beforeEach(() => { 3 | cy.visit("/auth"); 4 | }); 5 | 6 | it("should redirect unauthenticated users to the /auth page", () => { 7 | cy.visit("/dashboard"); 8 | cy.location("pathname").should("equal", "/auth"); 9 | }); 10 | 11 | /** 12 | * Skip until test users are available as per https://twitter.com/ikscodes/status/1313637273498337281 13 | */ 14 | it.skip("should authenticate and redirect to the /onboarding page", () => { 15 | cy.get('input[name="email"]').type( 16 | "hello+ya-automated-tests@mikenikles.com" 17 | ); 18 | cy.get("button[type=submit]").click(); 19 | cy.location("pathname", { 20 | timeout: 1000 * 60, // Time to manually click the auth link in the email 21 | }).should("equal", "/onboarding"); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /services/website/cypress/integration/onboarding.spec.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | describe("/onboarding", () => { 4 | beforeEach(() => { 5 | cy.task("db:reset"); 6 | cy.loginWithApi(); 7 | cy.visit("/onboarding"); 8 | }); 9 | 10 | it("should load the /onboarding page", () => { 11 | cy.location("pathname").should("equal", "/onboarding"); 12 | }); 13 | 14 | it("should create a new website", () => { 15 | const websiteUrl = "dev.com"; 16 | cy.findByLabelText("What's your first name?").type("Tester"); 17 | cy.findByText("Next: Configure your website").click(); 18 | cy.findByLabelText("What's your website URL?").type(websiteUrl); 19 | cy.findByLabelText("What's your preferred reporting timezone?").select( 20 | "America/Toronto" 21 | ); 22 | cy.findByText("Let's go").click(); 23 | 24 | cy.get("#script").should("contain.value", `data-domain="${websiteUrl}"`); 25 | cy.get(`a[href="https://${websiteUrl}"]`).contains(websiteUrl); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /services/website/cypress/plugins/db.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | 3 | const tasks = { 4 | async "db:reset"() { 5 | await fetch("http://localhost:3000/api/admin/tests/db-reset", { 6 | method: "POST", 7 | }); 8 | return null; 9 | }, 10 | }; 11 | 12 | export default (on, config) => { 13 | on("task", tasks); 14 | return config; 15 | }; 16 | -------------------------------------------------------------------------------- /services/website/cypress/plugins/index.ts: -------------------------------------------------------------------------------- 1 | // *********************************************************** 2 | // This example plugins/index.js can be used to load plugins 3 | // 4 | // You can change the location of this file or turn off loading 5 | // the plugins file with the 'pluginsFile' configuration option. 6 | // 7 | // You can read more here: 8 | // https://on.cypress.io/plugins-guide 9 | // *********************************************************** 10 | import dbTasks from "./db"; 11 | 12 | // This function is called when a project is opened or re-opened (e.g. due to 13 | // the project's config changing) 14 | 15 | export default (on, config) => { 16 | // `on` is used to hook into various events Cypress emits 17 | // `config` is the resolved Cypress config 18 | dbTasks(on, config); 19 | }; 20 | -------------------------------------------------------------------------------- /services/website/cypress/support/commands.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | declare namespace Cypress { 4 | interface Chainable { 5 | /** 6 | * Custom command to log in with an API call. 7 | * This allows bypassing the Magic link authentication step. 8 | * 9 | * @example cy.loginWithApi("test-user@your-analytics.org") 10 | */ 11 | loginWithApi(): Chainable; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /services/website/cypress/support/commands.ts: -------------------------------------------------------------------------------- 1 | // *********************************************** 2 | // This example commands.js shows you how to 3 | // create various custom commands and overwrite 4 | // existing commands. 5 | // 6 | // For more comprehensive examples of custom 7 | // commands please read more here: 8 | // https://on.cypress.io/custom-commands 9 | // *********************************************** 10 | // 11 | // 12 | // -- This is a parent command -- 13 | // Cypress.Commands.add("login", (email, password) => { ... }) 14 | // 15 | // 16 | // -- This is a child command -- 17 | // Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... }) 18 | // 19 | // 20 | // -- This is a dual command -- 21 | // Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... }) 22 | // 23 | // 24 | // -- This is will overwrite an existing command -- 25 | // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) 26 | 27 | import "@testing-library/cypress/add-commands"; 28 | 29 | Cypress.Commands.add("loginWithApi", () => { 30 | return cy.request("POST", "/api/admin/tests/user/login-with-api", { 31 | email: "hello+ya-automated-tests@mikenikles.com", 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /services/website/cypress/support/index.ts: -------------------------------------------------------------------------------- 1 | // *********************************************************** 2 | // This example support/index.js is processed and 3 | // loaded automatically before your test files. 4 | // 5 | // This is a great place to put global configuration and 6 | // behavior that modifies Cypress. 7 | // 8 | // You can change the location of this file or turn off 9 | // automatically serving support files with the 10 | // 'supportFile' configuration option. 11 | // 12 | // You can read more here: 13 | // https://on.cypress.io/configuration 14 | // *********************************************************** 15 | 16 | // Import commands.js using ES2015 syntax: 17 | import "./commands"; 18 | 19 | // Alternatively you can use CommonJS syntax: 20 | // require('./commands') 21 | -------------------------------------------------------------------------------- /services/website/cypress/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["es5", "dom"], 5 | "types": ["cypress", "@testing-library/cypress"] 6 | }, 7 | "include": ["**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /services/website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@your-analytics/website", 3 | "description": "The website for www.your-analytics.org", 4 | "version": "0.0.1", 5 | "type": "module", 6 | "scripts": { 7 | "build": "run-s clean && npm run build:css && NODE_ENV=production svelte-kit build", 8 | "build:css": "NODE_ENV=production postcss src/tailwind.css -o static/tailwind.css", 9 | "clean": "rm -fr .svelte-kit", 10 | "cy:install": "cypress install", 11 | "cy:open": "cypress open", 12 | "cy:run": "cypress run", 13 | "dev": "run-s clean && run-p watch:*", 14 | "start": "svelte-kit preview", 15 | "test": "run-p --race dev cy:run", 16 | "validate": "svelte-check --ignore **/world-map.svelte", 17 | "watch:css": "postcss src/tailwind.css -o static/tailwind.css -w", 18 | "watch:dev": "svelte-kit dev --host" 19 | }, 20 | "dependencies": { 21 | "chart.js": "^3.7.0", 22 | "chartjs-chart-geo": "^3.6.0", 23 | "cookie": "^0.4.1", 24 | "cookie-parser": "^1.4.6", 25 | "date-fns": "^2.27.0", 26 | "jsonwebtoken": "^8.5.1", 27 | "magic-sdk": "^2.3.1" 28 | }, 29 | "devDependencies": { 30 | "@sveltejs/adapter-auto": "^1.0.0-next.5", 31 | "@sveltejs/kit": "^1.0.0-next.203", 32 | "@testing-library/cypress": "^8.0.2", 33 | "@tsconfig/svelte": "^1.0.10", 34 | "@types/chart.js": "^2.9.24", 35 | "@types/node": "^14.11.2", 36 | "autoprefixer": "^10.4.0", 37 | "cssnano": "^5.0.14", 38 | "cypress": "^9.2.0", 39 | "node-fetch": "^2.6.1", 40 | "npm-run-all": "^4.1.5", 41 | "postcss": "^8.4.5", 42 | "postcss-cli": "^9.1.0", 43 | "postcss-preset-env": "^7.1.0", 44 | "svelte": "^3.44.3", 45 | "svelte-check": "^2.2.11", 46 | "svelte-preprocess": "^4.10.1", 47 | "tailwindcss": "^3.0.7", 48 | "tslib": "^2.0.1", 49 | "typescript": "^4.0.3" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /services/website/postcss.config.cjs: -------------------------------------------------------------------------------- 1 | const tailwind = require("tailwindcss"); 2 | const cssnano = require("cssnano"); 3 | const presetEnv = require("postcss-preset-env")(); 4 | const autoprefixer = require("autoprefixer"); 5 | 6 | const plugins = 7 | process.env.NODE_ENV === "production" 8 | ? [tailwind, presetEnv, autoprefixer, cssnano] 9 | : [tailwind, presetEnv]; 10 | 11 | module.exports = { plugins }; 12 | -------------------------------------------------------------------------------- /services/website/src/api/onboarding.ts: -------------------------------------------------------------------------------- 1 | import { ADMIN_API_BASE_URL } from "../config"; 2 | 3 | export const setFirstName = async (firstName) => { 4 | const url = `${ADMIN_API_BASE_URL}/user`; 5 | const response = await fetch(url, { 6 | headers: new Headers({ 7 | "Content-Type": "application/json", 8 | }), 9 | body: JSON.stringify({ firstName }), 10 | method: "PUT", 11 | }); 12 | return response.status; 13 | }; 14 | 15 | export const addNewWebsite = async (info) => { 16 | const url = `${ADMIN_API_BASE_URL}/website`; 17 | const response = await fetch(url, { 18 | headers: new Headers({ 19 | "Content-Type": "application/json", 20 | }), 21 | body: JSON.stringify(info), 22 | method: "POST", 23 | }); 24 | return response.status; 25 | }; 26 | -------------------------------------------------------------------------------- /services/website/src/api/settings.ts: -------------------------------------------------------------------------------- 1 | import { ADMIN_API_BASE_URL } from "../config"; 2 | 3 | export const setVisibility = async (website, visibility) => { 4 | const url = `/${ADMIN_API_BASE_URL}/website/${website}/settings/visibility`; 5 | const response = await fetch(url, { 6 | headers: new Headers({ 7 | "Content-Type": "application/json", 8 | }), 9 | body: JSON.stringify({ 10 | visibility, 11 | }), 12 | method: "PUT", 13 | }); 14 | 15 | return response.status === 200; 16 | }; 17 | 18 | export const getVisibility = async (fetch, host, website) => { 19 | const response = await fetch( 20 | `https://${host}/${ADMIN_API_BASE_URL}/website/${website}/settings/visibility` 21 | ); 22 | 23 | if (response.status === 200) { 24 | return await response.json(); 25 | } 26 | return null; 27 | }; 28 | 29 | export const getSettings = async (fetch, host, website) => { 30 | const response = await fetch( 31 | `https://${host}/${ADMIN_API_BASE_URL}/website/${website}/settings`, 32 | { 33 | headers: { 34 | "Content-Type": "application/json", 35 | }, 36 | method: "GET", 37 | credentials: "include", 38 | } 39 | ); 40 | 41 | if (response.status === 200) { 42 | return await response.json(); 43 | } 44 | return null; 45 | }; 46 | -------------------------------------------------------------------------------- /services/website/src/api/stats.ts: -------------------------------------------------------------------------------- 1 | import { get, writable } from "svelte/store"; 2 | import { QUERY_API_BASE_URL } from "../config"; 3 | import statsFilterQueryString from "../stores/stats-filters-query-string"; 4 | 5 | const fetchStats = (fetch, host, site) => async (path, storeName) => { 6 | const url = `https://${host}/${QUERY_API_BASE_URL}/${site}/${path}?${get( 7 | statsFilterQueryString 8 | )}`; 9 | const response = await fetch(url, { 10 | credentials: "include", 11 | }); 12 | 13 | if (response.status === 200) { 14 | const data = (await response.json()).data; 15 | statsStores[storeName].set(data); // Needed on the client to reload the dashboard when the site changes. 16 | return { 17 | storeName, 18 | data, 19 | }; 20 | } 21 | statsStores[storeName].set({}); // Needed on the client to reload the dashboard when the site changes. 22 | return { 23 | storeName, 24 | data: {}, 25 | }; 26 | }; 27 | 28 | interface IStringNumber { 29 | [key: string]: number; 30 | } 31 | 32 | export const browser = writable(null); 33 | export const os = writable(null); 34 | export const screen = writable(null); 35 | export const topPages = writable(null); 36 | export const topReferrers = writable(null); 37 | export const totalPageviews = writable(null); 38 | export const uniqueVisitors = writable(null); 39 | export const visitors = writable(null); 40 | export const worldMap = writable(null); 41 | 42 | export const statsStores = { 43 | browser, 44 | os, 45 | screen, 46 | topPages, 47 | topReferrers, 48 | totalPageviews, 49 | uniqueVisitors, 50 | visitors, 51 | worldMap, 52 | }; 53 | 54 | const resetAllStats = () => { 55 | browser.set(null); 56 | os.set(null); 57 | screen.set(null); 58 | topPages.set(null); 59 | topReferrers.set(null); 60 | totalPageviews.set(null); 61 | uniqueVisitors.set(null); 62 | visitors.set({}); // `null` destroys the chart 63 | worldMap.set({}); // `null` destroys the chart 64 | }; 65 | 66 | export const fetchAllStats = (fetch, host, site) => { 67 | resetAllStats(); 68 | const fetcher = fetchStats(fetch, host, site); 69 | 70 | return Promise.allSettled([ 71 | fetcher("browser", "browser"), 72 | fetcher("os", "os"), 73 | fetcher("screen", "screen"), 74 | fetcher("top-pages", "topPages"), 75 | fetcher("top-referrers", "topReferrers"), 76 | fetcher("total-pageviews", "totalPageviews"), 77 | fetcher("unique-visitors", "uniqueVisitors"), 78 | fetcher("visitors", "visitors"), 79 | fetcher("world-map", "worldMap"), 80 | ]); 81 | }; 82 | 83 | export const fetchUniqueVisitorsOnOnboardingPage = (fetch, host, site) => { 84 | uniqueVisitors.set(null); 85 | const fetcher = fetchStats(fetch, host, site); 86 | return fetcher("unique-visitors", "uniqueVisitors"); 87 | }; 88 | -------------------------------------------------------------------------------- /services/website/src/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | %svelte.head% 16 | 17 | 18 | 20 |
%svelte.body%
21 | 22 | 23 | -------------------------------------------------------------------------------- /services/website/src/auth/magic.ts: -------------------------------------------------------------------------------- 1 | import { Magic } from "magic-sdk"; 2 | import { ADMIN_API_BASE_URL, MAGIC_PUBLIC_KEY } from "../config"; 3 | 4 | let magic; 5 | 6 | const init = () => { 7 | magic = new Magic(MAGIC_PUBLIC_KEY); 8 | }; 9 | 10 | export const login = async (event) => { 11 | if (!magic) { 12 | init(); 13 | } 14 | const email = new FormData(event.target).get("email"); 15 | if (email) { 16 | const didToken = await magic.auth.loginWithMagicLink({ email }); 17 | const response = await fetch(`${ADMIN_API_BASE_URL}/user/login`, { 18 | headers: new Headers({ 19 | Authorization: "Bearer " + didToken, 20 | }), 21 | method: "POST", 22 | }); 23 | 24 | if (response.status === 200) { 25 | window.location.href = "dashboard"; 26 | } 27 | } 28 | }; 29 | 30 | export const logout = async () => { 31 | await fetch(`${ADMIN_API_BASE_URL}/user/logout`, { 32 | method: "POST", 33 | }); 34 | }; 35 | -------------------------------------------------------------------------------- /services/website/src/components/card.svelte: -------------------------------------------------------------------------------- 1 | 4 | 5 |
6 | 7 |
8 | -------------------------------------------------------------------------------- /services/website/src/components/date-range.svelte: -------------------------------------------------------------------------------- 1 | 44 | 45 |
46 |
47 | 52 |
53 | 54 | {#if datePreset === "custom"} 55 |
56 | 57 |
58 | 59 |
60 |
61 |
62 | 63 |
64 | 65 |
66 |
67 |
68 | 69 |
70 | {/if} 71 |
72 | -------------------------------------------------------------------------------- /services/website/src/components/footer.svelte: -------------------------------------------------------------------------------- 1 | 4 | 5 |
6 |
7 |
8 | 9 | Twitter 10 | 11 | 12 | 13 | 14 | 15 | GitHub 16 | 17 | 18 | 19 | 20 |
21 |
22 |

23 | © 2020 - {currentYear} Your Analytics, Inc. All rights reserved. 24 |

25 |
26 |
27 |
28 | -------------------------------------------------------------------------------- /services/website/src/components/header/index.svelte: -------------------------------------------------------------------------------- 1 | 18 | 19 |
20 | 21 | Your Analytics logo 22 | 23 | {#if $session.user} 24 |
25 | 32 | 33 | 34 |
35 | {/if} 36 |
37 | -------------------------------------------------------------------------------- /services/website/src/components/header/nav-item.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 | 12 | 13 |
  • 14 | {label} 15 |
    • 16 | -------------------------------------------------------------------------------- /services/website/src/components/header/nav-mobile-menu-button.svelte: -------------------------------------------------------------------------------- 1 | 4 | 5 |
    6 | 37 |
    38 | -------------------------------------------------------------------------------- /services/website/src/components/header/nav-mobile.svelte: -------------------------------------------------------------------------------- 1 | 14 | 15 |
    16 | {#if $isOpen} 17 | {#if false} 18 |
    19 | Your websites 20 | Team 21 |
    22 | {/if} 23 |
    24 |
    25 |
    26 | Profile avatar 27 |
    28 |
    29 |
    {userFirstName}
    30 |
    {userEmail}
    31 |
    32 |
    33 |
    34 | {#if hasMultipleSites} 35 | Your websites 36 | {:else} 37 | {userWebsites[0]} 38 | {/if} 39 | 40 |
    41 |
    42 | {/if} 43 |
    44 | -------------------------------------------------------------------------------- /services/website/src/components/header/nav.svelte: -------------------------------------------------------------------------------- 1 | 13 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/feature-card.svelte: -------------------------------------------------------------------------------- 1 | 8 | 9 | 10 |

    {title}

    11 |
    12 | {title} 13 |
    14 |

    {description}

    15 |     16 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/features.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
    7 |
    8 | 9 | 10 | 11 |
    12 |
    13 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/frontend-integrations.svelte: -------------------------------------------------------------------------------- 1 | 31 | 32 | 47 | 48 |
    49 |
    50 | 51 |

    52 | Supports your frontend stack of choice 53 |

    54 |
    55 | {#each Object.entries(stacks) as [key, stack]} 56 |
    57 | {stack.label} 63 |
    64 | {/each} 65 |
    66 | {#if false} 67 | 68 | {#if stacks[selectedStack]} 69 |
    70 |
    71 |
    72 | Copy to clipboard 76 | Copy to clipboard 77 |
    78 |
    79 |
    80 | {@html stacks[selectedStack].instructions} 81 | 82 |
    83 |
    84 | {/if} 85 | {/if} 86 |     87 |
    88 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/hero.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 |
    8 |
    9 |
    10 |

    Your web analytics
    with privacy and simplicity in mind

    11 |

    Your Analytics - an open source, privacy-focused web analytics platform. Focus on the basics and respect visitor privacy.

    12 |
    13 | 14 | Get Started 15 | 16 | 17 | 18 | 19 | 20 | 21 | Live Demo 22 | 23 |
    24 |
    25 |
    26 |
    -------------------------------------------------------------------------------- /services/website/src/components/landing-page/pricing.svelte: -------------------------------------------------------------------------------- 1 | 20 | 21 |
    22 |
    23 | 24 |

    Self-hosted

    25 |

    Your Analytics is open source and available for you to host on your own servers.For Free!

    26 |

    $0

    27 | {#if false} 28 | 29 | 30 | {/if} 31 |     32 | 33 |

    Hosted

    34 |

    Hassle-free, hosted by the team who develops Your Analytics. Pay based on how many pages your visitors look at on your website.

    35 |
    36 |
    37 |

    {pricingOptions[selectedPricingIndex].price}/m

    38 |

    39 | {pricingOptions[selectedPricingIndex].pageViews} page views 40 |

    41 |
    42 |
    43 |
    44 |

    Page views per month:

    45 | {#each pricingOptions as {pageViews, price}, index} 46 |
    47 | 48 | 51 |
    52 | {/each} 53 |
    54 | {#if false} 55 |

    For 10M+ page views, please get in touch with us. Contact us.

    56 | {/if} 57 |
    58 |
    59 | {#if false} 60 | 61 | {/if} 62 |     63 |
    64 |
    65 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/section.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
    7 |
    8 |

    {title}

    9 |

    {description}

    10 |
    11 | 12 |
    13 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/stats-card.svelte: -------------------------------------------------------------------------------- 1 | 7 | 8 | 9 |
    10 |

    {title}

    11 |

    {description}

    12 |
    13 |
    14 | 15 |
    16 |     17 | -------------------------------------------------------------------------------- /services/website/src/components/landing-page/stats.svelte: -------------------------------------------------------------------------------- 1 | 14 | 15 |
    16 |
    17 | 18 |
    19 | 20 | 21 |
    22 |     23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
    47 |
    48 | -------------------------------------------------------------------------------- /services/website/src/components/main-content.svelte: -------------------------------------------------------------------------------- 1 |
    2 | 3 |
    4 | -------------------------------------------------------------------------------- /services/website/src/components/onboarding/step-1.svelte: -------------------------------------------------------------------------------- 1 | 19 | 20 | 21 |
    22 | 25 |
    26 | 27 |
    28 |
    29 | 30 |
    31 | 32 | 35 | 36 |
    37 |     38 | -------------------------------------------------------------------------------- /services/website/src/components/onboarding/step-2.svelte: -------------------------------------------------------------------------------- 1 | 46 | 47 | 53 | 54 | 55 |
    56 | 59 |
    60 | 61 | http(s)://www. 62 | 63 | $onboarding.isSiteAlreadyConfiguredGlobally = false} type="text" name="url" required class="px-3 py-2 w-full border border-gray-300 rounded-none rounded-r-md focus:outline-none focus:shadow-outline-blue focus:border-blue-300 sm:text-sm sm:leading-5"> 64 | {#if isUrlInvalid && !$onboarding.isSiteAdded} 65 |
    66 | 67 | 68 | 69 |
    70 | {/if} 71 |
    72 |

    73 | {#if !$onboarding.isSiteAdded && hasUserAlreadyConfiguredSite} 74 | You've already configured this website. 75 | {:else if !$onboarding.isSiteAdded && $onboarding.isSiteAlreadyConfiguredGlobally} 76 | This site has already been configured. If it belongs to you, please contact us. 77 | {/if} 78 |

    79 |
    80 | 81 |
    82 | 85 |
    86 | 87 |
    88 |
    89 | 90 |
    91 | 92 | 95 | 96 |
    97 |     98 | -------------------------------------------------------------------------------- /services/website/src/components/onboarding/step-3.svelte: -------------------------------------------------------------------------------- 1 | 15 | 16 | 17 |

    Add the following script to all pages you want to track on your website.

    18 |
    19 | 20 | 23 |
    24 |

    Once added, please visit {$onboarding.url} to see the first few visits logged below.

    25 |
    26 | 27 |
    28 |

    Don't want to wait? Go to your dashboard.

    29 |     30 | -------------------------------------------------------------------------------- /services/website/src/components/onboarding/step-header.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | 17 |
  • 18 |
    19 |

    Step {stepNumber}

    20 |

    {label}

    21 |
    22 |
    • 23 | -------------------------------------------------------------------------------- /services/website/src/components/onboarding/step-wrapper.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
    7 | 8 | 9 | 10 |
    11 | -------------------------------------------------------------------------------- /services/website/src/components/stats/devices.svelte: -------------------------------------------------------------------------------- 1 | 19 | 20 | 25 | 26 | {#if $browser && $os && $screen} 27 | {#if showTitle} 28 |

    Devices

    29 | {/if} 30 | 31 |
    32 | 33 | 34 | 35 |
    36 | 37 | 38 | 39 | {#each tabData[selectedTabIndex] as [label, total], rowIndex} 40 | 41 | {label} 42 | 43 | 44 | {/each} 45 | 46 |
    47 | {:else} 48 | 49 | {/if} 50 | -------------------------------------------------------------------------------- /services/website/src/components/stats/elements/comparison-numbers.svelte: -------------------------------------------------------------------------------- 1 | 8 | 9 |
    10 |
    11 |
    12 |
    13 | {title} 14 |
    15 |
    16 |
    17 | {#if currentValue} 18 | 19 | {:else} 20 | 21 | {/if} 22 |
    23 |
    24 |
    25 |
    26 |
    27 | -------------------------------------------------------------------------------- /services/website/src/components/stats/elements/number.svelte: -------------------------------------------------------------------------------- 1 | 11 | 12 | {formattedNumber} 13 | -------------------------------------------------------------------------------- /services/website/src/components/stats/loading.svelte: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /services/website/src/components/stats/top-pages.svelte: -------------------------------------------------------------------------------- 1 | 18 | 19 | {#if $topPages} 20 | {#if showTitle || showShowMore} 21 |
    22 | {#if showTitle} 23 |

    Top Pages

    24 | {/if} 25 | {#if showShowMore && !showRest && rest.length > 0} 26 | 27 | {/if} 28 |
    29 | {/if} 30 | 31 | 32 | 33 | {#each topTen as [page, total], rowIndex} 34 | 35 | {page} 36 | 37 | 38 | {/each} 39 | {#if showRest} 40 | {#each rest as [page, total], rowIndex} 41 | 42 | {page} 43 | 44 | 45 | {/each} 46 | {/if} 47 | 48 |
    49 | {:else} 50 | 51 | {/if} 52 | -------------------------------------------------------------------------------- /services/website/src/components/stats/top-referrers.svelte: -------------------------------------------------------------------------------- 1 | 18 | 19 | {#if $topReferrers} 20 | {#if showTitle || showShowMore} 21 |
    22 | {#if showTitle} 23 |

    Top Referrers

    24 | {/if} 25 | {#if showShowMore && !showRest && rest.length > 0} 26 | 27 | {/if} 28 |
    29 | {/if} 30 | 31 | 32 | 33 | {#each topTen as [domain, total], rowIndex} 34 | 35 | 36 |
    37 | {domain} favicon 38 |
    39 |     40 | {domain} 41 | 42 |     43 | {/each} 44 | {#if showRest} 45 | {#each rest as [domain, total], rowIndex} 46 | 47 | {domain} favicon 48 | {domain} 49 | 50 | 51 | {/each} 52 | {/if} 53 | 54 |
    55 | {:else} 56 | 57 | {/if} 58 | -------------------------------------------------------------------------------- /services/website/src/components/stats/total-pageviews.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /services/website/src/components/stats/unique-visitors.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /services/website/src/components/stats/visitors.svelte: -------------------------------------------------------------------------------- 1 | 119 | 120 |
    121 | {#if $visitors} 122 |
    124 | 125 |
    126 | {:else} 127 | 128 | {/if} 129 |
    130 | -------------------------------------------------------------------------------- /services/website/src/components/stats/world-map.svelte: -------------------------------------------------------------------------------- 1 | 62 | 63 | {#if $worldMap} 64 | {#if showTitle} 65 |

    World map

    66 | {/if} 67 | 68 |
    70 | 71 |
    72 | {:else} 73 | 74 | {/if} 75 | -------------------------------------------------------------------------------- /services/website/src/components/table/cell.svelte: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | 17 | -------------------------------------------------------------------------------- /services/website/src/components/table/index.svelte: -------------------------------------------------------------------------------- 1 |
    2 | 3 | 4 |
    5 |
    6 | -------------------------------------------------------------------------------- /services/website/src/components/table/row.svelte: -------------------------------------------------------------------------------- 1 | 4 | 5 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /services/website/src/components/website-select.svelte: -------------------------------------------------------------------------------- 1 | 15 | 16 | {#if Object.keys(websites).length >= 2} 17 |
    18 | 23 |
    24 | {/if} 25 | -------------------------------------------------------------------------------- /services/website/src/config.ts: -------------------------------------------------------------------------------- 1 | const isDevelopment = 2 | typeof window !== "undefined" 3 | ? window.location.hostname.endsWith(".gitpod.io") || 4 | window.location.hostname === "localhost" 5 | : process.env.NODE_ENV === "development"; 6 | 7 | export const ADMIN_API_BASE_URL = "api/admin"; 8 | export const QUERY_API_BASE_URL = "api/query"; 9 | export const MAGIC_PUBLIC_KEY = isDevelopment 10 | ? "pk_test_517AC93805DD89CB" 11 | : "pk_live_BA455263710CC21F"; 12 | -------------------------------------------------------------------------------- /services/website/src/hooks.ts: -------------------------------------------------------------------------------- 1 | import { sequence } from "@sveltejs/kit/hooks"; 2 | import cookie from "cookie"; 3 | import cookieParser from "cookie-parser"; 4 | import jwt from "jsonwebtoken"; 5 | import { dev } from "$app/env"; 6 | 7 | const { COOKIE_SECRET = "cookie-dev-secret" } = process.env; 8 | 9 | const baseDevApiUrls = { 10 | admin: "http://localhost:8082/", 11 | query: "http://localhost:8081/", 12 | }; 13 | 14 | const handleDevApiRedirects: import("@sveltejs/kit").Handle = async ({ 15 | request, 16 | resolve, 17 | }) => { 18 | if (dev && request.path.match(/^\/api\/(query|admin)\//)) { 19 | const [_, apiType, path] = request.path.match( 20 | /^\/api\/(query|admin)\/(.*)/ 21 | ); 22 | 23 | const apiBaseUrl = baseDevApiUrls[apiType]; 24 | const res = await fetch(`${apiBaseUrl}${path}?${request.query}`, { 25 | body: request.body && JSON.stringify(request.body), 26 | headers: request.headers, 27 | method: request.method, 28 | }); 29 | 30 | const responseHeaders = {}; 31 | res.headers.forEach((value, key) => { 32 | responseHeaders[key] = value; 33 | }); 34 | 35 | return { 36 | headers: responseHeaders, 37 | status: res.status, 38 | body: await res.text(), 39 | }; 40 | } 41 | return await resolve(request); 42 | }; 43 | 44 | const handleCookies: import("@sveltejs/kit").Handle = async ({ 45 | request, 46 | resolve, 47 | }) => { 48 | const cookies = cookie.parse(request.headers.cookie || ""); 49 | let signedCookies = cookieParser.signedCookies(cookies, [COOKIE_SECRET]); 50 | signedCookies = cookieParser.JSONCookies(signedCookies); 51 | const jwtToken = signedCookies["jwt"]; 52 | const { user } = jwtToken ? jwt.decode(jwtToken) : false; 53 | request.locals.user = user; 54 | return await resolve(request); 55 | }; 56 | 57 | export const handle = sequence(handleDevApiRedirects, handleCookies); 58 | 59 | export const getSession: import("@sveltejs/kit").GetSession = (request) => { 60 | return request.locals.user 61 | ? { 62 | user: { 63 | // only include properties needed client-side — 64 | // exclude anything else attached to the user 65 | // like access tokens etc 66 | email: request.locals.user.email, 67 | emailHash: request.locals.user.emailHash, 68 | firstName: request.locals.user.firstName, 69 | issuer: request.locals.user.issuer, 70 | sites: request.locals.user.sites, 71 | }, 72 | } 73 | : {}; 74 | }; 75 | -------------------------------------------------------------------------------- /services/website/src/routes/[site]/index.svelte: -------------------------------------------------------------------------------- 1 | 35 | 36 | 67 | 68 | 69 | Web analytics for {$page.params.site} | Your Analytics 70 | 71 | 72 |
    73 |
    74 | 75 | 76 | 77 | 78 |
    79 | 80 | 81 |
    82 |     83 | 84 | 85 | 86 |
    87 |
    88 | 89 | 90 | 91 |
    92 |
    93 | 94 | 95 | 96 |
    97 |
    98 |
    99 |
    100 | 101 | 102 | 103 |
    104 |
    105 | 106 | 107 | 108 |
    109 |
    110 |     111 |
    112 | -------------------------------------------------------------------------------- /services/website/src/routes/[site]/settings.svelte: -------------------------------------------------------------------------------- 1 | 21 | 22 | 48 | 49 | 54 | 55 | 56 | Settings for {$page.params.site} | Your Analytics 57 | 58 | 59 |
    60 | 61 |
    62 |
    63 |
    64 |
    65 |

    Website

    66 |

    67 | Configure settings specifically for {$page.params.site} 68 |

    69 |
    70 |
    71 |
    72 |
    73 |
    74 |
    75 |
    76 | Visibility 77 |
    78 | {#each visibilities as visibility, index} 79 |
    0} class="flex items-center"> 80 | 81 | 84 |
    85 |

    {visibility.description}

    86 | {/each} 87 |
    88 |
    89 |
    90 |
    91 | 94 |
    95 |
    96 |
    97 |
    98 |
    99 |
    100 |     -------------------------------------------------------------------------------- /services/website/src/routes/__error.svelte: -------------------------------------------------------------------------------- 1 | 11 | 12 | 15 | 16 |

    {title}

    17 | -------------------------------------------------------------------------------- /services/website/src/routes/__layout.svelte: -------------------------------------------------------------------------------- 1 | 8 | 9 | 14 | 15 | 16 | 17 | 18 | 19 |
    20 | 21 |
    22 |