├── .clj-kondo └── config.edn ├── .dir-locals.el ├── .dockerignore ├── .github └── workflows │ ├── build.yml │ ├── deps.yml │ └── test.yml ├── .gitignore ├── .java_modules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── bin ├── machine.sh ├── run_postgres.sh ├── run_sqlite.sh └── run_sqlite_ephemeral.sh ├── deps.edn ├── dev-resources ├── bench │ ├── insert_input.json │ └── query_input.json ├── calibration │ ├── insert_input.json │ └── query_input.json ├── clamav │ ├── attachment_body.txt │ ├── docker-compose.yml │ └── safe.txt ├── erd │ └── 0.7.26 │ │ ├── erd_diagram │ │ └── erd_diagram.png ├── keycloak_demo │ ├── docker-compose.yml │ └── test-realm.json ├── load_balanced │ ├── docker-compose.yml │ └── nginx.conf ├── oidc │ └── .well-known │ │ └── openid-configuration ├── proxied_example │ ├── docker-compose.yml │ └── nginx.conf ├── superset_demo │ ├── docker-compose.yml │ ├── keycloak │ │ └── test-realm.json │ └── superset │ │ ├── .env │ │ ├── docker-bootstrap.sh │ │ ├── docker-ci.sh │ │ ├── docker-frontend.sh │ │ ├── docker-init.sh │ │ ├── frontend-mem-nag.sh │ │ ├── pythonpath_dev │ │ ├── .gitignore │ │ ├── client_secret.example.json │ │ ├── client_secret.json │ │ ├── keycloak_security_manager.py │ │ └── superset_config.py │ │ ├── requirements-local.txt │ │ └── run-server.sh ├── template │ ├── 0_vpc.yml │ ├── 1_db.yml │ └── 2_lrs.yml └── tla-demo │ └── docker-compose.yml ├── doc ├── authority.md ├── aws.md ├── dev.md ├── docker.md ├── endpoints.md ├── env_vars.md ├── general_faq.md ├── https.md ├── images │ ├── delete_seed.png │ ├── doc_logo.png │ ├── login.png │ ├── mac_sec_0.png │ ├── mac_sec_1.png │ ├── mac_sec_2.png │ ├── oidc │ │ └── auth0 │ │ │ ├── 00_apis.png │ │ │ ├── 01_create_api.png │ │ │ ├── 02_api_settings.png │ │ │ ├── 03_api_rbac.png │ │ │ ├── 04_api_permissions.png │ │ │ ├── 05_roles_list.png │ │ │ ├── 06_new_role.png │ │ │ ├── 07_role_permissions.png │ │ │ ├── 08_add_permissions_0.png │ │ │ ├── 09_add_permissions_1.png │ │ │ ├── 10_permissions_added.png │ │ │ ├── 11_create_user_0.png │ │ │ ├── 12_create_user_1.png │ │ │ ├── 13_assign_role_0.png │ │ │ ├── 14_assign_role_1.png │ │ │ ├── 15_assign_role_2.png │ │ │ ├── 16_applications_list.png │ │ │ ├── 17_create_application.png │ │ │ ├── 18_callback_url.png │ │ │ ├── 19_get_client_id.png │ │ │ ├── 20_oidc_login_0.png │ │ │ ├── 21_oidc_login_1.png │ │ │ ├── 22_oidc_grant.png │ │ │ └── 23_oidc_logged_in.png │ ├── postman_auth.png │ ├── postman_body.png │ ├── postman_headers.png │ ├── postman_requesttype.png │ ├── reactions │ │ ├── edit_condition_alpha.png │ │ ├── edit_condition_beta.png │ │ ├── edit_condition_beta_ref.png │ │ ├── edit_dynamic_vars.png │ │ ├── edit_identity_path.png │ │ ├── edit_intro.png │ │ ├── edit_template.png │ │ ├── edit_title.png │ │ ├── table.png │ │ ├── view_1.png │ │ ├── view_2.png │ │ └── view_3.png │ ├── startup.png │ ├── superset │ │ ├── 0_landing.png │ │ ├── 1_login.png │ │ ├── 2_db_select.png │ │ ├── 3_db_conns.png │ │ ├── 4_pg_connect.png │ │ ├── 5_pg_connected.png │ │ ├── 6_sql_explorer.png │ │ └── 7_create_chart.png │ └── win_sec.png ├── index.md ├── oidc.md ├── oidc │ └── auth0.md ├── other_demos.md ├── overview.md ├── postgres.md ├── postman.md ├── reactions.md ├── reactions │ └── spec.md ├── sqlite.md ├── startup.md ├── superset.md └── troubleshooting.md ├── docker-compose.yml ├── exe ├── config.xml ├── config_pg.xml ├── lrsql.exe ├── lrsql.ico └── lrsql_pg.exe ├── resources └── lrsql │ ├── config │ ├── authority.json.template │ ├── config.edn │ ├── lrsql.json.example │ ├── oidc_authority.json.template │ ├── oidc_client.json.template │ ├── prod │ │ ├── default │ │ │ ├── connection.edn │ │ │ ├── lrs.edn │ │ │ ├── tuning.edn │ │ │ └── webserver.edn │ │ ├── postgres │ │ │ ├── connection.edn │ │ │ ├── database.edn │ │ │ └── tuning.edn │ │ ├── sqlite │ │ │ └── database.edn │ │ └── sqlite_mem │ │ │ ├── connection.edn │ │ │ └── database.edn │ └── test │ │ ├── default │ │ ├── connection.edn │ │ ├── lrs.edn │ │ ├── tuning.edn │ │ └── webserver.edn │ │ ├── oidc │ │ ├── lrs.edn │ │ └── webserver.edn │ │ ├── postgres │ │ ├── connection.edn │ │ └── tuning.edn │ │ └── sqlite_mem │ │ └── connection.edn │ ├── doc │ └── docs.html.template │ └── localization │ └── language.json └── src ├── bench └── lrsql │ └── bench.clj ├── build └── lrsql │ └── build.clj ├── db ├── postgres │ └── lrsql │ │ └── postgres │ │ ├── data.clj │ │ ├── main.clj │ │ ├── record.clj │ │ └── sql │ │ ├── ddl.sql │ │ ├── delete.sql │ │ ├── insert.sql │ │ ├── query.sql │ │ └── update.sql └── sqlite │ └── lrsql │ └── sqlite │ ├── data.clj │ ├── main.clj │ ├── record.clj │ └── sql │ ├── ddl.sql │ ├── delete.sql │ ├── insert.sql │ ├── query.sql │ └── update.sql ├── dev └── lrsql │ └── user.clj ├── main ├── logback.xml └── lrsql │ ├── admin │ ├── interceptors │ │ ├── account.clj │ │ ├── credentials.clj │ │ ├── csv_download.clj │ │ ├── jwt.clj │ │ ├── lrs_management.clj │ │ ├── oidc.clj │ │ ├── openapi.clj │ │ ├── reaction.clj │ │ ├── status.clj │ │ ├── ui.clj │ │ └── xapi_credentials_override.clj │ ├── protocol.clj │ └── routes.clj │ ├── backend │ ├── data.clj │ └── protocol.clj │ ├── init.clj │ ├── init │ ├── authority.clj │ ├── clamav.clj │ ├── config.clj │ ├── git_data.clj │ ├── localization.clj │ ├── log.clj │ ├── oidc.clj │ └── reaction.clj │ ├── input │ ├── activity.clj │ ├── actor.clj │ ├── admin.clj │ ├── admin │ │ ├── jwt.clj │ │ └── status.clj │ ├── attachment.clj │ ├── auth.clj │ ├── document.clj │ ├── reaction.clj │ └── statement.clj │ ├── ops │ ├── command │ │ ├── admin.clj │ │ ├── auth.clj │ │ ├── document.clj │ │ ├── reaction.clj │ │ └── statement.clj │ ├── query │ │ ├── activity.clj │ │ ├── actor.clj │ │ ├── admin.clj │ │ ├── auth.clj │ │ ├── document.clj │ │ ├── reaction.clj │ │ └── statement.clj │ ├── util.clj │ └── util │ │ └── reaction.clj │ ├── reaction │ └── protocol.clj │ ├── spec │ ├── activity.clj │ ├── actor.clj │ ├── admin.clj │ ├── admin │ │ ├── jwt.clj │ │ └── status.clj │ ├── attachment.clj │ ├── auth.clj │ ├── authority.clj │ ├── common.clj │ ├── config.clj │ ├── document.clj │ ├── oidc.clj │ ├── reaction.clj │ ├── statement.clj │ └── util.clj │ ├── system.clj │ ├── system │ ├── database.clj │ ├── logger.clj │ ├── lrs.clj │ ├── reactor.clj │ ├── tuning.clj │ ├── util.clj │ └── webserver.clj │ ├── util.clj │ └── util │ ├── activity.clj │ ├── actor.clj │ ├── admin.clj │ ├── auth.clj │ ├── cert.clj │ ├── concurrency.clj │ ├── document.clj │ ├── headers.clj │ ├── interceptor.clj │ ├── logging.clj │ ├── oidc.clj │ ├── path.clj │ ├── reaction.clj │ └── statement.clj └── test ├── logback-test.xml └── lrsql ├── admin ├── cors_test.clj ├── protocol_test.clj └── route_test.clj ├── bench_test.clj ├── concurrency_test.clj ├── conformance_test.clj ├── https_test.clj ├── init └── oidc_test.clj ├── input └── input_test.clj ├── lrs_test.clj ├── ops ├── query │ └── reaction_test.clj └── util │ └── reaction_test.clj ├── params_test.clj ├── reaction └── protocol_test.clj ├── scan_test.clj ├── scope_test.clj ├── test_constants.clj ├── test_runner.clj ├── test_support.clj ├── test_support_test.clj └── util ├── admin_test.clj ├── auth_test.clj ├── concurrency_test.clj ├── config_test.clj ├── database_test.clj ├── oidc_test.clj ├── path_test.clj ├── reaction_test.clj ├── statement_test.clj └── util_test.clj /.clj-kondo/config.edn: -------------------------------------------------------------------------------- 1 | {:lint-as 2 | {lrsql.util.concurrency/with-rerunable-txn next.jdbc/with-transaction 3 | lrsql.system.lrs/with-rerunable-txn next.jdbc/with-transaction}} 4 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;;; Directory Local Variables 2 | ;;; For more information see (info "(emacs) Directory Variables") 3 | 4 | ((nil . 5 | ((cider-clojure-cli-global-options . "-A:test:bench:build-dev") 6 | ))) 7 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /target/bundle/runtimes/** 2 | -------------------------------------------------------------------------------- /.github/workflows/deps.yml: -------------------------------------------------------------------------------- 1 | name: Deps 2 | 3 | on: 4 | push: 5 | branches: 6 | - '*' 7 | jobs: 8 | deps: 9 | runs-on: ubuntu-latest 10 | timeout-minutes: 10 11 | steps: 12 | - name: Checkout project 13 | uses: actions/checkout@v4 14 | 15 | - name: Setup CI Environment 16 | uses: yetanalytics/action-setup-env@v2 17 | 18 | - name: Cache Deps 19 | uses: actions/cache@v4 20 | with: 21 | path: | 22 | ~/.m2 23 | ~/.gitlibs 24 | key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} 25 | restore-keys: | 26 | ${{ runner.os }}-deps- 27 | 28 | - name: Make a POM 29 | run: make clean pom.xml 30 | 31 | - name: Submit Dependency Snapshot 32 | uses: advanced-security/maven-dependency-submission-action@v4 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: push 4 | 5 | jobs: 6 | lint: 7 | uses: yetanalytics/workflow-linter/.github/workflows/linter.yml@v2024.08.01 8 | with: 9 | lint-directories: "src/bench src/build src/db src/main src/test" 10 | 11 | test: 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | target: 17 | - test-sqlite 18 | - test-postgres-13 19 | - test-postgres-14 20 | - test-postgres-15 21 | - test-postgres-16 22 | - test-postgres-17 23 | 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@v4 27 | 28 | - name: Setup CI Environment 29 | uses: yetanalytics/action-setup-env@v2 30 | 31 | - name: Run Makefile Target ${{ matrix.target }} 32 | run: make ${{ matrix.target }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | pom.xml 3 | pom.xml.asc 4 | *.jar 5 | *.class 6 | *.pem 7 | /lib/ 8 | /classes/ 9 | /target/ 10 | /logs/ 11 | /checkouts/ 12 | .lein-deps-sum 13 | .lein-repl-history 14 | .lein-plugins/ 15 | .lein-failures 16 | .nrepl-port 17 | .cpcache/ 18 | *.db 19 | *.sqlite 20 | /config/ 21 | results/ 22 | Test.java 23 | .DS_Store 24 | # VSCode 25 | .clj-kondo/ 26 | !.clj-kondo/config.edn 27 | .calva/ 28 | .lsp/ 29 | .vscode/ 30 | # Admin SPA 31 | lrs-admin-ui-*.zip 32 | /resources/public/admin/ 33 | /tmp/ 34 | # Templates 35 | node_modules/ 36 | -------------------------------------------------------------------------------- /.java_modules: -------------------------------------------------------------------------------- 1 | java.base,java.logging,java.naming,java.xml,java.sql,java.transaction.xa,java.security.sasl,java.management,jdk.crypto.ec 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Yet Analytics Open Source Contribution Guidelines 2 | 3 | ## Welcome to the Yet Analytics Open Source Community! 4 | 5 | Thank you for your interest in contributing to Yet Analytics Open Source projects. It is our goal in maintaining these Open Source projects to provide useful tools for the entire xAPI Community. We welcome feedback and contributions from users, stakeholders and engineers alike. 6 | 7 | The following document outlines the policies, methodology, and guidelines for contributing to our open source projects. 8 | 9 | ## Code of Conduct 10 | 11 | The Yet Analytics Open Source Community has a [Code of Conduct](CODE_OF_CONDUCT.md) which should be read and followed when contributing in any way. 12 | 13 | ## Issue Reporting 14 | 15 | Yet Analytics encourages users to contribute by reporting any issues or enhancement suggestions via [GitHub Issues](https://github.com/yetanalytics/lrsql/issues). 16 | 17 | Before submission, we encourage you to read through the existing [Documentation](doc/index.md) to ensure that the issue has not been addressed or explained. 18 | 19 | ### Issue Templates 20 | 21 | If the repository has an Issue Template, please follow the template as much as possible in your submission as this helps our team more quickly triage and understand the issues you are seeing or enhancements you are suggesting. 22 | 23 | ### Security Issues 24 | 25 | If you believe you have found a potential security issue in the codebase of a Yet Analytics project, please do NOT open an issue. Email [team@yetanalytics.com](mailto:team@yetanalytics.com) directly instead. 26 | 27 | ## Code Contributions 28 | 29 | ### Methodology 30 | 31 | For community contribution to the codebase of a Yet Analytics project we ask that you follow the [Fork and Pull](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) methodology for proposing changes. In short, this method requires you to do the following: 32 | 33 | - Fork the repository 34 | - Clone your Fork and perform the appropriate changes on a branch 35 | - Push the changes back to your Fork 36 | - Make sure your Fork is up to date with the latest `main` branch in the central repository (merge upstream changes) 37 | - Submit a Pull Request using your fork's branch 38 | 39 | The Yet Analytics team will then review the changes, and may make suggestions on GitHub before a final decision is made. The Yet team reviews Pull Requests regularly and we will be notified of its creation and all updates immediately. 40 | 41 | ### Style 42 | 43 | For contributions in Clojure, we would suggest you read this [Clojure Style Guide](https://github.com/bbatsov/clojure-style-guide) as it is one that we generally follow in our codebases. 44 | 45 | ### Tests 46 | 47 | In order for us to merge a Pull Request it must pass the `make ci` Makefile target. This target runs a set of unit, integration and/or conformance tests which verify the build's behavior. Please run this target and remediate any issues before submitting a Pull Request. 48 | 49 | We ask that when adding or changing functionality in the system that you examine whether it is a candidate for additional or modified test coverage and add it if so. You can see what sort of tests are in place currently by exploring the namespaces in `src/test`. 50 | 51 | ## License and Copyright 52 | 53 | By contributing to a Yet Analytics Open Source project you agree that your contributions will be licensed under its [Apache License 2.0](LICENSE). 54 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.19.1 2 | 3 | ADD target/bundle /lrsql 4 | ADD .java_modules /lrsql/.java_modules 5 | 6 | # replace the linux runtime via jlink 7 | RUN apk update \ 8 | && apk upgrade \ 9 | && apk add ca-certificates \ 10 | && update-ca-certificates \ 11 | && apk add --no-cache openjdk11 \ 12 | && mkdir -p /lrsql/runtimes \ 13 | && jlink --output /lrsql/runtimes/linux/ --add-modules $(cat /lrsql/.java_modules) \ 14 | && apk del openjdk11 \ 15 | && rm -rf /var/cache/apk/* 16 | 17 | # delete bench utils for leaner container 18 | RUN rm -rf /lrsql/bench* 19 | 20 | WORKDIR /lrsql 21 | EXPOSE 8080 22 | EXPOSE 8443 23 | CMD ["/lrsql/bin/run_sqlite.sh"] 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![SQL LRS Logo](doc/images/doc_logo.png) 2 | 3 | # Yet Analytics SQL LRS 4 | 5 | [![CI](https://github.com/yetanalytics/lrsql/actions/workflows/test.yml/badge.svg)](https://github.com/yetanalytics/lrsql/actions/workflows/test.yml) 6 | [![CD](https://github.com/yetanalytics/lrsql/actions/workflows/build.yml/badge.svg)](https://github.com/yetanalytics/lrsql/actions/workflows/build.yml) 7 | [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/yetanalytics/lrsql?label=docker&style=plastic&color=blue)](https://hub.docker.com/r/yetanalytics/lrsql) 8 | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-5e0b73.svg)](CODE_OF_CONDUCT.md) 9 | 10 | A SQL-based Learning Record Store. 11 | 12 | ## What is SQL LRS? 13 | 14 | A Learning Record Store (LRS) is a persistent store for xAPI statements and associated attachments and documents. The full LRS specification can be found in Part 3 of the [xAPI specification](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Communication.md). SQL LRS is distinct from other LRSs developed at Yet Analytics for being SQL-based and supporting multiple SQL database management systems (DBMSs) like SQLite and Postgres. 15 | 16 | ## Releases 17 | 18 | For releases and release notes, see the [Releases](https://github.com/yetanalytics/lrsql/releases) page. 19 | 20 | ## Documentation 21 | 22 | 23 | 24 | [SQL LRS Overview](doc/overview.md) 25 | 26 | ### FAQ 27 | 28 | - [General Questions](doc/general_faq.md) 29 | - [Troubleshooting](doc/troubleshooting.md) 30 | 31 | ### Basic Configuration 32 | 33 | - [Getting Started](doc/startup.md) 34 | - [Setting up TLS/HTTPS](doc/https.md) 35 | - [Authority Configuration](doc/authority.md) 36 | - [Docker Image](doc/docker.md) 37 | - [OpenID Connect Support](doc/oidc.md) 38 | - [Auth0 Setup Guide](doc/oidc/auth0.md) 39 | 40 | ### DBMS-specific Sections 41 | 42 | - [Postgres](doc/postgres.md) 43 | - [SQLite](doc/sqlite.md) 44 | 45 | ### Reference 46 | 47 | - [Configuration Variables](doc/env_vars.md) 48 | - [HTTP Endpoints](doc/endpoints.md) 49 | - [Developer Documentation](doc/dev.md) 50 | - [Example AWS Deployment](doc/aws.md) 51 | - [Reactions](doc/reactions.md) 52 | - [JSON Spec](doc/reactions/spec.md) 53 | - [Sending xAPI statement(s) with Postman](doc/postman.md) 54 | 55 | ### Demos 56 | 57 | - [Visualization with Apache Superset](doc/superset.md) 58 | - [Additional Configuration Demos](doc/other_demos.md) 59 | 60 | ## Contribution 61 | 62 | Before contributing to this project, please read the [Contribution Guidelines](CONTRIBUTING.md) and the [Code of Conduct](CODE_OF_CONDUCT.md). 63 | 64 | ## License 65 | 66 | Copyright © 2021-2025 Yet Analytics, Inc. 67 | 68 | Distributed under the Apache License version 2.0. 69 | -------------------------------------------------------------------------------- /bin/machine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #TODO: Add windows, possibly better macos targeting 4 | 5 | unameOut="$(uname -s)" 6 | case "${unameOut}" in 7 | Linux*) machine=linux;; 8 | *) machine=macos;; 9 | esac 10 | 11 | echo ${machine} 12 | -------------------------------------------------------------------------------- /bin/run_postgres.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MACHINE=`bin/machine.sh` 4 | 5 | runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.postgres.main $@ 6 | -------------------------------------------------------------------------------- /bin/run_sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MACHINE=`bin/machine.sh` 4 | 5 | runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.sqlite.main $@ 6 | -------------------------------------------------------------------------------- /bin/run_sqlite_ephemeral.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MACHINE=`bin/machine.sh` 4 | 5 | runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.sqlite.main --ephemeral true $@ 6 | -------------------------------------------------------------------------------- /dev-resources/bench/query_input.json: -------------------------------------------------------------------------------- 1 | [ 2 | {}, 3 | {"ascending": "true"}, 4 | {"agent": "{\"mbox\": \"mailto:bob@example.com\"}"}, 5 | {"agent": "{\"mbox\": \"mailto:bob@example.com\"}", "related_agents": "true"}, 6 | {"activity": "https://w3id.org/xapi/video/activity-type/video"}, 7 | {"activity": "https://w3id.org/xapi/video/activity-type/video", "related_activities": "true"}, 8 | {"registration": "00000000-4000-8000-0000-000000000000"}, 9 | {"since": "2020-02-20T01:01:01Z"}, 10 | {"until": "2020-02-20T01:01:01Z"} 11 | ] 12 | -------------------------------------------------------------------------------- /dev-resources/calibration/query_input.json: -------------------------------------------------------------------------------- 1 | [ 2 | {}, 3 | {"activity": "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-2", 4 | "related_activities": "true"}, 5 | {"agent": "{\"name\":\"Penny Mayer\",\"mbox\":\"mailto:1@yetanalytics.com\"}"}, 6 | {"registration": "491cbeb0-8f93-40c8-9d4f-1b34a4708561"}, 7 | {"since" : "2021-08-03T15:23:50.078000000Z"} 8 | ] 9 | -------------------------------------------------------------------------------- /dev-resources/clamav/attachment_body.txt: -------------------------------------------------------------------------------- 1 | --105423a5219f5a63362a375ba7a64a8f234da19c7d01e56800c3c64b26bb2fa0 2 | Content-Type:application/json 3 | 4 | { 5 | "actor": { 6 | "mbox": "mailto:sample.agent@example.com", 7 | "name": "Sample Agent", 8 | "objectType": "Agent" 9 | }, 10 | "verb": { 11 | "id": "http://adlnet.gov/expapi/verbs/answered", 12 | "display": { 13 | "en-US": "answered" 14 | } 15 | }, 16 | "object": { 17 | "id": "http://www.example.com/tincan/activities/multipart", 18 | "objectType": "Activity", 19 | "definition": { 20 | "name": { 21 | "en-US": "Multi Part Activity" 22 | }, 23 | "description": { 24 | "en-US": "Multi Part Activity Description" 25 | } 26 | } 27 | }, 28 | "attachments": [ 29 | { 30 | "usageType": "http://example.com/attachment-usage/test", 31 | "display": { "en-US": "A test attachment" }, 32 | "description": { "en-US": "A test attachment (description)" }, 33 | "contentType": "text/plain; charset=ascii", 34 | "length": 27, 35 | "sha2": "495395e777cd98da653df9615d09c0fd6bb2f8d4788394cd53c56a3bfdcd848a" 36 | } 37 | ] 38 | } 39 | --105423a5219f5a63362a375ba7a64a8f234da19c7d01e56800c3c64b26bb2fa0 40 | Content-Type:text/plain 41 | Content-Transfer-Encoding:binary 42 | X-Experience-API-Hash:495395e777cd98da653df9615d09c0fd6bb2f8d4788394cd53c56a3bfdcd848a 43 | 44 | here is a simple attachment 45 | --105423a5219f5a63362a375ba7a64a8f234da19c7d01e56800c3c64b26bb2fa0-- 46 | -------------------------------------------------------------------------------- /dev-resources/clamav/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # To run: docker compose up 2 | # See the Docker Compose docs for more info: https://docs.docker.com/compose/ 3 | services: 4 | clamav: 5 | image: clamav/clamav:1.2.1 6 | ports: 7 | - "3310:3310" 8 | -------------------------------------------------------------------------------- /dev-resources/clamav/safe.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/dev-resources/clamav/safe.txt -------------------------------------------------------------------------------- /dev-resources/erd/0.7.26/erd_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/dev-resources/erd/0.7.26/erd_diagram.png -------------------------------------------------------------------------------- /dev-resources/keycloak_demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | configs: 4 | test_realm: 5 | file: ./test-realm.json 6 | 7 | services: 8 | keycloak: 9 | image: quay.io/keycloak/keycloak:16.1.0 10 | environment: 11 | KEYCLOAK_USER: admin 12 | KEYCLOAK_PASSWORD: changeme123 13 | KEYCLOAK_IMPORT: /tmp/test-realm.json 14 | configs: 15 | - source: test_realm 16 | target: /tmp/test-realm.json 17 | ports: 18 | - "8081:8080" 19 | -------------------------------------------------------------------------------- /dev-resources/load_balanced/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | # README: 3 | # Runs *two* SQL LRS containers next to each other behind a load balancer(nginx). 4 | 5 | # To run: 6 | # 1. From the load_balanced dir: docker compose up 7 | # 2. Access on port 8083 in your browser. 8 | # 3. Comment in/out the `build` and `image` lines in lrs_1 and lrs_2 to switch between building from local source, or pulling the published lrsql image from the docker repository. 9 | 10 | 11 | # See the Docker Compose docs for more info: https://docs.docker.com/compose/ 12 | configs: 13 | lb_config: 14 | file: ./nginx.conf 15 | 16 | volumes: 17 | db_data: 18 | 19 | services: 20 | db: 21 | image: postgres 22 | volumes: 23 | - db_data:/var/lib/postgresql/data 24 | environment: 25 | POSTGRES_USER: lrsql_user 26 | POSTGRES_PASSWORD: lrsql_password 27 | POSTGRES_DB: lrsql_db 28 | lrs_1: 29 | # build: ../.. # switch to this for active dev 30 | image: yetanalytics/lrsql:latest 31 | command: 32 | - /lrsql/bin/run_postgres.sh 33 | ports: 34 | - "8080:8080" 35 | depends_on: 36 | - db 37 | healthcheck: 38 | test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 39 | interval: 5s 40 | timeout: 5s 41 | retries: 10 42 | environment: 43 | LRSQL_API_KEY_DEFAULT: my_key 44 | LRSQL_API_SECRET_DEFAULT: my_secret 45 | LRSQL_ADMIN_USER_DEFAULT: my_username 46 | LRSQL_ADMIN_PASS_DEFAULT: my_password 47 | LRSQL_ALLOW_ALL_ORIGINS: "true" 48 | LRSQL_DB_HOST: db 49 | LRSQL_DB_NAME: lrsql_db 50 | LRSQL_DB_USER: lrsql_user 51 | LRSQL_DB_PASSWORD: lrsql_password 52 | # If Postgres is too slow to start, increase this 53 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 54 | LRSQL_JWT_COMMON_SECRET: sandwich #in production, this should be between 32 and 64 chars for security 55 | 56 | lrs_2: 57 | # build: ../.. # switch to this for active dev 58 | image: yetanalytics/lrsql:latest 59 | command: 60 | - /lrsql/bin/run_postgres.sh 61 | ports: 62 | - "8081:8080" 63 | depends_on: 64 | lrs_1: 65 | condition: service_healthy 66 | healthcheck: 67 | test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 68 | interval: 5s 69 | timeout: 5s 70 | retries: 10 71 | environment: 72 | LRSQL_API_KEY_DEFAULT: my_key 73 | LRSQL_API_SECRET_DEFAULT: my_secret 74 | LRSQL_ADMIN_USER_DEFAULT: my_username 75 | LRSQL_ADMIN_PASS_DEFAULT: my_password 76 | LRSQL_ALLOW_ALL_ORIGINS: "true" 77 | LRSQL_DB_HOST: db 78 | LRSQL_DB_NAME: lrsql_db 79 | LRSQL_DB_USER: lrsql_user 80 | LRSQL_DB_PASSWORD: lrsql_password 81 | # If Postgres is too slow to start, increase this 82 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 83 | LRSQL_JWT_COMMON_SECRET: sandwich #in production, this should be between 32 and 64 chars for security 84 | lb: 85 | image: nginx:stable-alpine 86 | configs: 87 | - source: lb_config 88 | target: /etc/nginx/conf.d/default.conf 89 | ports: 90 | - "8083:8083" 91 | depends_on: 92 | lrs_2: 93 | condition: service_healthy 94 | -------------------------------------------------------------------------------- /dev-resources/load_balanced/nginx.conf: -------------------------------------------------------------------------------- 1 | upstream backend { 2 | server lrs_1:8080; 3 | server lrs_2:8080; 4 | } 5 | 6 | server { 7 | listen 8083; 8 | 9 | include /etc/nginx/mime.types; 10 | 11 | location / { 12 | proxy_pass http://backend/; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /dev-resources/oidc/.well-known/openid-configuration: -------------------------------------------------------------------------------- 1 | { 2 | "issuer": "https://server.example.com", 3 | "authorization_endpoint": "https://server.example.com/connect/authorize", 4 | "token_endpoint": "https://server.example.com/connect/token", 5 | "token_endpoint_auth_methods_supported": [ 6 | "client_secret_basic", 7 | "private_key_jwt" 8 | ], 9 | "token_endpoint_auth_signing_alg_values_supported": [ 10 | "RS256", 11 | "ES256" 12 | ], 13 | "userinfo_endpoint": "https://server.example.com/connect/userinfo", 14 | "check_session_iframe": "https://server.example.com/connect/check_session", 15 | "end_session_endpoint": "https://server.example.com/connect/end_session", 16 | "jwks_uri": "https://server.example.com/jwks.json", 17 | "registration_endpoint": "https://server.example.com/connect/register", 18 | "scopes_supported": [ 19 | "openid", 20 | "profile", 21 | "email", 22 | "address", 23 | "phone", 24 | "offline_access" 25 | ], 26 | "response_types_supported": [ 27 | "code", 28 | "code id_token", 29 | "id_token", 30 | "token id_token" 31 | ], 32 | "acr_values_supported": [ 33 | "urn:mace:incommon:iap:silver", 34 | "urn:mace:incommon:iap:bronze" 35 | ], 36 | "subject_types_supported": [ 37 | "public", 38 | "pairwise" 39 | ], 40 | "userinfo_signing_alg_values_supported": [ 41 | "RS256", 42 | "ES256", 43 | "HS256" 44 | ], 45 | "userinfo_encryption_alg_values_supported": [ 46 | "RSA1_5", 47 | "A128KW" 48 | ], 49 | "userinfo_encryption_enc_values_supported": [ 50 | "A128CBC-HS256", 51 | "A128GCM" 52 | ], 53 | "id_token_signing_alg_values_supported": [ 54 | "RS256", 55 | "ES256", 56 | "HS256" 57 | ], 58 | "id_token_encryption_alg_values_supported": [ 59 | "RSA1_5", 60 | "A128KW" 61 | ], 62 | "id_token_encryption_enc_values_supported": [ 63 | "A128CBC-HS256", 64 | "A128GCM" 65 | ], 66 | "request_object_signing_alg_values_supported": [ 67 | "none", 68 | "RS256", 69 | "ES256" 70 | ], 71 | "display_values_supported": [ 72 | "page", 73 | "popup" 74 | ], 75 | "claim_types_supported": [ 76 | "normal", 77 | "distributed" 78 | ], 79 | "claims_supported": [ 80 | "sub", 81 | "iss", 82 | "auth_time", 83 | "acr", 84 | "name", 85 | "given_name", 86 | "family_name", 87 | "nickname", 88 | "profile", 89 | "picture", 90 | "website", 91 | "email", 92 | "email_verified", 93 | "locale", 94 | "zoneinfo", 95 | "http://example.info/claims/groups" 96 | ], 97 | "claims_parameter_supported": true, 98 | "service_documentation": "http://server.example.com/connect/service_documentation.html", 99 | "ui_locales_supported": [ 100 | "en-US", 101 | "en-GB", 102 | "en-CA", 103 | "fr-FR", 104 | "fr-CA" 105 | ] 106 | } 107 | -------------------------------------------------------------------------------- /dev-resources/proxied_example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | # README: 3 | # Runs a SQL LRS container and a proxy (nginx) which serves the LRS on a special path (/foo). 4 | 5 | # To run: 6 | # 1. From the load_balanced dir: docker compose up 7 | # 2. Access on port 8083 in your browser. 8 | 9 | # See the Docker Compose docs for more info: https://docs.docker.com/compose/ 10 | configs: 11 | px_config: 12 | file: ./nginx.conf 13 | 14 | volumes: 15 | db_data: 16 | 17 | services: 18 | db: 19 | image: postgres 20 | volumes: 21 | - db_data:/var/lib/postgresql/data 22 | environment: 23 | POSTGRES_USER: lrsql_user 24 | POSTGRES_PASSWORD: lrsql_password 25 | POSTGRES_DB: lrsql_db 26 | lrs: 27 | # build: ../.. # switch to this for active dev 28 | image: yetanalytics/lrsql:latest 29 | command: 30 | - /lrsql/bin/run_postgres.sh 31 | ports: 32 | - "8080:8080" 33 | depends_on: 34 | - db 35 | healthcheck: 36 | test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 37 | interval: 5s 38 | timeout: 5s 39 | retries: 10 40 | environment: 41 | LRSQL_API_KEY_DEFAULT: my_key 42 | LRSQL_API_SECRET_DEFAULT: my_secret 43 | LRSQL_ADMIN_USER_DEFAULT: my_username 44 | LRSQL_ADMIN_PASS_DEFAULT: my_password 45 | LRSQL_ALLOW_ALL_ORIGINS: "true" 46 | #NOTE: this path var is needed to inform the frontend behavior once the app is proxied 47 | LRSQL_PROXY_PATH: /foo 48 | LRSQL_DB_HOST: db 49 | LRSQL_DB_NAME: lrsql_db 50 | LRSQL_DB_USER: lrsql_user 51 | LRSQL_DB_PASSWORD: lrsql_password 52 | # If Postgres is too slow to start, increase this 53 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 54 | LRSQL_JWT_COMMON_SECRET: sandwich #in production, this should be between 32 and 64 chars for security 55 | proxy: 56 | image: nginx:stable-alpine 57 | configs: 58 | - source: px_config 59 | target: /etc/nginx/conf.d/default.conf 60 | ports: 61 | - "8083:8083" 62 | depends_on: 63 | lrs: 64 | condition: service_healthy 65 | -------------------------------------------------------------------------------- /dev-resources/proxied_example/nginx.conf: -------------------------------------------------------------------------------- 1 | upstream backend { 2 | server lrs:8080; 3 | } 4 | 5 | server { 6 | listen 8083; 7 | 8 | include /etc/nginx/mime.types; 9 | 10 | location /foo { 11 | rewrite /foo/(.*) /$1 break; 12 | proxy_pass http://backend/; 13 | proxy_redirect off; 14 | proxy_set_header Host $host; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=superset 2 | 3 | # Generate secret key using this command: openssl rand -base64 42 4 | SECRET_KEY=6+QrAihvb1UBP9DcUEoZg18WecbxGXL4aRvvc7WhURNJ9rgi+2oISOpT 5 | BABEL_DEFAULT_LOCALE=en_US 6 | PUBLIC_ROLE_LIKE=Gamma 7 | 8 | # database configurations 9 | DATABASE_DB=superset 10 | DATABASE_HOST=supersetdb 11 | DATABASE_PASSWORD=superset 12 | DATABASE_USER=superset 13 | 14 | # database engine specific environment variables 15 | # change the below if you prefers another database engine 16 | DATABASE_PORT=5432 17 | DATABASE_DIALECT=postgresql 18 | POSTGRES_DB=superset 19 | POSTGRES_USER=superset 20 | POSTGRES_PASSWORD=superset 21 | 22 | # Add the mapped in /app/pythonpath_dev which allows devs to override stuff 23 | PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev 24 | 25 | REDIS_HOST=redis 26 | REDIS_PORT=6379 27 | 28 | # FLASK_ENV=development 29 | # SUPERSET_ENV=development 30 | FLASK_ENV=production 31 | SUPERSET_ENV=production 32 | # SUPERSET_LOAD_EXAMPLES=yes 33 | CYPRESS_CONFIG=false 34 | SUPERSET_PORT=8088 35 | HOST_PORT=8088 36 | 37 | # keycloak 38 | OIDC_OPENID_REALM=test 39 | 40 | # Dashboard embed 41 | SUPERSET_FEATURE_EMBEDDED_SUPERSET=true 42 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/docker-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -eo pipefail 20 | 21 | REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt" 22 | # If Cypress run – overwrite the password for admin and export env variables 23 | if [ "$CYPRESS_CONFIG" == "true" ]; then 24 | export SUPERSET_CONFIG=tests.integration_tests.superset_test_config 25 | export SUPERSET_TESTENV=true 26 | export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset 27 | fi 28 | # 29 | # Make sure we have dev requirements installed 30 | # 31 | if [ -f "${REQUIREMENTS_LOCAL}" ]; then 32 | echo "Installing local overrides at ${REQUIREMENTS_LOCAL}" 33 | pip install -r "${REQUIREMENTS_LOCAL}" 34 | else 35 | echo "Skipping local overrides" 36 | fi 37 | 38 | if [[ "${1}" == "worker" ]]; then 39 | echo "Starting Celery worker..." 40 | celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO 41 | elif [[ "${1}" == "beat" ]]; then 42 | echo "Starting Celery beat..." 43 | celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule 44 | elif [[ "${1}" == "app" ]]; then 45 | echo "Starting web app..." 46 | flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 47 | elif [[ "${1}" == "app-gunicorn" ]]; then 48 | echo "Starting web app..." 49 | /usr/bin/run-server.sh 50 | fi 51 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/docker-ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | /app/docker/docker-init.sh 19 | 20 | # TODO: copy config overrides from ENV vars 21 | 22 | # TODO: run celery in detached state 23 | export SERVER_THREADS_AMOUNT=8 24 | # start up the web server 25 | 26 | /usr/bin/run-server.sh 27 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/docker-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -e 19 | 20 | cd /app/superset-frontend 21 | npm install -g npm@7 22 | npm install -f --no-optional --global webpack webpack-cli 23 | npm install -f --no-optional 24 | 25 | echo "Running frontend" 26 | npm run dev 27 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/docker-init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | set -e 19 | 20 | # 21 | # Always install local overrides first 22 | # 23 | /app/docker/docker-bootstrap.sh 24 | 25 | STEP_CNT=4 26 | 27 | echo_step() { 28 | cat </auth/realms/", 4 | "auth_uri": "http:///auth/realms//protocol/openid-connect/auth", 5 | "client_id": "", 6 | "client_secret": "", 7 | "redirect_uris": ["http:///*"], 8 | "userinfo_uri": "http:///auth/realms//protocol/openid-connect/userinfo", 9 | "token_uri": "http:///auth/realms//protocol/openid-connect/token", 10 | "token_introspection_uri": "http:///auth/realms//protocol/openid-connect/token/introspect" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/pythonpath_dev/client_secret.json: -------------------------------------------------------------------------------- 1 | { 2 | "web": { 3 | "issuer": "http://host.docker.internal:8081/auth/realms/test", 4 | "auth_uri": "http://host.docker.internal:8081/auth/realms/test/protocol/openid-connect/auth", 5 | "client_id": "superset", 6 | "client_secret": "SQp7KIc7ZuNpsTk594Aqs1xOuHXC9RpF", 7 | "redirect_uris": ["http://localhost:8088/*"], 8 | "userinfo_uri": "http://host.docker.internal:8081/auth/realms/test/protocol/openid-connect/userinfo", 9 | "token_uri": "http://host.docker.internal:8081/auth/realms/test/protocol/openid-connect/token", 10 | "token_introspection_uri": "http://host.docker.internal:8081/auth/realms/test/protocol/openid-connect/token/introspect" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/pythonpath_dev/keycloak_security_manager.py: -------------------------------------------------------------------------------- 1 | from flask_appbuilder.security.manager import AUTH_OID 2 | from superset.security import SupersetSecurityManager 3 | from flask_oidc import OpenIDConnect 4 | from flask_appbuilder.security.views import AuthOIDView 5 | from flask_login import login_user 6 | from urllib.parse import quote 7 | from flask_appbuilder.views import expose 8 | from flask import request, redirect 9 | 10 | 11 | class OIDCSecurityManager(SupersetSecurityManager): 12 | 13 | def __init__(self, appbuilder): 14 | super(OIDCSecurityManager, self).__init__(appbuilder) 15 | if self.auth_type == AUTH_OID: 16 | self.oid = OpenIDConnect(self.appbuilder.get_app) 17 | self.authoidview = AuthOIDCView 18 | 19 | 20 | class AuthOIDCView(AuthOIDView): 21 | 22 | @expose('/login/', methods=['GET', 'POST']) 23 | def login(self, flag=True): 24 | sm = self.appbuilder.sm 25 | oidc = sm.oid 26 | superset_roles = ["Admin", "Alpha", "Gamma", "Public", "granter", "sql_lab"] 27 | default_role = "Gamma" 28 | 29 | @self.appbuilder.sm.oid.require_login 30 | def handle_login(): 31 | user = sm.auth_user_oid(oidc.user_getfield('email')) 32 | 33 | if user is None: 34 | info = oidc.user_getinfo(['preferred_username', 'given_name', 'family_name', 'email', 'roles']) 35 | roles = [role for role in superset_roles if role in info.get('roles', [])] 36 | roles += [default_role, ] if not roles else [] 37 | user = sm.add_user(info.get('preferred_username'), info.get('given_name', ''), info.get('family_name', ''), 38 | info.get('email'), [sm.find_role(role) for role in roles]) 39 | 40 | login_user(user, remember=False) 41 | return redirect(self.appbuilder.get_url_for_index) 42 | 43 | return handle_login() 44 | 45 | @expose('/logout/', methods=['GET', 'POST']) 46 | def logout(self): 47 | oidc = self.appbuilder.sm.oid 48 | 49 | oidc.logout() 50 | super(AuthOIDCView, self).logout() 51 | redirect_url = request.url_root.strip('/') 52 | # redirect_url = request.url_root.strip('/') + self.appbuilder.get_url_for_login 53 | 54 | return redirect( 55 | oidc.client_secrets.get('issuer') + '/protocol/openid-connect/logout?redirect_uri=' + quote(redirect_url)) 56 | -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/requirements-local.txt: -------------------------------------------------------------------------------- 1 | itsdangerous==2.0.1 2 | flask-oidc 3 | flask_openid -------------------------------------------------------------------------------- /dev-resources/superset_demo/superset/run-server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | HYPHEN_SYMBOL='-' 21 | 22 | gunicorn \ 23 | --bind "${SUPERSET_BIND_ADDRESS:-0.0.0.0}:${SUPERSET_PORT:-8088}" \ 24 | --access-logfile "${ACCESS_LOG_FILE:-$HYPHEN_SYMBOL}" \ 25 | --error-logfile "${ERROR_LOG_FILE:-$HYPHEN_SYMBOL}" \ 26 | --workers ${SERVER_WORKER_AMOUNT:-1} \ 27 | --worker-class ${SERVER_WORKER_CLASS:-gthread} \ 28 | --threads ${SERVER_THREADS_AMOUNT:-20} \ 29 | --timeout ${GUNICORN_TIMEOUT:-60} \ 30 | --limit-request-line ${SERVER_LIMIT_REQUEST_LINE:-0} \ 31 | --limit-request-field_size ${SERVER_LIMIT_REQUEST_FIELD_SIZE:-0} \ 32 | "${FLASK_APP}" 33 | -------------------------------------------------------------------------------- /doc/authority.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Authority Configuration 4 | 5 | The SQL LRS allows for configuration of the Authority included in xAPI statements that are written to the LRS. This authority will overwrite any authority present on incoming statements. 6 | 7 | ### Configuring a custom Authority template 8 | 9 | On startup the SQL LRS looks for the file `config/authority.json.template` and bases the authority on the template it finds there. 10 | 11 | #### Static Custom Authority 12 | 13 | The authority template file should contain an xAPI Agent object representing the desired authority. The object may contain static values, which would result in all statements in the LRS having the same authority. 14 | 15 | Here is an example of a static Authority template file 16 | 17 | ```JSON 18 | { 19 | "account": { 20 | "homePage":"https://www.yetanalytics.com", 21 | "name":"Yet Analytics" 22 | }, 23 | "objectType":"Agent" 24 | } 25 | ``` 26 | 27 | #### Dynamic Custom Authority 28 | 29 | Alternatively, it can make use a few provided variables to make the Authority more dynamic with the Account or API Key used to write the statement. The following table contains the variables available for use in the Authority template: 30 | 31 | | Variable | Description | 32 | | --- | --- | 33 | | `authority-url` | `LRSQL_AUTHORITY_URL` (`authorityUrl`) config variable value set by an environment variable or `config/lrsql.json`. Default is `http://example.org`. | 34 | | `cred-id` | LRS Credential Pair ID (UUID). This can be used to form a unique Authority for each API Key. | 35 | | `account-id` | LRS Admin Account ID (UUID). This can be used to make a unique Authority for each Account, but not necessarily for each API Key. | 36 | 37 | Here is an example of an Authority template making use of some of these variables: 38 | 39 | ```json 40 | { 41 | "account": { 42 | "homePage":"{{authority-url}}", 43 | "name":"{{cred-id}}" 44 | }, 45 | "objectType":"Agent" 46 | } 47 | ``` 48 | 49 | ### Default Authority 50 | 51 | If you do not configure a template, the default is to use the `cred-id` and `authority-url` variables like so: 52 | 53 | ```json 54 | { 55 | "account": { 56 | "homePage":"{{authority-url}}", 57 | "name":"{{cred-id}}" 58 | }, 59 | "objectType":"Agent" 60 | } 61 | ``` 62 | 63 | ### OIDC Authority 64 | 65 | When SQL LRS is [configured for OIDC](oidc.md) an alternative template is used that forms the authority from token claims: 66 | 67 | ``` json 68 | { 69 | "objectType": "Group", 70 | "member": [ 71 | { 72 | "account": { 73 | "homePage": "{{iss}}", 74 | "name": "{{lrsql/resolved-client-id}}" 75 | } 76 | }, 77 | { 78 | "account": { 79 | "homePage": "{{iss}}", 80 | "name": "{{sub}}" 81 | } 82 | } 83 | ] 84 | } 85 | ``` 86 | 87 | Possible claims vary by identity provider, see the [comprehensive list here](https://www.iana.org/assignments/jwt/jwt.xhtml). SQL LRS provides one additional value, `lrsql/resolved-client-id` that resolves to the following claims in order of precedence: 88 | 89 | 1. `client_id` 90 | 2. `azp` 91 | 3. `aud` (if `aud` is a string) 92 | 4. `aud[0]` (if `aud` is an array) 93 | 94 | Note that the request will fail if expected claims are not present on the token. 95 | 96 | As with the normal authority template, you can provide a custom version by setting the `LRSQL_OIDC_AUTHORITY_TEMPLATE` (`lrs.oidcAuthorityTemplate`) config variable. 97 | 98 | [<- Back to Index](index.md) 99 | -------------------------------------------------------------------------------- /doc/general_faq.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # General Questions 4 | 5 | ### Is SQL LRS a Learning Record Provider (LRP), Learning Management System (LMS), or content store? 6 | 7 | No, SQL LRS is strictly a [Learning Record Store](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-About.md#def-learning-record-store) (LRS), and is not an application that includes its own LRP or LMS. Unlike a [Learning Record Provider](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-About.md#def-learning-record-provider) (LRP), SQL LRS does not produce learning data on its own, and unlike a [Learning Management System](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-About.md#def-learning-management-system) (LMS), it does not host, distribute, author, or serve as a repository for learning content. 8 | 9 | As an LRS, SQL LRS is strictly a database application for xAPI statements, attachments, and documents. It receives, validates, stores, and makes available xAPI data produced by external LRPs or LMSs. 10 | 11 | ### I've used Yet's Cloud LRS products in the past. What are the differences? 12 | 13 | If you previously used Yet's Cloud LRS products, it is important to be aware of certain differences: 14 | 15 | - Tenancy is not supported in SQL LRS; the entire database can be considered to be a single default tenant. 16 | - All operations in SQL LRS are synchronous; async operations are not supported. 17 | - `stored` timestamps are not strictly monotonic in SQL LRS; two or more Statements may be assigned the same timestamp if stored in quick succession. 18 | - If a Statement voids a target Statement that is itself voiding, SQL LRS will accept it upon insertion, though it will not update the state of the target Statement as per the [xAPI spec](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Communication.md#214-voided-statements). (The Cloud LRS, on the other hand, will simply reject the voiding Statement.) 19 | 20 | [<- Back to Index](index.md) 21 | -------------------------------------------------------------------------------- /doc/https.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Setting Up TLS/HTTPS 4 | 5 | ### Certificate Management 6 | 7 | SQL LRS has a number of ways to configure a certificate for HTTPS. The system will attempt to use certificates in the following order when it starts up, based on configuration variables: 8 | 9 | #### 1. Custom Keystore 10 | 11 | If you have created a keystore containing a certificate you wish to use with the SQL LRS, specify the following variables in `config/lrsql.json` (or as environment variables). See the guide on [configuration variables](env_vars.md) for more information. 12 | 13 | - Set `LRSQL_KEY_FILE` (`keyFile` in `config/lrsql.json`) to the location of a valid keystore on disk 14 | - Set `LRSQL_KEY_ALIAS` and `LRSQL_KEY_PASSWORD` (`keyAlias` and `keyPassword` respectively in config file) 15 | 16 | Your `config/lrsql.json` should resemble the following: 17 | 18 | ```JSON 19 | { 20 | ... 21 | "webserver" : { 22 | ... 23 | "keyFile" : "my_keystore_location.jks", 24 | "keyAlias" : "my_certificate_alias", 25 | "keyPassword" : "my_key_password" 26 | } 27 | } 28 | ``` 29 | 30 | #### 2. Custom PEM Files 31 | 32 | If you did not set the keystore variables in the previous section, the SQL LRS will then look for pem files set with the following variables: 33 | 34 | - Set `LRSQL_KEY_PKEY_FILE` (`keyPkeyFile` in config file) to the location of your PEM private key 35 | - Set `LRSQL_KEY_CERT_CHAIN` (`keyCertChain` in config file) to the location of the certificate PEM file and optionally additional cert chain pems (comma separated) provided by your registrar. 36 | 37 | ```json 38 | { 39 | ... 40 | "webserver" : { 41 | ... 42 | "keyPkeyFile" : "config/my_private.key.pem", 43 | "keyCertChain" : "config/my_certificate.crt.pem,config/my_cert_chain.pem" 44 | } 45 | } 46 | ``` 47 | 48 | #### 3. Self-Signed Temporary TLS Certificate 49 | 50 | If no keystore or cert files are found, the SQL LRS will create a self-signed cert by default and log a warning. This is not intended to be used in a production setting, but can be used for testing and development. See below for how to disable certificate generation. 51 | 52 | ### HTTPS Configuration 53 | 54 | Additional variables can be set in `config/lrsql.json` that configure SSL behavior in the SQL LRS. 55 | 56 | - If you would like to change the HTTPS port (default `8443`) you can use `LRSQL_SSL_PORT` (`sslPort` in the config file). 57 | 58 | - If you would like to disable HTTP so that only HTTPS is served by the SQL LRS, you can do so by setting `LRSQL_ENABLE_HTTP` (`enableHttp` in config) to `false`. 59 | 60 | - If you would like to disable the generation of self-signed certificates entirely you can set `LRSQL_KEY_ENABLE_SELFIE` (`keyEnableSelfie` in config) to `false`. 61 | 62 | For more information on these and other options see [Configuration Variables](env_vars.md). 63 | 64 | ### Generating Dev Certs with `mkcert` 65 | 66 | If you install [mkcert](https://github.com/FiloSottile/mkcert) you can generate stable "valid" certs to use while developing the app. These should only be used locally for development purposes: 67 | 68 | ```shell 69 | 70 | $ cp "$(mkcert -CAROOT)"/rootCA.pem config/cacert.pem 71 | $ mkcert -key-file config/server.key.pem \ 72 | -cert-file config/server.crt.pem \ 73 | example.com "*.example.com" example.test localhost 127.0.0.1 ::1 74 | $ clojure -Mdb-sqlite -m lrsql.sqlite.main --ephemeral true 75 | ... 76 | 11:25:54.085 [main] INFO lrsql.util.cert - Generated keystore from key and cert(s)... 77 | 78 | ``` 79 | 80 | [<- Back to Index](index.md) 81 | -------------------------------------------------------------------------------- /doc/images/delete_seed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/delete_seed.png -------------------------------------------------------------------------------- /doc/images/doc_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/doc_logo.png -------------------------------------------------------------------------------- /doc/images/login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/login.png -------------------------------------------------------------------------------- /doc/images/mac_sec_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/mac_sec_0.png -------------------------------------------------------------------------------- /doc/images/mac_sec_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/mac_sec_1.png -------------------------------------------------------------------------------- /doc/images/mac_sec_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/mac_sec_2.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/00_apis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/00_apis.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/01_create_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/01_create_api.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/02_api_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/02_api_settings.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/03_api_rbac.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/03_api_rbac.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/04_api_permissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/04_api_permissions.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/05_roles_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/05_roles_list.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/06_new_role.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/06_new_role.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/07_role_permissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/07_role_permissions.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/08_add_permissions_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/08_add_permissions_0.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/09_add_permissions_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/09_add_permissions_1.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/10_permissions_added.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/10_permissions_added.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/11_create_user_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/11_create_user_0.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/12_create_user_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/12_create_user_1.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/13_assign_role_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/13_assign_role_0.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/14_assign_role_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/14_assign_role_1.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/15_assign_role_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/15_assign_role_2.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/16_applications_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/16_applications_list.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/17_create_application.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/17_create_application.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/18_callback_url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/18_callback_url.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/19_get_client_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/19_get_client_id.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/20_oidc_login_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/20_oidc_login_0.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/21_oidc_login_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/21_oidc_login_1.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/22_oidc_grant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/22_oidc_grant.png -------------------------------------------------------------------------------- /doc/images/oidc/auth0/23_oidc_logged_in.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/oidc/auth0/23_oidc_logged_in.png -------------------------------------------------------------------------------- /doc/images/postman_auth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/postman_auth.png -------------------------------------------------------------------------------- /doc/images/postman_body.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/postman_body.png -------------------------------------------------------------------------------- /doc/images/postman_headers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/postman_headers.png -------------------------------------------------------------------------------- /doc/images/postman_requesttype.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/postman_requesttype.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_condition_alpha.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_condition_alpha.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_condition_beta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_condition_beta.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_condition_beta_ref.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_condition_beta_ref.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_dynamic_vars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_dynamic_vars.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_identity_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_identity_path.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_intro.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_template.png -------------------------------------------------------------------------------- /doc/images/reactions/edit_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/edit_title.png -------------------------------------------------------------------------------- /doc/images/reactions/table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/table.png -------------------------------------------------------------------------------- /doc/images/reactions/view_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/view_1.png -------------------------------------------------------------------------------- /doc/images/reactions/view_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/view_2.png -------------------------------------------------------------------------------- /doc/images/reactions/view_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/reactions/view_3.png -------------------------------------------------------------------------------- /doc/images/startup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/startup.png -------------------------------------------------------------------------------- /doc/images/superset/0_landing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/0_landing.png -------------------------------------------------------------------------------- /doc/images/superset/1_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/1_login.png -------------------------------------------------------------------------------- /doc/images/superset/2_db_select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/2_db_select.png -------------------------------------------------------------------------------- /doc/images/superset/3_db_conns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/3_db_conns.png -------------------------------------------------------------------------------- /doc/images/superset/4_pg_connect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/4_pg_connect.png -------------------------------------------------------------------------------- /doc/images/superset/5_pg_connected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/5_pg_connected.png -------------------------------------------------------------------------------- /doc/images/superset/6_sql_explorer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/6_sql_explorer.png -------------------------------------------------------------------------------- /doc/images/superset/7_create_chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/superset/7_create_chart.png -------------------------------------------------------------------------------- /doc/images/win_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/doc/images/win_sec.png -------------------------------------------------------------------------------- /doc/index.md: -------------------------------------------------------------------------------- 1 | # Documentation Index 2 | 3 | 4 | 5 | [SQL LRS Overview](overview.md) 6 | 7 | ### FAQ 8 | 9 | - [General Questions](general_faq.md) 10 | - [Troubleshooting](troubleshooting.md) 11 | 12 | ### Basic Configuration 13 | 14 | - [Getting Started](startup.md) 15 | - [Setting up TLS/HTTPS](https.md) 16 | - [Authority Configuration](authority.md) 17 | - [Docker Image](docker.md) 18 | - [OpenID Connect Support](oidc.md) 19 | - [Auth0 Setup Guide](oidc/auth0.md) 20 | 21 | ### DBMS-specific Sections 22 | 23 | - [Postgres](postgres.md) 24 | - [SQLite](sqlite.md) 25 | 26 | ### Reference 27 | 28 | - [Configuration Variables](env_vars.md) 29 | - [HTTP Endpoints](endpoints.md) 30 | - [Developer Documentation](dev.md) 31 | - [Example AWS Deployment](aws.md) 32 | - [Reactions](reactions.md) 33 | - [Sending xAPI statement(s) with Postman](postman.md) 34 | 35 | ### Demos 36 | 37 | - [Visualization with Apache Superset](superset.md) 38 | - [Additional Configuration Demos](other_demos.md) 39 | 40 | ### Releases 41 | 42 | For releases and release notes see the [Releases](https://github.com/yetanalytics/lrsql/releases) page on the SQL LRS GitHub repository. 43 | -------------------------------------------------------------------------------- /doc/other_demos.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Additional Deployment Configuration Examples 4 | 5 | ## Load Balanced LRS Demo 6 | 7 | This demo illustrates how SQL LRS can be configured with multiple load balanced application servers with a single PostgreSQL database. The important configuration variable to pay attention to for multiple nodes in a single cluster is `LRSQL_JWT_COMMON_SECRET`, which allows the servers to share JWTs. Alternatively you may be able to implement session-sticky server rotation at the load balancer level, depending on your load balancer. 8 | 9 | ### Run the Docker Stack 10 | 11 | cd dev-resources/load_balanced 12 | docker compose up 13 | 14 | ## Proxied LRS Demo 15 | 16 | This demo illustrates how SQL LRS must be configured if you are using a proxy (like nginx) to serve SQL LRS on a custom path. This is useful, for instance, for when you cannot use a dedicated domain/subdomain and need to serve SQL LRS from a path like `https://www.yetanalytics.com/my-lrs/...`. The important configuration variable to note for this situation is `LRSQL_PROXY_PATH` which tells the frontend to look for the server at that path. *NOTE: This variable does not actually move the location of SQL LRS endpoints, that must be done with a proxy, instead it just makes the components aware that that is happening*. 17 | 18 | ### Run the Docker Stack 19 | 20 | cd dev-resources/proxied_example 21 | docker compose up 22 | 23 | ## TLA Demo 24 | 25 | This demo illustrates a configuration similar to the Total Learning Architecture, wherein multiple Noisy LRS instances are feeding a single Transactional LRS. In this demo, three PostgreSQL-backed LRSs will be launched, three LRS Pipe processes will consume their data, and a Transactional LRS will receive the aggregation of that data. 26 | 27 | ### Run the Docker Stack 28 | 29 | cd dev-resources/tla-demo 30 | docker compose up 31 | 32 | [<- Back to Index](index.md) 33 | -------------------------------------------------------------------------------- /doc/overview.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # SQL LRS Overview 4 | 5 | SQL LRS is a Learning Record Store (LRS) application, which is a persistent store for xAPI statements and associated attachments and documents. The full LRS specification can be found in Part 3 of the [xAPI specification](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Communication.md). SQL LRS is distinct from other LRSs developed at Yet Analytics for being SQL-based and supporting multiple SQL database management systems (DBMSs) like SQLite, and Postgres. 6 | 7 | Features include: 8 | - A user interface that features API credential management, admin account management, statement browsing and management, and LRS monitoring 9 | - A [Docker image](docker.md) 10 | - Custom [Statement authority configuration](authority.md) 11 | - Authentication via [OpenID](oidc.md) 12 | - Support for BI platforms like [Apache Superset](superset.md) 13 | - [Reactions](reactions.md) 14 | 15 | ### How to use SQL LRS? 16 | 17 | SQL LRS admin accounts can be created using the user interface (see [Getting Started](startup.md)) or API calls (see [Endpoints](endpoints.md)). Accounts can be used to create or access SQL LRS credentials. These credentials, which consist of an API key, a secret API key, and their scopes ([described in the xAPI spec](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Communication.md#42-oauth-10-authorization-scope)), are then used as headers for LRS-specific methods to authenticate and authorize the request sender. 18 | 19 | [<- Back to Index](index.md) 20 | -------------------------------------------------------------------------------- /doc/postman.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Postman 4 | 5 | This page describes how to send an xAPI statement through [Postman](https://www.postman.com/), a well-known API platform for building and using APIs. 6 | 7 | First, install and run Postman. You may need to sign up if this is your first time running Postman. 8 | 9 | #### 1. Set Request type 10 | Open POSTMAN and create a new request. 11 | 12 | 1) Set the request type to POST. 13 | 2) Enter the URL: http://localhost:8080/xapi/statements. 14 | 15 | ![postman request type](images/postman_requesttype.png) 16 | 17 | #### 2. Set Headers 18 | 19 | Go to the "Headers" section and add two headers: 20 | 21 | | Key | Value | 22 | |------------|----------------| 23 | |Content-Type|application/json| 24 | |X-Experience-API-Version|1.0.3| 25 | 26 | 27 | ![postman headers](images/postman_headers.png) 28 | 29 | #### 3. Set Authorization 30 | 31 | Go to the "Authorization" section, choose the type "Basic Auth", and enter your credentials: 32 | 33 | * Username: my_key (or created key in the **Credential Management** page in the yetXApiSQLRS) 34 | * Password: my_secret (or create secret in the **Credential Management** page in the yetXApiSQLRS) 35 | 36 | ![postman autho](images/postman_auth.png) 37 | 38 | #### 4. Set Body 39 | 40 | Go to the "Body" section, select the "raw" option, and choose the "JSON (application/json)" content type. 41 | 42 | Copy and paste your valid JSON xAPI Statement(s) into the body: 43 | 44 | ![postman body](images/postman_body.png) 45 | 46 | 47 | Click the "Send" button to make the request. 48 | 49 | [<- Back to Index](index.md) 50 | -------------------------------------------------------------------------------- /doc/sqlite.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # SQLite Database 4 | 5 | When running SQL LRS in SQLite mode (via `bin/run_sqlite.sh` or `lrsql.exe`), the application will store all of its data in a local file it creates. This makes it very easy to manage the database. You can also direct any tools which are able to connect to or query SQLite databases to this file. By default the name of this file is `lrsql.sqlite.db` and it will appear in the root of the SQL LRS directory upon first startup. 6 | 7 | ### Changing the DB Name 8 | 9 | If you wish to change the name of the db file that SQL LRS connects to, you can do so using a configuration variable. Changing `LRSQL_DB_NAME` (`dbName` in `config/lrsql.json`) will accomplish this. The example below shows how to direct SQL LRS to another file. 10 | 11 | ```json 12 | { 13 | ... 14 | "database": { 15 | "dbName": "new-file-name.db" 16 | } 17 | } 18 | ``` 19 | 20 | ### Deleting the Database 21 | Since the database is just a file, it should be no surprise that you can delete it and this will reset the LRS, starting over again from scratch. Alternatively changing the `dbName` variable as in the section above will keep the old file around but similarly allow you to start over fresh with a new one. 22 | 23 | *WARNING:* Keep in mind there will be no ability to recover the LRS data if the file is deleted! This includes xAPI data as well as all accounts and credentials. 24 | 25 | [<- Back to Index](index.md) 26 | -------------------------------------------------------------------------------- /doc/troubleshooting.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Troubleshooting 4 | 5 | ### I am unable to connect to the LRS when opening the UI 6 | 7 | First of all, ensure that the SQL LRS app is running and that the host and port are configured correctly. If you are running SQL LRS as a Docker image, ensure that the port is exposed. 8 | 9 | In addition, if you are using a proper domain name (either via DNS or via a `hosts` file) or using a proxy server, you may need to adjust [configuration for CORS](env_vars.md#cors) (Cross-Origin Resource Sharing). CORS restricts which endpoints SQL LRS will accept requests from; requests from disallowed endpoints will result in a 403 Forbidden response. Either specify allowed endpoints via `LRSQL_ALLOWED_ORIGINS` (the recommended method for production) or allow all endpoints via setting `LRSQL_ALLOW_ALL_ORIGINS` to `true`. 10 | 11 | ### I am unable to run the Docker image in Postgres mode 12 | 13 | First of all, ensure that you are indeed executing the SQL LRS image in Postgres mode. The command `/lrsql/bin/run_postgres.sh` needs to be run as a custom command in order to override the default command, which runs the app in SQLite mode. 14 | 15 | In addition, check that you have the appropriate values of `POSTGRES_DB`, `POSTGRES_USER`, and `POSTGRES_PASSWORD` for your Postgres Docker image, and that the respective config values (`LRSQL_DB_NAME`, `LRSQL_DB_USER`, and `LRSQL_DB_PASSWORD`) match up. 16 | 17 | See the `docker-compose.yml` file as a reference for running Postgres SQL LRS via Docker/Docker Compose. 18 | 19 | ### My Postgres connections don't get released when not in use 20 | 21 | You may want to adjust the `LRSQL_POOL_MINIMUM_IDLE` config var, as it is set to 10 by default in Postgres mode. (See [here](env_vars.md#hikaricp-properties) for more info on connection pool configuration.) 22 | 23 | [<- Back to Index](index.md) 24 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Runs SQL LRS with Postgres - Provided for demonstration purposes only! 2 | # To run: docker compose up 3 | # See the Docker Compose docs for more info: https://docs.docker.com/compose/ 4 | volumes: 5 | db_data: 6 | services: 7 | db: 8 | image: postgres 9 | volumes: 10 | - db_data:/var/lib/postgresql/data 11 | environment: 12 | POSTGRES_USER: lrsql_user 13 | POSTGRES_PASSWORD: lrsql_password 14 | POSTGRES_DB: lrsql_db 15 | ports: 16 | - "5432:5432" # Useful if we only want to run the DB w/o other services 17 | clamav: 18 | image: clamav/clamav:1.2.1 19 | lrs: 20 | # build: . # switch to this for active dev 21 | image: yetanalytics/lrsql:latest 22 | command: 23 | - /lrsql/bin/run_postgres.sh 24 | ports: 25 | - "8080:8080" 26 | depends_on: 27 | - db 28 | environment: 29 | LRSQL_API_KEY_DEFAULT: my_key 30 | LRSQL_API_SECRET_DEFAULT: my_secret 31 | LRSQL_ADMIN_USER_DEFAULT: my_username 32 | LRSQL_ADMIN_PASS_DEFAULT: my_password 33 | LRSQL_DB_HOST: db 34 | LRSQL_DB_NAME: lrsql_db 35 | LRSQL_DB_USER: lrsql_user 36 | LRSQL_DB_PASSWORD: lrsql_password 37 | # If Postgres is too slow to start, increase this 38 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 39 | # Set to true if using dev UI, domain name, proxy server, etc. 40 | LRSQL_ALLOW_ALL_ORIGINS: false 41 | # Enable ClamAV Scanning 42 | LRSQL_ENABLE_CLAMAV: true 43 | LRSQL_CLAMAV_HOST: clamav 44 | -------------------------------------------------------------------------------- /exe/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | true 4 | console 5 | 6 | lrsql.sqlite.main 7 | lrsql.jar 8 | 9 | lrsql.exe 10 | You must have Java 11+ installed to run SQL LRS 11 | . 12 | normal 13 | http://java.com/download 14 | https://yet.zendesk.com/hc/en-us 15 | false 16 | false 17 | 18 | lrsql.ico 19 | 20 | runtimes/windows 21 | false 22 | false 23 | 11 24 | 25 | preferJre 26 | 64/32 27 | -Dfile.encoding=UTF-8 28 | 29 | 30 | -------------------------------------------------------------------------------- /exe/config_pg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | true 4 | console 5 | 6 | lrsql.postgres.main 7 | lrsql.jar 8 | 9 | lrsql_pg.exe 10 | You must have Java 11+ installed to run SQL LRS 11 | . 12 | normal 13 | http://java.com/download 14 | https://yet.zendesk.com/hc/en-us 15 | false 16 | false 17 | 18 | lrsql.ico 19 | 20 | runtimes/windows 21 | false 22 | false 23 | 11 24 | 25 | preferJre 26 | 64/32 27 | -Dfile.encoding=UTF-8 28 | 29 | 30 | -------------------------------------------------------------------------------- /exe/lrsql.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/exe/lrsql.exe -------------------------------------------------------------------------------- /exe/lrsql.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/exe/lrsql.ico -------------------------------------------------------------------------------- /exe/lrsql_pg.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/lrsql/7129a40edb87ef459704964a75bcd2985b628138/exe/lrsql_pg.exe -------------------------------------------------------------------------------- /resources/lrsql/config/authority.json.template: -------------------------------------------------------------------------------- 1 | {# for template syntax see: https://github.com/yogthos/Selmer#templates 2 | Variables 3 | ================================================= 4 | authority-url : LRSQL_AUTHORITY_URL env var value 5 | cred-id : LRS Credential Pair ID (UUID) 6 | account-id : LRS Admin Account ID (UUID) 7 | #} 8 | { 9 | "account": { 10 | "homePage":"{{authority-url}}", 11 | "name":"{{cred-id}}" 12 | }, 13 | "objectType":"Agent" 14 | } 15 | -------------------------------------------------------------------------------- /resources/lrsql/config/config.edn: -------------------------------------------------------------------------------- 1 | {:database 2 | #profile 3 | {;; Test/Dev 4 | :test-sqlite {:db-type "sqlite" 5 | :db-name "example.sqlite.db"} 6 | :test-sqlite-mem {:db-type "sqlite" 7 | :db-name ":memory:"} 8 | :test-postgres {:db-type "postgres" 9 | :db-name "lrsql_db" 10 | :db-host "0.0.0.0" 11 | :db-port 5432 12 | :db-user "lrsql_user" 13 | :db-password "lrsql_password" 14 | ;; Schemas are commented out for testing (since all tests 15 | ;; happen in the default `public` schema). We can uncomment 16 | ;; these properties for manual schema testing. 17 | ;; :db-schema "lrsql" 18 | ;; :db-properties "currentSchema=lrsql" 19 | 20 | ;; Testing Only! Specify the version used with testcontainers 21 | :test-db-version #or [#env LRSQL_TEST_DB_VERSION "11"]} 22 | :test-oidc {:db-type "sqlite" 23 | :db-name ":memory:"} 24 | 25 | ;; Production 26 | :prod-sqlite-mem #include "prod/sqlite_mem/database.edn" 27 | :prod-sqlite #include "prod/sqlite/database.edn" 28 | :prod-postgres #include "prod/postgres/database.edn"} 29 | :connection 30 | #profile 31 | {;; Test/Dev 32 | :test-sqlite #include "test/default/connection.edn" 33 | :test-sqlite-mem #include "test/sqlite_mem/connection.edn" 34 | :test-postgres #include "test/postgres/connection.edn" 35 | :test-oidc #include "test/default/connection.edn" 36 | ;; Production 37 | :prod-sqlite #include "prod/default/connection.edn" 38 | :prod-sqlite-mem #include "prod/sqlite_mem/connection.edn" 39 | :prod-postgres #include "prod/postgres/connection.edn"} 40 | :tuning 41 | #profile 42 | {; Test/Dev 43 | :test-sqlite #include "test/default/tuning.edn" 44 | :test-sqlite-mem #include "test/default/tuning.edn" 45 | :test-postgres #include "test/postgres/tuning.edn" 46 | :test-oidc #include "test/default/tuning.edn" 47 | ;; Production 48 | :prod-sqlite #include "prod/default/tuning.edn" 49 | :prod-sqlite-mem #include "prod/default/tuning.edn" 50 | :prod-postgres #include "prod/postgres/tuning.edn"} 51 | :lrs 52 | #profile 53 | {;; Test/Dev 54 | :test-sqlite #include "test/default/lrs.edn" 55 | :test-sqlite-mem #include "test/default/lrs.edn" 56 | :test-postgres #include "test/default/lrs.edn" 57 | :test-oidc #include "test/oidc/lrs.edn" 58 | ;; Production 59 | :prod-sqlite #include "prod/default/lrs.edn" 60 | :prod-sqlite-mem #include "prod/default/lrs.edn" 61 | :prod-postgres #include "prod/default/lrs.edn"} 62 | :webserver 63 | #profile 64 | {;; Test/Dev 65 | :test-sqlite #include "test/default/webserver.edn" 66 | :test-sqlite-mem #include "test/default/webserver.edn" 67 | :test-postgres #include "test/default/webserver.edn" 68 | :test-oidc #include "test/oidc/webserver.edn" 69 | ;; Production 70 | :prod-sqlite #include "prod/default/webserver.edn" 71 | :prod-sqlite-mem #include "prod/default/webserver.edn" 72 | :prod-postgres #include "prod/default/webserver.edn"} 73 | ;; A user-provided JSON file for merge-with merge into this map 74 | :user-config-json #or [#env LRSQL_USER_CONFIG_JSON "config/lrsql.json"] 75 | ;; Set logging params on system start 76 | :logger 77 | {:log-level #or [#env LRSQL_LOG_LEVEL nil]}} 78 | -------------------------------------------------------------------------------- /resources/lrsql/config/lrsql.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "dbHost": "0.0.0.0", 4 | "dbPort": 5432, 5 | "dbName": "lrsql_db", 6 | "dbUser": "lrsql_user", 7 | "dbPassword": "changeme" 8 | }, 9 | "connection": { 10 | "poolName": "my-pool-name" 11 | }, 12 | "lrs" : { 13 | "adminUserDefault": "myUsername", 14 | "adminPassDefault": "changeme", 15 | "authorityUrl": "http://mydomain.com" 16 | }, 17 | "webserver": { 18 | "httpHost": "0.0.0.0", 19 | "httpPort": 8080, 20 | "sslPort": 8443, 21 | "allowAllOrigins": true 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /resources/lrsql/config/oidc_authority.json.template: -------------------------------------------------------------------------------- 1 | {# for template syntax see: https://github.com/yogthos/Selmer#templates 2 | 3 | Use any variable present in token claims 4 | https://www.iana.org/assignments/jwt/jwt.xhtml 5 | 6 | This default template uses the "iss" claim to represent the IDP and the "sub" 7 | claim to represent the user. 8 | 9 | Special Variables: 10 | 11 | * lrsql/resolved-client-id - Replacement for aud that is always a string. 12 | 13 | #} 14 | { 15 | "objectType": "Group", 16 | "member": [ 17 | { 18 | "account": { 19 | "homePage": "{{iss}}", 20 | "name": "{{lrsql/resolved-client-id}}" 21 | } 22 | }, 23 | { 24 | "account": { 25 | "homePage": "{{iss}}", 26 | "name": "{{sub}}" 27 | } 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /resources/lrsql/config/oidc_client.json.template: -------------------------------------------------------------------------------- 1 | {# for template syntax see: https://github.com/yogthos/Selmer#templates 2 | 3 | This template maps config to oidc-client-js configuration. 4 | 5 | Variables: 6 | * lrs - The LRS configuration. 7 | * webserver - The webserver configuration. 8 | 9 | #} 10 | { 11 | "authority": "{{webserver.oidc-issuer}}", 12 | "client_id": "{{webserver.oidc-client-id}}", 13 | "response_type": "code", 14 | "scope": "openid profile {{lrs.oidc-scope-prefix}}admin", 15 | "automaticSilentRenew": true, 16 | "monitorSession": false, 17 | "filterProtocolClaims": false, 18 | "extraQueryParams": { 19 | "audience": "{{webserver.oidc-audience}}" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/default/connection.edn: -------------------------------------------------------------------------------- 1 | ;; All values here are Hikari defaults except for `:pool-minimum-idle` and 2 | ;; `:pool-maximum-size`, since the default config is geared for single-threaded 3 | ;; databases. 4 | {:pool-auto-commit #boolean #or [#env LRSQL_POOL_AUTO_COMMIT true] 5 | :pool-keepalive-time #long #or [#env LRSQL_POOL_KEEPALIVE_TIME 0] 6 | :pool-connection-timeout #long #or [#env LRSQL_POOL_CONNECTION_TIMEOUT 30000] 7 | :pool-idle-timeout #long #or [#env LRSQL_POOL_IDLE_TIMEOUT 600000] 8 | :pool-validation-timeout #long #or [#env LRSQL_POOL_VALIDATION_TIMEOUT 5000] 9 | :pool-max-lifetime #long #or [#env LRSQL_POOL_MAX_LIFETIME 1800000] 10 | :pool-minimum-idle #long #or [#env LRSQL_POOL_MINIMUM_IDLE 1] 11 | :pool-maximum-size #long #or [#env LRSQL_POOL_MAXIMUM_SIZE 1] 12 | :pool-initialization-fail-timeout #long #or [#env LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT 1] 13 | :pool-isolate-internal-queries #boolean #or [#env LRSQL_POOL_ISOLATE_INTERNAL_QUERIES false] 14 | :pool-leak-detection-threshold #long #or [#env LRSQL_POOL_LEAK_DETECTION_THRESHOLD 0] 15 | :pool-transaction-isolation #or [#env LRSQL_POOL_TRANSACTION_ISOLATION nil] 16 | :pool-name #or [#env LRSQL_POOL_NAME nil] 17 | :pool-enable-jmx #boolean #or [#env LRSQL_POOL_ENABLE_JMX false]} 18 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/default/lrs.edn: -------------------------------------------------------------------------------- 1 | {:admin-user-default #or [#env LRSQL_ADMIN_USER_DEFAULT nil] 2 | :admin-pass-default #or [#env LRSQL_ADMIN_PASS_DEFAULT nil] 3 | :api-key-default #or [#env LRSQL_API_KEY_DEFAULT nil] 4 | :api-secret-default #or [#env LRSQL_API_SECRET_DEFAULT nil] 5 | :stmt-get-default #long #or [#env LRSQL_STMT_GET_DEFAULT 50] 6 | :stmt-get-max #long #or [#env LRSQL_STMT_GET_MAX 50] 7 | :stmt-get-max-csv #long #or [#env LRSQL_STMT_GET_MAX_CSV -1] 8 | :stmt-url-prefix "/xapi" ; overriden by ::webserver/url-prefix 9 | :authority-template #or [#env LRSQL_AUTHORITY_TEMPLATE "config/authority.json.template"] 10 | :authority-url #or [#env LRSQL_AUTHORITY_URL "http://example.org"] 11 | :oidc-authority-template #or [#env LRSQL_OIDC_AUTHORITY_TEMPLATE "config/oidc_authority.json.template"] 12 | :oidc-scope-prefix #or [#env LRSQL_OIDC_SCOPE_PREFIX ""] 13 | :stmt-retry-limit #or [#env LRSQL_STMT_RETRY_LIMIT 10] 14 | :stmt-retry-budget #or [#env LRSQL_STMT_RETRY_BUDGET 1000] 15 | :enable-reactions #boolean #or [#env LRSQL_ENABLE_REACTIONS false] 16 | :reaction-buffer-size #long #or [#env LRSQL_REACTION_BUFFER_SIZE 10000]} 17 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/default/tuning.edn: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/default/webserver.edn: -------------------------------------------------------------------------------- 1 | {:key-file #or [#env LRSQL_KEY_FILE "config/keystore.jks"] 2 | :key-alias #or [#env LRSQL_KEY_ALIAS "lrsql_keystore"] 3 | :key-password #or [#env LRSQL_KEY_PASSWORD "lrsql_pass"] 4 | :key-pkey-file #or [#env LRSQL_KEY_PKEY_FILE "config/server.key.pem"] 5 | :key-cert-chain #or [#env LRSQL_KEY_CERT_FILE "config/server.crt.pem,config/cacert.pem"] 6 | :key-enable-selfie #boolean #or [#env LRSQL_KEY_ENABLE_SELFIE true] 7 | :jwt-exp-time #long #or [#env LRSQL_JWT_EXP_TIME 3600] 8 | :jwt-exp-leeway #long #or [#env LRSQL_JWT_EXP_LEEWAY 1] 9 | :jwt-refresh-exp-time #long #or [#env LRSQL_JWT_REFRESH_EXP_TIME 86400] 10 | :jwt-refresh-interval #long #or [#env LRSQL_JWT_REFRESH_INTERVAL 3540] 11 | :jwt-interaction-window #long #or [#env LRSQL_JWT_INTERACTION_WINDOW 600] 12 | :jwt-no-val #boolean #or [#env LRSQL_JWT_NO_VAL false] 13 | :jwt-no-val-uname #or [#env LRSQL_JWT_NO_VAL_UNAME nil] 14 | :jwt-no-val-issuer #or [#env LRSQL_JWT_NO_VAL_ISSUER nil] 15 | :jwt-no-val-role-key #or [#env LRSQL_JWT_NO_VAL_ROLE_KEY nil] 16 | :jwt-no-val-role #or [#env LRSQL_JWT_NO_VAL_ROLE nil] 17 | :jwt-no-val-logout-url #or [#env LRSQL_JWT_NO_VAL_LOGOUT_URL nil] 18 | :jwt-common-secret #or [#env LRSQL_JWT_COMMON_SECRET nil] 19 | :sec-head-hsts #or [#env LRSQL_SEC_HEAD_HSTS nil] 20 | :sec-head-frame #or [#env LRSQL_SEC_HEAD_FRAME nil] 21 | :sec-head-content-type #or [#env LRSQL_SEC_HEAD_CONTENT_TYPE nil] 22 | :sec-head-xss #or [#env LRSQL_SEC_HEAD_XSS nil] 23 | :sec-head-download #or [#env LRSQL_SEC_HEAD_DOWNLOAD nil] 24 | :sec-head-cross-domain #or [#env LRSQL_SEC_HEAD_CROSS_DOMAIN nil] 25 | :sec-head-content #or [#env LRSQL_SEC_HEAD_CONTENT nil] 26 | :enable-http #boolean #or [#env LRSQL_ENABLE_HTTP true] 27 | :enable-http2 #boolean #or [#env LRSQL_ENABLE_HTTP2 true] 28 | :http-host #or [#env LRSQL_HTTP_HOST "0.0.0.0"] 29 | :http-port #long #or [#env LRSQL_HTTP_PORT 8080] 30 | :allow-all-origins #boolean #or [#env LRSQL_ALLOW_ALL_ORIGINS false] 31 | :allowed-origins #array #or [#env LRSQL_ALLOWED_ORIGINS nil] 32 | :ssl-port #long #or [#env LRSQL_SSL_PORT 8443] 33 | :url-prefix #or [#env LRSQL_URL_PREFIX "/xapi"] 34 | :proxy-path #or [#env LRSQL_PROXY_PATH nil] 35 | :enable-admin-delete-actor #boolean #or [#env LRSQL_ENABLE_ADMIN_DELETE_ACTOR false] 36 | :enable-admin-ui #boolean #or [#env LRSQL_ENABLE_ADMIN_UI true] 37 | :admin-language #or [#env LRSQL_ADMIN_LANGUAGE "en-US"] 38 | :enable-admin-status #boolean #or [#env LRSQL_ENABLE_ADMIN_STATUS true] 39 | :enable-stmt-html #boolean #or [#env LRSQL_ENABLE_STMT_HTML true] 40 | :oidc-issuer #env LRSQL_OIDC_ISSUER 41 | :oidc-audience #env LRSQL_OIDC_AUDIENCE 42 | :oidc-client-id #env LRSQL_OIDC_CLIENT_ID 43 | :oidc-client-template #or [#env LRSQL_OIDC_CLIENT_TEMPLATE "config/oidc_client.json.template"] 44 | :oidc-verify-remote-issuer #boolean #or [#env LRSQL_OIDC_VERIFY_REMOTE_ISSUER true] 45 | :oidc-enable-local-admin #boolean #or [#env LRSQL_OIDC_ENABLE_LOCAL_ADMIN false] 46 | :enable-clamav #boolean #or [#env LRSQL_ENABLE_CLAMAV false] 47 | :clamav-host #or [#env LRSQL_CLAMAV_HOST "localhost"] 48 | :clamav-port #long #or [#env LRSQL_CLAMAV_PORT 3310] 49 | :auth-by-cred-id #boolean #or [#env LRSQL_AUTH_BY_CRED_ID false]} 50 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/postgres/connection.edn: -------------------------------------------------------------------------------- 1 | #merge 2 | [#include "prod/default/connection.edn" 3 | {:pool-minimum-idle #long #or [#env LRSQL_POOL_MINIMUM_IDLE 10] 4 | :pool-maximum-size #long #or [#env LRSQL_POOL_MAXIMUM_SIZE 10]}] 5 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/postgres/database.edn: -------------------------------------------------------------------------------- 1 | {:db-type #or [#env LRSQL_DB_TYPE "postgres"] 2 | :db-name #or [#env LRSQL_DB_NAME "lrsql_pg"] 3 | :db-host #or [#env LRSQL_DB_HOST "0.0.0.0"] 4 | :db-port #long #or [#env LRSQL_DB_PORT 5432] 5 | :db-properties #or [#env LRSQL_DB_PROPERTIES nil] 6 | :db-jdbc-url #or [#env LRSQL_DB_JDBC_URL nil] 7 | :db-user #or [#env LRSQL_DB_USER nil] 8 | :db-password #or [#env LRSQL_DB_PASSWORD nil] 9 | :db-schema #or [#env LRSQL_DB_SCHEMA nil] 10 | :db-catalog #or [#env LRSQL_DB_CATALOG nil]} 11 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/postgres/tuning.edn: -------------------------------------------------------------------------------- 1 | {:enable-jsonb #boolean #or [#env LRSQL_ENABLE_JSONB false]} 2 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/sqlite/database.edn: -------------------------------------------------------------------------------- 1 | {:db-type #or [#env LRSQL_DB_TYPE "sqlite"] 2 | :db-name #or [#env LRSQL_DB_NAME "lrsql.sqlite.db"] 3 | :db-host #or [#env LRSQL_DB_HOST nil] 4 | :db-port #long #or [#env LRSQL_DB_PORT -1] ; Placeholder to prevent error 5 | :db-properties #or [#env LRSQL_DB_PROPERTIES nil] 6 | :db-jdbc-url #or [#env LRSQL_DB_JDBC_URL nil] 7 | :db-user #or [#env LRSQL_DB_USER nil] 8 | :db-password #or [#env LRSQL_DB_PASSWORD nil] 9 | :db-schema #or [#env LRSQL_DB_SCHEMA nil] 10 | :db-catalog #or [#env LRSQL_DB_CATALOG nil]} 11 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/sqlite_mem/connection.edn: -------------------------------------------------------------------------------- 1 | ;; SQLite deletes the in-mem DB whenever a connection closes, which by default 2 | ;; happens after 30 min/1800000 ms, so we set it to 0 to make the lifetime 3 | ;; infinite. 4 | #merge 5 | [#include "prod/default/connection.edn" 6 | {:pool-max-lifetime #long #or [#env LRSQL_POOL_MAX_LIFETIME 0]}] 7 | -------------------------------------------------------------------------------- /resources/lrsql/config/prod/sqlite_mem/database.edn: -------------------------------------------------------------------------------- 1 | {:db-type #or [#env LRSQL_DB_TYPE "sqlite"] 2 | :db-name #or [#env LRSQL_DB_NAME ":memory:"] 3 | :db-host #or [#env LRSQL_DB_HOST nil] 4 | :db-port #long #or [#env LRSQL_DB_PORT -1] ; Placeholder to prevent error 5 | :db-properties #or [#env LRSQL_DB_PROPERTIES nil] 6 | :db-jdbc-url #or [#env LRSQL_DB_JDBC_URL nil] 7 | :db-user #or [#env LRSQL_DB_USER nil] 8 | :db-password #or [#env LRSQL_DB_PASSWORD nil] 9 | :db-schema #or [#env LRSQL_DB_SCHEMA nil] 10 | :db-catalog #or [#env LRSQL_DB_CATALOG nil]} 11 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/default/connection.edn: -------------------------------------------------------------------------------- 1 | {:pool-auto-commit true 2 | :pool-keepalive-time 0 3 | :pool-connection-timeout 30000 4 | :pool-idle-timeout 600000 5 | :pool-validation-timeout 5000 6 | :pool-max-lifetime 1800000 7 | :pool-minimum-idle 1 8 | :pool-maximum-size 1 9 | :pool-initialization-fail-timeout 1 10 | :pool-isolate-internal-queries false 11 | :pool-leak-detection-threshold 0 12 | :pool-transaction-isolation nil 13 | :pool-name nil 14 | :pool-enable-jmx true} 15 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/default/lrs.edn: -------------------------------------------------------------------------------- 1 | {:admin-user-default "username" 2 | :admin-pass-default "password" 3 | :api-key-default "username" 4 | :api-secret-default "password" 5 | :stmt-get-default 50 6 | :stmt-get-max 50 7 | :stmt-get-max-csv nil 8 | :stmt-url-prefix "/xapi" 9 | :authority-template "config/authority.json.template" 10 | :authority-url "http://example.org" 11 | :oidc-authority-template "config/oidc_authority.json.template" 12 | :oidc-scope-prefix "" 13 | :stmt-retry-limit 20 14 | :stmt-retry-budget 10000 15 | :enable-reactions true 16 | :reaction-buffer-size 10000} 17 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/default/tuning.edn: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/default/webserver.edn: -------------------------------------------------------------------------------- 1 | {:key-file "config/keystore.jks" 2 | :key-alias "lrsql_keystore" 3 | :key-password "lrsql_pass" 4 | :key-enable-selfie true 5 | :jwt-exp-time 3600 6 | :jwt-exp-leeway 1 7 | :jwt-refresh-exp-time 86400 8 | :jwt-refresh-interval 3540 9 | :jwt-interaction-window 600 10 | :jwt-no-val false 11 | :jwt-no-val-uname nil 12 | :jwt-no-val-issuer nil 13 | :jwt-no-val-role-key nil 14 | :jwt-no-val-role nil 15 | :jwt-no-val-logout-url nil 16 | :jwt-common-secret nil 17 | :enable-http true 18 | :enable-http2 true 19 | :ssl-port 8443 20 | :http-host "0.0.0.0" 21 | :http-port 8080 22 | :allow-all-origins false 23 | :allowed-origins ["http://localhost:8080" 24 | ;; Allow access from admin-ui dev 25 | "http://localhost:9500"] 26 | :url-prefix "/xapi" 27 | :proxy-path nil 28 | :enable-admin-delete-actor true 29 | :enable-admin-ui true 30 | :enable-admin-status true 31 | :admin-language "en-US" 32 | :enable-stmt-html true 33 | :oidc-verify-remote-issuer true 34 | :oidc-client-template "config/oidc_client.json.template" 35 | :oidc-enable-local-admin false 36 | :enable-clamav false 37 | :clamav-host "localhost" 38 | :clamav-port 3310 39 | :auth-by-cred-id false} 40 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/oidc/lrs.edn: -------------------------------------------------------------------------------- 1 | #merge 2 | [#include "test/default/lrs.edn" 3 | {:oidc-scope-prefix "lrs:"}] 4 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/oidc/webserver.edn: -------------------------------------------------------------------------------- 1 | #merge 2 | [#include "test/default/webserver.edn" 3 | {:oidc-issuer "http://localhost:8081/auth/realms/test" 4 | :oidc-audience "http://localhost:8080" 5 | :oidc-client-id "lrs_admin_ui"}] 6 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/postgres/connection.edn: -------------------------------------------------------------------------------- 1 | #merge 2 | [#include "test/default/connection.edn" 3 | {:pool-minimum-idle 10 4 | :pool-maximum-size 10}] 5 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/postgres/tuning.edn: -------------------------------------------------------------------------------- 1 | {:enable-jsonb false} 2 | -------------------------------------------------------------------------------- /resources/lrsql/config/test/sqlite_mem/connection.edn: -------------------------------------------------------------------------------- 1 | ;; SQLite deletes the in-mem DB whenever a connection closes, which by default 2 | ;; happens after 30 min/1800000 ms, so we set it to 0 to make the lifetime 3 | ;; infinite. 4 | #merge 5 | [#include "test/default/connection.edn" 6 | {:pool-max-lifetime 0}] 7 | -------------------------------------------------------------------------------- /resources/lrsql/doc/docs.html.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Yet SQL LRS Documentation 5 | 119 | 120 | 121 |
122 |
123 | 126 |
127 | {{content|safe}} 128 |
129 | © 2021-2025 Yet Analytics Inc. | 130 | Licensed under the Apache 2.0 License | 131 | Contribute on GitHub 132 | Build: {{sha | safe}} 133 |
134 |
135 | 136 | 137 | -------------------------------------------------------------------------------- /resources/lrsql/localization/language.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /src/build/lrsql/build.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.build 2 | "Build utils for LRSQL artifacts" 3 | (:require [clojure.tools.build.api :as b])) 4 | 5 | ;; We add the `src/db` subdirectories separately to ensure that all `src` paths 6 | ;; in the code start w/ `lrsql/` (see: the calls to `hugsql.core/def-db-fns`) 7 | (def src-dirs 8 | ["src/main" 9 | "resources" 10 | "src/db/sqlite" 11 | "src/db/postgres"]) 12 | 13 | (def class-dir 14 | "target/classes/") 15 | 16 | ;; Don't ship crypto - shouldn't be included in the build path anyways, 17 | ;; but we exclude them here as extra defense. 18 | ;; On the other hand, we keep the unobfuscated OSS source code so that users 19 | ;; have easy access to it. 20 | (def ignored-file-regexes 21 | ["^.*jks$" 22 | "^.*key$" 23 | "^.*pem$"]) 24 | 25 | (def uberjar-file 26 | "target/bundle/lrsql.jar") 27 | 28 | (defn- create-basis [] 29 | (b/create-basis 30 | {:project "deps.edn" 31 | :aliases [:db-sqlite :db-postgres]})) 32 | 33 | (defn write-git-data! [] 34 | (when-let [version (b/git-process {:git-args "describe --exact-match --tags"})] 35 | (b/write-file {:path "target/classes/lrsql/config/git-details.edn" 36 | :content version}))) 37 | 38 | ;; We create a single JAR for all DB backends in order to minimize artifact 39 | ;; download size, since all backends share most of the app code 40 | (defn uber 41 | "Create an Uberjar at `target/bundle/lrsql.jar` that can be executed to 42 | run the SQL LRS app, for any DB backend." 43 | [_] 44 | (let [basis (create-basis)] 45 | (b/copy-dir 46 | {:src-dirs src-dirs 47 | :target-dir class-dir 48 | :ignores ignored-file-regexes}) 49 | (write-git-data!) 50 | (b/compile-clj 51 | {:basis basis 52 | :src-dirs src-dirs 53 | :class-dir class-dir}) 54 | (b/uber 55 | {:basis basis 56 | :class-dir class-dir 57 | :uber-file uberjar-file}))) 58 | 59 | ;; Alternate Jar for just bencher 60 | (def src-dirs-bench 61 | ["src/bench"]) 62 | 63 | (def uberjar-file-bench 64 | "target/bundle/bench.jar") 65 | 66 | (defn- create-basis-bench [] 67 | (b/create-basis 68 | {:project "deps.edn" 69 | :aliases [:bench]})) 70 | 71 | (defn uber-bench 72 | "Create Benchmark uberjar." 73 | [_] 74 | (let [basis (create-basis-bench)] 75 | (b/copy-dir 76 | {:src-dirs src-dirs-bench 77 | :target-dir class-dir 78 | :ignores ignored-file-regexes}) 79 | (b/compile-clj 80 | {:basis basis 81 | :src-dirs src-dirs-bench 82 | :class-dir class-dir}) 83 | (b/uber 84 | {:basis basis 85 | :class-dir class-dir 86 | :uber-file uberjar-file-bench}))) 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/db/postgres/lrsql/postgres/data.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.postgres.data 2 | (:require [next.jdbc.prepare :refer [SettableParameter]] 3 | [next.jdbc.result-set :refer [ReadableColumn]] 4 | [lrsql.util :as u]) 5 | (:import [clojure.lang IPersistentMap] 6 | [org.postgresql.util PGobject] 7 | [java.sql PreparedStatement ResultSetMetaData])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; PGObject 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (defn- json->pg-object 14 | [type jsn] 15 | (doto (PGobject.) 16 | (.setType type) 17 | (.setValue (u/write-json-str jsn)))) 18 | 19 | (defn- pg-object->json 20 | [kw-labels label ^PGobject pg-obj] 21 | (let [type (.getType pg-obj) 22 | value (.getValue pg-obj)] 23 | (if (#{"jsonb" "json"} type) 24 | (u/parse-json value :keyword-keys? (some? (kw-labels label))) 25 | (throw (ex-info "Invalid PostgreSQL JSON type" 26 | {:type ::invalid-postgres-json 27 | :json-type type 28 | :json-value value}))))) 29 | 30 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 31 | ;; Read 32 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 33 | 34 | (defn set-read-pgobject->json! 35 | [kw-labels] 36 | (extend-protocol ReadableColumn 37 | PGobject 38 | (read-column-by-label [^PGobject pg-obj ^String label] 39 | (pg-object->json kw-labels label pg-obj)) 40 | (read-column-by-index [^PGobject pg-obj ^ResultSetMetaData rsmeta ^long i] 41 | (pg-object->json kw-labels (.getColumnLabel rsmeta i) pg-obj)))) 42 | 43 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 44 | ;; Write 45 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 46 | 47 | (defn set-write-json->pgobject! 48 | [type] 49 | (extend-protocol SettableParameter 50 | IPersistentMap 51 | (set-parameter [^IPersistentMap m ^PreparedStatement stmt ^long i] 52 | (.setObject stmt i (json->pg-object type m))))) 53 | 54 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 55 | ;; Timezone Input 56 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 57 | 58 | (def local-tz-input 59 | "Returns a properly formatted hug input map to inject a timezone id into a 60 | query needing a timezone id" 61 | {:tz-id (str "'" u/local-zone-id "'")}) 62 | 63 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 64 | ;; JSON Field Coercion 65 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 66 | 67 | (def type->pg-type 68 | {:bool "BOOLEAN" 69 | :int "INTEGER" 70 | :dec "DECIMAL" 71 | :string "TEXT" 72 | :json "JSONB"}) 73 | -------------------------------------------------------------------------------- /src/db/postgres/lrsql/postgres/main.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.postgres.main 2 | (:require [com.stuartsierra.component :as component] 3 | [lrsql.system :as system] 4 | [lrsql.system.util :as su] 5 | [lrsql.postgres.record :as pr]) 6 | (:gen-class)) 7 | 8 | (def postgres-backend (pr/map->PostgresBackend {})) 9 | 10 | (defn run-test-postgres 11 | "Run a Postgres-backed LRSQL instance based on the `:test-postgres` 12 | config profile. For use with `clojure -X:db-postgres`." 13 | [_] ; Need to pass in a map for -X 14 | (-> (system/system postgres-backend :test-postgres) 15 | component/start 16 | su/add-shutdown-hook!)) 17 | 18 | (defn -main [& _args] 19 | (-> (system/system postgres-backend :prod-postgres) 20 | component/start 21 | su/add-shutdown-hook!)) 22 | -------------------------------------------------------------------------------- /src/db/sqlite/lrsql/sqlite/main.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.sqlite.main 2 | (:require [com.stuartsierra.component :as component] 3 | [lrsql.system :as system] 4 | [lrsql.system.util :as su] 5 | [lrsql.sqlite.record :as sr]) 6 | (:gen-class)) 7 | 8 | (def sqlite-backend (sr/map->SQLiteBackend {})) 9 | 10 | (defn run-test-sqlite 11 | "Run a SQLite-backed LRSQL instance based on the `:test-sqlite` (if 12 | `:ephemeral?` is set to `false`) or `:test-sqlite-mem` (if `true`) config 13 | profile. For use with `clojure -X:db-sqlite`." 14 | [{:keys [ephemeral? 15 | override-profile]}] 16 | (let [profile (or override-profile 17 | (if ephemeral? :test-sqlite-mem :test-sqlite))] 18 | (-> (system/system sqlite-backend profile) 19 | component/start 20 | su/add-shutdown-hook!))) 21 | 22 | (defn -main 23 | "Main entrypoint for SQLite-backed LRSQL instances. Passing `--ephemeral true` 24 | will spin up an in-mem SQLLite instance; otherwise, a persistent SQLite db 25 | will be stored on disk." 26 | [& args] 27 | (let [{?per-str "--ephemeral"} args 28 | ephemeral? (Boolean/parseBoolean ?per-str) 29 | profile (if ephemeral? :prod-sqlite-mem :prod-sqlite)] 30 | (-> (system/system sqlite-backend profile) 31 | component/start 32 | su/add-shutdown-hook!))) 33 | -------------------------------------------------------------------------------- /src/main/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | logs/lrsql.log 4 | 5 | 6 | logs/lrsql.%d{yyyy-MM-dd}.log 7 | 60 8 | 1GB 9 | 10 | 11 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 12 | 13 | 14 | 15 | 16 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/interceptors/csv_download.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.interceptors.csv-download 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.edn :as edn] 4 | [clojure.java.io :as io] 5 | [io.pedestal.interceptor :refer [interceptor]] 6 | [io.pedestal.interceptor.chain :as chain] 7 | [lrsql.admin.protocol :as adp] 8 | [lrsql.util.admin :as admin-u] 9 | [lrsql.admin.interceptors.jwt :as jwt] 10 | [com.yetanalytics.lrs.pedestal.interceptor.xapi :as i-xapi] 11 | [com.yetanalytics.lrs-reactions.spec :as rs]) 12 | (:import [javax.servlet ServletOutputStream])) 13 | 14 | ;; See also: `admin.interceptors.account/generate-jwt` 15 | (defn generate-one-time-jwt 16 | [secret exp] 17 | (interceptor 18 | {:name ::convert-jwt 19 | :enter 20 | (fn convert-jwt [ctx] 21 | (let [{lrs :com.yetanalytics/lrs 22 | {:keys [account-id] :as jwt-claim} ::jwt/data} 23 | ctx 24 | {new-jwt :jwt exp :exp one-time-id :oti} 25 | (admin-u/one-time-jwt jwt-claim secret exp)] 26 | (adp/-create-one-time-jwt lrs new-jwt exp one-time-id) 27 | (assoc (chain/terminate ctx) 28 | :response 29 | {:status 200 30 | :body {:account-id account-id 31 | :json-web-token new-jwt}})))})) 32 | 33 | (def validate-property-paths 34 | (interceptor 35 | {:name ::validate-property-paths 36 | :enter 37 | (fn validate-property-paths [ctx] 38 | (let [property-paths (-> ctx 39 | (get-in [:request 40 | :params 41 | :property-paths]) 42 | edn/read-string)] 43 | (if-some [e (s/explain-data (s/every ::rs/path) property-paths)] 44 | (assoc (chain/terminate ctx) 45 | :response 46 | {:status 400 47 | :body {:error (format "Invalid property paths:\n%s" 48 | (-> e s/explain-out with-out-str))}}) 49 | ;; Need to dissoc since lrs.pedestal.interceptor.xapi/params-interceptor 50 | ;; restricts allowed keys in the query param map. 51 | (-> ctx 52 | (update-in [:request :params] dissoc :property-paths) 53 | (update-in [:request :query-params] dissoc :property-paths) 54 | (assoc-in [:request :property-paths] property-paths)))))})) 55 | 56 | (def validate-query-params 57 | (interceptor 58 | (i-xapi/params-interceptor :xapi.statements.GET.request/params))) 59 | 60 | (def csv-response-header 61 | {"Content-Type" "text/csv" 62 | "Content-Disposition" "attachment"}) 63 | 64 | (def download-statement-csv 65 | (interceptor 66 | {:name ::download-statement-csv 67 | :enter 68 | (fn download-statement-csv [ctx] 69 | (let [{lrs :com.yetanalytics/lrs 70 | request :request} 71 | ctx 72 | {:keys [property-paths]} 73 | request 74 | query-params 75 | (get-in ctx [:xapi :xapi.statements.GET.request/params])] 76 | (assoc ctx 77 | :response 78 | {:status 200 79 | :headers csv-response-header 80 | :body (fn [^ServletOutputStream os] 81 | (with-open [writer (io/writer os)] 82 | (adp/-get-statements-csv lrs 83 | writer 84 | property-paths 85 | query-params)))})))})) 86 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/interceptors/lrs_management.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.interceptors.lrs-management 2 | (:require [clojure.spec.alpha :as s] 3 | [io.pedestal.interceptor :refer [interceptor]] 4 | [io.pedestal.interceptor.chain :as chain] 5 | [lrsql.admin.protocol :as adp] 6 | [lrsql.spec.admin :as ads])) 7 | 8 | (def validate-delete-actor-params 9 | (interceptor 10 | {:name ::validate-delete-actor-params 11 | :enter (fn validate-delete-params [ctx] 12 | (let [params (get-in ctx [:request :json-params])] 13 | (if-some [err (s/explain-data 14 | ads/delete-actor-spec 15 | params)] 16 | (assoc (chain/terminate ctx) 17 | :response 18 | {:status 400 19 | :body {:error (format "Invalid parameters:\n%s" 20 | (-> err s/explain-out with-out-str))}}) 21 | (assoc ctx ::data params))))})) 22 | 23 | (def delete-actor 24 | (interceptor 25 | {:name ::delete-actor 26 | :enter (fn delete-actor [ctx] 27 | (let [{lrs :com.yetanalytics/lrs 28 | params ::data} 29 | ctx] 30 | (adp/-delete-actor lrs params) 31 | (assoc ctx 32 | :response {:status 200 33 | :body params})))})) 34 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/interceptors/openapi.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.interceptors.openapi 2 | (:require [com.yetanalytics.gen-openapi.core :as oa-core] 3 | [com.yetanalytics.gen-openapi.generate.schema :as gs] 4 | [com.yetanalytics.lrs.pedestal.openapi :as lrs-oa] 5 | [io.pedestal.interceptor :refer [interceptor]])) 6 | 7 | (defn add-lrsql-specifics [lrs-oa-spec] 8 | (-> lrs-oa-spec 9 | (update :schemas merge {:KeyPair (gs/o {:api-key :t#string 10 | :secret-key :t#string}) 11 | :ScopedKeyPair {:allOf [:r#KeyPair 12 | :r#Scopes]} 13 | :Scopes (gs/o {:scopes (gs/a :t#string)})}) 14 | (update :securitySchemes {:bearerAuth {:type :http 15 | :scheme :bearer 16 | :bearerFormat :JWT}}))) 17 | 18 | (defn openapi [routes version] 19 | (let [m {:status 200 20 | :body (oa-core/make-oa-map 21 | {:openapi "3.0.0" 22 | :info {:title "LRSQL" 23 | :version version} 24 | :externalDocs {:url "https://github.com/yetanalytics/lrsql/blob/main/doc/endpoints.md"} 25 | :components (gs/dsl (add-lrsql-specifics 26 | lrs-oa/components))} 27 | routes)}] 28 | (interceptor 29 | {:name ::openapi 30 | :enter (fn openapi [ctx] 31 | (assoc ctx :response m))}))) 32 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/interceptors/status.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.interceptors.status 2 | (:require [clojure.spec.alpha :as s] 3 | [io.pedestal.interceptor :refer [interceptor]] 4 | [io.pedestal.interceptor.chain :as chain] 5 | [lrsql.admin.protocol :as adp] 6 | [lrsql.spec.admin.status :as adss])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Validation Interceptors 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (def validate-params 13 | "Validate the JSON params for admin status." 14 | (interceptor 15 | {:name ::validate-params 16 | :enter 17 | (fn validate-params [ctx] 18 | (let [params (get-in ctx [:request :query-params] {})] 19 | (if-some [err (s/explain-data adss/get-status-params-spec params)] 20 | ;; Invalid parameters - Bad Request 21 | (assoc (chain/terminate ctx) 22 | :response 23 | {:status 400 24 | :body {:error (format "Invalid parameters:\n%s" 25 | (-> err s/explain-out with-out-str))}}) 26 | ;; Valid params - continue 27 | (assoc ctx ::params params))))})) 28 | 29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | ;; Terminal Interceptors 31 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 32 | 33 | (def get-status 34 | "Return LRS status information for visualization in the UI" 35 | (interceptor 36 | {:name ::get-status 37 | :enter 38 | (fn get-status [ctx] 39 | (let [{lrs :com.yetanalytics/lrs} ctx] 40 | (assoc ctx 41 | :response 42 | {:status 200 43 | :body (adp/-get-status lrs (::params ctx))})))})) 44 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/interceptors/ui.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.interceptors.ui 2 | (:require [ring.util.response :as resp] 3 | [selmer.parser :as selm-parser] 4 | [io.pedestal.interceptor :refer [interceptor]] 5 | [com.yetanalytics.lrs.pedestal.interceptor :as i] 6 | [lrsql.admin.interceptors.oidc :as oidc-i] 7 | [lrsql.init.localization :refer [custom-language-map]])) 8 | 9 | (defn get-spa 10 | "Handler function that returns the index.html file." 11 | [path-prefix] 12 | (fn [_] 13 | (-> (selm-parser/render-file "public/admin/index.html" 14 | {:prefix path-prefix}) 15 | resp/response 16 | (resp/content-type "text/html")))) 17 | 18 | (defn admin-ui-redirect 19 | "Handler function to redirect to the admin UI." 20 | [path-prefix] 21 | (fn [_] 22 | (resp/redirect (str path-prefix "/admin/ui")))) 23 | 24 | (defn get-env 25 | "Provide select config data to client upon request. Takes a map with static 26 | config to inject: 27 | :enable-admin-status - boolean, determines if the admin status endpoint is 28 | enabled." 29 | [{:keys [jwt-refresh-interval 30 | jwt-interaction-window 31 | enable-admin-delete-actor 32 | enable-admin-status 33 | admin-language-code 34 | enable-reactions 35 | no-val? 36 | no-val-logout-url 37 | stmt-get-max 38 | proxy-path 39 | auth-by-cred-id] 40 | :or {enable-admin-delete-actor false 41 | enable-admin-status false 42 | enable-reactions false 43 | no-val? false}}] 44 | (interceptor 45 | {:name ::get-env 46 | :enter 47 | (fn get-env [ctx] 48 | (let [{url-prefix ::i/path-prefix 49 | oidc-env ::oidc-i/admin-env} ctx] 50 | (assoc ctx 51 | :response 52 | {:status 200 53 | :body 54 | (merge 55 | (cond-> {:jwt-refresh-interval jwt-refresh-interval 56 | :jwt-interaction-window jwt-interaction-window 57 | :url-prefix url-prefix 58 | :proxy-path proxy-path 59 | :enable-admin-delete-actor enable-admin-delete-actor 60 | :enable-admin-status enable-admin-status 61 | :enable-reactions enable-reactions 62 | :no-val? no-val? 63 | :admin-language-code admin-language-code 64 | :custom-language (custom-language-map) 65 | :stmt-get-max stmt-get-max 66 | :auth-by-cred-id auth-by-cred-id} 67 | (and no-val? 68 | (not-empty no-val-logout-url)) 69 | (assoc :no-val-logout-url no-val-logout-url)) 70 | oidc-env)})))})) 71 | -------------------------------------------------------------------------------- /src/main/lrsql/admin/protocol.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.admin.protocol) 2 | 3 | (defprotocol AdminAccountManager 4 | (-create-account [this username password] 5 | "Create a new account with `username` and `password`.") 6 | (-get-accounts [this] 7 | "Get all admin user accounts") 8 | (-authenticate-account [this username password] 9 | "Authenticate by looking up if the account exists in the account table.") 10 | (-existing-account? [this account-id] 11 | "Check that the account with the given ID exists in the account table. Returns a boolean.") 12 | (-get-account [this account-id] 13 | "Get the account with the given ID exists in the account table. Returns a boolean.") 14 | (-delete-account [this account-id oidc-enabled?] 15 | "Delete the account and all associated creds. Assumes the account has already been authenticated. Requires OIDC status to prevent sole account deletion.") 16 | (-ensure-account-oidc [this username oidc-issuer] 17 | "Create or verify an existing admin account with the given username and oidc-issuer.") 18 | (-update-admin-password [this account-id old-password new-password] 19 | "Update the password for an admin account given old and new passwords.")) 20 | 21 | (defprotocol AdminJWTManager 22 | (-purge-blocklist [this leeway] 23 | "Purge the blocklist of any JWTs that have expired since they were added.") 24 | (-create-one-time-jwt [this jwt exp one-time-id] 25 | "Add a one-time JWT that will be blocked after it is validated.") 26 | (-block-jwt [this jwt expiration] 27 | "Block `jwt` and apply an associated `expiration` number of seconds. Returns an error if `jwt` is already in the blocklist.") 28 | (-block-one-time-jwt [this jwt one-time-id] 29 | "Similar to `-block-jwt` but specific to blocking one-time JWTs. Returns an error if `jwt` and `one-time-id` cannot be found or updated.") 30 | (-jwt-blocked? [this jwt] 31 | "Is `jwt` on the blocklist?")) 32 | 33 | (defprotocol APIKeyManager 34 | (-create-api-keys [this account-id label scopes] 35 | "Create a new API key pair with the associated scopes.") 36 | (-get-api-keys [this account-id] 37 | "Get all API key pairs associated with the account.") 38 | (-update-api-keys [this account-id api-key secret-key label scopes] 39 | "Update the key pair associated with the account with new scopes.") 40 | (-delete-api-keys [this account-id api-key secret-key] 41 | "Delete the key pair associated with the account.")) 42 | 43 | (defprotocol AdminStatusProvider 44 | (-get-status [this params] 45 | "Get various LRS metrics.")) 46 | 47 | (defprotocol AdminReactionManager 48 | (-create-reaction [this title ruleset active] 49 | "Create a new reaction with the given title, ruleset and status.") 50 | (-get-all-reactions [this] 51 | "Return all reactions with any status.") 52 | (-update-reaction [this reaction-id title ruleset active] 53 | "Update a reaction with a new title, ruleset and/or active status") 54 | (-delete-reaction [this reaction-id] 55 | "Soft-delete a reaction.")) 56 | 57 | (defprotocol AdminLRSManager 58 | (-delete-actor [this params] 59 | "Delete actor by `:actor-id`") 60 | (-get-statements-csv [this writer property-paths params] 61 | "Retrieve statements by CSV. Instead of returning a sequence of 62 | statements, streams them to `writer` as a side effect, in order to 63 | avoid storing them in memory. `property-paths` are defined in the 64 | Reactions API, while `params` are the same query params for 65 | `-get-statements`.")) 66 | -------------------------------------------------------------------------------- /src/main/lrsql/backend/data.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.backend.data 2 | "Namespace for type conversions between SQL and Clojure datatypes 3 | during DB interaction. All public functions extend either 4 | the SettableParameter or ResultColumn protocols from next.jdbc 5 | for reading or writing data, respectively." 6 | (:require [clojure.string :as cstr] 7 | [next.jdbc.date-time :refer [read-as-instant]] 8 | [next.jdbc.prepare :refer [SettableParameter]] 9 | [next.jdbc.result-set :refer [ReadableColumn]] 10 | [lrsql.util :as u]) 11 | (:import [clojure.lang IPersistentMap] 12 | [java.sql PreparedStatement ResultSetMetaData])) 13 | 14 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 15 | ;; Read 16 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 17 | 18 | ;; Instant 19 | 20 | (defn set-read-time->instant! 21 | "Set reading java.sql.Data and java.sql.Timestamp data as Instants." 22 | [] 23 | (read-as-instant)) 24 | 25 | ;; JSON 26 | 27 | (defn- parse-json-payload 28 | [json-labels json-kw-labels label ^"[B" b] 29 | (let [label (cstr/lower-case label)] 30 | (cond 31 | (json-labels label) (u/parse-json b) 32 | (json-kw-labels label) (u/parse-json b :keyword-keys? true) 33 | :else b))) 34 | 35 | (defn set-read-bytes->json! 36 | "Set reading byte arrays as JSON data if the column label is a member of the 37 | set `json-labels`." 38 | [json-labels json-kw-labels] 39 | ;; Note: due to a long-standing bug, the byte array extension needs to come 40 | ;; first: https://clojure.atlassian.net/browse/CLJ-1381#icft=CLJ-1381 41 | ;; 42 | ;; Note: clj-kondo does not recognize the use of a non-simple-symbol in 43 | ;; `extend-protocol` so we add the ignore (make sure to comment it out if 44 | ;; you're editing this code). 45 | #_{:clj-kondo/ignore [:syntax]} 46 | (extend-protocol ReadableColumn 47 | (Class/forName "[B") ; Evaulates to `[B` 48 | (read-column-by-label [^"[B" b ^String label] 49 | (parse-json-payload json-labels json-kw-labels label b)) 50 | (read-column-by-index [^"[B" b ^ResultSetMetaData rsmeta ^long i] 51 | (parse-json-payload json-labels json-kw-labels (.getColumnLabel rsmeta i) b)))) 52 | 53 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 54 | ;; Write 55 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 56 | 57 | (defn set-write-json->bytes! 58 | "Set writing JSON data (i.e. any IPersistentMap instance) as a byte array." 59 | [] 60 | (extend-protocol SettableParameter 61 | IPersistentMap 62 | (set-parameter [^IPersistentMap m ^PreparedStatement s ^long i] 63 | (.setBytes s i (u/write-json m))))) 64 | -------------------------------------------------------------------------------- /src/main/lrsql/init.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init 2 | "Initialize HugSql functions and state." 3 | (:require [clojure.tools.logging :as log] 4 | [hugsql.core :as hugsql] 5 | [hugsql.adapter.next-jdbc :as next-adapter] 6 | [lrsql.backend.protocol :as bp] 7 | [lrsql.input.admin :as admin-input] 8 | [lrsql.input.auth :as auth-input] 9 | [lrsql.ops.command.admin :as admin-cmd] 10 | [lrsql.ops.command.auth :as auth-cmd] 11 | [lrsql.ops.query.admin :as admin-q] 12 | [lrsql.ops.query.auth :as auth-q])) 13 | 14 | (defn init-hugsql-adapter! 15 | "Initialize HugSql to use the next-jdbc adapter." 16 | [] 17 | (hugsql/set-adapter! (next-adapter/hugsql-adapter-next-jdbc))) 18 | 19 | (defn init-backend! 20 | "Init the functionality of `backend`, including IO data conversion and 21 | setting up the DB tables and indexes." 22 | [backend tx] 23 | ;; Init IO data conversion 24 | (bp/-set-read! backend) 25 | (bp/-set-write! backend) 26 | ;; Init DDL 27 | (log/debug "Ensuring Tables...") 28 | (bp/-create-all! backend tx) 29 | (log/debug "Running Migrations...") 30 | (bp/-update-all! backend tx) 31 | (log/debug "SQL backend initialization complete!")) 32 | 33 | (defn insert-default-creds! 34 | "Seed the credential table with the default API key and secret, as well as 35 | the admin account table; the default API key and secret are set by the 36 | environmental variables. The scope of the default credentials would be 37 | hardcoded as \"all\". Does not seed the tables when the username 38 | or password is nil, or if the tables were already seeded." 39 | [backend tx ?username ?password ?api-key ?secret-key] 40 | ;; Seed Admin Account 41 | (when (and ?username ?password) 42 | (let [admin-in (admin-input/insert-admin-input 43 | ?username 44 | ?password)] 45 | ;; Don't insert account or creds if reconnecting to a DB previously 46 | ;; seeded with an account 47 | (when-not (admin-q/query-admin backend tx admin-in) 48 | ;; Insert admin account 49 | (admin-cmd/insert-admin! backend tx admin-in) 50 | ;; Seed Credentials 51 | (when (and ?api-key ?secret-key) 52 | (let [key-pair {:api-key ?api-key 53 | :secret-key ?secret-key} 54 | acc-id (:primary-key admin-in) 55 | cred-in (auth-input/insert-credential-input 56 | acc-id key-pair nil) 57 | scope-in (auth-input/insert-credential-scopes-input 58 | key-pair #{"all"})] 59 | ;; Don't insert creds if reconnecting to a DB previously seeded 60 | ;; with a cred 61 | (when-not (auth-q/query-credential-scopes* backend tx cred-in) 62 | (auth-cmd/insert-credential! backend tx cred-in) 63 | (auth-cmd/insert-credential-scopes! backend tx scope-in))))))) 64 | ;; Migration: deonte seed creds as such if it exists in the DB 65 | (when (and ?api-key ?secret-key) 66 | (let [key-pair {:api-key ?api-key 67 | :secret-key ?secret-key} 68 | seed-in (auth-input/update-credential-is-seed-input 69 | key-pair true)] 70 | (auth-cmd/update-credential-is-seed! backend tx seed-in)))) 71 | -------------------------------------------------------------------------------- /src/main/lrsql/init/clamav.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init.clamav 2 | "ClamAV virus scanning" 3 | (:require [clojure.string :as cs] 4 | [clojure.java.io :as io]) 5 | (:import [xyz.capybara.clamav ClamavClient] 6 | [xyz.capybara.clamav.commands.scan.result ScanResult ScanResult$OK])) 7 | 8 | (defn init-file-scanner 9 | "Given ClamAV config, creates a client and returns a function that reads in 10 | the input and scans it with ClamAV. 11 | Compatible with build-routes' :file-scanner argument" 12 | [{:keys [clamav-host 13 | clamav-port] 14 | :or {clamav-host "localhost" 15 | clamav-port 3310}}] 16 | (let [client (new ClamavClient clamav-host clamav-port)] 17 | (fn [input] 18 | (with-open [in (io/input-stream input)] 19 | (let [^ScanResult scan-result (.scan client in)] 20 | (when-not (instance? ScanResult$OK scan-result) 21 | (let [virus-list (-> scan-result 22 | ;; bean access seems to work, could probably be 23 | ;; improved with a better understanding of 24 | ;; kotlin-java interop 25 | bean 26 | :foundViruses 27 | (get "stream") 28 | (->> (into [])))] 29 | {:message (format "Submitted file failed scan. Found: %s" 30 | (cs/join ", " virus-list))}))))))) 31 | -------------------------------------------------------------------------------- /src/main/lrsql/init/config.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init.config 2 | (:require [cheshire.core :as json] 3 | [clojure.java.io :as io] 4 | [aero.core :as aero] 5 | [clojure.string :refer [split]] 6 | [camel-snake-kebab.core :as csk]) 7 | (:import [java.io File])) 8 | 9 | (defn- merge-user-config 10 | "Given a static config map from aero and a path where a user config json file 11 | might reside, deep merge in valid json." 12 | [{:keys [user-config-json] 13 | :as static-config}] 14 | (let [;; place handle on the config file at path 15 | ^File config-file (io/file user-config-json)] 16 | (merge-with 17 | merge 18 | static-config 19 | ;; merge with user configuration if one is provided 20 | (when (.exists config-file) 21 | (try 22 | (with-open [rdr (io/reader config-file)] 23 | (json/parse-stream-strict rdr csk/->kebab-case-keyword)) 24 | (catch Exception ex 25 | (throw 26 | (ex-info "Invalid JSON in Config File" 27 | {:type ::invalid-user-config-json 28 | :path user-config-json} 29 | ex)))))))) 30 | 31 | ;; The default aero `#include` resolver does not work with JARs, so we 32 | ;; need to resolve the root dirs manually. 33 | 34 | (def config-path-prefix "lrsql/config/") 35 | 36 | (defn- resolver 37 | [_ include] 38 | (io/resource (str config-path-prefix include))) 39 | 40 | (defmethod aero/reader 'array 41 | [_ _ value] 42 | (if (empty? value) 43 | nil 44 | (split value #","))) 45 | 46 | (defn read-config* 47 | "Read `config.edn` with the given value of `profile`. Valid 48 | profiles are `:test-[db-type]` and `:prod-[db-type]`. 49 | Based on the :config-file-json key found will attempt to merge in properties 50 | from the given path, if the file is present." 51 | [profile] 52 | (let [;; Read in and process aero config 53 | {:keys [database 54 | connection 55 | tuning 56 | lrs 57 | webserver 58 | logger]} (-> (str config-path-prefix "config.edn") 59 | io/resource 60 | (aero/read-config 61 | {:profile profile 62 | :resolver resolver}) 63 | merge-user-config)] 64 | ;; form the final config the app will use 65 | {:connection (assoc connection :database database) 66 | :lrs (assoc lrs :stmt-url-prefix (:url-prefix webserver)) 67 | :webserver webserver 68 | :logger logger 69 | :tuning tuning})) 70 | 71 | (def read-config 72 | "Memoized version of `read-config*`." 73 | (memoize read-config*)) 74 | -------------------------------------------------------------------------------- /src/main/lrsql/init/git_data.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init.git-data 2 | (:require [clojure.edn :as edn] 3 | [clojure.java.io :as io] 4 | [clojure.java.shell :refer [sh]] 5 | [clojure.string])) 6 | 7 | (defn read-version [] 8 | (try (edn/read-string (slurp (io/resource "lrsql/config/git-details.edn"))) 9 | (catch Exception _ 10 | (try 11 | (let [hash (:out (sh "git" "rev-parse" "HEAD"))] 12 | (->> hash 13 | (take 7) 14 | (apply str) 15 | (str "DEV-") 16 | clojure.string/trim)) 17 | (catch Exception _ "DEV"))))) 18 | -------------------------------------------------------------------------------- /src/main/lrsql/init/localization.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init.localization 2 | "Utilities for white labeling/localization overrides" 3 | (:require [cheshire.core :as json] 4 | [clojure.java.io :as io])) 5 | 6 | (def lang-path 7 | "lrsql/localization/language.json") 8 | 9 | (defn custom-language-map* 10 | "The language map function to render customized admin frontend language maps" 11 | [] 12 | (-> lang-path 13 | io/resource 14 | slurp 15 | json/parse-string)) 16 | 17 | (def custom-language-map 18 | (memoize custom-language-map*)) 19 | -------------------------------------------------------------------------------- /src/main/lrsql/init/log.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.init.log 2 | (:require [clojure.string :as cs]) 3 | (:import [ch.qos.logback.classic Logger Level] 4 | [org.slf4j LoggerFactory])) 5 | 6 | (defn set-log-level! 7 | "Set logging to the desired level or throw." 8 | [level] 9 | (.setLevel 10 | ^Logger (LoggerFactory/getLogger (Logger/ROOT_LOGGER_NAME)) 11 | (case (cs/upper-case level) 12 | "OFF" Level/OFF 13 | "ERROR" Level/ERROR 14 | "WARN" Level/WARN 15 | "INFO" Level/INFO 16 | "DEBUG" Level/DEBUG 17 | "TRACE" Level/TRACE 18 | "ALL" Level/ALL 19 | (throw (ex-info (format "Unknown LRSQL_LOG_LEVEL: %s" level) 20 | {:type ::unknown-log-level 21 | :level level}))))) 22 | -------------------------------------------------------------------------------- /src/main/lrsql/input/activity.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.activity 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec :as xs] 4 | [com.yetanalytics.lrs.protocol :as lrsp] 5 | [lrsql.spec.common :as c] 6 | [lrsql.spec.activity :as as] 7 | [lrsql.util :as u])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Activity Insertion 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (s/fdef insert-activity-input 14 | :args (s/cat :activity ::xs/activity) 15 | :ret ::as/activity-input) 16 | 17 | (defn insert-activity-input 18 | "Given the xAPI `activity`, construct an entry for the `:activity-inputs` 19 | vec in the `insert-statement!` input param map." 20 | [activity] 21 | {:table :activity 22 | :primary-key (u/generate-squuid) 23 | :activity-iri (get activity "id") 24 | :payload activity}) 25 | 26 | (s/fdef insert-statement-to-activity-input 27 | :args (s/cat :statement-id ::c/statement-id 28 | :activity-usage ::as/usage 29 | :activity-input ::as/activity-input) 30 | :ret ::as/stmt-activity-input) 31 | 32 | (defn insert-statement-to-activity-input 33 | "Given `statement-id`, `activity-usage` (e.g. \"Object\"), and the return 34 | value of `activity-insert-input`, construct an entry for the 35 | `:stmt-activity-inputs` vec in the `insert-statement!` input param map." 36 | [statement-id activity-usage {activity-id :activity-iri}] 37 | {:table :statement-to-activity 38 | :primary-key (u/generate-squuid) 39 | :statement-id statement-id 40 | :usage activity-usage 41 | :activity-iri activity-id}) 42 | 43 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 44 | ;; Activity Query 45 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 46 | 47 | (s/fdef query-activity-input 48 | :args (s/cat :params ::lrsp/get-activity-params) 49 | :ret as/query-activity-spec) 50 | 51 | (defn query-activity-input 52 | "Given activity query params, construct the input param map for 53 | `query-activity`." 54 | [{activity-id :activityId}] 55 | {:activity-iri activity-id}) 56 | -------------------------------------------------------------------------------- /src/main/lrsql/input/actor.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.actor 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec :as xs] 4 | [com.yetanalytics.lrs.protocol :as lrsp] 5 | [lrsql.spec.common :as c] 6 | [lrsql.spec.actor :as as] 7 | [lrsql.util :as u] 8 | [lrsql.util.actor :as au])) 9 | 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | ;; Actor Insertion 12 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 13 | 14 | (s/fdef insert-actor-input 15 | :args (s/cat :actor (s/alt :agent ::xs/agent 16 | :group ::xs/group)) 17 | :ret (s/nilable ::as/actor-input)) 18 | 19 | (defn insert-actor-input 20 | "Given the xAPI `actor`, construct an entry for the `:actor-inputs` vec in 21 | the `insert-statement!` input param map, or `nil` if `actor` does not have 22 | an IFI." 23 | [actor] 24 | (when-some [ifi-str (au/actor->ifi actor)] 25 | {:table :actor 26 | :primary-key (u/generate-squuid) 27 | :actor-ifi ifi-str 28 | :actor-type (get actor "objectType" "Agent") 29 | :payload actor})) 30 | 31 | (s/fdef insert-group-input 32 | :args (s/cat :actor (s/alt :agent ::xs/agent 33 | :group ::xs/group)) 34 | :ret (s/nilable ::as/actor-inputs)) 35 | 36 | (defn insert-group-input 37 | "Given the xAPI `actor`, return a coll of `:actor-inputs` entries for its 38 | member actors, or nil if `actor` is not a Group or has no members. Both 39 | Anonymous and Identified Group members count." 40 | [actor] 41 | ;; Use let-binding in order to avoid cluttering args list 42 | (let [{obj-type "objectType" members "member"} actor] 43 | (when (and (= "Group" obj-type) (not-empty members)) 44 | (map insert-actor-input members)))) 45 | 46 | (s/fdef insert-statement-to-actor-input 47 | :args (s/cat :statement-id ::c/statement-id 48 | :actor-usage ::as/usage 49 | :actor-input ::as/actor-input) 50 | :ret ::as/stmt-actor-input) 51 | 52 | (defn insert-statement-to-actor-input 53 | "Given `statement-id`, `actor-usage` (e.g. \"Actor\") and the return value 54 | of `actor-insert-input`, return the input param map for 55 | `f/insert-statement-to-actor!`." 56 | [statement-id actor-usage {:keys [actor-ifi actor-type]}] 57 | {:table :statement-to-actor 58 | :primary-key (u/generate-squuid) 59 | :statement-id statement-id 60 | :usage actor-usage 61 | :actor-ifi actor-ifi 62 | :actor-type actor-type}) 63 | 64 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 65 | ;; Actor Delete 66 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 67 | 68 | (s/fdef delete-actor-input 69 | :args (s/cat :actor-ifi :lrsql.spec.actor/actor-ifi) 70 | :ret (s/keys :req-un [:lrsql.spec.actor/actor-ifi])) 71 | 72 | (defn delete-actor-input [actor-ifi] 73 | {:actor-ifi actor-ifi}) 74 | 75 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 76 | ;; Actor Query 77 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 78 | 79 | (s/fdef query-agent-input 80 | :args (s/cat :params ::lrsp/get-person-params) 81 | :ret as/query-agent-spec) 82 | 83 | (defn query-agent-input 84 | "Given agent query params, create the input param map for `query-agent`." 85 | [{agent :agent}] 86 | {:actor-ifi (au/actor->ifi agent) 87 | :actor-type "Agent"}) ; Cannot query Groups 88 | -------------------------------------------------------------------------------- /src/main/lrsql/input/admin/jwt.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.admin.jwt 2 | (:require [clojure.spec.alpha :as s] 3 | [lrsql.spec.admin.jwt :as jwts] 4 | [lrsql.util :as u])) 5 | 6 | (defn- eviction-time 7 | "Generate the current time, offset by `exp` number of seconds later." 8 | [exp] 9 | (-> (u/current-time) 10 | (u/offset-time exp :seconds))) 11 | 12 | (defn- current-time 13 | "Generate the current time, offset by `leeway` number of seconds earlier. 14 | 15 | See: `buddy.sign.jwt/validate-claims`" 16 | [leeway] 17 | (-> (u/current-time) 18 | (u/offset-time (* -1 leeway) :seconds))) 19 | 20 | (s/fdef query-blocked-jwt-input 21 | :args (s/cat :jwt ::jwts/jwt) 22 | :ret jwts/query-blocked-jwt-input-spec) 23 | 24 | (defn query-blocked-jwt-input 25 | [jwt] 26 | {:jwt jwt}) 27 | 28 | (s/fdef insert-blocked-jwt-input 29 | :args (s/cat :jwt ::jwts/jwt 30 | :exp ::jwts/exp) 31 | :ret jwts/insert-blocked-jwt-input-spec) 32 | 33 | (defn insert-blocked-jwt-input 34 | [jwt exp] 35 | {:jwt jwt 36 | :eviction-time (eviction-time exp)}) 37 | 38 | (s/fdef purge-blocklist-input 39 | :args (s/cat :leeway ::jwts/leeway) 40 | :ret jwts/purge-blocklist-input-spec) 41 | 42 | (defn purge-blocklist-input 43 | [leeway] 44 | {:current-time (current-time leeway)}) 45 | 46 | ;; One-time JWTs 47 | 48 | (s/fdef insert-one-time-jwt-input 49 | :args (s/cat :jwt ::jwts/jwt 50 | :exp ::jwts/exp 51 | :oti ::jwts/one-time-id) 52 | :ret jwts/insert-one-time-jwt-input-spec) 53 | 54 | (defn insert-one-time-jwt-input 55 | [jwt exp oti] 56 | {:jwt jwt 57 | :eviction-time (eviction-time exp) 58 | :one-time-id oti}) 59 | 60 | (s/fdef update-one-time-jwt-input 61 | :args (s/cat :jwt ::jwts/jwt 62 | :oti ::jwts/one-time-id)) 63 | 64 | (defn update-one-time-jwt-input 65 | [jwt oti] 66 | {:jwt jwt 67 | :one-time-id oti}) 68 | -------------------------------------------------------------------------------- /src/main/lrsql/input/admin/status.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.admin.status 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.squuid :as squuid] 4 | [lrsql.spec.admin.status :as stat] 5 | [lrsql.util :as u])) 6 | 7 | (s/fdef query-timeline-input 8 | :args (s/cat :params stat/get-status-params-spec) 9 | :ret stat/query-timeline-input-spec) 10 | 11 | (def unit-for 12 | "Mapping of timeline bucket time unit to suitable FOR sql substring arg." 13 | {"year" 4 14 | "month" 7 15 | "day" 10 16 | "hour" 13 17 | "minute" 16 18 | "second" 19}) 19 | 20 | (defn query-timeline-input 21 | "Transform parameters for timeline into values suitable for query." 22 | [{:keys [timeline-unit 23 | timeline-since 24 | timeline-until] 25 | :or {timeline-unit "day"}}] 26 | (let [since' (-> (or (some-> timeline-since u/str->time) 27 | (u/offset-time (u/current-time) -14 :days)) 28 | ;; acount for random bits that push the edge of the range 29 | ;; TODO: update if arg is added to squuid/time->uuid 30 | (u/offset-time -1 :millis)) 31 | until' (or (some-> timeline-until u/str->time) 32 | (u/current-time))] 33 | {:unit-for (get unit-for timeline-unit) 34 | :since-id (squuid/time->uuid since') 35 | :until-id (squuid/time->uuid until')})) 36 | 37 | (s/fdef query-status-input 38 | :args (s/cat :params stat/get-status-params-spec) 39 | :ret stat/query-status-input-spec) 40 | 41 | (defn query-status-input 42 | "Transform status query params into input suitable for query" 43 | [{:keys [include] 44 | :or {include ["statement-count" 45 | "actor-count" 46 | "platform-frequency" 47 | "last-statement-stored" 48 | "timeline"]} 49 | :as params}] 50 | (let [include' (if (coll? include) 51 | (set include) 52 | #{include})] 53 | (cond-> {:include include'} 54 | (include' "timeline") (assoc :timeline (query-timeline-input params))))) 55 | -------------------------------------------------------------------------------- /src/main/lrsql/input/attachment.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.attachment 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.lrs.xapi.statements :as ss] 4 | [lrsql.spec.common :as c] 5 | [lrsql.spec.attachment :as as] 6 | [lrsql.util :as u])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Attachment Insertion 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (s/fdef insert-attachment-input 13 | :args (s/cat :statement-id ::c/statement-id 14 | :attachment ::ss/attachment) 15 | :ret ::as/attachment-input) 16 | 17 | (defn insert-attachment-input 18 | "Given `statement-id` and `attachment`, construct an entry in the 19 | `:stmt-stmt-inputs` vec in the `insert-statement!` input param map. 20 | `statement-id` will serve as a foreign key reference in the DB." 21 | [statement-id attachment] 22 | (let [{contents :content 23 | content-type :contentType 24 | length :length 25 | sha2 :sha2} 26 | attachment] 27 | {:table :attachment 28 | :primary-key (u/generate-squuid) 29 | :statement-id statement-id 30 | :attachment-sha sha2 31 | :content-type content-type 32 | :content-length length 33 | :contents (u/data->bytes contents)})) 34 | -------------------------------------------------------------------------------- /src/main/lrsql/input/reaction.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.input.reaction 2 | (:require [clojure.spec.alpha :as s] 3 | [lrsql.spec.reaction :as rs] 4 | [lrsql.util :as u])) 5 | 6 | (s/fdef insert-reaction-input 7 | :args (s/cat :title ::rs/title :ruleset ::rs/ruleset :active ::rs/active) 8 | :ret rs/insert-reaction-input-spec) 9 | 10 | (defn insert-reaction-input 11 | "Given `ruleset` and `active`, construct the input map for `insert-reaction!`." 12 | [title ruleset active] 13 | (let [{squuid :squuid 14 | squuid-ts :timestamp} (u/generate-squuid*)] 15 | {:primary-key squuid 16 | :title title 17 | :ruleset ruleset 18 | :active active 19 | :created squuid-ts 20 | :modified squuid-ts})) 21 | 22 | (s/fdef update-reaction-input 23 | :args (s/cat :reaction-id ::rs/reaction-id 24 | :title (s/nilable ::rs/title) 25 | :ruleset (s/nilable ::rs/ruleset) 26 | :active (s/nilable ::rs/active)) 27 | :ret rs/update-reaction-input-spec) 28 | 29 | (defn update-reaction-input 30 | "Given `reaction-id`, `ruleset` and `active`, construct the input map for 31 | `update-reaction!`." 32 | [reaction-id title ruleset active] 33 | (merge 34 | {:reaction-id reaction-id 35 | :modified (u/current-time)} 36 | (when title 37 | {:title title}) 38 | (when ruleset 39 | {:ruleset ruleset}) 40 | (when (some? active) 41 | {:active active}))) 42 | 43 | (s/fdef delete-reaction-input 44 | :args (s/cat :reaction-id ::rs/reaction-id) 45 | :ret rs/delete-reaction-input-spec) 46 | 47 | (defn delete-reaction-input 48 | "Given `reaction-id`, construct the input map for `delete-reaction!`." 49 | [reaction-id] 50 | {:reaction-id reaction-id 51 | :modified (u/current-time)}) 52 | 53 | (s/fdef error-reaction-input 54 | :args (s/cat :reaction-id ::rs/reaction-id 55 | :error ::rs/error) 56 | :ret rs/error-reaction-input-spec) 57 | 58 | (defn error-reaction-input 59 | "Given `reaction-id` and `error`, construct the input map for `error-reaction!`" 60 | [reaction-id error] 61 | {:reaction-id reaction-id 62 | :modified (u/current-time) 63 | :error error}) 64 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/command/auth.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.command.auth 2 | (:require [clojure.spec.alpha :as s] 3 | [lrsql.backend.protocol :as bp] 4 | [lrsql.spec.common :refer [transaction?]] 5 | [lrsql.spec.auth :as as])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Credential Insertion 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | ;; TODO: Dupe checking 12 | 13 | (s/fdef insert-credential! 14 | :args (s/cat :bk as/credential-backend? 15 | :tx transaction? 16 | :input as/insert-cred-input-spec) 17 | :ret nil?) 18 | 19 | (defn insert-credential! 20 | "Insert the credential keys in `input` into the DB. Returns `nil`." 21 | [bk tx input] 22 | (bp/-insert-credential! bk tx input) 23 | nil) 24 | 25 | (s/fdef insert-credential-scopes! 26 | :args (s/cat :bk as/credential-backend? 27 | :tx transaction? 28 | :inputs as/insert-cred-scopes-input-spec) 29 | :ret nil?) 30 | 31 | (defn insert-credential-scopes! 32 | "Insert `input`, a seq of maps where each API key pair is associated 33 | with a different scope." 34 | [bk tx input] 35 | (run! (partial bp/-insert-credential-scope! bk tx) input)) 36 | 37 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 38 | ;; Credential Deletion 39 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 40 | 41 | ;; TODO: Existence checking 42 | 43 | (s/fdef delete-credential-scopes! 44 | :args (s/cat :bk as/credential-backend? 45 | :tx transaction? 46 | :inputs as/delete-cred-scopes-input-spec) 47 | :ret nil?) 48 | 49 | (defn delete-credential-scopes! 50 | "Delete the scopes associated with the credential in the `input` seq 51 | Returns `nil`." 52 | [bk tx input] 53 | (run! (partial bp/-delete-credential-scope! bk tx) input)) 54 | 55 | (s/fdef delete-credential! 56 | :args (s/cat :bk as/credential-backend? 57 | :tx transaction? 58 | :input as/delete-cred-input-spec) 59 | :ret nil?) 60 | 61 | (defn delete-credential! 62 | "Delete the credential and all of its scopes associated with the key pair 63 | in `input`. Returns `nil`." 64 | [bk tx input] 65 | (bp/-delete-credential! bk tx input) 66 | nil) 67 | 68 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 69 | ;; Credential Label Update 70 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 71 | 72 | (s/fdef update-credential-label! 73 | :args (s/cat :bk as/credential-backend? 74 | :tx transaction? 75 | :input as/update-cred-label-input-spec) 76 | :ret nil?) 77 | 78 | (defn update-credential-label! 79 | "Update the credential label. Returns `nil`." 80 | [bk tx input] 81 | (bp/-update-credential-label! bk tx input) 82 | nil) 83 | 84 | (s/fdef update-credential-is-seed! 85 | :args (s/cat :bk as/credential-backend? 86 | :tx transaction? 87 | :input as/update-cred-is-seed-input-spec) 88 | :ret nil?) 89 | 90 | (defn update-credential-is-seed! 91 | "Update the `is_seed` flag for the credential, i.e. is the credential a 92 | seed credential? Returns `nil`." 93 | [bk tx input] 94 | (bp/-update-credential-is-seed! bk tx input) 95 | nil) 96 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/command/reaction.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.command.reaction 2 | (:require [clojure.spec.alpha :as s] 3 | [lrsql.backend.protocol :as bp] 4 | [lrsql.spec.reaction :as rs] 5 | [lrsql.spec.common :refer [transaction?]])) 6 | 7 | (s/fdef insert-reaction! 8 | :args (s/cat :bk rs/reaction-backend? 9 | :tx transaction? 10 | :input rs/insert-reaction-input-spec) 11 | :ret rs/insert-reaction-ret-spec) 12 | 13 | (defn insert-reaction! 14 | "Insert a new reaction." 15 | [bk tx {:keys [primary-key] :as input}] 16 | {:result 17 | (if (= (bp/-insert-reaction! bk tx input) 18 | :lrsql.reaction/title-conflict-error) 19 | :lrsql.reaction/title-conflict-error 20 | primary-key)}) 21 | 22 | (s/fdef update-reaction! 23 | :args (s/cat :bk rs/reaction-backend? 24 | :tx transaction? 25 | :input rs/update-reaction-input-spec) 26 | :ret rs/update-reaction-ret-spec) 27 | 28 | (defn update-reaction! 29 | "Update an existing reaction." 30 | [bk tx {:keys [reaction-id] :as input}] 31 | (let [result (bp/-update-reaction! bk tx input)] 32 | {:result (cond 33 | (= 1 result) reaction-id 34 | (= 0 result) :lrsql.reaction/reaction-not-found-error 35 | (= :lrsql.reaction/title-conflict-error result) 36 | :lrsql.reaction/title-conflict-error)})) 37 | 38 | (s/fdef delete-reaction! 39 | :args (s/cat :bk rs/reaction-backend? 40 | :tx transaction? 41 | :input rs/delete-reaction-input-spec) 42 | :ret rs/delete-reaction-ret-spec) 43 | 44 | (defn delete-reaction! 45 | "Delete a reaction." 46 | [bk tx {:keys [reaction-id] :as input}] 47 | (let [result (bp/-delete-reaction! bk tx input)] 48 | {:result (if (= 1 result) 49 | reaction-id 50 | :lrsql.reaction/reaction-not-found-error)})) 51 | 52 | (s/fdef error-reaction! 53 | :args (s/cat :bk rs/reaction-backend? 54 | :tx transaction? 55 | :input rs/error-reaction-input-spec) 56 | :ret rs/error-reaction-ret-spec) 57 | 58 | (defn error-reaction! 59 | "Set error on and deactivate a reaction." 60 | [bk tx {:keys [reaction-id] :as input}] 61 | (let [result (bp/-error-reaction! bk tx input)] 62 | {:result (if (= 1 result) 63 | reaction-id 64 | :lrsql.reaction/reaction-not-found-error)})) 65 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/query/activity.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.query.activity 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.lrs.protocol :as lrsp] 4 | [lrsql.backend.protocol :as bp] 5 | [lrsql.spec.common :refer [transaction?]] 6 | [lrsql.spec.activity :as as])) 7 | 8 | (s/fdef query-activity 9 | :args (s/cat :bk as/activity-backend? 10 | :tx transaction? 11 | :input as/query-activity-spec) 12 | :ret ::lrsp/get-activity-ret) 13 | 14 | (defn query-activity 15 | "Query an Activity from the DB. Returns a map between `:activity` and the 16 | activity found, which is `nil` if not found." 17 | [bk tx input] 18 | (let [{activity :payload} (bp/-query-activity bk tx input)] 19 | {:activity activity})) 20 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/query/actor.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.query.actor 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.lrs.protocol :as lrsp] 4 | [lrsql.backend.protocol :as bp] 5 | [lrsql.spec.common :refer [transaction?]] 6 | [lrsql.spec.actor :as as] 7 | [lrsql.util.actor :as au])) 8 | 9 | (s/fdef query-agent 10 | :args (s/cat :bk as/actor-backend? 11 | :tx transaction? 12 | :input as/query-agent-spec) 13 | :ret ::lrsp/get-person-ret) 14 | 15 | (defn query-agent 16 | "Query an Agent from the DB. Returns a map between `:person` and the 17 | resulting Person object. Throws an exception if not found. Does not 18 | query Groups." 19 | [bk tx input] 20 | ;; If agent is not found, return the original input 21 | (let [agent (if-some [{result :payload} (bp/-query-actor bk tx input)] 22 | result 23 | (:payload input))] 24 | {:person (->> agent au/actor->person)})) 25 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/query/document.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.query.document 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.lrs.protocol :as lrsp] 4 | [lrsql.backend.protocol :as bp] 5 | [lrsql.spec.common :refer [transaction?]] 6 | [lrsql.spec.document :as ds] 7 | [lrsql.util :as u] 8 | [lrsql.ops.util :refer [throw-invalid-table-ex]])) 9 | 10 | (s/fdef query-document 11 | :args (s/cat :bk ds/document-backend? 12 | :tx transaction? 13 | :input ds/document-input-spec) 14 | :ret ::lrsp/get-document-ret) 15 | 16 | (defn query-document 17 | "Query a single document from the DB. Returns a map where the value of 18 | `:document` is either a map with the contents as a byte array, or `nil` 19 | if not found." 20 | [bk tx {:keys [table] :as input}] 21 | (if-some [res (case table 22 | :state-document 23 | (bp/-query-state-document bk tx input) 24 | :agent-profile-document 25 | (bp/-query-agent-profile-document bk tx input) 26 | :activity-profile-document 27 | (bp/-query-activity-profile-document bk tx input) 28 | ;; Else 29 | (throw-invalid-table-ex "query-document" input))] 30 | (let [{contents :contents 31 | content-type :content_type 32 | content-len :content_length 33 | state-id :state_id 34 | profile-id :profile_id 35 | updated :last_modified} 36 | res] 37 | {:document 38 | {:contents contents 39 | :content-length content-len 40 | :content-type content-type 41 | :id (or state-id profile-id) 42 | :updated (u/time->str updated)}}) 43 | ;; Not found 44 | {:document nil})) 45 | 46 | (s/fdef query-document-ids 47 | :args (s/cat :bk ds/document-backend? 48 | :tx transaction? 49 | :input ds/document-ids-query-spec) 50 | :ret ::lrsp/get-document-ids-ret) 51 | 52 | ;; TODO: The LRS should also return last modified bko. 53 | ;; However, this is not supported in Milt's LRS spec. 54 | (defn query-document-ids 55 | "Query multiple document IDs from the DB. Returns a map containing the 56 | vector of IDs." 57 | [bk tx {:keys [table] :as input}] 58 | (let [ids (case table 59 | :state-document 60 | (->> input 61 | (bp/-query-state-document-ids bk tx) 62 | (map :state_id)) 63 | :agent-profile-document 64 | (->> input 65 | (bp/-query-agent-profile-document-ids bk tx) 66 | (map :profile_id)) 67 | :activity-profile-document 68 | (->> input 69 | (bp/-query-activity-profile-document-ids bk tx) 70 | (map :profile_id)) 71 | ;; Else 72 | (throw-invalid-table-ex "query-document-ids" input))] 73 | {:document-ids (vec ids)})) 74 | -------------------------------------------------------------------------------- /src/main/lrsql/ops/util.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.ops.util) 2 | 3 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 4 | ;; Macros 5 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 6 | 7 | (defmacro throw-invalid-table-ex 8 | "Throw an exception with the following error data: 9 | :type ::invalid-table 10 | :table 11 | :input `input` 12 | :fn-name `fn-name`" 13 | [fn-name input] 14 | (let [table-kw# (:table input) 15 | table-name# (when table-kw# (name table-kw#))] 16 | `(throw 17 | (ex-info (format "`%s` is not supported for table type `%s`" 18 | ~fn-name 19 | ~table-name#) 20 | {:type ::invalid-table 21 | :table ~table-kw# 22 | :input ~input 23 | :fn-name ~fn-name})))) 24 | -------------------------------------------------------------------------------- /src/main/lrsql/reaction/protocol.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.reaction.protocol) 2 | 3 | (defprotocol StatementReactor 4 | (-react-to-statement [this statement-id] 5 | "Given a trigger statement ID, check active reactions and possibly issue reaction statements to the LRS or add reaction errors.")) 6 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/activity.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.activity 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec :as xs] 4 | [lrsql.backend.protocol :as bp] 5 | [lrsql.spec.common :as c])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Interface 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (defn activity-backend? 12 | [bk] 13 | (satisfies? bp/ActivityBackend bk)) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Axioms 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (s/def ::activity-iri :activity/id) 20 | 21 | (s/def ::usage 22 | #{"Object", "Category", "Grouping", "Parent", "Other" 23 | "SubObject" "SubCategory" "SubGrouping" "SubParent" "SubOther"}) 24 | 25 | ;; JSON string version: (make-str-spec ::xs/activity u/parse-json u/write-json) 26 | (s/def ::payload ::xs/activity) 27 | 28 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 29 | ;; Insertion spec 30 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 31 | 32 | ;; Activity 33 | ;; - id: SEQUENTIAL UUID NOT NULL PRIMARY KEY 34 | ;; - activity_iri: STRING NOT NULL UNIQUE KEY 35 | ;; - payload: JSON NOT NULL 36 | 37 | (s/def ::activity-input 38 | (s/keys :req-un [::c/primary-key 39 | ::activity-iri 40 | ::payload])) 41 | 42 | (s/def ::activity-inputs 43 | (s/coll-of ::activity-input :gen-max 5)) 44 | 45 | ;; Statement-to-Activity 46 | ;; - id: SEQUENTIAL UUID NOT NULL PRIMARY KEY 47 | ;; - statement_id: UUID NOT NULL FOREIGN KEY 48 | ;; - usage: ENUM ('Object', 'Category', 'Grouping', 'Parent', 'Other', 49 | ;; 'SubObject', 'SubCategory', 'SubGrouping', 'SubParent', 'SubOther') 50 | ;; NOT NULL 51 | ;; - activity_iri: STRING NOT NULL FOREIGN KEY 52 | 53 | (s/def ::stmt-activity-input 54 | (s/keys :req-un [::c/primary-key 55 | ::c/statement-id 56 | ::usage 57 | ::activity-iri])) 58 | 59 | (s/def ::stmt-activity-inputs 60 | (s/coll-of ::stmt-activity-input :gen-max 5)) 61 | 62 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 63 | ;; Query spec 64 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 65 | 66 | (def query-activity-spec 67 | (s/keys :req-un [::activity-iri])) 68 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/admin/jwt.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.admin.jwt 2 | (:require [clojure.spec.alpha :as s] 3 | [lrsql.backend.protocol :as bp] 4 | [lrsql.spec.common :as c] 5 | [lrsql.spec.config :as config])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Interface 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (defn admin-jwt-backend? 12 | [bk] 13 | (satisfies? bp/JWTBlocklistBackend bk)) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Inputs 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (s/def ::jwt string?) 20 | (s/def ::exp ::config/jwt-exp-time) 21 | (s/def ::leeway ::config/jwt-exp-leeway) 22 | (s/def ::eviction-time c/instant-spec) 23 | (s/def ::current-time c/instant-spec) 24 | (s/def ::one-time-id uuid?) 25 | 26 | ;; Blocked JWTs 27 | 28 | (def query-blocked-jwt-input-spec 29 | (s/keys :req-un [::jwt])) 30 | 31 | (def insert-blocked-jwt-input-spec 32 | (s/keys :req-un [::jwt ::eviction-time])) 33 | 34 | (def purge-blocklist-input-spec 35 | (s/keys :req-un [::current-time])) 36 | 37 | ;; One-time JWTs 38 | 39 | (def query-one-time-jwt-input-spec 40 | (s/keys :req-un [::jwt ::one-time-id])) 41 | 42 | (def insert-one-time-jwt-input-spec 43 | (s/keys :req-un [::jwt ::eviction-time ::one-time-id])) 44 | 45 | (def update-one-time-jwt-input-spec 46 | (s/keys :req-un [::jwt ::one-time-id])) 47 | 48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 49 | ;; Results 50 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 51 | 52 | (s/def :lrsql.spec.admin.jwt.command/result ::jwt) 53 | 54 | (def blocked-jwt-op-result-spec 55 | (s/keys :req-un [:lrsql.spec.admin.jwt.command/result])) 56 | 57 | (def blocked-jwt-query-result-spec 58 | boolean?) 59 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/attachment.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.attachment 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec] 4 | [lrsql.backend.protocol :as bp] 5 | [lrsql.spec.common :as c])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Interface 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (defn attachment-backend? 12 | [bk] 13 | (satisfies? bp/AttachmentBackend bk)) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Axioms 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (s/def ::attachment-sha :attachment/sha2) 20 | (s/def ::attachment-shas 21 | (s/coll-of ::attachment-sha :kind set? :gen-max 5)) 22 | 23 | (s/def ::content-type :attachment/contentType) 24 | (s/def ::content-length :attachment/length) 25 | (s/def ::contents bytes?) 26 | 27 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 28 | ;; Insertion spec 29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | 31 | ;; Attachment 32 | ;; - id: SEQUENTIAL UUID NOT NULL PRIMARY KEY 33 | ;; - statement_key: UUID NOT NULL FOREIGN KEY 34 | ;; - attachment_sha: STRING NOT NULL 35 | ;; - content_type: STRING NOT NULL 36 | ;; - content_length: INTEGER NOT NULL 37 | ;; - contents: BINARY NOT NULL 38 | 39 | (s/def ::attachment-input 40 | (s/keys :req-un [::c/primary-key 41 | ::c/statement-id 42 | ::attachment-sha 43 | ::content-type 44 | ::content-length 45 | ::contents])) 46 | 47 | (s/def ::attachment-inputs 48 | (s/coll-of ::attachment-input :gen-max 5)) 49 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/authority.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.authority 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec :as xs] 4 | [lrsql.spec.common :as c] 5 | [lrsql.spec.actor :as hs-actor])) 6 | 7 | (s/def ::authority-url ::xs/irl) 8 | (s/def ::cred-id ::c/primary-key) 9 | (s/def ::account-id ::c/primary-key) 10 | 11 | (s/def ::context-map 12 | (s/keys :req-un [::authority-url 13 | ::cred-id 14 | ::account-id])) 15 | 16 | (s/def ::authority-fn 17 | (s/fspec 18 | :args (s/cat :context-map ::context-map) 19 | :ret ::xs/agent)) 20 | 21 | (def query-authority-spec 22 | :statement/authority) 23 | 24 | (s/def ::authority-ifis 25 | (s/coll-of ::hs-actor/actor-ifi :min-count 1)) 26 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/common.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.common 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen] 4 | [next.jdbc.protocols :as jp] 5 | [clojure.core.async.impl.protocols :as ap]) 6 | (:import [java.time Instant])) 7 | 8 | ;; UUIDs 9 | 10 | (s/def ::primary-key uuid?) 11 | (s/def ::statement-id uuid?) 12 | 13 | ;; Transactions 14 | 15 | (defn transaction? 16 | [tx] 17 | (satisfies? jp/Transactable tx)) 18 | 19 | ;; Exceptions 20 | 21 | (defn exception? 22 | [ex] 23 | (instance? Exception ex)) 24 | 25 | (s/def ::error exception?) 26 | 27 | ;; Instants 28 | 29 | ;; Need to use this since `inst?` also returns true for java.util.Date 30 | ;; instances, not just java.time.Instant ones. 31 | 32 | (def instant-spec 33 | (s/with-gen 34 | (partial instance? Instant) 35 | (fn [] 36 | (sgen/fmap 37 | #(Instant/ofEpochSecond %) 38 | (sgen/large-integer* {:min 0}))))) 39 | 40 | ;; Core.async channels 41 | (s/def ::channel #(satisfies? ap/Channel %)) 42 | 43 | ;; JSON 44 | 45 | ;; Like :xapi-schema.spec/any-json BUT allows simple keyword keys. 46 | (s/def ::any-json 47 | (s/nilable 48 | (s/or :scalar 49 | (s/or :string 50 | string? 51 | :number 52 | (s/or :double 53 | (s/double-in :infinite? false :NaN? false) 54 | :int 55 | int?) 56 | :boolean 57 | boolean?) 58 | :coll 59 | (s/or :map 60 | (s/map-of 61 | (s/or :string string? 62 | :keyword simple-keyword?) 63 | ::any-json 64 | :gen-max 4) 65 | :vector 66 | (s/coll-of 67 | ::any-json 68 | :kind vector? 69 | :into [] 70 | :gen-max 4))))) 71 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/oidc.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.oidc 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen])) 4 | 5 | ;; Claims https://www.iana.org/assignments/jwt/jwt.xhtml 6 | 7 | (s/def ::scope 8 | (s/with-gen string? 9 | (fn [] 10 | (sgen/return 11 | "openid all")))) 12 | 13 | (s/def ::iss string?) 14 | (s/def ::sub string?) 15 | (s/def ::aud 16 | (s/or :scalar string? 17 | :array (s/every string? :min-count 1))) 18 | 19 | (s/def ::azp string?) 20 | (s/def ::client_id string?) 21 | 22 | (s/def ::claims 23 | (s/with-gen 24 | (s/keys :req-un [::scope 25 | ::iss 26 | ::sub 27 | ::aud] 28 | :opt-un [::azp 29 | ::client_id]) 30 | (fn [] 31 | (sgen/return {:scope "openid all" 32 | :iss "http://example.com/realm" 33 | :aud "someapp" 34 | :sub "1234"})))) 35 | 36 | ;; This is a special namespaced keyword with a single segment (so we don't need 37 | ;; to escape the dots) for inclusion in the OIDC authority template 38 | ;; https://github.com/yogthos/Selmer#namespaced-keys 39 | (s/def :lrsql/resolved-client-id string?) 40 | 41 | (s/def ::authority-claims 42 | (s/merge 43 | (s/keys :req [:lrsql/resolved-client-id]) 44 | ::claims)) 45 | -------------------------------------------------------------------------------- /src/main/lrsql/spec/util.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.spec.util 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen])) 4 | 5 | (defn remove-nil-vals 6 | "Remove nil values from `m`. Useful when values could either be nil or 7 | not present in `m`." 8 | [m] 9 | (into {} (filter #(-> % second some?) m))) 10 | 11 | (defn remove-neg-vals 12 | "Remove negative integers from `m`. Useful when all integer values 13 | in the map should be natural integers (i.e. 0 or more)." 14 | [m] 15 | (into {} (filter #(-> % second neg-int? not) m))) 16 | 17 | (defn make-str-spec 18 | "Make a spec w/ gen capability for strings of a particular format." 19 | [spec conform-fn unform-fn] 20 | (s/with-gen 21 | (s/and string? 22 | (s/conformer conform-fn unform-fn) 23 | spec) 24 | #(sgen/fmap unform-fn 25 | (s/gen spec)))) 26 | -------------------------------------------------------------------------------- /src/main/lrsql/system.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.system 2 | (:require [com.stuartsierra.component :as component] 3 | [lrsql.system.database :as db] 4 | [lrsql.system.logger :as logger] 5 | [lrsql.system.tuning :as tuning] 6 | [lrsql.system.lrs :as lrs] 7 | [lrsql.system.webserver :as webserver] 8 | [lrsql.system.reactor :as reactor] 9 | [lrsql.init.config :refer [read-config]])) 10 | 11 | (defn system 12 | "Return a lrsql system with configuration specified by the `profile` 13 | keyword. Optional kwarg ``conf-overrides`` takes a map where the key is a vec 14 | of a key location in config map, and the value is an override." 15 | [backend profile & {:keys [conf-overrides]}] 16 | (let [profile-config 17 | (read-config profile) 18 | config (reduce (fn [agg [key val]] 19 | (assoc-in agg key val)) 20 | profile-config 21 | (seq conf-overrides)) 22 | initial-sys ; init without configuration 23 | (component/system-map 24 | ;; Logger is required by backend so it happens first 25 | :logger (logger/map->Logger {}) 26 | :tuning (tuning/map->Tuning {}) 27 | :backend (component/using 28 | backend 29 | [:logger :tuning]) 30 | :connection (component/using 31 | (db/map->Connection {}) 32 | [:backend]) 33 | :lrs (component/using 34 | (lrs/map->LearningRecordStore {}) 35 | [:connection :backend]) 36 | :reactor (component/using 37 | (reactor/map->Reactor {}) 38 | [:backend :lrs]) 39 | :webserver (component/using 40 | (webserver/map->Webserver {}) 41 | [:lrs :reactor])) 42 | assoc-config 43 | (fn [m config-m] (assoc m :config config-m))] 44 | ;; This code can be confusing. What is happening is that the above creates 45 | ;; a system map with empty maps and then based on the key of the system, 46 | ;; populates the corresponding config (by key) from the overall aero config 47 | (-> (merge-with assoc-config initial-sys config) 48 | (component/system-using {})))) 49 | -------------------------------------------------------------------------------- /src/main/lrsql/system/logger.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.system.logger 2 | (:require [com.stuartsierra.component :as component] 3 | [lrsql.init.log :refer [set-log-level!]])) 4 | 5 | (defrecord Logger [config] 6 | component/Lifecycle 7 | (start [this] 8 | (when-some [level (:log-level config)] 9 | (set-log-level! level)) 10 | this) 11 | (stop [this] this)) 12 | -------------------------------------------------------------------------------- /src/main/lrsql/system/reactor.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.system.reactor 2 | (:require [com.stuartsierra.component :as component] 3 | [next.jdbc :as jdbc] 4 | [com.yetanalytics.lrs.protocol :as lrsp] 5 | [lrsql.reaction.protocol :as rp] 6 | [lrsql.init.reaction :as react-init] 7 | [lrsql.input.reaction :as react-input] 8 | [lrsql.ops.command.reaction :as react-cmd] 9 | [lrsql.ops.query.reaction :as react-q])) 10 | 11 | (defrecord Reactor [backend 12 | lrs 13 | reaction-executor] 14 | component/Lifecycle 15 | (start [this] 16 | (assoc this 17 | :reaction-executor 18 | (react-init/reaction-executor 19 | (:reaction-channel lrs) 20 | this))) 21 | (stop [this] 22 | (react-init/shutdown-reactions! 23 | (:reaction-channel lrs) 24 | reaction-executor) 25 | (assoc this 26 | :backend nil 27 | :lrs nil 28 | :reaction-executor nil)) 29 | rp/StatementReactor 30 | (-react-to-statement [_ statement-id] 31 | (let [conn (-> lrs 32 | :connection 33 | :conn-pool) 34 | statement-results 35 | (jdbc/with-transaction [tx conn] 36 | (reduce 37 | (fn [acc {:keys [reaction-id 38 | error] 39 | :as result}] 40 | (if error 41 | (let [input (react-input/error-reaction-input 42 | reaction-id error)] 43 | (react-cmd/error-reaction! backend tx input) 44 | acc) 45 | (conj acc (select-keys result [:statement :authority])))) 46 | [] 47 | (:result 48 | (react-q/query-statement-reactions 49 | backend tx {:trigger-id statement-id}))))] 50 | ;; Submit statements one at a time with varying authority 51 | {:statement-ids 52 | (reduce 53 | (fn [acc {:keys [statement authority]}] 54 | (into acc 55 | (:statement-ids 56 | (lrsp/-store-statements 57 | lrs 58 | {:agent authority 59 | :scopes #{:scope/statements.write}} 60 | [statement] 61 | [])))) 62 | [] 63 | statement-results)}))) 64 | -------------------------------------------------------------------------------- /src/main/lrsql/system/tuning.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.system.tuning 2 | (:require [com.stuartsierra.component :as component])) 3 | 4 | (defrecord Tuning [config] 5 | component/Lifecycle 6 | (start [this] 7 | this) 8 | (stop [this] this)) 9 | -------------------------------------------------------------------------------- /src/main/lrsql/system/util.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.system.util 2 | (:require [com.stuartsierra.component :as component] 3 | [clojure.spec.alpha :as s] 4 | [clojure.walk :as w] 5 | [clojure.tools.logging :as log] 6 | [next.jdbc.connection :as jdbc-conn] 7 | [lrsql.util :refer [form-encode form-decode]])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Helpers and Macros 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (def redactable-config-vars 14 | #{:db-password 15 | :admin-pass-default 16 | :api-secret-default 17 | :key-password}) 18 | 19 | (defn- redact-at-node 20 | [node] 21 | (if (map-entry? node) 22 | (let [[k v] node] 23 | (if (contains? redactable-config-vars k) 24 | ;; We only consider strings and similar because 25 | ;; other vals should immediately cause an assertion 26 | ;; error and never be meaningful passwords. 27 | (cond 28 | (string? v) [k "[REDACTED]"] 29 | (keyword? v) [k :redacted] 30 | (symbol? v) [k 'redacted] 31 | :else [k v]) 32 | [k v])) 33 | node)) 34 | 35 | (defn redact-config-vars 36 | "Given a `config` map, replace redactable values with \"[REDACTED]\" 37 | (if it is a string; keywords and symbols are treated similarly). 38 | See `redactable-config-vars` for which vars are redactable." 39 | [config] 40 | (w/postwalk redact-at-node config)) 41 | 42 | (defmacro assert-config 43 | [spec component-name config] 44 | `(when-some [err# (s/explain-data ~spec ~(redact-config-vars config))] 45 | (do 46 | (log/errorf "Configuration errors:\n%s" 47 | (with-out-str (s/explain-out err#))) 48 | (throw (ex-info (format "Invalid %s configuration!" 49 | ~component-name) 50 | {:type ::invalid-config 51 | :error-data err#}))))) 52 | 53 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 54 | ;; JDBC Config 55 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 56 | 57 | (defn parse-db-props 58 | "Given `prop-str` of the form \"key=value&key=value...\", return a 59 | keyword-key map of property names to values." 60 | [prop-str] 61 | (reduce-kv (fn [m k v] (assoc m (keyword k) (form-encode v))) 62 | {} 63 | (form-decode prop-str))) 64 | 65 | (defn make-jdbc-url 66 | "Construct the JDBC URL from DB properties; if `:db-jdbc-url` is already 67 | present, return that." 68 | [{:keys [db-type 69 | db-name 70 | db-host 71 | db-port 72 | db-properties 73 | db-jdbc-url]}] 74 | (if db-jdbc-url 75 | ;; If JDBC URL is given directly, this overrides all 76 | db-jdbc-url 77 | ;; Construct a new JDBC URL from config vars 78 | (jdbc-conn/jdbc-url 79 | (cond-> {:dbtype db-type 80 | :dbname db-name 81 | :host db-host 82 | :port db-port} 83 | db-properties 84 | (merge (parse-db-props db-properties)))))) 85 | 86 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 87 | ;; Graceful Shutdown 88 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 89 | 90 | (defn add-shutdown-hook! 91 | "Given a running system, add a shutdown hook to gracefully stop it." 92 | [system] 93 | (.addShutdownHook (Runtime/getRuntime) 94 | (Thread. ^Runnable 95 | (fn [] 96 | (component/stop system)))) 97 | system) 98 | -------------------------------------------------------------------------------- /src/main/lrsql/util/activity.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.activity 2 | (:require [com.yetanalytics.lrs.xapi.activities :as as])) 3 | 4 | (defn merge-activities 5 | "Given the Activity objects `old-activity` and `new-activity`, merge 6 | them such that their lang maps are merged." 7 | [old-activity new-activity] 8 | (as/merge-activity old-activity new-activity)) 9 | -------------------------------------------------------------------------------- /src/main/lrsql/util/actor.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.actor 2 | (:require [com.yetanalytics.lrs.xapi.agents :as agnt])) 3 | 4 | ;; NOTE: Actors and Groups are encoded without any intention of being parsed 5 | ;; back, hence this sort of simple encoding is sufficient. 6 | (defn actor->ifi 7 | "Returns string of the format `::`. 8 | Returns `nil` if `actor` doesn't have an IFI (e.g. Anonymous Group)." 9 | [actor] 10 | (let [{?mbox "mbox" 11 | ?sha "mbox_sha1sum" 12 | ?openid "openid" 13 | ?account "account"} 14 | actor] 15 | (cond 16 | ?mbox (str "mbox::" ?mbox) 17 | ?sha (str "mbox_sha1sum::" ?sha) 18 | ?openid (str "openid::" ?openid) 19 | ?account (let [{acc-name "name" 20 | acc-page "homePage"} 21 | ?account] 22 | (str "account::" acc-name "@" acc-page)) 23 | :else nil))) 24 | 25 | (defn actor->ifi-coll 26 | "Similar to `actor->ifi`, except that it returns a vector of IFIs. If 27 | `actor` has a `member` field (i.e. it's a group), return the vector 28 | of those IFIs (even if it's an identified group)." 29 | [actor] 30 | (if-some [member (get actor "member")] 31 | (mapv actor->ifi member) 32 | [(actor->ifi actor)])) 33 | 34 | (defn actor->person 35 | "Given the Agent or Group `actor`, return an equivalent Person object with 36 | the IFI and name wrapped in vectors." 37 | [actor] 38 | ;; agnt/person takes zero or more args 39 | (if actor 40 | (agnt/person actor) 41 | (agnt/person))) 42 | -------------------------------------------------------------------------------- /src/main/lrsql/util/concurrency.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.concurrency 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.tools.logging :as log] 4 | [next.jdbc :refer [with-transaction]])) 5 | 6 | ;; `j-range` and `initial` are internal opts and should not be set by the user. 7 | ;; We also need to enforce MAX_VALUE constraints since `int?`, `nat-int?`, etc 8 | ;; don't do so on their own. 9 | 10 | (s/def ::budget (s/and int? (fn [n] (< 0 n Integer/MAX_VALUE)))) 11 | (s/def ::max-attempt (s/and int? (fn [n] (< 0 n Integer/MAX_VALUE)))) 12 | (s/def ::j-range (s/and int? (fn [n] (<= 0 n Integer/MAX_VALUE)))) 13 | (s/def ::initial (s/and int? (fn [n] (<= 0 n Integer/MAX_VALUE)))) 14 | 15 | (def backoff-opts-spec 16 | (s/keys :req-un [::budget ::max-attempt] 17 | :opt-un [::j-range ::initial])) 18 | 19 | (s/fdef backoff-ms 20 | :args (s/cat :attempt nat-int? 21 | :opts backoff-opts-spec) 22 | :ret (s/nilable nat-int?)) 23 | 24 | (defn backoff-ms 25 | "Take an `attempt` number and an opts map containing the total `:budget` 26 | in ms and an `:max-attempt` number and return a backoff time in ms. 27 | Can also optionally provide a jitter in `j-range` ms and an `initial` ms 28 | amount of delay to be used first in the opts map." 29 | [attempt {:keys [budget max-attempt j-range initial] 30 | :or {j-range 10}}] 31 | (let [jitter (rand-int j-range)] 32 | (cond 33 | (= attempt 0) 0 34 | (> attempt max-attempt) nil 35 | (and (some? initial) 36 | (= attempt 1)) (+ initial jitter) 37 | :else (int (+ (* budget (/ (Math/pow 2 (- attempt 1)) 38 | (Math/pow 2 max-attempt))) 39 | jitter))))) 40 | 41 | (defn rerunable-txn* 42 | "Take a `txn-expr` thunk, a positive number of `attempt`s, and an `opts` map, 43 | and run `txn-expr`. If it throws an exception and `:retry-test` passes, 44 | attempt to retry until `:max-attempt` has been reached, with the backoff 45 | time exponentially increasing and scaled by `:budget`." 46 | [txn-expr attempt {:keys [retry-test max-attempt] :as opts}] 47 | (try 48 | (txn-expr) 49 | (catch Exception e 50 | (if (and (< attempt max-attempt) 51 | (retry-test e)) 52 | ;; Type hinted to long for JDK16+ which won't accept it without 53 | (let [^long sleep (backoff-ms (inc attempt) opts)] 54 | (Thread/sleep sleep) 55 | (rerunable-txn* txn-expr (inc attempt) opts)) 56 | (do 57 | (log/warn "Rerunable Transaction exhausted attempts or could not be retried") 58 | (throw e)))))) 59 | 60 | (defmacro rerunable-txn 61 | "Macro to create a rerunable version of `next.jdbc/transact`. `f` needs to 62 | be a one-arity function that takes a transaction arg and runs the body. 63 | `opts` include `:retry-test`, `:max-attempt`, and `:budget`, as well as the 64 | usual options for `transact`." 65 | [transactable f opts] 66 | `(rerunable-txn* 67 | (fn [] (transact ~transactable ~f ~opts)) 68 | 0 69 | ~opts)) 70 | 71 | (defmacro with-rerunable-txn 72 | "Macro to create a rerunable version of `next.jdbc/with-transaction`. Binds 73 | `sym` to `transactable` and executes `body` in a rerunable manner. `opts` 74 | include `:retry-test`, `:max-attempt`, and `:budget`, as well as the usual 75 | options for `with-transaction`." 76 | [[sym transactable opts] & body] 77 | `(rerunable-txn* 78 | (fn [] (with-transaction [~sym ~transactable ~opts] ~@body)) 79 | 0 80 | ~opts)) 81 | -------------------------------------------------------------------------------- /src/main/lrsql/util/document.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.document) 2 | 3 | (defn document-dispatch 4 | "Return either `:state-document`, `:agent-profile-document`, or 5 | `:activity-profile-document` depending on the fields in `params`. Works for 6 | both ID params and query params." 7 | [{?state-id :stateId 8 | ?profile-id :profileId 9 | ?activity-id :activityId 10 | ?agent :agent 11 | :as params}] 12 | (cond 13 | ;; ID params 14 | ?state-id 15 | :state-document 16 | (and ?profile-id ?agent) 17 | :agent-profile-document 18 | (and ?profile-id ?activity-id) 19 | :activity-profile-document 20 | ;; Query params 21 | (and ?activity-id ?agent) 22 | :state-document 23 | ?activity-id 24 | :activity-profile-document 25 | ?agent 26 | :agent-profile-document 27 | ;; Error 28 | :else 29 | (throw (ex-info "Invalid document ID or query parameters!" 30 | {:type ::invalid-document-resource-params 31 | :params params})))) 32 | -------------------------------------------------------------------------------- /src/main/lrsql/util/headers.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.headers 2 | (:require [io.pedestal.http.secure-headers :as hsh] 3 | [io.pedestal.interceptor :refer [interceptor]])) 4 | 5 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 6 | ;; General 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | 9 | (defn headers-interceptor 10 | "Takes a map of header names to values and creates an interceptor to inject 11 | them in response." 12 | [headers] 13 | (interceptor 14 | {:leave (fn [{response :response :as context}] 15 | (assoc-in context [:response :headers] 16 | (merge headers (:headers response))))})) 17 | 18 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 19 | ;; Security Headers 20 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 21 | 22 | (def default-value "[default]") 23 | 24 | (def sec-head-defaults 25 | {:sec-head-hsts (hsh/hsts-header) 26 | :sec-head-frame (hsh/frame-options-header) 27 | :sec-head-content-type (hsh/content-type-header) 28 | :sec-head-xss (hsh/xss-protection-header) 29 | :sec-head-download (hsh/download-options-header) 30 | :sec-head-cross-domain (hsh/cross-domain-policies-header) 31 | :sec-head-content (hsh/content-security-policy-header)}) 32 | 33 | (def sec-head-names 34 | {:sec-head-hsts "Strict-Transport-Security" 35 | :sec-head-frame "X-Frame-Options" 36 | :sec-head-content-type "X-Content-Type-Options" 37 | :sec-head-xss "X-XSS-Protection" 38 | :sec-head-download "X-Download-Options" 39 | :sec-head-cross-domain "X-Permitted-Cross-Domain-Policies" 40 | :sec-head-content "Content-Security-Policy"}) 41 | 42 | (defn build-sec-headers 43 | [sec-header-opts] 44 | (reduce-kv 45 | (fn [agg h-key h-val] 46 | (if (string? h-val) 47 | (assoc agg (get sec-head-names h-key) 48 | (if (= default-value h-val) 49 | (get sec-head-defaults h-key) 50 | h-val)) 51 | agg)) {} sec-header-opts)) 52 | 53 | (defn secure-headers 54 | "Iterate header-opts, generating values for each header and returning an 55 | interceptor" 56 | [sec-header-opts] 57 | (let [sec-headers (build-sec-headers sec-header-opts)] 58 | (headers-interceptor sec-headers))) 59 | -------------------------------------------------------------------------------- /src/main/lrsql/util/interceptor.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.interceptor 2 | "Common interceptors shared by LRS and admin routes." 3 | (:require [clojure.string :as cstr] 4 | [io.pedestal.interceptor.error :refer [error-dispatch]] 5 | [io.pedestal.interceptor.chain :as chain])) 6 | 7 | (defn- get-message 8 | [ex] 9 | (.getMessage ^Throwable ex)) 10 | 11 | ;; clj-kondo/VSCode does not recognize complex `error-dispatch` macro 12 | #_{:clj-kondo/ignore [:unresolved-symbol]} 13 | (defn handle-json-parse-exn 14 | "Return an interceptor to handle Jackson JsonParseExceptions (which 15 | are thrown by the `body-params` Pedestal interceptor). `redact?`, 16 | if true, will attempt to redact any personal identifying info." 17 | ([] 18 | (handle-json-parse-exn false)) 19 | ([redact?] 20 | (error-dispatch 21 | [ctx ex] 22 | ;; JSON Parse failure 23 | [{:exception-type :com.fasterxml.jackson.core.JsonParseException 24 | :exception exception}] 25 | (let [msg (cond-> exception 26 | true get-message 27 | redact? (cstr/replace #"Unrecognized token '.*'" 28 | "Unrecognized token '[REDACTED]'"))] 29 | (assoc ctx 30 | :response 31 | {:status 400 32 | :body {:error msg}})) 33 | ;; JSON EOF failure (subclass of the above) 34 | [{:exception-type :com.fasterxml.jackson.core.io.JsonEOFException 35 | :exception exception}] 36 | (let [msg (get-message exception)] 37 | (assoc ctx 38 | :response 39 | {:status 400 40 | :body {:error msg}})) 41 | ;; Other error (incl. non-parsing-related Jackson errors); 42 | ;; continue as need be. 43 | :else 44 | (assoc ctx ::chain/error ex)))) 45 | -------------------------------------------------------------------------------- /src/main/lrsql/util/logging.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.logging) 2 | 3 | (def logo 4 | "\n __ ______ _ _ 5 | |\\ \\ \\ \\ | | | | | _ _ _ _ 6 | | \\ \\_\\ \\ | | |_| | | |_ / _ \\ | |_ _| |_ _ 7 | | / __ \\ | \\ / ____| __| | _ | _ ____| | |_| | _|_| ___ ___ 8 | |/ /\\ \\ \\ \\ | | | | / /| | | | | |/ _ \\ / /_ | |\\ /| | _ / __/ / __| 9 | / \\ \\ \\ \\ | | | | /_| |_ | | | | | | | |_| | | | | | |_| | |___ _| | 10 | /____\\ \\_\\ \\| |_| |____|\\__| |_| |_|_| |_|\\__|_|_| |_| \\__|_|\\____/___/ 11 | _______ _______ _ _ _______ _______ 12 | | _____| | ___ | | | | | | ___ | | _____| 13 | | |_____ | | | | | | | | | |___| | | |_____ 14 | |_____ | | | | | | | | | | ____| |_____ | 15 | _____| | | |__ | | | |_____ | |_____ | |\\ \\ _____| | 16 | |_______| |_____\\_\\ |_______| |_______| |_| \\__\\ |_______| 17 | (c) 2021-2025 Yet Analytics Inc.") 18 | -------------------------------------------------------------------------------- /src/main/lrsql/util/path.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.path 2 | "Utilities for property paths to query data from Statements. 3 | Property path specs are defined in the lrs-reactions library." 4 | (:require [clojure.spec.alpha :as s] 5 | [clojure.string :as cstr] 6 | [com.yetanalytics.lrs-reactions.spec :as rs])) 7 | 8 | (defn- alpha-only? 9 | [s] 10 | (re-matches #"[A-Za-z]*" s)) 11 | 12 | (s/fdef path->sqlpath-string 13 | :args (s/cat :path ::rs/path 14 | :qstring (s/? string?)) 15 | :ret string?) 16 | 17 | (defn path->sqlpath-string 18 | "Given a vector of keys and/or indices, return a JSONPath-like string suitable 19 | for SQL JSON access. Unlike JSONPath strings themselves, all string keys 20 | require the dot syntax." 21 | ([path] 22 | (path->sqlpath-string path "$")) 23 | ([[seg & rpath] s] 24 | (if seg 25 | (recur rpath 26 | (cond 27 | (and (string? seg) 28 | (alpha-only? seg)) 29 | (format "%s.%s" s seg) 30 | 31 | (string? seg) ; URLs and other special-character containing strs 32 | (format "%s.\"%s\"" s seg) 33 | 34 | (nat-int? seg) 35 | (format "%s[%d]" s seg) 36 | 37 | :else 38 | (throw (ex-info "Invalid path segement" 39 | {:type ::invalid-path-segment 40 | :segment seg})))) 41 | s))) 42 | 43 | (s/fdef path->jsonpath-vec 44 | :args (s/cat :path ::rs/path) 45 | :ret vector?) 46 | 47 | (defn path->jsonpath-vec 48 | "Given a vector of keys and/or indices, return a vector that is one 49 | entry of a Pathetic-parsed JSONPath vector of vectors. 50 | Calling `(mapv path->jsonpath-vec [path1 path2])` is equivalent to 51 | calling `(pathetic/parse-paths \"jsonpath1 | jsonpath2\")`." 52 | [path] 53 | (mapv (fn [seg] [seg]) path)) 54 | 55 | (s/fdef path->csv-header 56 | :args (s/cat :path ::rs/path) 57 | :ret string?) 58 | 59 | (defn path->csv-header 60 | "Given a vector of keys and/or indices, return a string of the keys 61 | separated by underscores, suitable for use as CSV headers." 62 | [path] 63 | (cstr/join "_" path)) 64 | -------------------------------------------------------------------------------- /src/test/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/test/lrsql/conformance_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.conformance-test 2 | (:require [clojure.test :as t :refer [deftest testing is]] 3 | [com.stuartsierra.component :as component] 4 | [com.yetanalytics.lrs.test-runner :as conf] 5 | [lrsql.test-support :as support])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Init 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (support/instrument-lrsql) 12 | 13 | (t/use-fixtures :each support/fresh-db-fixture) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Tests 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (deftest conformance-test 20 | (conf/with-test-suite 21 | (binding [conf/*print-logs* true] 22 | (testing "no regressions" 23 | (let [sys (support/test-system) 24 | sys' (component/start sys) 25 | pre (-> sys' :webserver :config :url-prefix) 26 | url (str "http://localhost:8080" pre)] 27 | (is (conf/conformant? 28 | "-e" url "-b" "-z" "-a" 29 | "-u" "username" 30 | "-p" "password" 31 | "-x" "1.0.3")) 32 | (component/stop sys')))))) 33 | -------------------------------------------------------------------------------- /src/test/lrsql/https_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.https-test 2 | (:require [clojure.test :refer [deftest testing is use-fixtures]] 3 | [babashka.curl :as curl] 4 | [com.stuartsierra.component :as component] 5 | [lrsql.test-support :as support])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Init 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (support/instrument-lrsql) 12 | 13 | (use-fixtures :each support/fresh-db-fixture) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Tests 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (deftest https-test 20 | (testing "HTTPS connection" 21 | (let [sys (support/test-system) 22 | sys' (component/start sys) 23 | pre (-> sys' :webserver :config :url-prefix)] 24 | ;; We need to pass the `--insecure` arg because curl would otherwise 25 | ;; not accept our generate selfie certs 26 | (is (= 200 27 | (:status (curl/get "https://0.0.0.0:8443/health" 28 | {:raw-args ["--insecure"]})))) 29 | (is (some? 30 | (:body (curl/get (format "https://0.0.0.0:8443%s/about" pre) 31 | {:raw-args ["--insecure"]})))) 32 | (testing "is not over the HTTP port" 33 | (is (thrown-with-msg? 34 | clojure.lang.ExceptionInfo 35 | #"curl: \(35\).+" 36 | (curl/get "https://0.0.0.0:8080/health" 37 | {:raw-args ["--insecure"]})))) 38 | (component/stop sys')))) 39 | -------------------------------------------------------------------------------- /src/test/lrsql/params_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.params-test 2 | "REST API parameter tests" 3 | (:require [clojure.test :refer [deftest testing is use-fixtures]] 4 | [babashka.curl :as curl] 5 | [com.stuartsierra.component :as component] 6 | [lrsql.test-support :as support]) 7 | (:import [clojure.lang ExceptionInfo])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Init 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (def get-map 14 | {:headers {"Content-Type" "application/json" 15 | "X-Experience-API-Version" "1.0.3"} 16 | :basic-auth ["username" "password"]}) 17 | 18 | (support/instrument-lrsql) 19 | 20 | (use-fixtures :each support/fresh-db-fixture) 21 | 22 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 23 | ;; Tests 24 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 25 | 26 | ;; There are three non-spec query params that are defined in the upstream lrs 27 | ;; lib: `page`, `from`, and `unwrap_html`. 28 | ;; `from` is actively used and needs to be validated, `unwrap_html` is used by 29 | ;; the UI but not the backend, and `page` is completely unused. 30 | ;; See: https://github.com/yetanalytics/lrs/blob/master/src/main/com/yetanalytics/lrs/pedestal/interceptor/xapi.cljc 31 | 32 | (deftest extra-params-test 33 | (testing "Extra parameters" 34 | (let [sys (support/test-system) 35 | sys' (component/start sys)] 36 | (is (= 400 37 | (try (curl/get "http://0.0.0.0:8080/xapi/statements?foo=bar" 38 | get-map) 39 | (catch ExceptionInfo e (-> e ex-data :status))))) 40 | (testing "- `from`" 41 | (is (= 200 42 | (:status 43 | (curl/get "http://0.0.0.0:8080/xapi/statements?from=00000000-4000-8000-0000-111122223333" 44 | get-map)))) 45 | (is (= 400 46 | (try (curl/get "http://0.0.0.0:8080/xapi/statements?from=2024-10-10T10:10:10Z" 47 | get-map) 48 | (catch ExceptionInfo e (-> e ex-data :status)))))) 49 | (testing "- `unwrap_html`" 50 | (is (= 200 51 | (:status 52 | (curl/get "http://0.0.0.0:8080/xapi/statements?unwrap_html=true" 53 | get-map)))) 54 | ;; TODO: Disallow non-boolean `unrwap_html` values? 55 | (is (= 200 56 | (:status 57 | (curl/get "http://0.0.0.0:8080/xapi/statements?unwrap_html=not-a-boolean" 58 | get-map))))) 59 | (testing "- `page`" 60 | ;; TODO: Disallow `page` param? 61 | (is (= 200 62 | (:status 63 | (curl/get "http://0.0.0.0:8080/xapi/statements?page=123" 64 | get-map))))) 65 | (component/stop sys')))) 66 | -------------------------------------------------------------------------------- /src/test/lrsql/test_runner.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.test-runner 2 | (:require [cognitect.test-runner.api :as runner] 3 | [lrsql.test-support :as support])) 4 | 5 | (defn -main 6 | [& args] 7 | (let [{db "--database" :or {db "sqlite"}} args] 8 | (with-redefs [support/fresh-db-fixture 9 | (case db 10 | "sqlite" support/fresh-sqlite-fixture 11 | "postgres" support/fresh-postgres-fixture)] 12 | (runner/test {:dirs ["src/test"]})))) 13 | -------------------------------------------------------------------------------- /src/test/lrsql/test_support_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.test-support-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [lrsql.init.config :refer [read-config]] 4 | [lrsql.test-support :refer [fresh-sqlite-fixture]])) 5 | 6 | (defn- get-db-name 7 | [config] 8 | (get-in config [:connection :database :db-name])) 9 | 10 | (deftest fresh-db-fixture-test 11 | (testing "sets the db name to a random uuid" 12 | (is 13 | (= ":memory:" 14 | (get-db-name 15 | (fresh-sqlite-fixture 16 | #(read-config :test-sqlite-mem))))))) 17 | -------------------------------------------------------------------------------- /src/test/lrsql/util/admin_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.admin-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [java-time.api :as jt] 4 | [lrsql.util.admin :as ua] 5 | [lrsql.util :as u])) 6 | 7 | (deftest password-test 8 | (testing "password hashing and verification" 9 | (let [pass-hash-m (ua/hash-password "foo")] 10 | (is (ua/valid-password? "foo" pass-hash-m)) 11 | (is (not (ua/valid-password? "pass" pass-hash-m)))))) 12 | 13 | (defn- account-id->jwt 14 | [test-id] 15 | (ua/account-id->jwt test-id "secret" 3600 86400)) 16 | 17 | (defn- account-id->jwt* 18 | [test-id ref-time] 19 | (ua/account-id->jwt* test-id "secret" 3600 ref-time)) 20 | 21 | (defn- one-time-jwt 22 | ([] 23 | (one-time-jwt {})) 24 | ([claim] 25 | (ua/one-time-jwt claim "secret" 3600))) 26 | 27 | (defn- jwt->payload 28 | [jwt] 29 | (ua/jwt->payload jwt "secret" 1)) 30 | 31 | (deftest jwt-test 32 | (let [test-id (u/str->uuid "00000000-0000-1000-0000-000000000001")] 33 | (testing "JSON web tokens" 34 | (is (re-matches #".*\..*\..*" (account-id->jwt test-id))) 35 | (is (= test-id 36 | (-> test-id 37 | account-id->jwt 38 | jwt->payload 39 | :account-id))) 40 | (is (inst? 41 | (-> test-id 42 | account-id->jwt 43 | jwt->payload 44 | :expiration))) 45 | (is (inst? 46 | (-> test-id 47 | account-id->jwt 48 | jwt->payload 49 | :refresh-exp))) 50 | (let [utime (-> test-id 51 | account-id->jwt 52 | jwt->payload 53 | :refresh-exp)] 54 | (is (= utime 55 | (-> (account-id->jwt* test-id utime) 56 | jwt->payload 57 | :refresh-exp)))) 58 | (is (= :lrsql.admin/unauthorized-token-error 59 | (jwt->payload nil))) 60 | (is (= :lrsql.admin/unauthorized-token-error 61 | (jwt->payload "not-a-jwt"))) 62 | (is (= :lrsql.admin/unauthorized-token-error 63 | (-> test-id 64 | account-id->jwt 65 | (ua/jwt->payload "different-secret" 1)))) 66 | (is (= :lrsql.admin/unauthorized-token-error 67 | (let [tok (ua/account-id->jwt test-id "secret" 1 100) 68 | _ (Thread/sleep 1001)] 69 | (ua/jwt->payload tok "secret" 0))))) 70 | (testing "One-time JWTs" 71 | (let [{:keys [jwt exp oti]} (one-time-jwt)] 72 | (is (int? exp)) 73 | (is (uuid? oti)) 74 | (is (inst? (-> jwt jwt->payload :expiration))) 75 | (is (uuid? (-> jwt jwt->payload :one-time-id))) 76 | (is (= oti (-> jwt jwt->payload :one-time-id)))) 77 | (let [expiration (jt/truncate-to (u/current-time) :seconds) 78 | {:keys [jwt exp oti]} (one-time-jwt {:account-id test-id 79 | :expiration expiration})] 80 | (is (int? exp)) 81 | (is (uuid? oti)) 82 | (is (= expiration 83 | (-> jwt jwt->payload :expiration))))))) 84 | -------------------------------------------------------------------------------- /src/test/lrsql/util/concurrency_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.concurrency-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [lrsql.util.concurrency :as conc] 4 | [lrsql.test-support :refer [check-validate]])) 5 | 6 | (deftest backoff-calc-test 7 | (testing "backoff calculation" 8 | ;; Generative testing 9 | (testing "(gentest)" 10 | (is (nil? (check-validate `conc/backoff-ms 1000)))) 11 | 12 | ;; Unit testing 13 | 14 | ;; 0th attempt should be zero backoff 15 | (is (= 0 (conc/backoff-ms 0 {:budget 10 16 | :max-attempt 10}))) 17 | 18 | ;; First retry with a set initial and 0 jitter 19 | (is (= 5 (conc/backoff-ms 1 {:budget 1000 20 | :initial 5 21 | :max-attempt 10 22 | :j-range 0}))) 23 | 24 | ;; Over max should return nil 25 | (is (nil? (conc/backoff-ms 10 {:budget 1000 26 | :max-attempt 5}))) 27 | 28 | ;; Exact values no jitter, over 1000 budget 29 | (is (= (mapv #(conc/backoff-ms % {:budget 1000 30 | :j-range 0 31 | :max-attempt 5}) 32 | [0 1 2 3 4 5 6]) 33 | [0 31 62 125 250 500 nil])) 34 | 35 | ;; Jitter included 36 | (is (let [value (conc/backoff-ms 3 {:budget 1000 37 | :max-attempt 5 38 | :j-range 5})] 39 | (and (>= value 125) 40 | (< value 131)))))) 41 | 42 | (deftest rerunable-txn-test 43 | (testing "rerunable-txn* function" 44 | (is (= 2 45 | (conc/rerunable-txn* (fn [] 2) 46 | 0 47 | {:retry-test (comp not int?) 48 | :max-attempt 10 49 | :budget 10}))) 50 | (is (= 2 ; max-attempt has no effect if transaction never fails 51 | (conc/rerunable-txn* (fn [] 2) 52 | 0 53 | {:retry-test (comp not int?) 54 | :max-attempt 11 55 | :budget 10}))) 56 | (is (thrown? 57 | clojure.lang.ExceptionInfo 58 | (conc/rerunable-txn* (fn [] (throw (ex-info "Bad!" {:bad? true}))) 59 | 0 60 | {:retry-test (fn [ex] (-> (ex-data ex) :bad?)) 61 | :max-attempt 10 62 | :budget 10}))) 63 | (testing "- `txn-expr` throws before `max-attempt` is reached" 64 | (is (= 2 65 | (let [ctr (atom 0)] 66 | (conc/rerunable-txn* 67 | (fn [] 68 | (if (< @ctr 10) 69 | (do (swap! ctr inc) 70 | (throw (ex-info "Bad!" {:bad? true}))) 71 | 2)) 72 | 0 73 | {:retry-test (comp not int?) 74 | :max-attempt 10 75 | :budget 10}))))) 76 | (testing "- `txn-expr` throws even as `max-attempt` is reached" 77 | (is (thrown? 78 | clojure.lang.ExceptionInfo 79 | (let [ctr (atom 0)] 80 | (conc/rerunable-txn* 81 | (fn [] 82 | (if (< @ctr 11) ; one more than max-attempt 83 | (do (swap! ctr inc) 84 | (throw (ex-info "Bad!" {:bad? true}))) 85 | 2)) 86 | 0 87 | {:retry-test (comp not int?) 88 | :max-attempt 10 89 | :budget 10}))))))) 90 | -------------------------------------------------------------------------------- /src/test/lrsql/util/database_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.database-test 2 | (:require [clojure.test :refer [deftest testing is are]] 3 | [clojure.spec.alpha :as s] 4 | [lrsql.spec.config :as cs] 5 | [lrsql.system.util :refer [parse-db-props]])) 6 | 7 | (deftest parse-db-props-test 8 | (testing "validation regex" 9 | (is (s/valid? ::cs/db-properties "foo=bar&baz=qux")) 10 | (is (s/valid? ::cs/db-properties "currentSchema=lrsql,public")) 11 | (is (s/valid? ::cs/db-properties "currentSchema=lrsql%2Cpublic")) 12 | (is (s/valid? ::cs/db-properties "currentSchema=foo%26bar")) 13 | (is (not (s/valid? ::cs/db-properties "currentSchema=foo&bar"))) 14 | (is (not (s/valid? ::cs/db-properties "currentSchema=foo&bar&baz=qux"))) 15 | (is (s/valid? ::cs/db-properties 16 | "options=-c%20search_path=lrsql,public,pgcatalog%20-c%20statement_timeout=90000"))) 17 | (testing "parsing DB property strings" 18 | (are [m s] (= m (parse-db-props s)) 19 | {:foo "bar"} "foo=bar" 20 | {:foo "bar" :baz "qux"} "foo=bar&baz=qux" 21 | {:FOO "bar%3BBAZ%3Dqux"} "FOO=bar;BAZ=qux" 22 | {:currentSchema "lrsql%2Cpublic"} "currentSchema=lrsql,public" 23 | {:currentSchema "lrsql%2Cpublic"} "currentSchema=lrsql%2Cpublic" 24 | ;; Taken from: 25 | ;; https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters 26 | {:options "-c+search_path%3Dlrsql%2Cpublic%2Cfoo%26bar"} 27 | "options=-c search_path=lrsql,public,foo%26bar" 28 | {:options "-c+search_path%3Dlrsql%2Cpublic%2Cfoo" :bar "baz"} 29 | "options=-c search_path=lrsql,public,foo&bar=baz" 30 | {:options "-c+search_path%3Dlrsql%2Cpublic%2Cpgcatalog+-c+statement_timeout%3D90000"} 31 | "options=-c%20search_path=lrsql,public,pgcatalog%20-c%20statement_timeout=90000" 32 | {:options "-c+search_path%3Dlrsql%2Cpublic%2Cpgcatalog+-c+statement_timeout%3D90000"} 33 | "options=-c search_path=lrsql,public,pgcatalog -c statement_timeout=90000"))) 34 | -------------------------------------------------------------------------------- /src/test/lrsql/util/path_test.clj: -------------------------------------------------------------------------------- 1 | (ns lrsql.util.path-test 2 | (:require [clojure.test :refer [deftest testing are use-fixtures]] 3 | [com.yetanalytics.pathetic :as pa] 4 | [lrsql.util.path :as p] 5 | [lrsql.test-support :as support])) 6 | 7 | (use-fixtures :once support/instrumentation-fixture) 8 | 9 | (deftest path->sqlpath-string-test 10 | (testing "Property path to JSONPath-like string for SQL" 11 | (are [input output] 12 | (= (p/path->sqlpath-string input) 13 | output) 14 | [] 15 | "$" 16 | 17 | ["object" "id"] 18 | "$.object.id" 19 | 20 | ["context" "contextActivities" "parent" 0 "id"] 21 | "$.context.contextActivities.parent[0].id" 22 | 23 | ["context" "extensions" "https://www.google.com/array"] 24 | "$.context.extensions.\"https://www.google.com/array\""))) 25 | 26 | (deftest path->jsonpath-vec-test 27 | (testing "Property path to parsed JSONPath vector" 28 | (are [input output] 29 | (= [(p/path->jsonpath-vec input)] 30 | (pa/parse-paths output)) 31 | [] 32 | "$" 33 | 34 | ["object" "id"] 35 | "$.object.id" 36 | 37 | ["context" "contextActivities" "parent" 0 "id"] 38 | "$.context.contextActivities.parent[0].id" 39 | 40 | ["context" "extensions" "https://www.google.com/array"] 41 | "$.context.extensions['https://www.google.com/array']"))) 42 | 43 | (deftest path->csv-header-test 44 | (testing "Property path to CSV header" 45 | (are [input output] 46 | (= (p/path->csv-header input) 47 | output) 48 | [] 49 | "" 50 | 51 | ["object" "id"] 52 | "object_id" 53 | 54 | ["context" "contextActivities" "parent" 0 "id"] 55 | "context_contextActivities_parent_0_id" 56 | 57 | ["context" "extensions" "https://www.google.com/array"] 58 | "context_extensions_https://www.google.com/array"))) 59 | --------------------------------------------------------------------------------