├── .dir-locals.el ├── .dockerignore ├── .github └── workflows │ ├── ci.yml │ ├── deps.yml │ ├── nvd_sched.yml │ └── release.yml ├── .gitignore ├── .java_modules ├── .nvd ├── config.json └── suppression.xml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── bin ├── machine.sh └── run.sh ├── demo ├── dashboard.yaml ├── datasource.yaml ├── docker-compose.yml ├── grafana.ini ├── prometheus.yml └── xapipe_dashboard.json ├── deps.edn ├── dev-resources ├── bench │ └── .gitkeep ├── jobs │ └── simple.json ├── lrs │ └── after_conf.edn ├── profiles │ ├── calibration.jsonld │ ├── calibration_a.jsonld │ ├── calibration_b.jsonld │ ├── calibration_c.jsonld │ ├── calibration_concept.jsonld │ ├── calibration_strict_pattern.jsonld │ ├── calibration_strict_pattern_alt.jsonld │ ├── calibration_strict_pattern_conflict.jsonld │ ├── tccc.jsonld │ └── test.jsonld ├── statements │ └── calibration_50.edn └── templates │ ├── 0_redis.yml │ └── 1_lrspipe_ec2.yml ├── doc ├── aws.md ├── demo.md ├── docker.md ├── img │ ├── logo.png │ ├── resources.png │ └── template-options.png ├── index.md ├── install.md ├── json.md ├── metrics.md ├── oauth.md ├── options.md ├── persistence.md └── usage.md ├── resources ├── .keep └── doc │ └── docs.html.template └── src ├── bench └── com │ └── yetanalytics │ └── xapipe │ ├── bench.clj │ └── bench │ └── maths.clj ├── build └── com │ └── yetanalytics │ └── xapipe │ └── build.clj ├── cli ├── com │ └── yetanalytics │ │ └── xapipe │ │ ├── cli.clj │ │ ├── cli │ │ └── options.clj │ │ ├── main.clj │ │ └── metrics │ │ └── impl │ │ └── prometheus.clj └── logback.xml ├── lib └── com │ └── yetanalytics │ ├── xapipe.clj │ └── xapipe │ ├── client.clj │ ├── client │ ├── json_only.clj │ ├── multipart_mixed.clj │ └── oauth.clj │ ├── filter.clj │ ├── filter │ ├── concept.clj │ └── path.clj │ ├── job.clj │ ├── job │ ├── config.clj │ ├── json.clj │ ├── state.clj │ └── state │ │ └── errors.clj │ ├── metrics.clj │ ├── spec │ └── common.clj │ ├── store.clj │ ├── store │ └── impl │ │ ├── file.clj │ │ ├── memory.clj │ │ ├── noop.clj │ │ └── redis.clj │ ├── util.clj │ ├── util │ ├── async.clj │ └── time.clj │ └── xapi.clj └── test ├── com └── yetanalytics │ ├── xapipe │ ├── cli │ │ └── options_test.clj │ ├── cli_test.clj │ ├── client_test.clj │ ├── filter │ │ └── path_test.clj │ ├── filter_test.clj │ ├── job │ │ ├── config_test.clj │ │ ├── json_test.clj │ │ └── state_test.clj │ ├── job_test.clj │ ├── main_test.clj │ ├── test_support.clj │ ├── test_support │ │ └── lrs.clj │ ├── util │ │ ├── async_test.clj │ │ └── time_test.clj │ └── xapi_test.clj │ └── xapipe_test.clj └── logback-test.xml /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;;; Directory Local Variables 2 | ;;; For more information see (info "(emacs) Directory Variables") 3 | 4 | ((nil . 5 | ((cider-clojure-cli-global-options . "-A:test:cli")))) 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # exclude runtimes as we build our own on alpine 2 | /target/bundle/runtimes 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - '*' 7 | tags: 8 | - 'v*' 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | timeout-minutes: 10 14 | steps: 15 | - name: Checkout project 16 | uses: actions/checkout@v4 17 | 18 | - name: Get an env 19 | uses: yetanalytics/action-setup-env@v2 20 | 21 | - name: Cache Deps 22 | uses: actions/cache@v4 23 | with: 24 | path: | 25 | ~/.m2 26 | ~/.gitlibs 27 | key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} 28 | restore-keys: | 29 | ${{ runner.os }}-deps- 30 | 31 | - name: Test Xapipe 32 | run: make test 33 | 34 | nvd_scan: 35 | uses: yetanalytics/actions/.github/workflows/nvd-scan.yml@v0.0.4 36 | with: 37 | nvd-clojure-version: '3.6.0' 38 | classpath-command: 'clojure -Spath -A:cli' 39 | nvd-config-filename: '.nvd/config.json' 40 | 41 | docker: 42 | needs: 43 | - test 44 | - nvd_scan 45 | runs-on: ubuntu-latest 46 | timeout-minutes: 10 47 | steps: 48 | - name: Checkout project 49 | uses: actions/checkout@v4 50 | 51 | - name: Get an env 52 | uses: yetanalytics/action-setup-env@v2 53 | 54 | - name: Cache Deps 55 | uses: actions/cache@v4 56 | with: 57 | path: | 58 | ~/.m2 59 | ~/.gitlibs 60 | key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} 61 | restore-keys: | 62 | ${{ runner.os }}-deps- 63 | 64 | - name: Build Xapipe 65 | run: make bundle BUNDLE_RUNTIMES=false 66 | 67 | - name: Extract metadata (tags, labels) for Docker 68 | id: meta 69 | uses: docker/metadata-action@v4 70 | with: 71 | images: yetanalytics/xapipe 72 | tags: | 73 | type=ref,event=branch 74 | type=ref,event=pr 75 | type=semver,pattern={{version}} 76 | type=semver,pattern={{major}}.{{minor}} 77 | 78 | - name: Log in to Docker Hub 79 | if: github.event_name != 'pull_request' 80 | uses: docker/login-action@v2 81 | with: 82 | username: ${{ secrets.DOCKERHUB_USERNAME }} 83 | password: ${{ secrets.DOCKERHUB_CI_TOKEN }} 84 | 85 | - name: Build and push Docker image 86 | uses: docker/build-push-action@v3 87 | with: 88 | context: . 89 | push: ${{ github.event_name != 'pull_request' }} 90 | tags: ${{ steps.meta.outputs.tags }} 91 | labels: ${{ steps.meta.outputs.labels }} 92 | -------------------------------------------------------------------------------- /.github/workflows/deps.yml: -------------------------------------------------------------------------------- 1 | name: Deps 2 | 3 | on: 4 | push: 5 | branches: 6 | - '*' 7 | 8 | jobs: 9 | deps: 10 | runs-on: ubuntu-latest 11 | timeout-minutes: 10 12 | steps: 13 | - name: Checkout project 14 | uses: actions/checkout@v4 15 | 16 | - name: Setup CI Environment 17 | uses: yetanalytics/action-setup-env@v2 18 | 19 | - name: Cache Deps 20 | uses: actions/cache@v4 21 | with: 22 | path: | 23 | ~/.m2 24 | ~/.gitlibs 25 | key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} 26 | restore-keys: | 27 | ${{ runner.os }}-deps- 28 | 29 | - name: Make a POM 30 | run: make clean pom.xml 31 | 32 | - name: Submit Dependency Snapshot 33 | uses: advanced-security/maven-dependency-submission-action@v3 34 | -------------------------------------------------------------------------------- /.github/workflows/nvd_sched.yml: -------------------------------------------------------------------------------- 1 | name: Periodic Vulnerability Scan 2 | 3 | on: 4 | schedule: 5 | - cron: '0 8 * * 1-5' # Every weekday at 8:00 AM 6 | 7 | jobs: 8 | nvd_scan: 9 | uses: yetanalytics/actions/.github/workflows/nvd-scan.yml@v0.0.4 10 | with: 11 | nvd-clojure-version: '3.6.0' 12 | classpath-command: 'clojure -Spath -A:cli' 13 | nvd-config-filename: '.nvd/config.json' 14 | 15 | notify_slack: 16 | runs-on: ubuntu-latest 17 | needs: nvd_scan 18 | if: ${{ always() && (needs.nvd_scan.result == 'failure') }} 19 | steps: 20 | - name: Notify Slack LRSPipe NVD Scan Reporter 21 | uses: slackapi/slack-github-action@v1.23.0 22 | with: 23 | payload: | 24 | { 25 | "run_link": "https://github.com/yetanalytics/xapipe/actions/runs/${{ github.run_id }}" 26 | } 27 | env: 28 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | # Draft Release on any tag 4 | on: 5 | push: 6 | tags: 7 | - 'v*' 8 | 9 | jobs: 10 | get_modules: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout project 14 | uses: actions/checkout@v4 15 | 16 | - name: List Java modules 17 | id: echo-modules 18 | run: echo "modules=$(cat .java_modules)" >> $GITHUB_OUTPUT 19 | 20 | outputs: 21 | modules: ${{ steps.echo-modules.outputs.modules }} 22 | 23 | build_jre: 24 | needs: get_modules 25 | uses: yetanalytics/workflow-runtimer/.github/workflows/runtimer.yml@v2 26 | with: 27 | java-version: '11' 28 | java-distribution: 'temurin' 29 | java-modules: ${{ needs.get_modules.outputs.modules }} 30 | 31 | build: 32 | needs: build_jre 33 | runs-on: ubuntu-latest 34 | steps: 35 | - name: Checkout project 36 | uses: actions/checkout@v4 37 | 38 | - name: Get an env 39 | uses: yetanalytics/action-setup-env@v2 40 | 41 | - name: Cache Deps 42 | uses: actions/cache@v4 43 | with: 44 | path: | 45 | ~/.m2 46 | ~/.gitlibs 47 | key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} 48 | restore-keys: | 49 | ${{ runner.os }}-deps- 50 | 51 | - name: Build Xapipe 52 | run: make bundle BUNDLE_RUNTIMES=false 53 | 54 | - name: Download ubuntu-latest Artifact 55 | uses: actions/download-artifact@v4 56 | with: 57 | name: ubuntu-22.04-jre 58 | 59 | - name: Download macOS-latest Artifact 60 | uses: actions/download-artifact@v4 61 | with: 62 | name: macos-14-jre 63 | 64 | - name: Download windows-latest Artifact 65 | uses: actions/download-artifact@v4 66 | with: 67 | name: windows-2022-jre 68 | 69 | - name: Unzip the runtimes 70 | run: | 71 | mkdir -p target/bundle/runtimes 72 | unzip ubuntu-22.04-jre.zip -d target/bundle/runtimes 73 | mv target/bundle/runtimes/ubuntu-22.04 target/bundle/runtimes/linux 74 | unzip macos-14-jre.zip -d target/bundle/runtimes 75 | mv target/bundle/runtimes/macos-14 target/bundle/runtimes/macos 76 | unzip windows-2022-jre.zip -d target/bundle/runtimes 77 | mv target/bundle/runtimes/windows-2022 target/bundle/runtimes/windows 78 | 79 | - name: Zip the bundle 80 | run: | 81 | cd target/bundle 82 | zip -r ../../xapipe.zip ./ 83 | 84 | - name: Craft Draft Release 85 | uses: softprops/action-gh-release@v1 86 | with: 87 | draft: true 88 | files: 'xapipe.zip' 89 | 90 | - name: Deploy Documentation (Tag Pushes) 91 | uses: JamesIves/github-pages-deploy-action@v4.4.1 92 | with: 93 | branch: gh-pages 94 | folder: target/bundle/doc 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | pom.xml.asc 3 | *.jar 4 | *.class 5 | /lib/ 6 | /classes/ 7 | /target/ 8 | /checkouts/ 9 | .lein-deps-sum 10 | .lein-repl-history 11 | .lein-plugins/ 12 | .lein-failures 13 | .nrepl-port 14 | .cpcache/ 15 | failures/ 16 | /store/ 17 | /.test_store/ 18 | /tmp/ 19 | /logs/ 20 | .DS_Store 21 | /dev-resources/bench/*.json 22 | .clj-kondo/ 23 | .lsp/ 24 | -------------------------------------------------------------------------------- /.java_modules: -------------------------------------------------------------------------------- 1 | java.base,java.logging,java.naming,java.sql,java.management,jdk.crypto.ec 2 | -------------------------------------------------------------------------------- /.nvd/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "nvd": {"suppression-file": ".nvd/suppression.xml"} 3 | } 4 | -------------------------------------------------------------------------------- /.nvd/suppression.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | ^pkg:maven\/(?!org\.clojure\/clojure).*$ 9 | cpe:/a:clojure:clojure 10 | CVE-2017-20189 11 | 12 | 13 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Yet Analytics Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | [team@yetanalytics.com](mailto:team@yetanalytics.com). 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available 126 | at [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq 132 | [translations]: https://www.contributor-covenant.org/translations 133 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Yet Analytics Open Source Contribution Guidelines 2 | 3 | ## Welcome to the Yet Analytics Open Source Community! 4 | 5 | Thank you for your interest in contributing to Yet Analytics Open Source projects. It is our goal in maintaining these Open Source projects to provide useful tools for the entire xAPI Community. We welcome feedback and contributions from users, stakeholders and engineers alike. 6 | 7 | The following document outlines the policies, methodology, and guidelines for contributing to our open source projects. 8 | 9 | ## Code of Conduct 10 | 11 | The Yet Analytics Open Source Community has a [Code of Conduct](CODE_OF_CONDUCT.md) which should be read and followed when contributing in any way. 12 | 13 | ## Issue Reporting 14 | 15 | Yet Analytics encourages users to contribute by reporting any issues or enhancement suggestions via [GitHub Issues](https://github.com/yetanalytics/xapipe/issues). 16 | 17 | Before submission, we encourage you to read through the existing [Documentation](README.md) to ensure that the issue has not been addressed or explained. 18 | 19 | ### Issue Templates 20 | 21 | If the repository has an Issue Template, please follow the template as much as possible in your submission as this helps our team more quickly triage and understand the issues you are seeing or enhancements you are suggesting. 22 | 23 | ### Security Issues 24 | 25 | If you believe you have found a potential security issue in the codebase of a Yet Analytics project, please do NOT open an issue. Email [team@yetanalytics.com](mailto:team@yetanalytics.com) directly instead. 26 | 27 | ## Code Contributions 28 | 29 | ### Methodology 30 | 31 | For community contribution to the codebase of a Yet Analytics project we ask that you follow the [Fork and Pull](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) methodology for proposing changes. In short, this method requires you to do the following: 32 | 33 | - Fork the repository 34 | - Clone your Fork and perform the appropriate changes on a branch 35 | - Push the changes back to your Fork 36 | - Make sure your Fork is up to date with the latest `main` branch in the central repository (merge upstream changes) 37 | - Submit a Pull Request using your fork's branch 38 | 39 | The Yet Analytics team will then review the changes, and may make suggestions on GitHub before a final decision is made. The Yet team reviews Pull Requests regularly and we will be notified of its creation and all updates immediately. 40 | 41 | ### Style 42 | 43 | For contributions in Clojure, we would suggest you read this [Clojure Style Guide](https://github.com/bbatsov/clojure-style-guide) as it is one that we generally follow in our codebases. 44 | 45 | ### Tests 46 | 47 | In order for us to merge a Pull Request it must pass the `make test-lib` Makefile target. This target runs a set of unit, integration and/or conformance tests which verify the build's behavior. Please run this target and remediate any issues before submitting a Pull Request. 48 | 49 | We ask that when adding or changing functionality in the system that you examine whether it is a candidate for additional or modified test coverage and add it if so. You can see what sort of tests are in place currently by exploring the namespaces in `src/test`. 50 | 51 | ## License and Copyright 52 | 53 | By contributing to a Yet Analytics Open Source project you agree that your contributions will be licensed under its [Apache License 2.0](LICENSE). 54 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.14 2 | 3 | ADD target/bundle /xapipe 4 | ADD .java_modules /xapipe/.java_modules 5 | 6 | # replace the linux runtime via jlink 7 | RUN apk update \ 8 | && apk upgrade \ 9 | && apk add ca-certificates \ 10 | && update-ca-certificates \ 11 | && apk add --no-cache openjdk11 \ 12 | && mkdir -p /xapipe/runtimes \ 13 | && jlink --output /xapipe/runtimes/linux/ --add-modules $(cat /xapipe/.java_modules) \ 14 | && apk del openjdk11 \ 15 | && rm -rf /var/cache/apk/* 16 | 17 | WORKDIR /xapipe 18 | ENTRYPOINT ["/xapipe/bin/run.sh"] 19 | CMD ["--help"] 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .phony: test bench clean bundle bundle-help ci 2 | 3 | clean: 4 | rm -rf target dev-resources/bench/*.json pom.xml 5 | 6 | JAVA_MODULES ?= $(shell cat .java_modules) 7 | 8 | target/nvd: 9 | clojure -Xnvd check :classpath '"'"$$(clojure -Spath -Acli)"'"' 10 | 11 | test: 12 | clojure -J--limit-modules -J$(JAVA_MODULES) -X:cli:test :dirs '["src/test"]' 13 | 14 | ci: test target/nvd 15 | 16 | BENCH_SIZE ?= 10000 17 | BENCH_PROFILE ?= dev-resources/profiles/calibration.jsonld 18 | 19 | dev-resources/bench/payload.json: 20 | clojure -Xtest:bench write-payload \ 21 | :num-statements $(BENCH_SIZE) \ 22 | :profile '"$(BENCH_PROFILE)"' \ 23 | :out '"dev-resources/bench/payload.json"' 24 | 25 | bench: dev-resources/bench/payload.json 26 | clojure -Xtest:bench run-bench-matrix \ 27 | :num-statements $(BENCH_SIZE) \ 28 | :payload-path '"dev-resources/bench/payload.json"' 29 | 30 | target/bundle/xapipe.jar: 31 | clojure -T:build uber 32 | 33 | target/bundle/bin: 34 | mkdir -p target/bundle/bin 35 | cp bin/*.sh target/bundle/bin 36 | chmod +x target/bundle/bin/*.sh 37 | 38 | # publish docs 39 | 40 | target/bundle/doc: 41 | clojure -X:doc 42 | 43 | # Make Runtime Environment (i.e. JREs) 44 | # Will only produce a single jre for macos/linux matching your machine 45 | MACHINE ?= $(shell bin/machine.sh) 46 | 47 | target/bundle/runtimes: 48 | mkdir -p target/bundle/runtimes 49 | jlink --output target/bundle/runtimes/${MACHINE}/ --add-modules ${JAVA_MODULES} 50 | 51 | BUNDLE_RUNTIMES ?= true 52 | 53 | ifeq ($(BUNDLE_RUNTIMES),true) 54 | target/bundle: target/bundle/xapipe.jar target/bundle/bin target/bundle/doc target/bundle/runtimes 55 | else 56 | target/bundle: target/bundle/xapipe.jar target/bundle/bin target/bundle/doc 57 | endif 58 | 59 | bundle: target/bundle 60 | 61 | # Run the bundle's help, used for compile-time sanity checks 62 | bundle-help: target/bundle 63 | cd target/bundle; bin/run.sh --help 64 | 65 | # Generate a POM for dependency graph resolution 66 | pom.xml: 67 | clojure -Acli -Spom 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![SQL LRS Logo](doc/img/logo.png) 2 | 3 | # Yet Analytics LRSPipe 4 | 5 | [![CI](https://github.com/yetanalytics/xapipe/actions/workflows/ci.yml/badge.svg)](https://github.com/yetanalytics/xapipe/actions/workflows/ci.yml) 6 | [![Docker Image Version (tag latest semver)](https://img.shields.io/docker/v/yetanalytics/xapipe/latest?color=blue&label=docker&style=plastic)](https://hub.docker.com/r/yetanalytics/xapipe) 7 | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-5e0b73.svg)](CODE_OF_CONDUCT.md) 8 | 9 | LRSPipe enables the Total Learning Architecture by acting as middleware between layers of data and by governing data flow directly based on xAPI Profiles. It’s more than an xAPI statement forwarder — it’s a forwarder that is governed by xAPI Profiles. 10 | 11 | ## How it Works 12 | 13 | LRSPipe is a standalone process which runs independently of any LRS. This process can run one or multiple "jobs" at a time, each of which contains information about a source LRS, a target LRS, and any applicable filters or options. While running, this process continually checks the source LRS(s) for xAPI Statements and attempts to replicate them into the target LRS(s). 14 | 15 | This process is one-way and any statements in the target LRS will not be replicated to the source LRS. These jobs can be paused, resumed, and modified, and LRSPipe tracks its progress using storage (either the local file system or a Redis server, depending on how it is configured) so a job can be interrupted at any time and pick right back up where it left off when it resumes. 16 | 17 | ### Filtering 18 | 19 | LRSPipe is capable of filtering which statements get forwarded. This filtering is performed using the components of provided xAPI Profiles. For more information see the [xAPI Profile Specification](https://github.com/adlnet/xapi-profiles). 20 | 21 | #### Full Forwarding 22 | 23 | In this mode all statements from the source LRS will be replicated into the target LRS. 24 | 25 | #### Statement Template Filtering 26 | 27 | In this mode LRSPipe is provided with the location of an xAPI Profile and will only forward statements that match the Statement Templates in the Profile. 28 | 29 | This can be made to be even more specific by providing a set of Statement Template Ids, which will cause it to only forward a subset of Statement Templates from a Profile. See [usage](doc/usage.md) for details and examples. 30 | 31 | #### Pattern Filtering 32 | 33 | In this mode LRSPipe is provided with the location of an xAPI Profile and will attempt to match statements to these Patterns, and will validate and forward matching statements. 34 | 35 | Much like Statement Template Filtering, this can also be limited to a set of Pattern Ids if one is provided. See [usage](doc/usage.md) for details and examples. 36 | 37 | ## Releases 38 | 39 | For releases and release notes, see the [Releases](https://github.com/yetanalytics/xapipe/releases/latest) page. 40 | 41 | ## Documentation 42 | 43 | - [Installation](doc/install.md) 44 | - [Usage](doc/usage.md) 45 | - [Persistence Config](doc/persistence.md) 46 | - [All Options](doc/options.md) 47 | - [JSON Config](doc/json.md) 48 | - [Metrics](doc/metrics.md) 49 | - [Docker Container](doc/docker.md) 50 | - [Demo](doc/demo.md) 51 | - [Sample AWS Deployment](doc/aws.md) 52 | 53 | ## Contribution 54 | 55 | Before contributing to this project, please read the [Contribution Guidelines](CONTRIBUTING.md) and the [Code of Conduct](CODE_OF_CONDUCT.md). 56 | 57 | ## License 58 | 59 | Copyright © 2021-2025 Yet Analytics, Inc. 60 | 61 | Distributed under the Apache License version 2.0. 62 | -------------------------------------------------------------------------------- /bin/machine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #TODO: Add windows, possibly better macos targeting 4 | 5 | unameOut="$(uname -s)" 6 | case "${unameOut}" in 7 | Linux*) machine=linux;; 8 | *) machine=macos;; 9 | esac 10 | 11 | echo ${machine} 12 | -------------------------------------------------------------------------------- /bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MACHINE=`bin/machine.sh` 4 | 5 | runtimes/$MACHINE/bin/java -server -jar xapipe.jar $@ 6 | -------------------------------------------------------------------------------- /demo/dashboard.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Prometheus' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /demo/datasource.yaml: -------------------------------------------------------------------------------- 1 | # config file version 2 | apiVersion: 1 3 | 4 | # list of datasources that should be deleted from the database 5 | deleteDatasources: 6 | - name: Prometheus 7 | orgId: 1 8 | 9 | # list of datasources to insert/update depending 10 | # whats available in the database 11 | datasources: 12 | # name of the datasource. Required 13 | - name: Prometheus 14 | # datasource type. Required 15 | type: prometheus 16 | # access mode. direct or proxy. Required 17 | access: proxy 18 | # org id. will default to orgId 1 if not specified 19 | orgId: 1 20 | # url 21 | url: http://prometheus:9090 22 | # database password, if used 23 | password: 24 | # database user, if used 25 | user: 26 | # database name, if used 27 | database: 28 | # enable/disable basic auth 29 | basicAuth: false 30 | # basic auth username, if used 31 | basicAuthUser: 32 | # basic auth password, if used 33 | basicAuthPassword: 34 | # enable/disable with credentials headers 35 | withCredentials: 36 | # mark as default datasource. Max one per org 37 | isDefault: true 38 | # fields that will be converted to json and stored in json_data 39 | jsonData: 40 | graphiteVersion: "1.1" 41 | tlsAuth: false 42 | tlsAuthWithCACert: false 43 | # json object of data that will be encrypted. 44 | secureJsonData: 45 | tlsCACert: "..." 46 | tlsClientCert: "..." 47 | tlsClientKey: "..." 48 | version: 1 49 | # allow users to edit datasources from the UI. 50 | editable: true 51 | -------------------------------------------------------------------------------- /demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | volumes: 4 | source_db_data: 5 | target_db_data: 6 | xapipe_store_data: 7 | prometheus_data: 8 | grafana_data: 9 | 10 | 11 | configs: 12 | prometheus_config: 13 | file: ./prometheus.yml 14 | grafana_config: 15 | file: ./grafana.ini 16 | grafana_datasource: 17 | file: ./datasource.yaml 18 | grafana_dashboard: 19 | file: ./dashboard.yaml 20 | grafana_dashboard_json: 21 | file: ./xapipe_dashboard.json 22 | 23 | services: 24 | # Source LRS 25 | source_db: 26 | image: postgres:14 27 | volumes: 28 | - source_db_data:/var/lib/postgresql/data 29 | environment: 30 | POSTGRES_USER: lrsql_user 31 | POSTGRES_PASSWORD: lrsql_password 32 | POSTGRES_DB: lrsql_db 33 | source_lrs: 34 | image: yetanalytics/lrsql:latest 35 | command: 36 | - /lrsql/bin/run_postgres.sh 37 | ports: 38 | - "8080:8080" 39 | depends_on: 40 | - source_db 41 | environment: 42 | LRSQL_API_KEY_DEFAULT: my_key 43 | LRSQL_API_SECRET_DEFAULT: my_secret 44 | LRSQL_ADMIN_USER_DEFAULT: my_username 45 | LRSQL_ADMIN_PASS_DEFAULT: my_password 46 | LRSQL_DB_HOST: source_db 47 | LRSQL_DB_NAME: lrsql_db 48 | LRSQL_DB_USER: lrsql_user 49 | LRSQL_DB_PASSWORD: lrsql_password 50 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 51 | restart: always 52 | 53 | # Target LRS 54 | target_db: 55 | image: postgres:14 56 | volumes: 57 | - target_db_data:/var/lib/postgresql/data 58 | environment: 59 | POSTGRES_USER: lrsql_user 60 | POSTGRES_PASSWORD: lrsql_password 61 | POSTGRES_DB: lrsql_db 62 | 63 | target_lrs: 64 | image: yetanalytics/lrsql:latest 65 | command: 66 | - /lrsql/bin/run_postgres.sh 67 | ports: 68 | - "8081:8081" 69 | depends_on: 70 | - target_db 71 | environment: 72 | LRSQL_HTTP_PORT: 8081 73 | LRSQL_API_KEY_DEFAULT: my_key 74 | LRSQL_API_SECRET_DEFAULT: my_secret 75 | LRSQL_ADMIN_USER_DEFAULT: my_username 76 | LRSQL_ADMIN_PASS_DEFAULT: my_password 77 | LRSQL_DB_HOST: target_db 78 | LRSQL_DB_NAME: lrsql_db 79 | LRSQL_DB_USER: lrsql_user 80 | LRSQL_DB_PASSWORD: lrsql_password 81 | LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 82 | restart: always 83 | 84 | # Xapipe 85 | redis: 86 | image: redis:6-alpine 87 | volumes: 88 | - xapipe_store_data:/data 89 | ports: 90 | - "6379" 91 | 92 | # Dashboards 93 | prometheus: 94 | image: prom/prometheus 95 | volumes: 96 | - prometheus_data:/prometheus 97 | ports: 98 | - 9090:9090 99 | configs: 100 | - source: prometheus_config 101 | target: /etc/prometheus/prometheus.yml 102 | 103 | grafana: 104 | image: grafana/grafana 105 | volumes: 106 | - grafana_data:/var/lib/grafana 107 | depends_on: 108 | - prometheus 109 | ports: 110 | - 3000:3000 111 | configs: 112 | - source: grafana_config 113 | target: /etc/grafana/grafana.ini 114 | - source: grafana_datasource 115 | target: /etc/grafana/provisioning/datasources/datasource.yaml 116 | - source: grafana_dashboard 117 | target: /etc/grafana/provisioning/dashboards/dashboard.yaml 118 | - source: grafana_dashboard_json 119 | target: /etc/grafana/provisioning/dashboards/xapipe_dashboard.json 120 | restart: always 121 | 122 | pushgateway: 123 | image: prom/pushgateway 124 | depends_on: 125 | - prometheus 126 | ports: 127 | - 9091:9091 128 | restart: always 129 | 130 | xapipe: 131 | image: yetanalytics/xapipe:${DEMO_VERSION:-latest} 132 | depends_on: 133 | - source_lrs 134 | - target_lrs 135 | - redis 136 | - pushgateway 137 | command: | 138 | -s redis 139 | --job-id link_source_target 140 | -f 141 | --redis-uri redis://redis:6379 142 | --source-url http://source_lrs:8080/xapi 143 | --source-username my_key 144 | --source-password my_secret 145 | --target-url http://target_lrs:8081/xapi 146 | --target-username my_key 147 | --target-password my_secret 148 | --metrics-reporter prometheus 149 | --prometheus-push-gateway pushgateway:9091 150 | restart: always 151 | -------------------------------------------------------------------------------- /demo/grafana.ini: -------------------------------------------------------------------------------- 1 | [dashboards] 2 | 3 | default_home_dashboard_path = /etc/grafana/provisioning/dashboards/xapipe_dashboard.json 4 | -------------------------------------------------------------------------------- /demo/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | 4 | scrape_configs: 5 | - job_name: "pushgateway" 6 | honor_labels: true 7 | static_configs: 8 | - targets: ["pushgateway:9091"] 9 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src/lib" "resources"] 2 | :deps {org.clojure/clojure {:mvn/version "1.11.2"} 3 | org.clojure/core.async {:mvn/version "1.3.618"} 4 | clj-http/clj-http {:mvn/version "3.12.3"} 5 | com.taoensso/carmine {:mvn/version "3.4.1"} 6 | commons-fileupload/commons-fileupload {:mvn/version "1.5"} 7 | commons-io/commons-io {:mvn/version "2.14.0"} 8 | com.yetanalytics/xapi-schema 9 | {:mvn/version "1.2.0" 10 | :exclusions [org.clojure/clojure 11 | org.clojure/clojurescript 12 | org.clojure/data.json]} 13 | org.clojure/tools.logging {:mvn/version "1.1.0"} 14 | com.yetanalytics/project-persephone 15 | {:mvn/version "0.8.1" 16 | :exclusions [org.clojure/clojurescript]} 17 | org.clojure/data.json {:mvn/version "2.4.0"} 18 | com.cognitect/transit-clj {:mvn/version "1.0.333" 19 | ;; clears CVE-2022-41719 20 | :exclusions [org.msgpack/msgpack]} 21 | cheshire/cheshire {:mvn/version "5.12.0"}} 22 | :aliases 23 | {:cli {:extra-paths ["src/cli"] 24 | :extra-deps {org.clojure/tools.cli {:mvn/version "1.0.206"} 25 | ch.qos.logback/logback-classic 26 | {:mvn/version "1.5.15" 27 | :exclusions [org.slf4j/slf4j-api]} 28 | org.slf4j/slf4j-api {:mvn/version "2.0.12"} 29 | org.slf4j/jul-to-slf4j {:mvn/version "2.0.12"} 30 | org.slf4j/jcl-over-slf4j {:mvn/version "2.0.12"} 31 | org.slf4j/log4j-over-slf4j {:mvn/version "2.0.12"} 32 | 33 | clj-commons/iapetos {:mvn/version "0.1.12"} 34 | io.prometheus/simpleclient_hotspot {:mvn/version "0.12.0"}}} 35 | :test {:extra-paths ["src/test" 36 | "src/bench"] 37 | :extra-deps 38 | {org.clojure/test.check {:mvn/version "1.1.0"} 39 | io.github.cognitect-labs/test-runner 40 | {:git/url "https://github.com/cognitect-labs/test-runner" 41 | :sha "dd6da11611eeb87f08780a30ac8ea6012d4c05ce"} 42 | com.yetanalytics/lrs {:mvn/version "1.2.11"} 43 | io.pedestal/pedestal.jetty {:mvn/version "0.5.9"} 44 | ;; Some integration tests use logback 45 | ch.qos.logback/logback-classic {:mvn/version "1.5.15" 46 | :exclusions [org.slf4j/slf4j-api]} 47 | org.slf4j/slf4j-api {:mvn/version "2.0.12"} 48 | org.slf4j/jul-to-slf4j {:mvn/version "2.0.12"} 49 | org.slf4j/jcl-over-slf4j {:mvn/version "2.0.12"} 50 | org.slf4j/log4j-over-slf4j {:mvn/version "2.0.12"} 51 | com.yetanalytics/datasim 52 | {:mvn/version "0.1.3" 53 | :exclusions [org.clojure/test.check 54 | com.yetanalytics/project-pan 55 | com.yetanalytics/xapi-schema]}} 56 | :exec-fn cognitect.test-runner.api/test} 57 | :bench {:extra-paths ["src/bench"] 58 | :ns-default com.yetanalytics.xapipe.bench} 59 | :build {:paths ["src/build"] 60 | :deps {io.github.clojure/tools.build {:git/tag "v0.6.6" 61 | :git/sha "4d41c26"}} 62 | :ns-default com.yetanalytics.xapipe.build} 63 | :nvd {:replace-deps {nvd-clojure/nvd-clojure {:mvn/version "1.9.0"}} 64 | :ns-default nvd.task} 65 | :doc {:replace-deps {com.yetanalytics/markdoc 66 | {:git/url "https://github.com/yetanalytics/markdoc" 67 | :sha "1a57b934dc92e539e858223ef33eb6a5fcf439a0"}} 68 | :exec-fn com.yetanalytics.markdoc/convert 69 | :exec-args {:in-root "doc/" 70 | :out-root "target/bundle/doc/" 71 | :template-file "resources/doc/docs.html.template"}}}} 72 | -------------------------------------------------------------------------------- /dev-resources/bench/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/xapipe/6af3d68299f88dd0dc4d87a63f567536c2ed7810/dev-resources/bench/.gitkeep -------------------------------------------------------------------------------- /dev-resources/jobs/simple.json: -------------------------------------------------------------------------------- 1 | {"id":"foo","config":{"get-buffer-size":10,"statement-buffer-size":500,"batch-buffer-size":10,"batch-timeout":200,"source":{"request-config":{"url-base":"http://0.0.0.0:8080","xapi-prefix":"/xapi"},"batch-size":50,"backoff-opts":{"budget":10000,"max-attempt":10},"poll-interval":1000,"get-params":{"limit":50}},"target":{"request-config":{"url-base":"http://0.0.0.0:8081","xapi-prefix":"/xapi"},"batch-size":50,"backoff-opts":{"budget":10000,"max-attempt":10}},"filter":{}},"state":{"status":"init","cursor":"1970-01-01T00:00:00.000000000Z","source":{"errors":[]},"target":{"errors":[]},"errors":[],"filter":{}}} 2 | -------------------------------------------------------------------------------- /dev-resources/profiles/calibration.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "https://xapinet.org/xapi/yet/calibration", 3 | "definition" : { 4 | "en" : "This xAPI Profile is intended for experimental purposes only and can be used to show the basic Pattern selection behavior of simulated datasets" 5 | }, 6 | "@context" : "https://w3id.org/xapi/profiles/context", 7 | "prefLabel" : { 8 | "en" : "Calibration - Experimental xAPI Profile" 9 | }, 10 | "type" : "Profile", 11 | "seeAlso" : "https://github.com/yetanalytics/datasim", 12 | "author" : { 13 | "url" : "https://www.yetanalytics.com/", 14 | "name" : "Yet Analytics", 15 | "type" : "Organization" 16 | }, 17 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0", 18 | "versions" : [ 19 | { 20 | "id" : "https://xapinet.org/xapi/yet/calibration/v1", 21 | "generatedAtTime" : "2020-03-24T19:16:07.395Z" 22 | } 23 | ], 24 | "patterns" : [ 25 | { 26 | "definition" : { 27 | "en" : "Pattern 1" 28 | }, 29 | "primary" : true, 30 | "prefLabel" : { 31 | "en" : "Learning Pattern 1" 32 | }, 33 | "type" : "Pattern", 34 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 35 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/patterns#pattern-1", 36 | "sequence" : [ 37 | "https://xapinet.org/xapi/yet/calibration/v1/patterns#pattern-2", 38 | "https://xapinet.org/xapi/yet/calibration/v1/patterns#pattern-3" 39 | ] 40 | }, 41 | { 42 | "definition" : { 43 | "en" : "Pattern 2" 44 | }, 45 | "primary" : false, 46 | "prefLabel" : { 47 | "en" : "Learning Pattern 2" 48 | }, 49 | "type" : "Pattern", 50 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 51 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/patterns#pattern-2", 52 | "optional" : "https://xapinet.org/xapi/yet/calibration/v1/templates#activity-1" 53 | }, 54 | { 55 | "definition" : { 56 | "en" : "Pattern 3" 57 | }, 58 | "primary" : false, 59 | "prefLabel" : { 60 | "en" : "Learning Pattern 3" 61 | }, 62 | "type" : "Pattern", 63 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 64 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/patterns#pattern-3", 65 | "optional" : "https://xapinet.org/xapi/yet/calibration/v1/templates#activity-2" 66 | } 67 | ], 68 | "concepts" : [ 69 | { 70 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-1", 71 | "type" : "Activity", 72 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 73 | "activityDefinition" : { 74 | "name" : { 75 | "en-US" : "Activity 1" 76 | }, 77 | "description" : { 78 | "en-US" : "The first activity" 79 | }, 80 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 81 | } 82 | }, 83 | { 84 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-2", 85 | "type" : "Activity", 86 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 87 | "activityDefinition" : { 88 | "name" : { 89 | "en-US" : "Activity 2" 90 | }, 91 | "description" : { 92 | "en-US" : "The second activity" 93 | }, 94 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 95 | } 96 | }, 97 | { 98 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-3", 99 | "type" : "Activity", 100 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 101 | "activityDefinition" : { 102 | "name" : { 103 | "en-US" : "Activity 3" 104 | }, 105 | "description" : { 106 | "en-US" : "The third activity" 107 | }, 108 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 109 | } 110 | }, 111 | { 112 | "id": "https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt", 113 | "inScheme": "https://xapinet.org/xapi/yet/calibration/v1", 114 | "type": "Verb", 115 | "definition": { 116 | "en": "Didn't do the thing" 117 | }, 118 | "prefLabel": { 119 | "en": "didnt" 120 | } 121 | }, 122 | { 123 | "id": "https://xapinet.org/xapi/yet/calibration/v1/concepts#did", 124 | "inScheme": "https://xapinet.org/xapi/yet/calibration/v1", 125 | "type": "Verb", 126 | "definition": { 127 | "en": "Did the thing" 128 | }, 129 | "prefLabel": { 130 | "en": "did" 131 | } 132 | } 133 | ], 134 | "templates" : [ 135 | { 136 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/templates#activity-1", 137 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 138 | "prefLabel" : { 139 | "en" : "Activity Template 1" 140 | }, 141 | "definition" : { 142 | "en" : "The statement template and rules associated with Activity 1 getting done." 143 | }, 144 | "type" : "StatementTemplate", 145 | "verb" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#did", 146 | "rules" : [ 147 | { 148 | "location" : "$.id", 149 | "presence" : "included" 150 | }, 151 | { 152 | "location" : "$.timestamp", 153 | "presence" : "included" 154 | }, 155 | { 156 | "any" : [ 157 | "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-1" 158 | ], 159 | "location" : "$.object.id", 160 | "presence" : "included" 161 | }, 162 | { 163 | "any" : [ 164 | "Activity 1" 165 | ], 166 | "location" : "$.object.definition.name.en-US", 167 | "presence" : "included" 168 | }, 169 | { 170 | "any" : [ 171 | "The first Activity" 172 | ], 173 | "location" : "$.object.definition.description.en-US", 174 | "presence" : "included" 175 | }, 176 | { 177 | "any" : [ 178 | "https://xapinet.org/xapi/yet/calibration/v1" 179 | ], 180 | "location" : "$.context.contextActivities.category[0].id", 181 | "presence" : "included" 182 | } 183 | ] 184 | }, 185 | { 186 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/templates#activity-2", 187 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 188 | "prefLabel" : { 189 | "en" : "Activity Template 2" 190 | }, 191 | "definition" : { 192 | "en" : "The statement template and rules associated with Activity 2 getting done." 193 | }, 194 | "type" : "StatementTemplate", 195 | "verb" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt", 196 | "rules" : [ 197 | { 198 | "location" : "$.id", 199 | "presence" : "included" 200 | }, 201 | { 202 | "location" : "$.timestamp", 203 | "presence" : "included" 204 | }, 205 | { 206 | "any" : [ 207 | "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-2" 208 | ], 209 | "location" : "$.object.id", 210 | "presence" : "included" 211 | }, 212 | { 213 | "any" : [ 214 | "Activity 2" 215 | ], 216 | "location" : "$.object.definition.name.en-US", 217 | "presence" : "included" 218 | }, 219 | { 220 | "any" : [ 221 | "The second Activity" 222 | ], 223 | "location" : "$.object.definition.description.en-US", 224 | "presence" : "included" 225 | }, 226 | { 227 | "any" : [ 228 | "https://xapinet.org/xapi/yet/calibration/v1" 229 | ], 230 | "location" : "$.context.contextActivities.category[0].id", 231 | "presence" : "included" 232 | } 233 | ] 234 | }, 235 | { 236 | "id" : "https://xapinet.org/xapi/yet/calibration/v1/templates#activity-3", 237 | "inScheme" : "https://xapinet.org/xapi/yet/calibration/v1", 238 | "prefLabel" : { 239 | "en" : "Activity Template 3" 240 | }, 241 | "definition" : { 242 | "en" : "The statement template and rules associated with Activity 3 getting done." 243 | }, 244 | "type" : "StatementTemplate", 245 | "verb" : "https://xapinet.org/xapi/yet/calibration/v1/concepts#did", 246 | "rules" : [ 247 | { 248 | "location" : "$.id", 249 | "presence" : "included" 250 | }, 251 | { 252 | "location" : "$.timestamp", 253 | "presence" : "included" 254 | }, 255 | { 256 | "any" : [ 257 | "https://xapinet.org/xapi/yet/calibration/v1/concepts#activity-3" 258 | ], 259 | "location" : "$.object.id", 260 | "presence" : "included" 261 | }, 262 | { 263 | "any" : [ 264 | "Activity 3" 265 | ], 266 | "location" : "$.object.definition.name.en-US", 267 | "presence" : "included" 268 | }, 269 | { 270 | "any" : [ 271 | "The third Activity" 272 | ], 273 | "location" : "$.object.definition.description.en-US", 274 | "presence" : "included" 275 | }, 276 | { 277 | "any" : [ 278 | "https://xapinet.org/xapi/yet/calibration/v1" 279 | ], 280 | "location" : "$.context.contextActivities.category[0].id", 281 | "presence" : "included" 282 | } 283 | ] 284 | } 285 | ] 286 | } -------------------------------------------------------------------------------- /dev-resources/profiles/calibration_a.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "https://xapinet.org/xapi/yet/calibration-a", 3 | "definition" : { 4 | "en" : "This xAPI Profile is intended for experimental purposes only and can be used to show the basic Pattern selection behavior of simulated datasets" 5 | }, 6 | "@context" : "https://w3id.org/xapi/profiles/context", 7 | "prefLabel" : { 8 | "en" : "Calibration A - Experimental xAPI Profile" 9 | }, 10 | "type" : "Profile", 11 | "seeAlso" : "https://github.com/yetanalytics/datasim", 12 | "author" : { 13 | "url" : "https://www.yetanalytics.com/", 14 | "name" : "Yet Analytics", 15 | "type" : "Organization" 16 | }, 17 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0", 18 | "versions" : [ 19 | { 20 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1", 21 | "generatedAtTime" : "2020-03-24T19:16:07.395Z" 22 | } 23 | ], 24 | "patterns" : [ 25 | { 26 | "definition" : { 27 | "en" : "Pattern 1" 28 | }, 29 | "primary" : true, 30 | "prefLabel" : { 31 | "en" : "Learning Pattern 1" 32 | }, 33 | "type" : "Pattern", 34 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 35 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/patterns#pattern-1", 36 | "sequence" : [ 37 | "https://xapinet.org/xapi/yet/calibration-a/v1/patterns#pattern-2", 38 | "https://xapinet.org/xapi/yet/calibration-a/v1/patterns#pattern-3" 39 | ] 40 | }, 41 | { 42 | "definition" : { 43 | "en" : "Pattern 2" 44 | }, 45 | "primary" : false, 46 | "prefLabel" : { 47 | "en" : "Learning Pattern 2" 48 | }, 49 | "type" : "Pattern", 50 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 51 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/patterns#pattern-2", 52 | "optional" : "https://xapinet.org/xapi/yet/calibration-a/v1/templates#activity-1" 53 | }, 54 | { 55 | "definition" : { 56 | "en" : "Pattern 3" 57 | }, 58 | "primary" : false, 59 | "prefLabel" : { 60 | "en" : "Learning Pattern 3" 61 | }, 62 | "type" : "Pattern", 63 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 64 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/patterns#pattern-3", 65 | "optional" : "https://xapinet.org/xapi/yet/calibration-a/v1/templates#activity-2" 66 | } 67 | ], 68 | "concepts" : [ 69 | { 70 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-1", 71 | "type" : "Activity", 72 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 73 | "activityDefinition" : { 74 | "name" : { 75 | "en-US" : "Activity 1" 76 | }, 77 | "description" : { 78 | "en-US" : "The first activity" 79 | }, 80 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 81 | } 82 | }, 83 | { 84 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-2", 85 | "type" : "Activity", 86 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 87 | "activityDefinition" : { 88 | "name" : { 89 | "en-US" : "Activity 2" 90 | }, 91 | "description" : { 92 | "en-US" : "The second activity" 93 | }, 94 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 95 | } 96 | }, 97 | { 98 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-3", 99 | "type" : "Activity", 100 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 101 | "activityDefinition" : { 102 | "name" : { 103 | "en-US" : "Activity 3" 104 | }, 105 | "description" : { 106 | "en-US" : "The third activity" 107 | }, 108 | "@context" : "https://w3id.org/xapi/profiles/activity-context" 109 | } 110 | }, 111 | { 112 | "id": "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#didnt", 113 | "inScheme": "https://xapinet.org/xapi/yet/calibration-a/v1", 114 | "type": "Verb", 115 | "definition": { 116 | "en": "Didn't do the thing" 117 | }, 118 | "prefLabel": { 119 | "en": "didnt" 120 | } 121 | }, 122 | { 123 | "id": "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#did", 124 | "inScheme": "https://xapinet.org/xapi/yet/calibration-a/v1", 125 | "type": "Verb", 126 | "definition": { 127 | "en": "Did the thing" 128 | }, 129 | "prefLabel": { 130 | "en": "did" 131 | } 132 | } 133 | ], 134 | "templates" : [ 135 | { 136 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/templates#activity-1", 137 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 138 | "prefLabel" : { 139 | "en" : "Activity Template 1" 140 | }, 141 | "definition" : { 142 | "en" : "The statement template and rules associated with Activity 1 getting done." 143 | }, 144 | "type" : "StatementTemplate", 145 | "verb" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#did", 146 | "rules" : [ 147 | { 148 | "location" : "$.id", 149 | "presence" : "included" 150 | }, 151 | { 152 | "location" : "$.timestamp", 153 | "presence" : "included" 154 | }, 155 | { 156 | "any" : [ 157 | "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-1" 158 | ], 159 | "location" : "$.object.id", 160 | "presence" : "included" 161 | }, 162 | { 163 | "any" : [ 164 | "Activity 1" 165 | ], 166 | "location" : "$.object.definition.name.en-US", 167 | "presence" : "included" 168 | }, 169 | { 170 | "any" : [ 171 | "The first Activity" 172 | ], 173 | "location" : "$.object.definition.description.en-US", 174 | "presence" : "included" 175 | }, 176 | { 177 | "any" : [ 178 | "https://xapinet.org/xapi/yet/calibration-a/v1" 179 | ], 180 | "location" : "$.context.contextActivities.category[0].id", 181 | "presence" : "included" 182 | } 183 | ] 184 | }, 185 | { 186 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/templates#activity-2", 187 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 188 | "prefLabel" : { 189 | "en" : "Activity Template 2" 190 | }, 191 | "definition" : { 192 | "en" : "The statement template and rules associated with Activity 2 getting done." 193 | }, 194 | "type" : "StatementTemplate", 195 | "verb" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#didnt", 196 | "rules" : [ 197 | { 198 | "location" : "$.id", 199 | "presence" : "included" 200 | }, 201 | { 202 | "location" : "$.timestamp", 203 | "presence" : "included" 204 | }, 205 | { 206 | "any" : [ 207 | "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-2" 208 | ], 209 | "location" : "$.object.id", 210 | "presence" : "included" 211 | }, 212 | { 213 | "any" : [ 214 | "Activity 2" 215 | ], 216 | "location" : "$.object.definition.name.en-US", 217 | "presence" : "included" 218 | }, 219 | { 220 | "any" : [ 221 | "The second Activity" 222 | ], 223 | "location" : "$.object.definition.description.en-US", 224 | "presence" : "included" 225 | }, 226 | { 227 | "any" : [ 228 | "https://xapinet.org/xapi/yet/calibration-a/v1" 229 | ], 230 | "location" : "$.context.contextActivities.category[0].id", 231 | "presence" : "included" 232 | } 233 | ] 234 | }, 235 | { 236 | "id" : "https://xapinet.org/xapi/yet/calibration-a/v1/templates#activity-3", 237 | "inScheme" : "https://xapinet.org/xapi/yet/calibration-a/v1", 238 | "prefLabel" : { 239 | "en" : "Activity Template 3" 240 | }, 241 | "definition" : { 242 | "en" : "The statement template and rules associated with Activity 3 getting done." 243 | }, 244 | "type" : "StatementTemplate", 245 | "verb" : "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#did", 246 | "rules" : [ 247 | { 248 | "location" : "$.id", 249 | "presence" : "included" 250 | }, 251 | { 252 | "location" : "$.timestamp", 253 | "presence" : "included" 254 | }, 255 | { 256 | "any" : [ 257 | "https://xapinet.org/xapi/yet/calibration-a/v1/concepts#activity-3" 258 | ], 259 | "location" : "$.object.id", 260 | "presence" : "included" 261 | }, 262 | { 263 | "any" : [ 264 | "Activity 3" 265 | ], 266 | "location" : "$.object.definition.name.en-US", 267 | "presence" : "included" 268 | }, 269 | { 270 | "any" : [ 271 | "The third Activity" 272 | ], 273 | "location" : "$.object.definition.description.en-US", 274 | "presence" : "included" 275 | }, 276 | { 277 | "any" : [ 278 | "https://xapinet.org/xapi/yet/calibration-a/v1" 279 | ], 280 | "location" : "$.context.contextActivities.category[0].id", 281 | "presence" : "included" 282 | } 283 | ] 284 | } 285 | ] 286 | } 287 | -------------------------------------------------------------------------------- /dev-resources/profiles/test.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "https://xapinet.org/xapi/yet/test-profile", 3 | "@context" : "https://w3id.org/xapi/profiles/context", 4 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0", 5 | "type" : "Profile", 6 | "prefLabel" : { 7 | "en" : "Profile Server Test xAPI Profile" 8 | }, 9 | "definition" : { 10 | "en" : "This xAPI Profile contains additional Concepts that can be used for testing purposes." 11 | }, 12 | "author" : { 13 | "url" : "https://www.yetanalytics.com/", 14 | "name" : "Yet Analytics", 15 | "type" : "Organization" 16 | }, 17 | "versions" : [ 18 | { 19 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1", 20 | "generatedAtTime" : "2021-04-20T14:06:32.000Z" 21 | }, 22 | { 23 | "id" : "https://xapinet.org/xapi/yet/test-profile/v0", 24 | "generatedAtTime" : "2020-03-24T19:16:07.395Z" 25 | } 26 | ], 27 | "concepts" : [ 28 | { 29 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-0", 30 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 31 | "type" : "Verb", 32 | "deprecated" : true, 33 | "prefLabel" : { 34 | "en" : "Verb 0" 35 | }, 36 | "definition" : { 37 | "en" : "This Verb Has Been Deprecated" 38 | } 39 | }, 40 | { 41 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-1", 42 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 43 | "type" : "Verb", 44 | "deprecated" : false, 45 | "prefLabel" : { 46 | "en" : "Verb 1" 47 | }, 48 | "definition" : { 49 | "lat" : "Lorem Ipsum." 50 | }, 51 | "broader" : ["https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-2"], 52 | "broadMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#did"], 53 | "relatedMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#did"], 54 | "exactMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#did"] 55 | }, 56 | { 57 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-2", 58 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 59 | "type" : "Verb", 60 | "deprecated" : true, 61 | "prefLabel" : { 62 | "en" : "Verb 2" 63 | }, 64 | "definition": { 65 | "lat" : "Lorem Ipsum." 66 | }, 67 | "narrower" : ["https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-1"], 68 | "related" : ["https://xapinet.org/xapi/yet/test-profile/v1/concepts#verb-1"], 69 | "narrowMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt"], 70 | "relatedMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt"], 71 | "exactMatch" : ["https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt"] 72 | }, 73 | { 74 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#activity-type", 75 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 76 | "type" : "ActivityType", 77 | "deprecated" : false, 78 | "prefLabel" : { 79 | "en" : "Activity Type 1" 80 | }, 81 | "definition" : { 82 | "lat" : "Lorem Ipsum." 83 | } 84 | }, 85 | { 86 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#activity-ext", 87 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 88 | "type" : "ActivityExtension", 89 | "prefLabel" : { 90 | "en" : "Activity Extension 1" 91 | }, 92 | "definition" : { 93 | "lat" : "Lorem Ipsum." 94 | }, 95 | "recommendedActivityTypes" : [ 96 | "https://xapinet.org/xapi/yet/test-profile/v1/concepts#activity-type" 97 | ], 98 | "context" : "https://example.org/context", 99 | "inlineSchema" : "{\"type\" : \"number\"}" 100 | }, 101 | { 102 | "id" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#activity", 103 | "inScheme" : "https://xapinet.org/xapi/yet/test-profile/v1", 104 | "type" : "Activity", 105 | "activityDefinition" : { 106 | "@context" : "https://w3id.org/xapi/profiles/activity-context", 107 | "name" : { 108 | "en" : "Activity 1" 109 | }, 110 | "description" : { 111 | "lat" : "Lorem Ipsum" 112 | }, 113 | "type" : "https://xapinet.org/xapi/yet/test-profile/v1/concepts#activity-type", 114 | "interactionType" : "choice", 115 | "correctResponsesPattern" : [ 116 | "choice-a", 117 | "choice-b", 118 | "choice-c", 119 | "choice-d" 120 | ], 121 | "choices" : [ 122 | { 123 | "id" : "choice-a", 124 | "description" : { 125 | "en" : "Choice A" 126 | } 127 | }, 128 | { 129 | "id" : "choice-b", 130 | "description" : { 131 | "en" : "Choice B" 132 | } 133 | }, 134 | { 135 | "id" : "choice-c", 136 | "description" : { 137 | "en" : "Choice C" 138 | } 139 | }, 140 | { 141 | "id" : "choice-d", 142 | "description" : { 143 | "en" : "Choice D" 144 | } 145 | } 146 | ] 147 | } 148 | } 149 | ] 150 | } -------------------------------------------------------------------------------- /dev-resources/templates/0_redis.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: Deploy Redis persistence for LRSPipe Jobs 3 | 4 | Parameters: 5 | # Networking 6 | VPCId: 7 | Description: VPC in which to run Redis 8 | Type: AWS::EC2::VPC::Id 9 | Subnets: 10 | Description: Subnets in which to run the Redis Cluster 11 | Type: List 12 | 13 | # Provisioning 14 | CacheNodeType: 15 | Description: Type of Elasticache Cluster Instance 16 | Type: String 17 | Default: cache.t3.micro 18 | NumCacheNodes: 19 | Type: Number 20 | Default: 1 21 | EngineVersion: 22 | Type: String 23 | Default: 5.0.6 24 | 25 | Resources: 26 | SubnetGroup: 27 | Type: AWS::ElastiCache::SubnetGroup 28 | Properties: 29 | Description: Subnet group for access to redis 30 | SubnetIds: !Ref Subnets 31 | RedisSecurityGroup: 32 | Type: AWS::EC2::SecurityGroup 33 | Properties: 34 | GroupDescription: Security group for Redis cache instances 35 | VpcId: !Ref VPCId 36 | RedisCluster: 37 | Type: AWS::ElastiCache::CacheCluster 38 | Properties: 39 | Engine: redis 40 | EngineVersion: !Ref EngineVersion 41 | CacheNodeType: !Ref CacheNodeType 42 | VpcSecurityGroupIds: 43 | - !Ref RedisSecurityGroup 44 | CacheSubnetGroupName: !Ref SubnetGroup 45 | NumCacheNodes: !Ref NumCacheNodes 46 | 47 | Outputs: 48 | EndpointAddress: 49 | Description: The redis endpoint 50 | Value: !GetAtt RedisCluster.RedisEndpoint.Address 51 | Export: 52 | Name: !Sub "${AWS::StackName}:EndpointAddress" 53 | EndpointPort: 54 | Description: The redis endpoint port 55 | Value: !GetAtt RedisCluster.RedisEndpoint.Port 56 | Export: 57 | Name: !Sub "${AWS::StackName}:EndpointPort" 58 | SecurityGroupId: 59 | Description: The security group for the Redis cluser 60 | Value: !Ref RedisSecurityGroup 61 | Export: 62 | Name: !Sub "${AWS::StackName}:SecurityGroup" 63 | -------------------------------------------------------------------------------- /doc/aws.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Example AWS Implementation 4 | 5 | This repo includes a sample AWS CloudFormation Template available at [`dev-resources/templates/lrspipe_ec2.yml`](https://github.com/yetanalytics/xapipe/blob/main/dev-resources/templates/lrspipe_ec2.yml). This template creates an EC2 Instance and launches an LRSPipe job on the server in Amazon Web Services. The template does not create the source or target LRS implementations and requires you to ensure that the Instance has proper access to both. In this section we will discuss how to deploy the LRSPipe template. 6 | 7 | __NOTE:__ *This configuration is not one-size-fits-all and you may require a different configuration for your particular needs. It is provided for demonstration purposes only and can be used as a reference to adapt to your particular enterprise's needs. If you apply these templates in your own AWS account, it can and will incur charges from Amazon Web Services. Yet Analytics is in no way responsible for any charges due to applying and implementing these templates, and is in no way responsible for any outcomes of applying these templates or implementing LRSPipe. If your team is interested in consulting or support in setting up or maintaining LRS Forwarding through LRSPipe or LRS infrastructure in general please [contact Yet here](https://www.sqllrs.com/contact)*. 8 | 9 | ## Deployment 10 | 11 | ### (Optional) Redis Persistence 12 | 13 | By default the LRSPipe CloudFormation template stores job state on the local disk of the EC2 instance running the LRSPipe process. If the instance is terminated all job state is lost. If you would like to persist job state outside of the instance an optional template is provided to do this with AWS ElastiCache Redis. 14 | 15 | To deploy the Redis template: 16 | 17 | - Go to AWS CloudFormation Service 18 | - Choose Create Stack (New Resources) 19 | - Choose 'Template is Ready' / 'Upload a template file' 20 | - Upload the Template `dev-resources/templates/0_redis.yml` 21 | - Click 'Next' 22 | - Choose a stack name and note it down for use in the `RedisStackName` parameter in the next section. 23 | 24 | #### Configuration 25 | 26 | Configure the Redis template parameters. Leave settings at the default where provided and set the following: 27 | 28 | - `Subnets`: Select the AWS VPC subnet(s) in which to run Redis. Note that these must be reachable from the subnet selected for `LrsPipeSubnet` below. 29 | - `VPCId`: Select the AWS VPC in which to run Redis. Make sure this VPC includes the subnet(s) you just selected and matches the `VPCId` you select in the next section. 30 | 31 | Now click 'Next' and proceed to deploy the template. When the template deployment is complete, proceed with the next section. 32 | 33 | ### LRSPipe Process 34 | 35 | Deploy the CloudFormation Template for the LRSPipe process: 36 | 37 | - Go to AWS CloudFormation Service 38 | - Choose Create Stack (New Resources) 39 | - Choose 'Template is Ready' / 'Upload a template file' 40 | - Upload the Template `dev-resources/templates/1_lrspipe_ec2.yml` 41 | - Click 'Next' 42 | 43 | #### Configuration 44 | 45 | ![LRSPipe Template Deployment Options](img/template-options.png) 46 | 47 | On the next page you will be presented with a number of deployment options. We will detail each one below: 48 | 49 | - `Stack Name`: Enter a name to identify this template. 50 | - `InstanceAmiId`: This is the operating system image of the EC2 Instance. We do not recommend changing this from default as this is the setup that has been tested with this template. 51 | - `InstanceKeyName`: If you would like to be able to SSH to the server for diagnostic purposes, enter the SSH key-pair name for the key you would like to use for authorization. If not leave it blank. 52 | - `InstanceSSHCidr`: Provide a whitelisted CIDR Range for access to SSH to the instance. 53 | - `InstanceType`: This is the size of the server to create. We recommend (and have tested) a c5.large for LRSPipe jobs, but you may want to go smaller or larger depending on use-case. 54 | - `LogRetentionInDays`: This tells CloudWatch how long to hold onto application logs. This can affect costs stemming from large amounts of log storage. 55 | - `LrsPipeConfig`: This is where you will paste in your LRSPipe JSON job configuration. This is where all of the actual job configuration takes place. See the [JSON-based Job Config](json.md) page for details and instructions on this step. 56 | - `LrsPipeSubnet`: This field, in conjunction with the `VPCId` determine where in your network the LRSPipe instance is created. This is important as it may impact access to the source and target LRS'. If your LRS' are hosted in the same AWS account make sure this Subnet has access to them. Alternatively if your LRS' are hosted externally, make sure this Subnet has internet access configured. 57 | - `LrsPipeVersion`: This is the version of LRSPipe software you want to deploy. This can be used to upgrade the version on a running instance as well. For a list of LRSPipe versions visit [releases](https://github.com/yetanalytics/xapipe/releases). 58 | - `RedisStackName`: If you are using Redis persistence, provide the CloudFormation stack name of the Redis stack you set up previously. Otherwise, leave this blank. 59 | - `VPCId`: This field controls which Virtual Private Cloud the Instance resides in. Make sure that the chosen VPC contains the Subnet from that previous step. 60 | 61 | Now click 'Next' and proceed to deploy the template. 62 | 63 | ## Monitoring 64 | 65 | ![LRSPipe Template Deployment Options](img/resources.png) 66 | 67 | This template comes with a CloudWatch configuration that allows you to view the logs directly in AWS. Once the template is deployed, in order to find the logs, go to the resources tab as pictured below and click the `LogGroup` link. This will take you to the index of logs available for viewing. *Note: Keep in mind that CloudWatch has a slight publishing delay from the time that an event occurs in the system.* These logs can be used to ensure that the LRSPipe configuration was deployed successfully, or to debug issues that may arise. 68 | 69 | [<- Back to Index](index.md) 70 | -------------------------------------------------------------------------------- /doc/demo.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Example Forwarding Demo 4 | 5 | This repo includes a Docker Compose file at [`demo/docker-compose.yml`](https://github.com/yetanalytics/xapipe/blob/main/demo/docker-compose.yml) that creates source and target LRS instances using [SQL LRS](https://github.com/yetanalytics/lrsql) and uses LRSPipe to forward data between them. This demo only requires having Docker 3.9+ installed. 6 | 7 | To run the demo: 8 | 9 | ``` shell 10 | cd demo 11 | docker compose up 12 | ``` 13 | 14 | This will create a source LRS at `http://0.0.0.0:8080` and a target LRS at `http://0.0.0.0:8081`. The credentials for each can be found (and changed) in the `docker-compose.yml` file. If you send xAPI data to the source it will be forwarded to the target. 15 | 16 | The demo includes a [Prometheus](https://prometheus.io/) metrics server and push gateway. When the demo is running you can navigate to [http://0.0.0.0:9090](http://0.0.0.0:9090) and explore xapipe metrics (see below). 17 | 18 | In addition to prometheus the demo creates a [Grafana](https://github.com/grafana/grafana) server at [http://0.0.0.0:3000](http://0.0.0.0:3000). Log in with username `admin` and password `admin` and set a password, then you can view a comprehensive dashboard with all metrics. See [metrics](metrics.md) for more details. 19 | 20 | [<- Back to Index](index.md) 21 | -------------------------------------------------------------------------------- /doc/docker.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe Docker Container 3 | 4 | For ease of deployment, LRSPipe is also distributed as a Docker container available on [DockerHub](https://hub.docker.com/r/yetanalytics/xapipe). 5 | 6 | The same [options](options.md) as used in the CLI are available as arguments to the container's run command. 7 | 8 | ## Invocation 9 | 10 | Start a job with a persistent volume to store job state: 11 | 12 | ``` shell 13 | docker run -v xapipe:/xapipe/store -it yetanalytics/xapipe \ 14 | --source-url http://host.docker.internal:8080/xapi \ 15 | --target-url http://host.docker.internal:8081/xapi \ 16 | --job-id myjob 17 | ``` 18 | 19 | Stop the job with `^C`. You can then resume it: 20 | 21 | ``` shell 22 | docker run -v xapipe:/xapipe/store -it yetanalytics/xapipe --job-id myjob 23 | ``` 24 | 25 | [<- Back to Index](index.md) 26 | -------------------------------------------------------------------------------- /doc/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/xapipe/6af3d68299f88dd0dc4d87a63f567536c2ed7810/doc/img/logo.png -------------------------------------------------------------------------------- /doc/img/resources.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/xapipe/6af3d68299f88dd0dc4d87a63f567536c2ed7810/doc/img/resources.png -------------------------------------------------------------------------------- /doc/img/template-options.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/xapipe/6af3d68299f88dd0dc4d87a63f567536c2ed7810/doc/img/template-options.png -------------------------------------------------------------------------------- /doc/index.md: -------------------------------------------------------------------------------- 1 | # LRSPipe 2 | 3 | LRSPipe is a standalone process which runs independently of any LRS. This process can run one or multiple "jobs" at a time, each of which contains information about a source LRS, a target LRS, and any applicable filters or options. While running, this process continually checks the source LRS(s) for xAPI Statements and attempts to replicate them into the target LRS(s). 4 | 5 | This process is one-way and any statements in the target LRS will not be replicated to the source LRS. These jobs can be paused, resumed, and modified, and LRSPipe tracks its progress using storage (either the local file system or a Redis server, depending on how it is configured) so a job can be interrupted at any time and pick right back up where it left off when it resumes. 6 | 7 | ## Documentation 8 | 9 | - [Installation](install.md) 10 | - [Usage](usage.md) 11 | - [Persistence Config](persistence.md) 12 | - [OAuth Support](oauth.md) 13 | - [All Options](options.md) 14 | - [JSON Config](json.md) 15 | - [Metrics](metrics.md) 16 | - [Docker Container](docker.md) 17 | - [Demo](demo.md) 18 | - [Sample AWS Deployment](aws.md) 19 | -------------------------------------------------------------------------------- /doc/install.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe Installation 3 | 4 | ## From Release Distribution 5 | 6 | LRSPipe can be downloaded as cross-platform standalone application. You can download xapipe from our [release page](https://github.com/yetanalytics/xapipe/releases/latest) or on the command line: 7 | 8 | ``` shell 9 | curl -L https://github.com/yetanalytics/xapipe/releases/latest/download/xapipe.zip -o xapipe.zip 10 | unzip xapipe.zip -d xapipe 11 | cd xapipe 12 | bin/run.sh --help 13 | ``` 14 | 15 | ## Build from Source 16 | 17 | If you would like to build LRSPipe from the source code in this repository, you will need Java JDK 11+ and Clojure 1.10+. Clone the repository and from the root directory run the following: 18 | 19 | ``` shell 20 | make bundle 21 | cd target/bundle 22 | bin/run.sh --help 23 | ``` 24 | 25 | For basic usage instructions please see the [usage](usage.md) page. 26 | 27 | [<- Back to Index](index.md) 28 | -------------------------------------------------------------------------------- /doc/json.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe JSON-based Job Config 3 | 4 | Instead of defining all job configuration in [CLI arguments](options.md) it is also possible to put all of your job config into a JSON file and launch jobs from it. In this section we will go over the types of configuration possible with a JSON file and the structure of the file. 5 | 6 | ## JSON Options 7 | 8 | To run a job with a JSON config file simply use: 9 | 10 | `bin/run.sh --json-file [location of JSON config file]` 11 | 12 | Alternatively if you would like to simply provide the JSON config inline as a string, you can use the following: 13 | 14 | `bin/run.sh --json [full JSON config string]` 15 | 16 | Below is an example of all of the JSON fields that can be used in a config file. We will break these down in sections below. 17 | 18 | ```json 19 | { 20 | "id": "NewJobID", 21 | "version": 0, 22 | "config": { 23 | "get-buffer-size": 10, 24 | "statement-buffer-size": 500, 25 | "batch-buffer-size": 10, 26 | "batch-timeout": 200, 27 | "cleanup-buffer-size": 50, 28 | "source": { 29 | "request-config": { 30 | "url-base": "http://localhost:8080", 31 | "xapi-prefix": "/xapi", 32 | "username": "username", 33 | "password": "password" 34 | }, 35 | "get-params": { 36 | "limit": 50 37 | }, 38 | "poll-interval": 1000, 39 | "batch-size": 50, 40 | "backoff-opts": { 41 | "budget": 10000, 42 | "max-attempt": 10 43 | } 44 | }, 45 | "target": { 46 | "request-config": { 47 | "url-base": "http://localhost:9080", 48 | "xapi-prefix": "/xapi", 49 | "username": "username", 50 | "password": "password" 51 | }, 52 | "batch-size": 50, 53 | "backoff-opts": { 54 | "budget": 10000, 55 | "max-attempt": 10 56 | } 57 | }, 58 | "filter": { 59 | "pattern": { 60 | "profile-urls": [ 61 | "https://xapinet.org/xapi/example-profile/v1" 62 | ], 63 | "pattern-ids": [ 64 | "https://xapinet.org/xapi/example-profile/v1/patterns#pattern-1" 65 | ] 66 | } 67 | } 68 | }, 69 | "state": { 70 | "status": "init", 71 | "cursor": "1970-01-01T00:00:00.000000000Z", 72 | "source": { 73 | "errors": [] 74 | }, 75 | "target": { 76 | "errors": [] 77 | }, 78 | "errors": [], 79 | "filter": {} 80 | } 81 | } 82 | ``` 83 | ### ID and Top-level Config Options 84 | ```json 85 | { 86 | "id": "NewJobID", 87 | "version": 0, 88 | "config": { 89 | "get-buffer-size": 10, 90 | "statement-buffer-size": 500, 91 | "batch-buffer-size": 10, 92 | "batch-timeout": 200, 93 | "cleanup-buffer-size": 50, 94 | ... 95 | } 96 | ... 97 | } 98 | ``` 99 | The `id` field can be specified in much the same way as with `--job-id` on the CLI. This will specify an ID for a new job, or allow you to resume a previously stored job. 100 | 101 | The optional `version` field denotes the version of the job specification itself. If not present it will be assumed to be the latest version. JSON jobs specifying older versions will be upgraded if possible or an error will be returned. 102 | 103 | The other options under the `config` map are optional and will default to the values specific on the [options guide](options.md). When restarting an existing job these values can be changed via giving them values here, or they will retain the previous job definition's values. 104 | 105 | ### Source and Target 106 | 107 | ```json 108 | { 109 | "config": { 110 | "source": { 111 | "request-config": { 112 | "url-base": "http://localhost:8080", 113 | "xapi-prefix": "/xapi", 114 | "username": "username", 115 | "password": "password" 116 | }, 117 | "get-params": { 118 | "limit": 50 119 | }, 120 | "poll-interval": 1000, 121 | "batch-size": 50, 122 | "backoff-opts": { 123 | "budget": 10000, 124 | "max-attempt": 10 125 | } 126 | }, 127 | "target": { 128 | "request-config": { 129 | "url-base": "http://localhost:9080", 130 | "xapi-prefix": "/xapi", 131 | "username": "username", 132 | "password": "password" 133 | }, 134 | "batch-size": 50, 135 | "backoff-opts": { 136 | "budget": 10000, 137 | "max-attempt": 10 138 | } 139 | }, 140 | ... 141 | } 142 | ... 143 | } 144 | ``` 145 | Source and Target are responsible for providing information about the respective source and target LRS configurations. All of the options correspond to the `--source-*` and `--target-*` arguments in the CLI reference. The only difference is you have to set `xapi-prefix` as a separate field. Any option with a default can be omitted from the JSON config and it will take the default value or the previously set value if resuming a stored job. 146 | 147 | These sections can be used to update LRS config for paused jobs, for instance in the case where LRS credentials are changed. 148 | 149 | ### Filter 150 | 151 | ```json 152 | { 153 | "config": { 154 | "filter": { 155 | "pattern": { 156 | "profile-urls": [ 157 | "https://xapinet.org/xapi/example-profile/v1" 158 | ], 159 | "pattern-ids": [ 160 | "https://xapinet.org/xapi/example-profile/v1/patterns#pattern-1" 161 | ] 162 | } 163 | }, 164 | ... 165 | } 166 | ... 167 | } 168 | ``` 169 | The filter map contains all of the filtering options passed to LRS Pipe. Much like the other sections these fields correspond to the many filter CLI args found on the [options page](options.md). We will cover a few basic scenarios here. 170 | 171 | #### Pattern Filtering 172 | 173 | In the snippet in the above section you can see how an xAPI Profile and a specific Pattern ID might be provided, analogous to the use of `--pattern-profile-url` and `--pattern-id` args. 174 | 175 | #### Template Filtering 176 | 177 | ```json 178 | { 179 | "config": { 180 | "filter": { 181 | "template": { 182 | "profile-urls": [ 183 | "https://xapinet.org/xapi/example-profile/v1" 184 | ], 185 | "template-ids": [ 186 | "https://xapinet.org/xapi/example-profile/v1/templates#template-1" 187 | ] 188 | } 189 | }, 190 | ... 191 | } 192 | ... 193 | } 194 | ``` 195 | Above you can see how you would apply the same structure but for a template-based filter. 196 | 197 | #### Concept Filtering 198 | 199 | ```json 200 | { 201 | "config": { 202 | "filter": { 203 | "concept": { 204 | "profile-urls": [ 205 | "https://xapinet.org/xapi/example-profile/v1" 206 | ], 207 | "concept-types": [ 208 | "Verb" 209 | ], 210 | "activity-type-ids": [], 211 | "verb-ids": [ 212 | "https://xapinet.org/xapi/example-profile/v1/concepts#verb-1" 213 | ], 214 | "attachment-usage-types": [] 215 | } 216 | }, 217 | ... 218 | }, 219 | ... 220 | } 221 | ``` 222 | 223 | Above you can see how you can choose a Profile, what types of concepts to filter, and specific IDs to filter for. 224 | 225 | #### JsonPath Filtering 226 | 227 | ```json 228 | { 229 | "config": { 230 | "filter": { 231 | "path": { 232 | "ensure-paths": [ 233 | [["result"],["score"],["scaled"]] 234 | ], 235 | "match-paths": [ 236 | [[["verb"],["id"]],"http://example.com/verb"] 237 | ] 238 | } 239 | } 240 | } 241 | } 242 | ``` 243 | The above example shows both an `ensure-path` (`$.result.score.scaled`) and `match-path` (`$.verb.id=http://example.com/verb`) filter. Note that in the JSON representation these are not stored as JSONPath syntax and are rather an internal format LRSPipe uses. If this is unintuitive, you can generate the appropriate syntax from JSONPath using the instructions below for Generating JSON Config. 244 | 245 | ### State 246 | 247 | ```json 248 | { 249 | "state": { 250 | "status": "init", 251 | "cursor": "1970-01-01T00:00:00.000000000Z", 252 | "source": { 253 | "errors": [] 254 | }, 255 | "target": { 256 | "errors": [] 257 | }, 258 | "errors": [], 259 | "filter": {} 260 | } 261 | } 262 | ``` 263 | Manipulating state in JSON config can become exceptionally complex and dangerous as this is the raw data representation of the running state of a job and out of scope of usage documentation. It is recommended that you simply preserve this section as-is. For resuming jobs this section will be entirely ignored in favor of the stored state. For that reason we will only cover one possibly relevant field here. 264 | 265 | - `cursor`: This field tracks the progress of where to load statements (by `stored` time) from the source LRS. By modifying it you can limit the synchronized statements by stored time. *NOTE: This cannot be changed for an existing stored job. Probably the safer way of doing this is to add the `since` key to `get-params` to make the source query start later than epoch.* 266 | 267 | ## Generating JSON Config 268 | 269 | If you would like to convert CLI job args into a config file, for debugging purposes or to help build a working saved config, there is an option to do this. When running LRSPipe from CLI simply add 270 | 271 | `--json-out [desired file location]` 272 | 273 | Rather than run the job, LRSPipe will write the JSON equivalent configuration to the filename provided. 274 | 275 | [<- Back to Index](index.md) 276 | -------------------------------------------------------------------------------- /doc/metrics.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | 3 | # Metrics 4 | 5 | The xapipe CLI supports prometheus metrics via a [push gateway](https://github.com/prometheus/pushgateway). With a push gateway set up, you can use it like so: 6 | 7 | ``` shell 8 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 9 | --target-url http://0.0.0.0:8081/xapi \ 10 | --metrics-reporter prometheus \ 11 | --prometheus-push-gateway 0.0.0.0:9091 12 | ``` 13 | 14 | The following Prometheus metrics are implemented: 15 | 16 | ## Counters 17 | 18 | * `xapipe_statements` 19 | * `xapipe_attachments` 20 | * `xapipe_job_errors` 21 | * `xapipe_source_errors` 22 | * `xapipe_target_errors` 23 | * `xapipe_all_errors` 24 | 25 | ## Histograms 26 | 27 | * `xapipe_source_request_time` 28 | * `xapipe_target_request_time` 29 | 30 | In addition the prometheus metrics collector provides various metrics under the `jvm_` prefix. 31 | 32 | [<- Back to Index](index.md) 33 | -------------------------------------------------------------------------------- /doc/oauth.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # OAuth Support 3 | 4 | LRSPipe supports the use of [OAuth 2.0](https://oauth.net/2/) with LRS endpoints that support it via the [Client Credentials Grant](https://tools.ietf.org/html/rfc6749#section-4.4) 5 | 6 | ## Client Credentials Grant 7 | 8 | To use OAuth, specify a source/target `auth-uri`, `client-id` and `client-secret`: 9 | 10 | ``` shell 11 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 12 | --source-auth-uri http://0.0.0.0:8083/auth/realms/test/protocol/openid-connect \ 13 | --source-client-id a_client_id \ 14 | --source-client-secret 1234 \ 15 | --target-url http://0.0.0.0:8081/xapi \ 16 | --target-auth-uri http://0.0.0.0:8083/auth/realms/test/protocol/openid-connect \ 17 | --target-client-id b_client_id \ 18 | --target-client-secret 1234 19 | ``` 20 | 21 | LRSPipe will connect to the specified auth provider(s) and provide up-to-date tokens for LRS requests as needed. 22 | 23 | ### Scope 24 | 25 | According to OAuth 2.0 an optional `scope` parameter can be provided on Client Credentials Grant requests. To set this value for the source/target LRS: 26 | 27 | ``` shell 28 | bin/run.sh ... \ 29 | --source-scope "lrs:read" \ 30 | --target-scope "lrs:write" 31 | ``` 32 | 33 | Note that the configuration of claims like scope should be done on the OAuth client itself. This option is provided for backwards compatibility only. 34 | 35 | ## Manual Bearer Token Usage 36 | 37 | If you have a bearer token that will be valid for the duration of your job, you can pass it directly: 38 | 39 | ``` shell 40 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 41 | --source-token eyJhbGciOi... 42 | --target-url http://0.0.0.0:8081/xapi \ 43 | --target-token eyJhbGciOi... 44 | ``` 45 | 46 | [<- Back to Index](index.md) 47 | -------------------------------------------------------------------------------- /doc/options.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe Options Reference 3 | 4 | Below is an reference and explanation of all options for running LRSPipe. This reference can be accessed from the CLI at any time by running `bin/run.sh --help`. 5 | 6 | ## CLI Options 7 | 8 | ``` 9 | Run a new job: 10 | --source-url http://0.0.0.0:8080/xapi --target-url http://0.0.0.0:8081/xapi 11 | 12 | Resume a paused job: 13 | --job-id 14 | 15 | Force Resume a job with errors: 16 | --job-id -f 17 | 18 | List All Jobs: 19 | --list-jobs 20 | 21 | Delete a Job: 22 | --delete-job 23 | 24 | All options: 25 | -h, --help Show the help and options summary 26 | --job-id ID Job ID 27 | --conn-timeout TIMEOUT Connection Manager Connection Timeout 28 | --conn-threads THREADS Connection Manager Max Threads 29 | --conn-default-per-route CONNS Connection Manager Simultaneous Connections Per Host 30 | --conn-insecure? Allow Insecure HTTPS Connections 31 | --conn-io-thread-count THREADS Connection Manager I/O Thread Pool Size, default is number of processors 32 | --show-job Show the job and exit 33 | --list-jobs List jobs in persistent storage 34 | --delete-job ID Delete the job specified and exit. 35 | -f, --force-resume If resuming a job, clear any errors and force it to resume. 36 | --json JSON Take a job specification as a JSON string 37 | --json-file FILE Take a job specification from a JSON file 38 | --json-out FILE Write JOB to a JSON file 39 | -s, --storage STORAGE :file Select storage backend, file (default), redis or noop, mem is for testing only 40 | --redis-uri URI redis://0.0.0.0:6379 Redis Connection URI 41 | --redis-prefix PREFIX xapipe Redis key prefix 42 | --file-store-dir PATH store Directory path for filesystem storage 43 | --metrics-reporter REPORTER noop Select a metrics reporter, noop (default) or prometheus 44 | --prometheus-push-gateway URL 0.0.0.0:9091 Address of prometheus push gateway server 45 | --source-url URL Source LRS xAPI Endpoint 46 | --source-batch-size SIZE 50 Source LRS GET limit param 47 | --source-poll-interval INTERVAL 1000 Source LRS GET poll timeout 48 | -p, --xapi-get-param KEY=VALUE {} xAPI GET Parameters 49 | --source-username USERNAME Source LRS BASIC Auth username 50 | --source-password PASSWORD Source LRS BASIC Auth password 51 | --json-only Only operate in JSON statement mode for data transfer, ignoring Attachments/multipart (for compatibility issues) 52 | --source-auth-uri URI Source LRS OAuth autentication URI 53 | --source-client-id ID Source LRS OAuth client ID 54 | --source-client-secret SECRET Source LRS OAuth client secret 55 | --source-scope SCOPE Source LRS OAuth scope 56 | --source-token TOKEN Source LRS OAuth Bearer token 57 | --source-backoff-budget BUDGET 10000 Source LRS Retry Backoff Budget in ms 58 | --source-backoff-max-attempt MAX 10 Source LRS Retry Backoff Max Attempts, set to -1 for no retry 59 | --source-backoff-j-range RANGE Source LRS Retry Backoff Jitter Range in ms 60 | --source-backoff-initial INITIAL Source LRS Retry Backoff Initial Delay 61 | --target-url URL Target LRS xAPI Endpoint 62 | --target-batch-size SIZE 50 Target LRS POST desired batch size 63 | --target-username USERNAME Target LRS BASIC Auth username 64 | --target-password PASSWORD Target LRS BASIC Auth password 65 | --target-auth-uri URI Target LRS OAuth autentication URI 66 | --target-client-id ID Target LRS OAuth client ID 67 | --target-client-secret SECRET Target LRS OAuth client secret 68 | --target-scope SCOPE Target LRS OAuth scope 69 | --target-token TOKEN Target LRS OAuth Bearer token 70 | --target-backoff-budget BUDGET 10000 Target LRS Retry Backoff Budget in ms 71 | --target-backoff-max-attempt MAX 10 Target LRS Retry Backoff Max Attempts, set to -1 for no retry 72 | --target-backoff-j-range RANGE Target LRS Retry Backoff Jitter Range in ms 73 | --target-backoff-initial INITIAL Target LRS Retry Backoff Initial Delay 74 | --get-buffer-size SIZE 10 Size of GET response buffer 75 | --batch-timeout TIMEOUT 200 Msecs to wait for a fully formed batch 76 | --template-profile-url URL [] Profile URL/location from which to apply statement template filters 77 | --template-id IRI [] Statement template IRIs to filter on 78 | --pattern-profile-url URL [] Profile URL/location from which to apply statement pattern filters 79 | --pattern-id IRI [] Pattern IRIs to filter on 80 | --ensure-path JSONPATH [] A JSONPath expression used to filter statements to only those with data at the given path 81 | --match-path JSONPATH=JSON [] A JSONPath expression and matching value used to filter statements to only those with data matching the value at the given path 82 | --concept-profile-url IRI [] Profile URL/location from which to apply concept filters 83 | --concept-type CONCEPT-TYPE [] Specific type of concept to filter on. If not set, it will match all concepts in the Profile. 84 | --activity-type-id IRI [] Activity Type IRIs to filter on. If left blank it will match all Activity Types in the Profile 85 | --verb-id IRI [] Verb IRIs to filter on. If left blank it will match all Verbs in the Profile 86 | --attachment-usage-type IRI [] Attachment Usage Type IRIs to filter on. If left blank it will match all Attachment usage types in the Profile 87 | --statement-buffer-size SIZE Desired size of statement buffer 88 | --batch-buffer-size SIZE Desired size of statement batch buffer 89 | --cleanup-buffer-size SIZE Desired size of tempfile cleanup buffer 90 | ``` 91 | 92 | [<- Back to Index](index.md) 93 | -------------------------------------------------------------------------------- /doc/persistence.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe Persistence Configuration 3 | 4 | LRSPipe uses storage to save job status and progress. This enables resuming paused jobs and finding out the status of a job. In this section we will look at a few configurable options for how LRSPipe stores it's state. 5 | 6 | ## File System (Default) 7 | 8 | Be default, LRSPipe uses the local filesystem for storage. It stores job data in individual files in a directory. Without specifying any options it will store these files in the directory `/store` at the root of where you run the job (presumably your unzipped LRSPipe release directory). You can change that location with the following argument: 9 | 10 | ``` shell 11 | bin/run.sh ... \ 12 | --file-store-dir "../desired-storage-dir" \ 13 | ... 14 | ``` 15 | 16 | The directory location specified will be the storage location for job files. 17 | 18 | _NOTE: If you have existing jobs and you change locations between runs LRSPipe will no longer be able to find them._ 19 | 20 | ## Redis 21 | 22 | If you wish to instead store job details in a Redis server you can do so by specifying the Redis server connection URI below: 23 | 24 | ``` shell 25 | bin/run.sh ... \ 26 | --storage redis 27 | --redis-uri URI \ 28 | ... 29 | ``` 30 | The URI should be in Redis format, which includes the following possible formats: 31 | 32 | ``` 33 | redis://HOST[:PORT][?db=DATABASE[&password=PASSWORD]] 34 | redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]] 35 | redis://[:PASSWORD@]HOST[:PORT][/DATABASE] 36 | redis://[:PASSWORD@]HOST[:PORT][?db=DATABASE] 37 | redis://HOST[:PORT]/DATABASE[?password=PASSWORD] 38 | ``` 39 | If you omit the URI it will default to `redis://0.0.0.0:6379`. 40 | 41 | Additionally if you wish to provide a custom string such that all LRSPipe keys contain it as a prefix (e.g. in the case of a shared-use Redis server), you can do so with the `--redis-prefix` flag. If you do not include this argument, all related Redis keys will be prefixed with `xapipe`. 42 | 43 | ## In-Memory / No-Op 44 | 45 | If you do not wish to store job status and progress at all, and would like LRSPipe to completely refresh on every restart, you can do so by specifying `noop` for the `--storage` flag. Keep in mind this will result in you not being able to resume a job at all and you will lose your progress any time the process is interrupted. 46 | 47 | [<- Back to Index](index.md) 48 | -------------------------------------------------------------------------------- /doc/usage.md: -------------------------------------------------------------------------------- 1 | [<- Back to Index](index.md) 2 | # LRSPipe Basic Usage 3 | 4 | In this section we'll illustrate a few examples of basic usage patterns of LRSPipe. 5 | 6 | ## Basic Forwarding 7 | 8 | ### Start a New Job 9 | 10 | In this example we are starting a basic forwarding job with no filters from a source LRS at `0.0.0.0:8080` to a target LRS at `0.0.0.0:8081`. We provide it with a `job-id` which we can reference later in the case that we need to stop, resume, or modify the job. Once initialized this job will forward all existing statements, and then remain active checking for new statements in the source LRS. 11 | 12 | ``` shell 13 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 14 | --target-url http://0.0.0.0:8081/xapi \ 15 | --job-id myjob \ 16 | --source-username my_key --source-password my_secret \ 17 | --target-username my_key --target-password my_secret 18 | ``` 19 | 20 | ### Resume a Paused Job 21 | 22 | If a job has been started in the past but is not actively running, it can be resumed using only the `job-id`. The system remembers the LRS details and how much it has already forwarded. 23 | 24 | ``` shell 25 | bin/run.sh --job-id myjob 26 | ``` 27 | 28 | ### Force-Resume a Job with Errors 29 | 30 | In some cases, when a job has been paused due to an error, it may need to be force-resumed with the `-f` flag below. 31 | 32 | ``` shell 33 | bin/run.sh --job-id myjob -f 34 | ``` 35 | 36 | ## Forwarding with Filtering 37 | 38 | ### Statement Template Filtering 39 | 40 | To filter statements based on xAPI Profile Statement Templates, use the `--template-profile-url` flag like so: 41 | 42 | ``` shell 43 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 44 | --target-url http://0.0.0.0:8081/xapi \ 45 | --job-id template-job-1 \ 46 | --template-profile-url "../location-of-profile.jsonld" \ 47 | --template-id "https://xapinet.org/xapi/yet/template-id-1" \ 48 | --template-id "https://xapinet.org/xapi/yet/template-id-2" \ 49 | --source-username my_key --source-password my_secret \ 50 | --target-username my_key --target-password my_secret 51 | 52 | ``` 53 | 54 | The profile url value can be either a web-accessible url, such as a Profile Server, or a local file. The `--template-id` flags are optional and further limit the forwarding to only the desired Templates. If the `--template-id` flag is omitted the job will filter on all available Statement Templates in the Profile. 55 | 56 | ### Pattern Filtering 57 | 58 | To filter statements based on xAPI Profile Patterns, use the `--pattern-profile-url` flag like so: 59 | 60 | ``` shell 61 | bin/run.sh --source-url http://0.0.0.0:8080/xapi \ 62 | --target-url http://0.0.0.0:8081/xapi \ 63 | --job-id pattern-job-1 \ 64 | --pattern-profile-url "../location-of-profile.jsonld" \ 65 | --pattern-id "https://xapinet.org/xapi/yet/pattern-id-1" \ 66 | --pattern-id "https://xapinet.org/xapi/yet/pattern-id-2" \ 67 | --source-username my_key --source-password my_secret \ 68 | --target-username my_key --target-password my_secret 69 | 70 | ``` 71 | 72 | As with Template Filtering, the `--pattern-id` flags are optional and further limit the forwarding to only the desired Patterns. If the `--pattern-id` flag is omitted the job will filter on all available Patterns in the Profile. 73 | 74 | ### Concept Filtering 75 | 76 | You can also filter on xAPI Profile Concepts alone, without matching specific Templates in the Profile. You can filter on `Verb`, `ActivityType` and/or `AttachmentUsageType`. You can either provide a Profile and specify the types of Concepts to filter on, or you can provide specific IDs. If you provide specific IDs they will override whatever concepts are in the Profile. 77 | 78 | Below is an example of filtering on all the Verbs in a Profile: 79 | 80 | ``` shell 81 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 82 | --target-url http://0.0.0.0:8081/xapi \ 83 | --job-id concept-job-1 \ 84 | --source-username my_key --source-password my_secret \ 85 | --target-username my_key --target-password my_secret \ 86 | --concept-profile-url "../location-of-profile.jsonld" \ 87 | --concept-type "Verb" 88 | ``` 89 | 90 | If you omit concept-type(s) it will filter on all concept types in the Profile. Additionally if you provide a concept-type it will not match any other concepts even if you provide specific IDs. 91 | 92 | This is an example of filtering on two specific verb ids: 93 | 94 | ``` shell 95 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 96 | --target-url http://0.0.0.0:8081/xapi \ 97 | --job-id concept-job-1 \ 98 | --source-username my_key --source-password my_secret \ 99 | --target-username my_key --target-password my_secret \ 100 | --verb-id "https://xapinet.org/xapi/yet/verb-id-1" 101 | --verb-id "https://xapinet.org/xapi/yet/verb-id-2" 102 | ``` 103 | 104 | ### JsonPath Presence Filtering 105 | 106 | In cases where you want to ensure that every statement posted to the target LRS has data at a given path, use the `--ensure-path` option: 107 | 108 | ``` shell 109 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 110 | --target-url http://0.0.0.0:8081/xapi \ 111 | --job-id path-job-1 \ 112 | --source-username my_key --source-password my_secret \ 113 | --target-username my_key --target-password my_secret \ 114 | --ensure-path $.result.score.scaled 115 | 116 | ``` 117 | 118 | Only statements with a value set at the given [JsonPath String](https://goessner.net/articles/JsonPath/) `$.result.score.scaled` will be passed to the target LRS. This simple filter is useful to ensure the density of an xAPI dataset. 119 | 120 | If the option is passed multiple times only statements that contain data at ALL paths will be written to the target LRS. 121 | 122 | To match a value at a given path see JsonPath Matching below. For more complex filters with features like negation use an xAPI Profile with Template and Pattern Filtering (see above). 123 | 124 | ### JsonPath Matching 125 | 126 | You can apply simple path-matching filters to LRSPipe using the `--match-path` option: 127 | 128 | ``` shell 129 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 130 | --target-url http://0.0.0.0:8081/xapi \ 131 | --job-id path-match-job-1 \ 132 | --source-username my_key --source-password my_secret \ 133 | --target-username my_key --target-password my_secret \ 134 | --match-path $.verb.id=http://example.com/verb 135 | 136 | ``` 137 | 138 | Only statements with the value `"http://example.com/verb"` at the path `$.verb.id` will be written to the target LRS. 139 | 140 | If the option is given multiple times, a statement must satisfy at least one given value for each path given: 141 | 142 | ``` shell 143 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 144 | --target-url http://0.0.0.0:8081/xapi \ 145 | --job-id path-match-job-2 \ 146 | --source-username my_key --source-password my_secret \ 147 | --target-username my_key --target-password my_secret \ 148 | --match-path $.verb.id=http://example.com/verb1 \ 149 | --match-path $.verb.id=http://example.com/verb2 150 | 151 | ``` 152 | 153 | Statements with an `$.verb.id` of `http://example.com/verb1` OR `http://example.com/verb2` will be passed. 154 | 155 | #### Json Values 156 | 157 | LRSPipe will attempt to parse path matches as JSON first, then as string. This means you can match a JSON object: 158 | 159 | ``` shell 160 | bin/run-sh --source-url http://0.0.0.0:8080/xapi \ 161 | --target-url http://0.0.0.0:8081/xapi \ 162 | --job-id path-match-job-3 \ 163 | --source-username my_key --source-password my_secret \ 164 | --target-username my_key --target-password my_secret \ 165 | --match-path $.actor='{"mbox":"mailto:bob@example.com","objectType":"Agent"}' 166 | 167 | ``` 168 | 169 | Statements with an `$.actor` matching the fields provided exactly will be passed. 170 | 171 | ## Job Management 172 | 173 | ### List Persisted Jobs 174 | 175 | To see the state of all jobs the command below can be used. 176 | 177 | ``` shell 178 | bin/run.sh --list-jobs -s redis 179 | 180 | Nov 03, 2021 4:41:48 PM com.yetanalytics.xapipe.cli invoke 181 | INFO: Page 0 182 | | job-id | status | cursor | 183 | |--------------------------------------+--------+--------------------------------| 184 | | d24de6cc-ade6-48e9-a23c-c7ee48ed53f9 | error | 1970-01-01T00:00:00.000000000Z | 185 | ``` 186 | 187 | ### Delete Job 188 | 189 | To delete a job entirely and have the system forget the job details, the `--delete-job` flag can be used. 190 | 191 | ``` shell 192 | bin/run.sh --delete-job myjob 193 | ``` 194 | 195 | For a more comprehensive reference of all LRSPipe options, see the [Options](options.md) page. 196 | 197 | [<- Back to Index](index.md) 198 | -------------------------------------------------------------------------------- /resources/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yetanalytics/xapipe/6af3d68299f88dd0dc4d87a63f567536c2ed7810/resources/.keep -------------------------------------------------------------------------------- /resources/doc/docs.html.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Yet LRS Pipe Documentation 5 | 119 | 120 | 121 |
122 |
123 | 126 |
127 | {{content|safe}} 128 |
129 | © 2021- Yet Analytics Inc. | 130 | Licensed under the Apache 2.0 License | 131 | Contribute on GitHub 132 | Build: {{sha | safe}} 133 |
134 |
135 | 136 | 137 | -------------------------------------------------------------------------------- /src/bench/com/yetanalytics/xapipe/bench/maths.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.bench.maths 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen] 4 | [clojure.set :as cset])) 5 | 6 | ;; Origninally Cribbed from Avery and Jean 7 | ;; https://github.com/clojure-cookbook/clojure-cookbook/blob/master/01_primitive-data/1-20_simple-statistics.asciidoc 8 | 9 | (defn mean ^Double [coll] 10 | (let [[sum count] (reduce (fn [[s c] x] 11 | [(+ s x) (inc c)]) 12 | [0.0 0] 13 | coll)] 14 | (if (pos? count) 15 | (double (/ sum count)) 16 | 0.0))) 17 | 18 | (s/fdef mean 19 | :args (s/cat :coll (s/coll-of number?)) 20 | :ret double?) 21 | 22 | (defn median ^Double [coll] 23 | (let [sorted (sort coll) 24 | cnt (count sorted)] 25 | (when (< 0 cnt) 26 | (let [halfway (quot cnt 2)] 27 | (double (if (odd? cnt) 28 | (nth sorted halfway) 29 | (let [bottom (dec halfway) 30 | bottom-val (nth sorted bottom) 31 | top-val (nth sorted halfway)] 32 | (mean [bottom-val top-val])))))))) 33 | 34 | (s/fdef median 35 | :args (s/cat :coll (s/coll-of number?)) 36 | :ret (s/nilable double?)) 37 | 38 | (defn mode [coll] 39 | (if-let [coll (not-empty coll)] 40 | (let [freqs (frequencies coll) 41 | occurrences (group-by val freqs) 42 | modes (last (sort occurrences)) 43 | modes (->> modes 44 | val 45 | (map key))] 46 | modes) 47 | (list))) 48 | 49 | (s/fdef mode 50 | :args (s/cat :coll (s/coll-of any?)) 51 | :ret (s/coll-of any?) 52 | :fn (fn [{{:keys [coll]} :args 53 | ret :ret}] 54 | (cond 55 | (= 1 (count coll)) 56 | (= coll ret) 57 | 58 | (seq coll) 59 | (not-empty (cset/intersection 60 | (set ret) 61 | (set coll))) 62 | 63 | (empty? coll) 64 | (empty? ret) 65 | :else (println coll ret)))) 66 | 67 | 68 | (defn stddev ^Double [coll & {:keys [complete-population?] 69 | :or {complete-population? false}}] 70 | (let [total (count coll)] 71 | (if (< 1 total) 72 | (let [avg (mean coll) 73 | squares (for [x coll] 74 | (let [x-avg (- x avg)] 75 | (* x-avg x-avg)))] 76 | (double 77 | (-> (/ (apply + squares) 78 | (if complete-population? 79 | total 80 | (- total 1))) 81 | (Math/sqrt)))) 82 | 0.0))) 83 | 84 | (s/def ::complete-population? 85 | boolean?) 86 | 87 | (s/fdef stddev 88 | :args (s/cat :coll (s/coll-of number?) 89 | :kwargs (s/? (s/keys* :opt-un [::complete-population?]))) 90 | :ret double?) 91 | -------------------------------------------------------------------------------- /src/build/com/yetanalytics/xapipe/build.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.build 2 | (:require [clojure.tools.build.api :as b])) 3 | 4 | (def lib 'com.yetanalytics/xapipe) 5 | 6 | (def class-dir "target/classes") 7 | 8 | (def basis 9 | (b/create-basis 10 | {:project "deps.edn" 11 | :aliases [:cli]})) 12 | 13 | (def uber-file (format "target/bundle/%s.jar" (name lib))) 14 | 15 | (defn uber [_] 16 | (b/copy-dir {:src-dirs ["src/lib" "src/cli" "resources"] 17 | :target-dir class-dir}) 18 | (b/compile-clj {:basis basis 19 | :src-dirs ["src/lib" "src/cli"] 20 | :class-dir class-dir}) 21 | (b/uber {:class-dir class-dir 22 | :uber-file uber-file 23 | :basis basis 24 | :main 'com.yetanalytics.xapipe.main})) 25 | -------------------------------------------------------------------------------- /src/cli/com/yetanalytics/xapipe/main.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.main 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.tools.logging :as log] 4 | [com.yetanalytics.xapipe.cli :as cli] 5 | [com.yetanalytics.xapipe.cli.options :as opts] 6 | [com.yetanalytics.xapipe.job :as job] 7 | [com.yetanalytics.xapipe.job.json :as jj] 8 | [com.yetanalytics.xapipe.job.state :as state] 9 | [com.yetanalytics.xapipe.store :as store]) 10 | (:gen-class)) 11 | 12 | (def usage 13 | " 14 | Run a new job: 15 | --source-url http://0.0.0.0:8080/xapi --target-url http://0.0.0.0:8081/xapi 16 | 17 | Resume a paused job: 18 | --job-id 19 | 20 | Force Resume a job with errors: 21 | --job-id -f 22 | 23 | List All Jobs: 24 | --list-jobs 25 | 26 | Delete a Job: 27 | --delete-job 28 | 29 | ") 30 | 31 | (defn main* 32 | [& args] 33 | (try 34 | (let [{{help? :help 35 | ?job-id-arg :job-id 36 | show-job? :show-job 37 | list-jobs? :list-jobs 38 | ?delete-job-id :delete-job 39 | force-resume? :force-resume 40 | ?json :json 41 | ?json-file :json-file 42 | ?json-out :json-out 43 | :as options} :options 44 | :keys [summary]} (opts/args->options args)] 45 | (if help? 46 | {:status 0 47 | :message (str 48 | usage 49 | "All options:\n" 50 | summary)} 51 | (let [store (cli/create-store options)] 52 | (cond 53 | ?delete-job-id 54 | (if (true? (store/delete-job store ?delete-job-id)) 55 | {:status 0 56 | :message "Job Deleted"} 57 | {:status 1 58 | :message "Job Not Deleted"}) 59 | 60 | list-jobs? 61 | (do 62 | (cli/list-store-jobs store) 63 | {:status 0}) 64 | 65 | :else 66 | (let [?from-json (or ?json ?json-file) 67 | _ (when ?from-json 68 | ;; If job-id arg is set, we make sure they match 69 | (when (and ?job-id-arg 70 | (not= ?job-id-arg 71 | (:id ?from-json))) 72 | (throw (ex-info (format "--job-id %s does not match JSON job id %s" 73 | ?job-id-arg (:id ?from-json)))))) 74 | ?job-id (or ?job-id-arg (:id ?from-json)) 75 | ?from-storage (and ?job-id 76 | (store/read-job 77 | store ?job-id)) 78 | 79 | 80 | {job-id :id 81 | :as job} 82 | (cond 83 | ;; Found in storage 84 | ?from-storage 85 | (-> ?from-storage 86 | job/upgrade-job 87 | (cond-> 88 | ;; If the user has requested force resume we clear 89 | force-resume? 90 | (-> (update :state state/clear-errors) 91 | (update :state state/set-status :paused))) 92 | (job/reconfigure-job 93 | (cli/reconfigure-with-options 94 | (:config (or ?from-json ?from-storage)) 95 | ;; reparse the args w/o defaults 96 | (:options 97 | (opts/args->options args 98 | :no-defaults true))))) 99 | ;; Json is provided 100 | ?from-json 101 | (-> ?from-json 102 | job/upgrade-job 103 | (update :config cli/reconfigure-with-options 104 | ;; reparse args w/o defaults 105 | (:options 106 | (opts/args->options args 107 | :no-defaults true)))) 108 | 109 | ;; New from options! 110 | :else 111 | (cli/create-job options)) 112 | reporter (cli/create-reporter job-id options) 113 | new? (not (some? ?from-storage))] 114 | 115 | (if new? 116 | (log/infof 117 | "Created new job %s: %s" 118 | job-id 119 | (jj/job->json 120 | (job/sanitize job))) 121 | (log/infof 122 | "Found existing job %s: %s" 123 | job-id 124 | (jj/job->json 125 | (job/sanitize job)))) 126 | 127 | (cond 128 | ;; Check job for validity! 129 | (not (s/valid? job/job-spec job)) 130 | {:status 1 131 | :message (s/explain-str job/job-spec (job/sanitize job))} 132 | 133 | ;; Check job for Errors! 134 | (job/errors? job) 135 | {:status 1 136 | :message (cli/errors->message (job/all-errors job))} 137 | 138 | show-job? {:status 0 139 | :message (jj/job->json 140 | (job/sanitize job))} 141 | (not-empty 142 | ?json-out) (do 143 | (jj/job->json-file! job ?json-out) 144 | {:status 0 145 | :message (format "Wrote job %s to %s" 146 | job-id ?json-out)}) 147 | :else (do 148 | 149 | (log/infof 150 | (if new? 151 | "Starting job %s" 152 | "Resuming job %s") 153 | job-id) 154 | (cli/handle-job store 155 | job 156 | (cli/options->client-opts options) 157 | reporter)))))))) 158 | (catch Exception ex 159 | {:status 1 160 | :message (ex-message ex)}))) 161 | 162 | (defn -main [& args] 163 | (let [{:keys [status message]} 164 | (try 165 | (apply main* args) 166 | (catch Exception ex 167 | {:status 1 168 | :message (ex-message ex)}))] 169 | (if (zero? status) 170 | (do 171 | (when (not-empty message) 172 | (log/info message)) 173 | (System/exit 0)) 174 | (do 175 | (when (not-empty message) 176 | (log/error message)) 177 | (System/exit status))))) 178 | -------------------------------------------------------------------------------- /src/cli/com/yetanalytics/xapipe/metrics/impl/prometheus.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.metrics.impl.prometheus 2 | "Prometheus metrics implementation for use by the prometheus CLI" 3 | (:require [clojure.spec.alpha :as s] 4 | [com.yetanalytics.xapipe.job :as job] 5 | [com.yetanalytics.xapipe.metrics :as metrics] 6 | [iapetos.core :as pro] 7 | [iapetos.export :as pro-exp] 8 | [iapetos.collector.jvm :as pro-jvm])) 9 | 10 | (s/def ::push-gateway string?) 11 | 12 | (s/fdef prometheus-push-reporter 13 | :args (s/cat :push-gateway ::push-gateway 14 | :job-id ::job/id) 15 | :ret ::metrics/reporter) 16 | 17 | (def collectors 18 | (concat 19 | (map 20 | pro/gauge 21 | metrics/gauge-keys) 22 | (map 23 | pro/counter 24 | metrics/counter-keys) 25 | (map 26 | pro/histogram 27 | metrics/histogram-keys) 28 | (map 29 | pro/summary 30 | metrics/summary-keys) 31 | [(pro-jvm/standard) 32 | (pro-jvm/gc) 33 | (pro-jvm/memory-pools) 34 | (pro-jvm/threads)])) 35 | 36 | (defn prometheus-push-reporter 37 | [push-gateway 38 | job-id] 39 | (let [registry (reduce 40 | pro/register 41 | (pro-exp/pushable-collector-registry 42 | {:push-gateway push-gateway 43 | :job job-id}) 44 | collectors)] 45 | (reify 46 | metrics/Reporter 47 | (-gauge [this k v] 48 | (pro/set registry k v)) 49 | (-counter [this k delta] 50 | (pro/inc registry k delta)) 51 | (-histogram [this k v] 52 | (pro/observe registry k v)) 53 | (-summary [this k v] 54 | (pro/observe registry k v)) 55 | (-flush! [this] 56 | (pro-exp/push! registry))))) 57 | -------------------------------------------------------------------------------- /src/cli/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | logs/lrspipe.log 4 | 5 | 6 | logs/lrspipe.%d{yyyy-MM-dd}.log 7 | 60 8 | 1GB 9 | 10 | 11 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 12 | 13 | 14 | 15 | 16 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/client/json_only.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.client.json-only 2 | "JSON-only statement response handling. Necessary over default JSON parsing 3 | to shim attachments which will not exist" 4 | (:require [clj-http.client :as http-client] 5 | [clojure.spec.alpha :as s] 6 | [com.yetanalytics.xapipe.client.multipart-mixed :as mm])) 7 | 8 | (s/def ::body 9 | (s/keys :req-un 10 | [:xapi.statements.GET.response/statement-result 11 | ::mm/attachments])) 12 | 13 | (s/fdef parse-response 14 | :args (s/cat :response map?) 15 | :ret (s/keys :req-un [::body])) 16 | 17 | (defn parse-response 18 | "Parse + close a json body" 19 | [req resp] 20 | (let [{:keys [body]} (http-client/coerce-json-body req resp false)] 21 | (assoc resp :body {:attachments [] 22 | :statement-result 23 | (reduce-kv 24 | (fn [m k v] 25 | (assoc m (keyword k) v)) 26 | {} 27 | body)}))) 28 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.client.multipart-mixed 2 | "multipart/mixed handling" 3 | (:require [cheshire.core :as json] 4 | [clj-http.util :as hutil] 5 | [clojure.java.io :as io] 6 | [clojure.spec.alpha :as s] 7 | [clojure.spec.gen.alpha :as sgen] 8 | [clojure.string :as cs] 9 | [xapi-schema.spec :as xs] 10 | [xapi-schema.spec.resources :as xsr]) 11 | (:import 12 | [java.io 13 | Writer 14 | PipedOutputStream 15 | PipedInputStream 16 | IOException 17 | InputStream 18 | OutputStream 19 | ByteArrayOutputStream 20 | File] 21 | [org.apache.commons.fileupload 22 | MultipartStream 23 | MultipartStream$MalformedStreamException])) 24 | 25 | (s/def ::tempfile 26 | (s/with-gen 27 | #(instance? File %) 28 | (fn [] 29 | (sgen/return 30 | (doto (File/createTempFile 31 | "xapipe_gen_attachment_" 32 | "") 33 | .deleteOnExit))))) 34 | 35 | (s/fdef create-tempfile! 36 | :args (s/cat :sha2 string?) 37 | :ret ::tempfile) 38 | 39 | (defn create-tempfile! 40 | "Create a unique but identifiable tempfile" 41 | [sha2] 42 | (doto (File/createTempFile 43 | "xapipe_attachment_" 44 | (format "_%s" sha2)) 45 | .deleteOnExit)) 46 | 47 | (s/fdef parse-headers 48 | :args (s/cat :headers string?) 49 | :ret (s/map-of string? string?)) 50 | 51 | (defn parse-headers 52 | "Parse a part's headers and return a map suitable for use in an attachment" 53 | [headers] 54 | (-> headers 55 | cs/split-lines 56 | (->> 57 | (into {} 58 | (map #(cs/split % #":" 2)))))) 59 | 60 | ;; an Attachment is the xapi properties + a temp file 61 | (s/def ::attachment 62 | (s/keys :req-un 63 | [:attachment/sha2 64 | :attachment/contentType 65 | ::tempfile])) 66 | 67 | (s/def ::attachments (s/every ::attachment :gen-max 1)) 68 | 69 | (s/fdef duplicate-attachment 70 | :args (s/cat :attachment ::attachment) 71 | :ret ::attachment) 72 | 73 | (defn duplicate-attachment 74 | "Given an attachment yield a copy" 75 | [{:keys [sha2 tempfile] :as original}] 76 | (let [dup-tempfile (create-tempfile! sha2)] 77 | (io/copy tempfile dup-tempfile) 78 | (assoc original :tempfile dup-tempfile))) 79 | 80 | (s/fdef clean-tempfiles! 81 | :args (s/cat :attachments ::attachments)) 82 | 83 | (defn clean-tempfiles! 84 | "Delete all tempfiles in a series of attachments" 85 | [attachments] 86 | (doseq [{:keys [^File tempfile]} attachments] 87 | (.delete tempfile))) 88 | 89 | ;; WE DONT DO THAT HERE 90 | #_(s/fdef unique-by-sha 91 | :args (s/cat :attachments ::attachments) 92 | :ret ::attachments) 93 | 94 | #_(defn unique-by-sha 95 | [attachments] 96 | (->> attachments 97 | (group-by :sha2) 98 | vals 99 | (mapv first))) 100 | 101 | (s/fdef parse-head 102 | :args (s/cat :stream #(instance? MultipartStream %)) 103 | :ret :xapi.statements.GET.response/statement-result) 104 | 105 | (defn parse-head 106 | "Parse out the head of the stream, a statement result object" 107 | [^MultipartStream stream] 108 | (let [_statement-headers (.readHeaders stream) 109 | result-baos (new ByteArrayOutputStream)] 110 | ;; write the body to the output stream 111 | (.readBodyData stream result-baos) 112 | ;; Return the statement result always 113 | (let [ss-result 114 | (with-open [r (io/reader (.toByteArray result-baos))] 115 | (json/parse-stream r))] 116 | (reduce-kv 117 | (fn [m k v] 118 | (assoc m (keyword k) v)) 119 | {} 120 | ss-result)))) 121 | 122 | (s/fdef parse-tail 123 | :args (s/cat :stream #(instance? MultipartStream %)) 124 | :ret ::attachments) 125 | 126 | (defn parse-tail 127 | "Parse any available attachments in the stream" 128 | [^MultipartStream stream] 129 | (loop [acc []] 130 | (if (.readBoundary stream) 131 | (let [{part-ctype "Content-Type" 132 | part-sha2 "X-Experience-API-Hash"} 133 | (-> stream 134 | .readHeaders 135 | parse-headers)] 136 | ;; xAPI Multipart Parts must have these 137 | (if (and part-ctype 138 | part-sha2) 139 | (let [tempfile (create-tempfile! part-sha2)] 140 | ;; Write body to a tempfile 141 | (with-open [os (io/output-stream tempfile)] 142 | (.readBodyData stream os)) 143 | (recur 144 | (conj acc 145 | {:sha2 part-sha2 146 | :contentType part-ctype 147 | :tempfile tempfile}))) 148 | (throw (ex-info "Invalid xAPI Part" 149 | {:type ::invalid-xapi-part})))) 150 | acc))) 151 | 152 | (s/def ::body 153 | (s/keys :req-un 154 | [:xapi.statements.GET.response/statement-result 155 | ::attachments])) 156 | 157 | (s/fdef parse-multipart-body 158 | :args (s/cat :input-stream #(instance? InputStream %) 159 | ;; contents of the Content-Type header 160 | :content-type-str string?) 161 | :ret ::body) 162 | 163 | (defn parse-multipart-body 164 | "Return a statement result and any attachments found" 165 | [^InputStream input-stream 166 | ^String content-type-str] 167 | (let [{:keys [content-type] 168 | {^String boundary-str :boundary 169 | :keys [^String charset] 170 | :or {charset "UTF-8"}} :content-type-params} 171 | (hutil/parse-content-type 172 | content-type-str) 173 | boundary (.getBytes boundary-str charset)] 174 | (with-open [input input-stream] 175 | (try 176 | (let [multipart-stream (new MultipartStream input-stream boundary)] 177 | (if (.skipPreamble multipart-stream) 178 | ;; The first bit should be statements 179 | {:statement-result (parse-head multipart-stream) 180 | ;; If there are attachments, find and coerce them 181 | :attachments (parse-tail multipart-stream)} 182 | (throw (ex-info "Empty Stream" 183 | {:type ::empty-stream})))) 184 | (catch MultipartStream$MalformedStreamException ex 185 | (throw (ex-info "Malformed Stream" 186 | {:type ::malformed-stream} 187 | ex))) 188 | (catch IOException ex 189 | (throw (ex-info "Read Error" 190 | {:type ::read-error} 191 | ex))))))) 192 | 193 | (s/fdef parse-response 194 | :args (s/cat :response map?) 195 | :ret (s/keys :req-un [::body])) 196 | 197 | (defn parse-response 198 | "Parse + close a multipart body" 199 | [{:keys [body] 200 | {content-type-str "Content-Type"} :headers 201 | :as resp}] 202 | (assoc resp :body (parse-multipart-body body content-type-str))) 203 | 204 | (s/fdef piped-streams 205 | :args (s/cat) 206 | :ret (s/tuple #(instance? OutputStream %) 207 | #(instance? InputStream %))) 208 | 209 | (defn piped-streams 210 | "Create an output stream and an input stream to which it is piped" 211 | [] 212 | (let [^PipedOutputStream posh (new PipedOutputStream)] 213 | [posh 214 | (new PipedInputStream posh)])) 215 | 216 | (s/fdef post-body 217 | :args (s/cat :boundary string? 218 | :statements (s/every ::xs/statement) 219 | :attachments ::attachments) 220 | :ret #(instance? InputStream %)) 221 | 222 | (def crlf "\r\n") 223 | 224 | (defn post-body 225 | "Return an input stream with the POST multipart body" 226 | [boundary 227 | statements 228 | attachments] 229 | (let [[posh pish] (piped-streams)] 230 | (future 231 | (with-open [^Writer posh-w (io/writer posh)] 232 | (.write posh-w (str "--" 233 | boundary 234 | crlf 235 | "Content-Type:application/json" 236 | crlf 237 | crlf 238 | )) 239 | (json/generate-stream statements posh-w) 240 | ;; Flush to notify 241 | (.flush posh-w) 242 | (doseq [{:keys [sha2 contentType ^File tempfile]} attachments] 243 | (.write posh-w (str crlf 244 | "--" 245 | boundary 246 | crlf 247 | (format "Content-Type:%s" contentType) 248 | crlf 249 | "Content-Transfer-Encoding:binary" 250 | crlf 251 | (format "X-Experience-API-Hash:%s" sha2) 252 | crlf 253 | crlf)) 254 | (io/copy tempfile posh-w) 255 | ;; flush after each part 256 | (.flush posh-w)) 257 | (.write posh-w 258 | (str crlf 259 | "--" 260 | boundary 261 | "--")))) 262 | pish)) 263 | 264 | ;; https://stackoverflow.com/a/67545577/3532563 265 | (defn gen-boundary 266 | "Generate a multipart boundary" 267 | [] 268 | (apply str (repeatedly 64 #(rand-nth "abcdefghijklmnopqrstuvwxyz0123456789")))) 269 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/client/oauth.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.client.oauth 2 | "OAuth Client Credentials Grant Support" 3 | (:require 4 | [clj-http.client :as client] 5 | [clojure.core.async :as a] 6 | [clojure.spec.alpha :as s] 7 | [clojure.string :as cs] 8 | [clojure.tools.logging :as log])) 9 | 10 | ;; Derive token URLs from OAuth2 endpoint/OIDC Issuer 11 | (s/fdef token-url 12 | :args (s/cat :auth-uri ::auth-uri) 13 | :ret string?) 14 | 15 | (defn token-url* 16 | [auth-uri] 17 | (str auth-uri 18 | (when-not (cs/ends-with? auth-uri "/") 19 | "/") 20 | "token")) 21 | 22 | (def token-url (memoize token-url*)) 23 | 24 | (s/def ::auth-uri string?) 25 | (s/def ::client-id string?) 26 | (s/def ::client-secret string?) 27 | (s/def ::scope string?) 28 | 29 | (s/def ::oauth-params 30 | (s/keys :req-un [::auth-uri 31 | ::client-id 32 | ::client-secret] 33 | :opt-un [::scope])) 34 | 35 | (s/fdef token-request 36 | :args (s/cat :params ::oauth-params) 37 | :ret map?) 38 | 39 | (defn token-request 40 | [{:keys [auth-uri 41 | client-id 42 | client-secret 43 | scope]}] 44 | {:url (token-url auth-uri) 45 | :method :post 46 | :basic-auth [client-id client-secret] 47 | :form-params (cond-> {:grant_type "client_credentials"} 48 | (not-empty scope) (assoc :scope scope)) 49 | :as :json}) 50 | 51 | (defonce token-cache 52 | (atom {})) 53 | 54 | (defn- token-cache-key 55 | [{:keys [auth-uri 56 | client-id]}] 57 | (format "%s|%s" auth-uri client-id)) 58 | 59 | (defn get-token! 60 | "Given client credentials grant params and kwarg options, attempt to get a 61 | token, either from the cache or remote. Expires cache entries based on 62 | :expires_in on the response. 63 | 64 | Returns a promise channel containing a tuple: 65 | [:result ] 66 | or 67 | [:exception ] 68 | 69 | Options: 70 | * bump-exp-ms time to bump up expiry of a token from the cache, with the 71 | assumption that some time has already passed since issuance. 72 | " 73 | [{:keys [auth-uri 74 | client-id 75 | scope] :as params} 76 | & {:keys [bump-exp-ms] 77 | :or {bump-exp-ms 500}}] 78 | (let [ret (a/promise-chan) 79 | cache-key (token-cache-key params)] 80 | (if-let [extant-token (get @token-cache cache-key)] 81 | ;; If a token is already cached, return it 82 | (a/put! ret [:result extant-token]) 83 | ;; If not, go get it 84 | (do 85 | (log/debugf "Token request for %s" 86 | cache-key) 87 | (client/request 88 | (merge (token-request params) 89 | {:async true}) 90 | (fn [{:keys [status 91 | body] 92 | :as resp}] 93 | (if (= 200 status) 94 | (let [{:keys [access_token 95 | expires_in]} body] 96 | ;; update the cache 97 | (swap! token-cache assoc cache-key access_token) 98 | ;; return to the user 99 | (a/put! ret [:result access_token]) 100 | (when expires_in 101 | ;; later, remove from cache when expired 102 | (let [remove-in (max 103 | (- (* expires_in 1000) bump-exp-ms) 104 | ;; don't go negative if exp is super 1s or below 105 | ;; for some weird reason 106 | 0)] 107 | (a/go 108 | (log/debugf "Waiting %d ms before removing token %s" 109 | remove-in cache-key) 110 | (a/ [(constantly true)] 57 | (not-empty ensure-paths) 58 | (conj 59 | (fn [{:keys [statement]}] 60 | (every? 61 | (fn [path] 62 | (not-empty (path/get-paths* statement [path]))) 63 | ensure-paths))) 64 | (not-empty match-paths) 65 | (into 66 | (for [[path path-matches] (group-by first match-paths) 67 | :let [match-vs (into #{} (map second path-matches))]] 68 | (fn [{:keys [statement]}] 69 | (if-let [s-paths (not-empty (path/get-paths* statement [path]))] 70 | (every? 71 | #(contains? match-vs (get-in statement %)) 72 | s-paths) 73 | false))))))) 74 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/job.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job 2 | "Immutable job configuration" 3 | (:require [clojure.spec.alpha :as s] 4 | [clojure.spec.gen.alpha :as sgen] 5 | [com.yetanalytics.xapipe.job.config :as config] 6 | [com.yetanalytics.xapipe.job.state :as state] 7 | [com.yetanalytics.xapipe.job.state.errors :as errors] 8 | [com.yetanalytics.xapipe.util.time :as t] 9 | [xapi-schema.spec :as xs])) 10 | 11 | (def current-version 0) 12 | 13 | (s/def ::config config/config-spec) 14 | (s/def ::state state/state-spec) 15 | (s/def ::version #{current-version}) 16 | (s/def ::id (s/and string? not-empty)) 17 | 18 | (defn valid-get-params-vs-state? 19 | "Verify that since and until params (if present) wrap the cursor." 20 | [{{{{?since :since 21 | ?until :until} :get-params} :source} :config 22 | {:keys [cursor]} :state}] 23 | (t/in-order? 24 | (concat 25 | (when ?since 26 | [?since]) 27 | [cursor] 28 | (when ?until 29 | [?until])))) 30 | 31 | (def job-spec-base 32 | (s/keys 33 | :req-un [::id 34 | ::config 35 | ::state] 36 | :opt-un [::version])) 37 | 38 | (def job-spec 39 | (s/with-gen 40 | (s/and 41 | job-spec-base 42 | valid-get-params-vs-state?) 43 | (fn [] 44 | (sgen/fmap 45 | (fn [[job stamps]] 46 | (let [[since cursor until] (sort stamps)] 47 | (cond-> (assoc-in job [:state :cursor] cursor) 48 | (get-in job [:config :source :get-params :since]) 49 | (assoc-in [:config :source :get-params :since] since) 50 | 51 | (get-in job [:config :source :get-params :until]) 52 | (assoc-in [:config :source :get-params :until] until)))) 53 | (sgen/tuple 54 | (s/gen job-spec-base) 55 | (sgen/vector-distinct 56 | (s/gen ::t/normalized-stamp) 57 | {:num-elements 3})))))) 58 | 59 | (defmulti inc-version 60 | "Upgrade a job to the next version up. 61 | Breaking changes to the job spec require implementations of this to be 62 | upgradeable." 63 | (fn [{:keys [version]}] 64 | [version (inc version)])) 65 | 66 | (defmethod inc-version :default 67 | [{:keys [version]}] 68 | (let [next-version (inc version)] 69 | (throw 70 | (ex-info (format "No known route from version %d to %d" 71 | version 72 | next-version) 73 | {:type ::no-known-upgrade 74 | :version version 75 | :next-version next-version})))) 76 | 77 | (s/fdef upgrade-job 78 | :args (s/cat :job job-spec) 79 | :ret job-spec) 80 | 81 | (defn upgrade-job 82 | "Attempt to upgrade a job to the current version or throw. 83 | Assumes no version to be the latest version." 84 | [{:keys [version] 85 | :as job 86 | :or {version current-version}}] 87 | (cond 88 | (= current-version version) (assoc job :version version) 89 | (< current-version version) (throw 90 | (ex-info (format "Unknown version %d" 91 | version) 92 | {:type ::unknown-version 93 | :version version 94 | :current-version current-version})) 95 | :else 96 | (recur (inc-version job)))) 97 | 98 | ;; Initialize a job 99 | (s/fdef init-job 100 | :args (s/cat :id ::id 101 | :config ::config) 102 | :ret job-spec) 103 | 104 | (defn init-job 105 | "Initialize a new job" 106 | [id 107 | config] 108 | (let [{{{?since :since} :get-params} :source 109 | filter-config :filter 110 | :as config} (config/ensure-defaults config)] 111 | {:id id 112 | :version current-version 113 | :config 114 | config 115 | :state 116 | {:status :init 117 | :cursor (or ?since 118 | "1970-01-01T00:00:00.000000000Z") 119 | :source {:errors []} 120 | :target {:errors []} 121 | :errors [] 122 | :filter (if (:pattern filter-config) 123 | {:pattern {}} 124 | {})}})) 125 | 126 | ;; Job-level state 127 | 128 | ;; State status 129 | (s/fdef get-status 130 | :args (s/cat :job job-spec) 131 | :ret ::state/status) 132 | 133 | (defn get-status 134 | "Get the current status of the job" 135 | [job] 136 | (get-in job [:state :status])) 137 | 138 | (s/fdef errors? 139 | :args (s/cat :job job-spec) 140 | :ret boolean?) 141 | 142 | (defn errors? 143 | "Check if a job has any errors" 144 | [{:keys [state]}] 145 | (state/errors? state)) 146 | 147 | (s/fdef all-errors 148 | :args (s/cat :job job-spec) 149 | :ret ::state/errors) 150 | 151 | (defn all-errors 152 | "Get all errors for the job of any type" 153 | [job] 154 | (apply concat (state/get-errors (:state job)))) 155 | 156 | (s/fdef sanitize 157 | :args (s/cat :job job-spec) 158 | :ret (s/and job-spec 159 | (fn [{{{{src-pw :password} :request-config} :source 160 | {{tgt-pw :password} :request-config} :target} 161 | :config}] 162 | (and (or (nil? src-pw) 163 | (= "************" src-pw)) 164 | (or (nil? tgt-pw) 165 | (= "************" tgt-pw)))))) 166 | 167 | (defn sanitize 168 | "Sanitize any sensitive info on a job for logging, etc" 169 | [job] 170 | (update job :config config/sanitize)) 171 | 172 | (s/fdef reconfigure-job 173 | :args (s/cat :job (s/with-gen job-spec 174 | (fn [] 175 | (sgen/fmap 176 | #(update % :config config/ensure-defaults) 177 | (s/gen job-spec)))) 178 | :config ::config) 179 | :ret job-spec) 180 | 181 | (defn reconfigure-job 182 | "Given a job and a new config, return the job with the config applied, and 183 | state adjusted if possible. 184 | If the resulting job would be invalid, we add an error to the job state." 185 | [{{{?old-since :since 186 | ?old-until :until} :get-params} :config 187 | {:keys [status cursor] 188 | filter-state :filter} :state 189 | :as job} 190 | {{{?new-since :since 191 | ?new-until :until} :get-params} :source 192 | filter-cfg :filter 193 | :as config}] 194 | (if (= status :error) 195 | (-> job 196 | (update :state 197 | state/add-error 198 | {:type :job 199 | :message "Cannot reconfigure job with errors."}) 200 | (update :state state/set-updated)) 201 | (let [?since (or ?new-since ?old-since) 202 | ?until (or ?new-until ?old-until)] 203 | (if (t/in-order? 204 | (concat 205 | (when ?since 206 | [?since]) 207 | [cursor] 208 | (when ?until 209 | [?until]))) 210 | (-> job 211 | (assoc :config config) 212 | (cond-> 213 | (and 214 | (:pattern filter-cfg) 215 | (not (:pattern filter-state))) 216 | (assoc-in [:state :filter :pattern] {}))) 217 | (-> job 218 | (update :state 219 | state/add-error 220 | {:type :job 221 | :message "since, cursor, until must be ordered!"}) 222 | (update :state state/set-updated)))))) 223 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/job/config.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.config 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen] 4 | [com.yetanalytics.xapipe.client :as client] 5 | [com.yetanalytics.xapipe.filter :as filt] 6 | [com.yetanalytics.xapipe.util :as u] 7 | [com.yetanalytics.xapipe.util.time :as t])) 8 | 9 | ;; TODO: The generated values here blow things up on large tests 10 | (s/def ::batch-size pos-int?) ;; limit param for get, batch size for post 11 | 12 | (s/def ::backoff-opts u/backoff-opts-spec) 13 | 14 | (s/def ::source 15 | (s/keys :req-un [::client/request-config] 16 | :opt-un [::client/get-params 17 | ::client/poll-interval 18 | ::batch-size 19 | ::backoff-opts])) 20 | 21 | (s/def ::target 22 | (s/keys :req-un [::client/request-config] 23 | :opt-un [::batch-size 24 | ::backoff-opts])) 25 | 26 | ;; How many get requests to read-ahead 27 | ;; default: 10 28 | (s/def ::get-buffer-size nat-int?) 29 | 30 | ;; How many statements to buffer 31 | ;; default: source batch-size * get-buffer-size 32 | (s/def ::statement-buffer-size nat-int?) 33 | 34 | ;; how many batches of (target batch size) to buffer 35 | ;; default: statement-buffer-size / target batch-size 36 | (s/def ::batch-buffer-size nat-int?) 37 | 38 | ;; How long will we wait for a batch to fill? 39 | (s/def ::batch-timeout pos-int?) 40 | 41 | ;; Buffer size for async tempfile cleanup 42 | (s/def ::cleanup-buffer-size nat-int?) 43 | 44 | ;; Filter config 45 | (s/def ::filter 46 | (s/with-gen filt/filter-config-spec 47 | (fn [] 48 | (sgen/return {})))) 49 | 50 | (def config-spec 51 | (s/keys :req-un [::source 52 | ::target] 53 | :opt-un [::get-buffer-size 54 | ::statement-buffer-size 55 | ::batch-buffer-size 56 | ::batch-timeout 57 | ::cleanup-buffer-size 58 | ::filter])) 59 | 60 | ;; ensure a config has optional keys w/defaults 61 | (s/fdef ensure-defaults 62 | :args (s/cat :config config-spec) 63 | :ret config-spec) 64 | 65 | (defn ensure-defaults 66 | "Apply configuration defaults" 67 | [{{get-batch-size :batch-size 68 | get-backoff-opts :backoff-opts 69 | poll-interval :poll-interval 70 | {?since :since 71 | ?until :until} :get-params 72 | :as source-config 73 | :or {get-batch-size 50 74 | get-backoff-opts {:budget 10000 75 | :max-attempt 10} 76 | poll-interval 1000}} 77 | :source 78 | {post-batch-size :batch-size 79 | post-backoff-opts :backoff-opts 80 | :as target-config 81 | :or {post-backoff-opts {:budget 10000 82 | :max-attempt 10}}} 83 | :target 84 | filter-config :filter 85 | :keys 86 | [get-buffer-size 87 | statement-buffer-size 88 | batch-buffer-size 89 | batch-timeout 90 | cleanup-buffer-size] 91 | :or 92 | {get-buffer-size 10 93 | batch-timeout 200}}] 94 | (let [post-batch-size 95 | (or post-batch-size 96 | get-batch-size) 97 | 98 | statement-buffer-size 99 | (or statement-buffer-size 100 | (* get-batch-size 101 | get-buffer-size)) 102 | 103 | batch-buffer-size 104 | (or batch-buffer-size 105 | (max 1 106 | (quot statement-buffer-size 107 | post-batch-size))) 108 | cleanup-buffer-size 109 | (or cleanup-buffer-size 110 | get-batch-size 111 | 0)] 112 | {:get-buffer-size get-buffer-size 113 | :statement-buffer-size statement-buffer-size 114 | :batch-buffer-size batch-buffer-size 115 | :batch-timeout batch-timeout 116 | :cleanup-buffer-size cleanup-buffer-size 117 | :source 118 | (-> source-config 119 | (assoc :batch-size get-batch-size 120 | :backoff-opts get-backoff-opts 121 | :poll-interval poll-interval) 122 | (assoc-in [:get-params :limit] get-batch-size) 123 | (cond-> 124 | ?since (update-in [:get-params :since] t/normalize-stamp) 125 | ?until (update-in [:get-params :until] t/normalize-stamp))) 126 | :target 127 | (assoc target-config 128 | :batch-size post-batch-size 129 | :backoff-opts post-backoff-opts) 130 | :filter 131 | (or filter-config {})})) 132 | 133 | (s/fdef sanitize-req-cfg 134 | :args (s/cat :rcfg ::client/request-config) 135 | :ret ::client/request-config) 136 | 137 | (defn sanitize-req-cfg 138 | "Sanitize a single request config" 139 | [{:keys [password] :as rcfg}] 140 | (if password 141 | (assoc rcfg :password "************") 142 | rcfg)) 143 | 144 | (s/fdef sanitize 145 | :args (s/cat :config config-spec) 146 | :ret (s/and 147 | config-spec 148 | (fn [{{{src-pw :password} :request-config} :source 149 | {{tgt-pw :password} :request-config} :target}] 150 | (and (or (nil? src-pw) 151 | (= "************" src-pw)) 152 | (or (nil? tgt-pw) 153 | (= "************" tgt-pw)))))) 154 | 155 | (defn sanitize 156 | "Sanitize a config, removing possibly sensitive values" 157 | [config] 158 | (-> config 159 | (update-in [:source :request-config] sanitize-req-cfg) 160 | (update-in [:target :request-config] sanitize-req-cfg))) 161 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/job/json.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.json 2 | "JSON Serialization/Deserialization for Jobs" 3 | (:require [cheshire.core :as json] 4 | [clojure.java.io :as io] 5 | [clojure.spec.alpha :as s] 6 | [clojure.spec.gen.alpha :as sgen] 7 | [cognitect.transit :as transit] 8 | [com.yetanalytics.xapipe.job :as job] 9 | [com.yetanalytics.xapipe.job.config :as config] 10 | [xapi-schema.spec :as xs]) 11 | (:import [java.io ByteArrayInputStream ByteArrayOutputStream File])) 12 | 13 | (s/fdef write-transit-str 14 | :args (s/cat :data any?) 15 | :ret string?) 16 | 17 | (defn write-transit-str 18 | [data] 19 | (with-open [baos (ByteArrayOutputStream.)] 20 | (let [w (transit/writer baos :json)] 21 | (transit/write w data) 22 | (.toString baos "UTF-8")))) 23 | 24 | (s/fdef read-transit-str 25 | :args (s/cat :t-str 26 | (s/with-gen string? 27 | (fn [] 28 | (sgen/fmap 29 | write-transit-str 30 | (s/gen any?))))) 31 | :ret any?) 32 | 33 | (defn read-transit-str 34 | [^String t-str] 35 | (with-open [bais (ByteArrayInputStream. (.getBytes t-str "UTF-8"))] 36 | (let [r (transit/reader bais :json)] 37 | (transit/read r)))) 38 | 39 | (defn- pack-paths 40 | "Transit encode a list of paths in data, if found" 41 | [data paths] 42 | (reduce 43 | (fn [d p] 44 | (cond-> d 45 | (some? (get-in d p)) 46 | (update-in p write-transit-str))) 47 | data 48 | paths)) 49 | 50 | (defn- unpack-paths 51 | "Transit decode a list of paths in data, if found" 52 | [data paths] 53 | (reduce 54 | (fn [d p] 55 | (cond-> d 56 | (some? (get-in d p)) 57 | (update-in p read-transit-str))) 58 | data 59 | paths)) 60 | 61 | (s/def ::job-json 62 | (s/with-gen 63 | (s/and string? 64 | not-empty) 65 | (fn [] 66 | (sgen/fmap 67 | ;; job->json is identical to cheshire generate so this is OK 68 | ;; EXCEPT for the need to pack stuff 69 | (comp 70 | json/generate-string 71 | #(pack-paths % [[:state :filter :pattern]])) 72 | (s/gen job/job-spec))))) 73 | 74 | (defn- keywordize-status 75 | [job] 76 | (update-in job [:state :status] (partial keyword nil))) 77 | 78 | (defn- keywordize-error-types 79 | [errors] 80 | (mapv 81 | (fn [error] 82 | (update error :type (partial keyword nil))) 83 | errors)) 84 | 85 | (defn- keywordize-job-error-types 86 | [job] 87 | (-> job 88 | (update-in [:state :errors] keywordize-error-types) 89 | (update-in [:state :source :errors] keywordize-error-types) 90 | (update-in [:state :target :errors] keywordize-error-types))) 91 | 92 | (s/fdef json->job 93 | :args (s/cat :json-str ::job-json) 94 | :ret job/job-spec) 95 | 96 | (defn json->job 97 | "Parse a job from JSON" 98 | [^String json-str] 99 | (-> (json/parse-string json-str (partial keyword nil)) 100 | keywordize-status 101 | keywordize-job-error-types 102 | (unpack-paths [[:state :filter :pattern]]) 103 | (update :config config/ensure-defaults))) 104 | 105 | (s/def ::pretty boolean?) 106 | 107 | (s/fdef job->json 108 | :args (s/cat :job job/job-spec 109 | :kwargs (s/keys* :opt-un [::pretty])) 110 | :ret ::job-json) 111 | 112 | (defn job->json [job & {:as kwargs}] 113 | (json/generate-string 114 | (pack-paths job [[:state :filter :pattern]]) 115 | kwargs)) 116 | 117 | (s/fdef job->json-file! 118 | :args (s/cat :job job/job-spec 119 | :out (s/with-gen 120 | (s/or :path string? 121 | :file #(instance? File %)) 122 | (fn [] 123 | (sgen/return "/dev/null"))) 124 | :kwargs (s/keys* :opt-un [::pretty])) 125 | :ret nil?) 126 | 127 | (defn job->json-file! 128 | [job 129 | out 130 | & {:as kwargs}] 131 | (with-open [w (io/writer out)] 132 | (json/generate-stream 133 | (pack-paths job [[:state :filter :pattern]]) 134 | w 135 | kwargs) 136 | nil)) 137 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/job/state.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.state 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen] 4 | [com.yetanalytics.xapipe.filter :as filt] 5 | [com.yetanalytics.xapipe.job.state.errors :as errors] 6 | [com.yetanalytics.xapipe.util.time :as t] 7 | [xapi-schema.spec :as xs])) 8 | 9 | ;; Error vectors are stored at 3 levels for the source, 10 | ;; target, and entire job. 11 | (s/def ::errors errors/errors-spec) 12 | 13 | ;; Cursor is a timestamp representing the latest stored time on a received 14 | ;; statement OR in the case of polling the last consistent-through given 15 | ;; without data. 16 | ;; It should only be persisted when it represents data successfully copied from 17 | ;; source to target! 18 | 19 | (s/def ::cursor ::t/normalized-stamp) 20 | 21 | (s/def ::source 22 | (s/keys :req-un [::errors])) 23 | 24 | (s/def ::target 25 | (s/keys :req-un [::errors])) 26 | 27 | (s/def ::status 28 | #{:init ;; not started 29 | :running ;; in progress 30 | :complete ;; all desired data transferred 31 | :error ;; stopped with errors 32 | :paused ;; manual stop/pause 33 | }) 34 | 35 | (s/def ::filter 36 | filt/filter-state-spec) 37 | 38 | (s/def ::updated ::t/normalized-stamp) 39 | 40 | (def state-spec 41 | (s/keys :req-un [::source 42 | ::target 43 | ::errors 44 | ::cursor 45 | ::status 46 | ::filter] 47 | ;; Timestamp is added/replaced at runtime 48 | :opt-un [::updated])) 49 | 50 | (s/fdef errors? 51 | :args (s/cat :state state-spec) 52 | :ret boolean?) 53 | 54 | (defn errors? 55 | "Check for errors left in the state" 56 | [{job-errors :errors 57 | {source-errors :errors} :source 58 | {target-errors :errors} :target}] 59 | (some? (not-empty (concat job-errors source-errors target-errors)))) 60 | 61 | (s/fdef get-errors 62 | :args (s/cat :state state-spec) 63 | :ret (s/tuple ::errors 64 | ::errors 65 | ::errors)) 66 | 67 | (defn get-errors 68 | "Get all errors for [:job :source :target]" 69 | [{job-errors :errors 70 | {source-errors :errors} :source 71 | {target-errors :errors} :target}] 72 | [job-errors 73 | source-errors 74 | target-errors]) 75 | 76 | ;; Update 77 | 78 | (s/fdef add-error 79 | :args (s/cat :state state-spec 80 | :error ::errors/error) 81 | :ret state-spec) 82 | 83 | (defn add-error 84 | "Add an error to the state" 85 | [state {etype :type 86 | :as error}] 87 | (-> state 88 | (update-in 89 | (case etype 90 | :job [:errors] 91 | :source [:source :errors] 92 | :target [:target :errors]) 93 | conj error) 94 | ;; Make sure the status is error 95 | (assoc :status :error))) 96 | 97 | (s/fdef add-errors 98 | :args (s/cat :state state-spec 99 | :errors ::errors) 100 | :ret state-spec) 101 | 102 | (defn add-errors 103 | "Add multiple errors" 104 | [state errors] 105 | (reduce (fn [s e] 106 | (add-error s e)) 107 | state 108 | errors)) 109 | 110 | (s/fdef clear-errors 111 | :args (s/cat :state state-spec) 112 | :ret state-spec) 113 | 114 | (defn clear-errors 115 | [state] 116 | (-> state 117 | (update :errors empty) 118 | (update-in [:source :errors] empty) 119 | (update-in [:target :errors] empty))) 120 | 121 | (s/fdef update-cursor 122 | :args (s/cat :state state-spec 123 | :new-cursor ::cursor) 124 | :ret state-spec) 125 | 126 | (defn update-cursor 127 | "Attempt to update the since cursor. Add error if we try to go backwards." 128 | [{old-cursor :cursor 129 | :as state} new-cursor] 130 | (let [[a b] (sort [old-cursor 131 | new-cursor])] 132 | (cond 133 | ;; no change, return 134 | (= a b) 135 | state 136 | 137 | ;; update 138 | (= [a b] [old-cursor new-cursor]) 139 | (assoc state :cursor new-cursor) 140 | 141 | ;; can't go backwards 142 | (= [b a] [old-cursor new-cursor]) 143 | (add-error 144 | state 145 | {:type :job 146 | :message (format "New cursor %s is before current cursor %s" 147 | new-cursor old-cursor)})))) 148 | 149 | (def valid-status-transitions 150 | #{[:init :running] ;; start 151 | [:init :error] ;; can't start 152 | [:init :complete] ;; no data 153 | [:init :paused] ;; immediate pause 154 | 155 | [:running :complete] ;; until reached/exit 156 | [:running :error] ;; runtime error 157 | [:running :paused] ;; user pause 158 | [:running :running] ;; cursor update 159 | 160 | [:paused :init] ;; resume init 161 | [:paused :error] ;; can't resume 162 | 163 | [:error :running] ;; if errors clear 164 | [:error :paused] ;; same 165 | [:error :error] ;; more/less errors 166 | 167 | [:complete :paused] 168 | }) 169 | 170 | (s/fdef set-status 171 | :args (s/cat :state state-spec 172 | :new-status #{:running ;; in progress 173 | :complete ;; complete 174 | :paused ;; manual stop/pause 175 | :init ;; reinit 176 | }) 177 | :ret state-spec) 178 | 179 | (defn set-status 180 | "Attempt to set the desired status, only on valid transitions" 181 | [{:keys [status] :as state} new-status] 182 | (cond 183 | ;; Invalid status transition error 184 | (not (contains? valid-status-transitions 185 | [status new-status])) 186 | (add-error state 187 | {:type :job 188 | :message (format "Invalid status transition: %s %s" 189 | status new-status)}) 190 | 191 | ;; Ensure errors are cleared before allowing state change 192 | (and 193 | (contains? #{[:error :running] 194 | [:error :paused]} [status new-status]) 195 | (not= [[] [] []] (get-errors state))) 196 | (add-error state 197 | {:type :job 198 | :message "Cannot start or pause job with errors."}) 199 | 200 | :else (assoc state :status new-status))) 201 | 202 | (s/fdef update-filter 203 | :args (s/cat :state state-spec 204 | :filter-state ::filter) 205 | :ret state-spec) 206 | 207 | (defn update-filter 208 | "Update filter state" 209 | [{:keys [status] :as state} filter-state] 210 | (if (= :error status) 211 | (add-error state 212 | {:type :job 213 | :message "Cannot update filter on job with errors"}) 214 | (assoc state :filter filter-state))) 215 | 216 | (s/fdef set-updated 217 | :args (s/cat :state state-spec) 218 | :ret (s/and state-spec 219 | #(:updated %))) 220 | 221 | (defn set-updated 222 | [state] 223 | (assoc state :updated (t/now-stamp))) 224 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/job/state/errors.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.state.errors 2 | (:require [clojure.spec.alpha :as s] 3 | [xapi-schema.spec :as xs])) 4 | 5 | (s/def ::message string?) 6 | (s/def ::type #{:job :source :target}) 7 | (s/def ::error (s/keys :req-un [::message 8 | ::type])) 9 | 10 | (def errors-spec (s/every ::error 11 | :gen-max 5)) 12 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/metrics.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.metrics 2 | "Metrics protocol called by xapipe lib. 3 | Inspired by https://github.com/pedestal/pedestal/blob/master/log/src/io/pedestal/log.clj" 4 | (:require [clojure.spec.alpha :as s] 5 | [clojure.spec.gen.alpha :as sgen] 6 | [clojure.tools.logging :as log])) 7 | 8 | (defprotocol Reporter 9 | (-gauge [this k v] 10 | "Set a gauge to an arbitrary value") 11 | (-counter [this k delta] 12 | "Increase a counter by delta") 13 | (-histogram [this k v] 14 | "Log an observation of a value") 15 | (-summary [this k v] 16 | "Log an observation of a value with calculated quatiles") 17 | (-flush! [this] 18 | "Flush metrics out if possible")) 19 | 20 | (deftype NoopReporter [] 21 | Reporter 22 | (-gauge [this _ _] nil) 23 | (-counter [this _ _] nil) 24 | (-histogram [this _ _] nil) 25 | (-summary [this _ _] nil) 26 | (-flush! [this] nil)) 27 | 28 | (s/def ::reporter 29 | (s/with-gen #(satisfies? Reporter %) 30 | (fn [] 31 | (sgen/return (->NoopReporter))))) 32 | 33 | ;; Keys describing gauges 34 | (def gauge-keys 35 | #{}) 36 | 37 | (s/def ::gauges 38 | gauge-keys) 39 | 40 | (s/fdef gauge 41 | :args (s/cat :reporter ::reporter 42 | :k ::gauges 43 | :v number?) 44 | :ret any?) 45 | 46 | (defn gauge 47 | "Set a gauge to an arbitrary value" 48 | [reporter k v] 49 | (-gauge reporter k v)) 50 | 51 | ;; Keys describing counters 52 | (def counter-keys 53 | #{:xapipe/statements 54 | :xapipe/attachments 55 | :xapipe/source-errors 56 | :xapipe/target-errors 57 | :xapipe/job-errors 58 | :xapipe/all-errors}) 59 | 60 | (s/def ::counters 61 | counter-keys) 62 | 63 | (s/fdef counter 64 | :args (s/cat :reporter ::reporter 65 | :k ::counters 66 | :delta nat-int?) 67 | :ret any?) 68 | 69 | (defn counter 70 | "Increase a counter by delta" 71 | [reporter k delta] 72 | (-counter reporter k delta)) 73 | 74 | ;; Keys describing histograms 75 | (def histogram-keys 76 | #{:xapipe/source-request-time 77 | :xapipe/target-request-time}) 78 | 79 | (s/def ::histograms 80 | histogram-keys) 81 | 82 | (s/fdef histogram 83 | :args (s/cat :reporter ::reporter 84 | :k ::histograms 85 | :v number?) 86 | :ret any?) 87 | 88 | (defn histogram 89 | "Log an observation of a value" 90 | [reporter k v] 91 | (-histogram reporter k v)) 92 | 93 | ;; Keys describing summaries 94 | (def summary-keys 95 | #{}) 96 | 97 | (s/def ::summaries 98 | summary-keys) 99 | 100 | (s/fdef summary 101 | :args (s/cat :reporter ::reporter 102 | :k ::summaries 103 | :v number?) 104 | :ret any?) 105 | 106 | (defn summary 107 | "Log an observation of a value for a summary" 108 | [reporter k v] 109 | (-summary reporter k v)) 110 | 111 | (s/fdef flush! 112 | :args (s/cat :reporter ::reporter) 113 | :ret any?) 114 | 115 | (defn flush! 116 | "Flush metrics out if possible. 117 | Catch all errors and just log them." 118 | [reporter] 119 | (try (-flush! reporter) 120 | (catch Exception ex 121 | (log/warnf ex 122 | "Metrics flush failed: %s" 123 | (ex-message ex))))) 124 | 125 | (s/fdef millis->frac-secs 126 | :args (s/cat :millis int?) 127 | :ret double?) 128 | 129 | (defn millis->frac-secs 130 | "Convert milliseconds to fractional seconds per: 131 | https://prometheus.io/docs/practices/naming/#base-units" 132 | [millis] 133 | (double (/ millis 1000))) 134 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/spec/common.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.spec.common 2 | "Common & Utility Specs" 3 | (:require [clojure.spec.alpha :as s] 4 | [clojure.core.async.impl.protocols :as aproto])) 5 | 6 | (defn channel? 7 | [x] 8 | (satisfies? aproto/Channel x)) 9 | 10 | (s/def ::channel channel?) 11 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/store.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.store 2 | "Persistent job state storage" 3 | (:require [clojure.spec.alpha :as s])) 4 | 5 | (defprotocol XapipeStore 6 | (read-job [store job-id] 7 | "Return the job if available, or nil") 8 | (write-job [store job] 9 | "Write a job to the store, overwriting previous state. Return true/false") 10 | (list-jobs [store] 11 | "Return a list of jobs") 12 | (delete-job [store job-id] 13 | "Delete the job from the store, returning true/false")) 14 | 15 | (s/def :com.yetanalytics.xapipe/store 16 | #(satisfies? XapipeStore %)) 17 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/store/impl/file.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.store.impl.file 2 | (:require [clojure.edn :as edn] 3 | [clojure.java.io :as io] 4 | [com.yetanalytics.xapipe.store :as store]) 5 | (:import [java.io File PushbackReader])) 6 | 7 | (defn read-file 8 | "Return file edn or nil" 9 | [f] 10 | (try 11 | (with-open [rdr (PushbackReader. (io/reader f))] 12 | (edn/read rdr)) 13 | (catch Exception _ 14 | nil))) 15 | 16 | (defn write-file 17 | "Write file and return true or false" 18 | [f content] 19 | (try 20 | (with-open [w (io/writer f)] 21 | (binding [*print-length* false 22 | *out* w] 23 | (pr content))) 24 | true 25 | (catch Exception _ 26 | false))) 27 | 28 | (defn delete-file 29 | "Delete file and return true or false" 30 | [f] 31 | (.delete ^File (io/file f))) 32 | 33 | (defn file-path 34 | [dirpath job-id] 35 | (format "%s/%s.edn" 36 | dirpath 37 | job-id)) 38 | 39 | (defn ensure-directory 40 | [dirpath] 41 | (let [^File f (io/file dirpath)] 42 | (or (and (.exists f) 43 | (.isDirectory f)) 44 | (.mkdir f)))) 45 | 46 | (deftype FileStore [dirpath] 47 | store/XapipeStore 48 | (read-job [store job-id] 49 | (read-file (file-path dirpath job-id))) 50 | (write-job [store {job-id :id 51 | :as job}] 52 | (write-file (file-path dirpath job-id) job)) 53 | (list-jobs [store] 54 | (for [^File f (file-seq (io/file dirpath)) 55 | :when (and (not (.isDirectory f)) 56 | (.endsWith (.getName f) ".edn"))] 57 | (read-file f))) 58 | (delete-job [store job-id] 59 | (delete-file (file-path dirpath job-id)))) 60 | 61 | (defn new-store 62 | "Make a new in-memory store" 63 | [& [dirpath]] 64 | (let [dirpath (or dirpath "store")] 65 | (when-not (ensure-directory dirpath) 66 | (throw (ex-info (format "Couldn't create directory %s" dirpath) 67 | {:type ::cannot-create-dir}))) 68 | (->FileStore dirpath))) 69 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/store/impl/memory.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.store.impl.memory 2 | (:require [com.yetanalytics.xapipe.store :as store])) 3 | 4 | (defprotocol DumpableMemoryStore 5 | (dump [this] 6 | "Reveal the store state")) 7 | 8 | (deftype MemoryStore [state-atom] 9 | store/XapipeStore 10 | (read-job [store job-id] 11 | (get @state-atom job-id)) 12 | (write-job [store {job-id :id 13 | :as job}] 14 | (-> (swap! state-atom assoc job-id job) 15 | (get job-id) 16 | (= job))) 17 | (list-jobs [store] 18 | (vals @state-atom)) 19 | (delete-job [store job-id] 20 | (let [before @state-atom] 21 | (swap! state-atom dissoc job-id) 22 | (some? (get before job-id)))) 23 | DumpableMemoryStore 24 | (dump [_] 25 | @state-atom)) 26 | 27 | (defn new-store 28 | "Make a new in-memory store" 29 | [] 30 | (let [state-atom (atom {})] 31 | (->MemoryStore state-atom))) 32 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/store/impl/noop.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.store.impl.noop 2 | "A stateless state store" 3 | (:require [com.yetanalytics.xapipe.store :as store])) 4 | 5 | (deftype NoopStore [] 6 | store/XapipeStore 7 | (read-job [_store _job-id] 8 | nil) 9 | (write-job [_store _job-id] 10 | true) 11 | (list-jobs [_] 12 | []) 13 | (delete-job [_ _] 14 | false)) 15 | 16 | (defn new-store 17 | "Make a new noop store" 18 | [& _] 19 | (->NoopStore)) 20 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/store/impl/redis.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.store.impl.redis 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.tools.logging :as log] 4 | [com.yetanalytics.xapipe.job :as job] 5 | [com.yetanalytics.xapipe.store :as store] 6 | [taoensso.carmine :as car])) 7 | 8 | (s/fdef scan-seq 9 | :args (s/cat :conn map? 10 | :prefix string? 11 | :cursor string?) 12 | :ret (s/every job/job-spec)) 13 | 14 | (defn- scan-seg! 15 | [conn prefix cursor] 16 | (let [[next-cursor ks] 17 | (car/wcar conn 18 | (car/scan cursor :match (format "%s:*" 19 | prefix)))] 20 | (when (not-empty ks) 21 | [next-cursor 22 | (car/wcar conn 23 | (apply car/mget ks))]))) 24 | 25 | (defn scan-seq 26 | "Return a (blocking) lazy seq of jobs in redis" 27 | ([conn prefix] 28 | (scan-seq conn prefix "0")) 29 | ([conn prefix cursor] 30 | (lazy-seq 31 | (when-let [[next-cursor jobs] (scan-seg! conn prefix (or cursor 32 | "0"))] 33 | (concat jobs 34 | (when (not= next-cursor "0") 35 | (scan-seq conn prefix next-cursor))))))) 36 | 37 | (deftype RedisStore [conn 38 | prefix] 39 | store/XapipeStore 40 | (read-job [store job-id] 41 | (car/wcar conn 42 | (car/get (format "%s:%s" prefix job-id)))) 43 | (write-job [store {job-id :id 44 | :as job}] 45 | (let [k (format "%s:%s" prefix job-id) 46 | [stat ret-job] (car/wcar conn 47 | (car/set k job) 48 | (car/get k))] 49 | (if (= "OK" stat) 50 | (= job ret-job) 51 | (do 52 | (log/errorf stat "Redis Exception") 53 | false)))) 54 | (list-jobs [store] 55 | (scan-seq conn prefix)) 56 | (delete-job [_ job-id] 57 | (= 1 58 | (car/wcar conn 59 | (car/del (format "%s:%s" prefix job-id)))))) 60 | 61 | (defn new-store 62 | "Make a new redis store" 63 | [conn & [?prefix]] 64 | (->RedisStore conn 65 | (or ?prefix 66 | "xapipe"))) 67 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/util.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.util 2 | "Utility functions" 3 | (:require [clojure.spec.alpha :as s])) 4 | 5 | ;; From `lrsql` 6 | ;; TODO: The generated values here blow things up on large tests 7 | 8 | (s/def ::budget (s/and int? (fn [n] (< 0 n Integer/MAX_VALUE)))) 9 | (s/def ::max-attempt (s/and int? (fn [n] (< 0 n Integer/MAX_VALUE)))) 10 | (s/def ::j-range (s/and int? (fn [n] (<= 0 n Integer/MAX_VALUE)))) 11 | (s/def ::initial (s/and int? (fn [n] (<= 0 n Integer/MAX_VALUE)))) 12 | 13 | (def backoff-opts-spec 14 | (s/keys :req-un [::budget ::max-attempt] 15 | :opt-un [::j-range ::initial])) 16 | 17 | (s/fdef backoff-ms 18 | :args (s/cat :attempt nat-int? 19 | :opts backoff-opts-spec) 20 | :ret (s/nilable nat-int?)) 21 | 22 | (defn backoff-ms 23 | "Take an `attempt` number and an opts map containing the total `:budget` 24 | in ms and an `:max-attempt` number and return a backoff time in ms. 25 | Can also optionally provide a jitter in `j-range` ms and an `initial` ms 26 | amount of delay to be used first in the opts map." 27 | [attempt {:keys [budget max-attempt j-range initial] 28 | :or {j-range 10}}] 29 | (let [jitter (rand-int j-range)] 30 | (cond 31 | (= attempt 0) 0 32 | (> attempt max-attempt) nil 33 | (and (some? initial) 34 | (= attempt 1)) (+ initial jitter) 35 | :else (int (+ (* budget (/ (Math/pow 2 (- attempt 1)) 36 | (Math/pow 2 max-attempt))) 37 | jitter))))) 38 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/util/async.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.util.async 2 | "Useful Async facilities" 3 | (:require [clojure.core.async :as a] 4 | [clojure.spec.alpha :as s] 5 | [com.yetanalytics.xapipe.metrics :as metrics] 6 | [com.yetanalytics.xapipe.spec.common :as cspec])) 7 | 8 | (s/def ::stateless-predicates 9 | (s/map-of 10 | keyword? 11 | ifn?)) 12 | 13 | (s/def ::stateful-predicates 14 | (s/map-of 15 | keyword? 16 | ifn?)) 17 | 18 | (s/def ::init-states 19 | (s/map-of 20 | keyword? 21 | any?)) 22 | 23 | (s/def ::emit-fn 24 | fn?) ;; TODO: fspec 25 | 26 | (s/def ::cleanup-chan cspec/channel?) 27 | 28 | (s/fdef batch-filter 29 | :args (s/cat :a ::cspec/channel 30 | :b ::cspec/channel 31 | :size pos-int? 32 | :timeout-ms nat-int? 33 | :kwargs (s/keys* 34 | :opt-un [::stateless-predicates 35 | ::stateful-predicates 36 | ::init-states 37 | ::metrics/reporter 38 | ::emit-fn 39 | ::cleanup-chan]))) 40 | 41 | (defn- default-emit-fn 42 | [batch filter-state last-dropped] 43 | (when (or (not-empty batch) last-dropped) 44 | {:batch batch 45 | :filter-state filter-state 46 | :last-dropped last-dropped})) 47 | 48 | (defn- apply-stateful-predicates 49 | [states stateful-predicates v] 50 | (if (not-empty stateful-predicates) 51 | (let [checks (into {} 52 | (for [[k p] stateful-predicates] 53 | [k (p (get states k) v)])) 54 | pass? (every? (comp true? second) 55 | (vals checks))] 56 | {:pass? pass? 57 | :states (reduce-kv 58 | (fn [m k v] 59 | (assoc m k (first v))) 60 | {} 61 | checks)}) 62 | {:pass? true 63 | :states states})) 64 | 65 | (defn- apply-predicates 66 | [states stateless-pred stateful-predicates v] 67 | (let [stateless-pass? (stateless-pred v) 68 | {stateful-pass? :pass? 69 | states :states} (apply-stateful-predicates 70 | states 71 | stateful-predicates 72 | v)] 73 | {:pass? (and stateless-pass? 74 | stateful-pass?) 75 | :states states})) 76 | 77 | (defn batch-filter 78 | "Given a channel a, get and attempt to batch records by size, sending them to 79 | channel b. If channel a is parked for longer than timeout-ms, send a partial 80 | batch. 81 | If :predicates or :stateful-predicates, maps of id keys to predicates, are 82 | provided, use them to filter statements. 83 | 84 | Stateful predicate state will be provided on the enclosing batch map by key. 85 | 86 | To provide initial state, supply a :init-states key, which should contain 87 | state for each stateful predicate. 88 | 89 | If :cleanup-chan is provided, send dropped records there 90 | 91 | Will remember the last record dropped for use in cursors and such 92 | 93 | Returns a channel that will close when done processing." 94 | [a b size timeout-ms 95 | & {:keys [stateless-predicates 96 | stateful-predicates 97 | init-states 98 | reporter 99 | emit-fn 100 | cleanup-chan] 101 | :or {stateless-predicates {} 102 | stateful-predicates {} 103 | reporter (metrics/->NoopReporter) 104 | emit-fn default-emit-fn}}] 105 | (let [stateless-pred (if (empty? stateless-predicates) 106 | (constantly true) 107 | (apply every-pred (vals stateless-predicates))) 108 | init-states (or init-states 109 | (into {} 110 | (for [[k p] stateful-predicates] 111 | [k {}]))) 112 | buffer (a/buffer size) 113 | buf-chan (a/chan buffer)] 114 | (a/go-loop [states init-states 115 | last-dropped nil] 116 | (let [buf-count (count buffer)] 117 | ;; Send if the buffer is full 118 | (if (= size buf-count) 119 | (do 120 | (when-let [emit-event (emit-fn 121 | (a/! b emit-event)) 127 | (recur states nil)) 128 | (let [timeout-chan (a/timeout timeout-ms) 129 | [v p] (a/alts! [a timeout-chan])] 130 | (if (identical? p timeout-chan) 131 | ;; We've timed out. Flush! 132 | (do 133 | (when-let [emit-event (emit-fn 134 | (a/! b emit-event)) 140 | (recur states nil)) 141 | (if-not (nil? v) 142 | ;; We have a record 143 | (let [{:keys [pass?] 144 | new-states :states} (apply-predicates 145 | states 146 | stateless-pred 147 | stateful-predicates 148 | v)] 149 | 150 | (if pass? 151 | (do 152 | (a/>! buf-chan v) 153 | (recur new-states nil)) 154 | (do 155 | (when cleanup-chan 156 | (a/>! cleanup-chan v)) 157 | (recur new-states v)))) 158 | ;; A is closed, we should close B 159 | (do 160 | ;; But only after draining anything in the buffer 161 | (when-let [emit-event (emit-fn 162 | (a/! b emit-event)) 168 | (a/close! b)))))))))) 169 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/util/time.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.util.time 2 | "Time(stamp) utilities. 3 | Normalization follows process from com.yetanalytics.lrs.xapi.statements.timestamp" 4 | (:require [clojure.spec.alpha :as s] 5 | [clojure.spec.gen.alpha :as sgen] 6 | [xapi-schema.spec :as xs]) 7 | (:import [java.time Instant ZoneId] 8 | [java.time.format DateTimeFormatter])) 9 | 10 | (defonce ^ZoneId UTC 11 | (ZoneId/of "UTC")) 12 | 13 | (defonce ^DateTimeFormatter in-formatter 14 | DateTimeFormatter/ISO_DATE_TIME) 15 | 16 | (defonce ^DateTimeFormatter out-formatter 17 | (DateTimeFormatter/ofPattern "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'")) 18 | 19 | (def instant-spec 20 | (s/with-gen #(instance? Instant %) 21 | (fn [] 22 | (sgen/fmap 23 | #(Instant/ofEpochMilli ^Instant %) 24 | (sgen/large-integer* 25 | {:min 0 :max 4102444800000}))))) 26 | 27 | (s/fdef parse-inst 28 | :args (s/cat :timestamp ::xs/timestamp) 29 | :ret instant-spec) 30 | 31 | (defn parse-inst 32 | ^Instant [^String timestamp] 33 | #_(Instant/parse timestamp) 34 | (-> (.parse in-formatter timestamp) 35 | (Instant/from) 36 | (.atZone UTC) 37 | .toInstant)) 38 | 39 | (s/fdef normalize-inst 40 | :args (s/cat :inst instant-spec) 41 | :ret ::xs/timestamp 42 | :fn (fn [{stamp-after :ret}] 43 | (= 30 (count stamp-after)))) 44 | 45 | (defn normalize-inst 46 | "Normalize an inst object, ensuring that it is a static length (nano), and 47 | UTC." 48 | [^Instant inst] 49 | (-> inst 50 | (Instant/from) 51 | (.atZone UTC) 52 | (->> (.format out-formatter)))) 53 | 54 | (s/fdef parse-stamp 55 | :args (s/cat :timestamp ::xs/timestamp) 56 | :ret (s/tuple string? string? (s/nilable string?) string?)) 57 | 58 | (defn parse-stamp 59 | "return a vector of [whole-stamp body ?frac-secs offset-or-Z]" 60 | [timestamp] 61 | (re-find 62 | #"^(\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d{1,9}))?(Z|[+-]\d\d:\d\d)$" 63 | timestamp)) 64 | 65 | (s/def ::normalized-stamp 66 | (s/with-gen 67 | (s/and ::xs/timestamp 68 | #(.endsWith ^String % "Z") 69 | #(= 30 (.length ^String %))) 70 | (fn [] 71 | (sgen/fmap 72 | normalize-inst 73 | (s/gen instant-spec))))) 74 | 75 | (s/fdef normalize-stamp 76 | :args (s/cat :timestamp ::xs/timestamp) 77 | :ret ::normalized-stamp) 78 | 79 | ;; TODO: also naive, replace 80 | (defn normalize-stamp 81 | ^String [^String timestamp] 82 | #_(.toString (parse-inst timestamp)) 83 | (let [zulu? (.endsWith timestamp "Z") 84 | char-count (count timestamp)] 85 | (cond 86 | ;; We can easily detect a stamp already normalized to 8601 zulu with nano 87 | ;; precision, and these we can let through. 88 | (and zulu? 89 | (= 30 char-count)) 90 | timestamp 91 | 92 | ;; if it has more than nano precision 93 | (and zulu? 94 | (< 30 char-count)) 95 | (format "%sZ" (subs timestamp 0 29)) 96 | 97 | ;; we have some kind of offset. We need to parse and re-add the frac-secs 98 | :else 99 | (let [[_ body ?frac-secs offset] (parse-stamp timestamp) 100 | ?frac-secs-str (when ?frac-secs 101 | (apply str 102 | ?frac-secs 103 | ;; pad 104 | (repeat (- 9 (count ?frac-secs)) 105 | "0")))] 106 | ;; zulu or zero-offset stamps can be handled mechanically 107 | (if (or zulu? (= "+00:00" offset)) 108 | (if ?frac-secs-str 109 | (format "%s.%sZ" 110 | body 111 | ?frac-secs-str) 112 | ;; let's add 'em 113 | (format "%s.000000000Z" 114 | body)) 115 | ;; if none of that is true, we have an offset, and need to parse with 116 | ;; the platform lib. In clojure instants are precise so we can just do 117 | ;; it. In cljs, we need to override it 118 | (normalize-inst (parse-inst timestamp))))))) 119 | 120 | (s/fdef now-stamp 121 | :args (s/cat) 122 | :ret ::normalized-stamp) 123 | 124 | (defn now-stamp 125 | "Return the current time as a normalized stamp" 126 | [] 127 | (normalize-inst (Instant/now))) 128 | 129 | (s/fdef in-order? 130 | :args (s/cat :stamps (s/every ::normalized-stamp)) 131 | :ret boolean?) 132 | 133 | (defn in-order? 134 | "Returns true if the stamps are provided in order" 135 | [stamps] 136 | (= stamps 137 | (sort stamps))) 138 | -------------------------------------------------------------------------------- /src/lib/com/yetanalytics/xapipe/xapi.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.xapi 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as sgen] 4 | [clojure.tools.logging :as log] 5 | [com.yetanalytics.xapipe.client :as client] 6 | [com.yetanalytics.xapipe.client.multipart-mixed :as multipart] 7 | [xapi-schema.spec :as xs])) 8 | 9 | (s/fdef attachment-hashes 10 | :args (s/cat :statement 11 | ::xs/statement) 12 | :ret (s/every (s/tuple :attachment/sha2 boolean?))) 13 | 14 | (defn attachment-hashes 15 | "Get any attachment hashes in a statement. 16 | Returns a tuple of [sha2, file-url?]" 17 | [{:strs [attachments 18 | object]}] 19 | (distinct 20 | (keep 21 | (fn [{:strs [sha2 fileUrl]}] 22 | [sha2 (some? fileUrl)]) 23 | (concat attachments 24 | (get object "attachments"))))) 25 | 26 | (s/def ::source-statement 27 | (s/keys :req-un [::xs/statement 28 | ::multipart/attachments])) 29 | 30 | (s/def ::atts-in 31 | (s/map-of ::xs/sha2 32 | (s/every ::multipart/attachment 33 | :gen-max 1) 34 | :gen-max 1)) 35 | 36 | (s/def ::atts-out 37 | (s/map-of ::xs/sha2 38 | (s/every ::multipart/attachment 39 | :gen-max 1) 40 | :gen-max 1)) 41 | 42 | (s/def ::atts-acc 43 | ::multipart/attachments) 44 | 45 | (def attachment-args-spec 46 | (s/with-gen 47 | (s/cat :acc-map (s/keys :req-un [::atts-in 48 | ::atts-out 49 | ::atts-acc]) 50 | :query (s/tuple ::xs/sha2 boolean?)) 51 | (fn [] 52 | (sgen/fmap (fn [[sha2 att]] 53 | [{:atts-in {sha2 [att]} 54 | :atts-out {} 55 | :atts-acc []} 56 | [sha2 false]]) 57 | (sgen/tuple 58 | (s/gen ::xs/sha2) 59 | (s/gen ::multipart/attachment)))))) 60 | 61 | (s/fdef find-attachments 62 | :args (s/with-gen 63 | (s/cat :acc-map (s/keys :req-un [::atts-in 64 | ::atts-out 65 | ::atts-acc]) 66 | :query (s/tuple ::xs/sha2 boolean?)) 67 | (fn [] 68 | (sgen/fmap (fn [[sha2 att]] 69 | [{:atts-in {sha2 [att]} 70 | :atts-out {} 71 | :atts-acc []} 72 | [sha2 false]]) 73 | (sgen/tuple 74 | (s/gen ::xs/sha2) 75 | (s/gen ::multipart/attachment))))) 76 | :ret (s/keys :req-un [::atts-in 77 | ::atts-out 78 | ::atts-acc])) 79 | 80 | (defn find-attachments 81 | [{a-i :atts-in 82 | a-o :atts-out 83 | aa :atts-acc 84 | :as acc-map} [sha2 file-url?]] 85 | (or 86 | ;; Match in unused attachments 87 | (when-let [att (some-> a-i (get sha2) first)] 88 | {:atts-in (if (< 1 (count (get a-i sha2))) 89 | (update a-i sha2 #(into [] (rest %))) 90 | (dissoc a-i sha2)) 91 | :atts-out (update a-o sha2 (fnil conj []) att) 92 | :atts-acc (conj aa att)}) 93 | ;; Match in used attachments 94 | (when-let [att (some-> a-o (get sha2) first)] 95 | (let [dup (multipart/duplicate-attachment att)] 96 | {:atts-in a-i 97 | :atts-out (update a-o sha2 conj att) 98 | :atts-acc (conj aa att)})) 99 | ;; No match. Only OK if a file url is present 100 | (when file-url? 101 | {:atts-in a-i 102 | :atts-out a-o 103 | :atts-acc aa}) 104 | (throw (ex-info "Invalid Multipart Response - No attachment found." 105 | {:type ::attachment-not-found 106 | :sha2 sha2 107 | :acc-map acc-map})))) 108 | 109 | (s/fdef response->statements 110 | :args 111 | (s/cat :response 112 | (s/with-gen 113 | (s/keys :req-un [::multipart/body]) 114 | (fn [] 115 | (sgen/fmap 116 | (fn [[s {:keys [sha2 contentType] :as att}]] 117 | {:body {:statement-result 118 | {:statements 119 | [(-> s 120 | (assoc 121 | "attachments" 122 | [{"usageType" "http://example.com/foo" 123 | "display" {"en-US" "Generated"} 124 | "contentType" contentType 125 | "sha2" sha2 126 | "length" 0}]) 127 | (update "object" dissoc "attachments"))]} 128 | :attachments [att]}}) 129 | (sgen/tuple 130 | (s/gen ::xs/lrs-statement) 131 | (s/gen ::multipart/attachment)))))) 132 | :ret (s/every ::source-statement)) 133 | 134 | (defn response->statements 135 | "Break a response down into statements paired with one or more attachments" 136 | [{{{:keys [statements]} :statement-result 137 | :keys [attachments]} :body}] 138 | ;; As we encounter statements that reference attachments they move from 139 | ;; atts-in to atts-out. If an attachment is not found in atts-in, it may 140 | ;; be copied from the first entry in atts-out 141 | (let [grouped (group-by :sha2 attachments) 142 | {source-statements :acc 143 | leftover :atts-in} (reduce 144 | (fn [state s] 145 | (if-let [hash-tuples (not-empty (attachment-hashes s))] 146 | (let [{:keys [atts-in 147 | atts-out 148 | atts-acc]} 149 | (reduce 150 | find-attachments 151 | (merge (select-keys state [:atts-in :atts-out]) 152 | {:atts-acc []}) 153 | hash-tuples)] 154 | {:atts-in atts-in 155 | :atts-out atts-out 156 | :acc (conj (:acc state) {:statement s 157 | :attachments atts-acc})}) 158 | ;; No sha2s to check 159 | (update state 160 | :acc conj 161 | {:statement s 162 | :attachments []}))) 163 | {:atts-in grouped 164 | :atts-out {} 165 | :acc []} 166 | statements)] 167 | (when (not-empty leftover) 168 | (log/warnf "Extra attachments for shas %s, cleaning..." (pr-str (keys leftover))) 169 | (multipart/clean-tempfiles! (mapcat identity (vals leftover)))) 170 | source-statements)) 171 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/client_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.client-test 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.test :refer :all] 4 | [com.yetanalytics.xapipe.client :refer :all] 5 | [com.yetanalytics.xapipe.test-support :as sup])) 6 | 7 | (sup/def-ns-check-tests 8 | com.yetanalytics.xapipe.client 9 | {;; Don't test stateful ones like this 10 | com.yetanalytics.xapipe.client/create-store ::sup/skip 11 | com.yetanalytics.xapipe.client/get-loop ::sup/skip 12 | com.yetanalytics.xapipe.client/shutdown ::sup/skip 13 | com.yetanalytics.xapipe.client/init-client ::sup/skip 14 | :default {sup/stc-opts {:num-tests 1}}}) 15 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/filter/path_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.filter.path-test 2 | (:require [clojure.test :refer :all] 3 | [com.yetanalytics.xapipe.filter.path :refer :all] 4 | [com.yetanalytics.xapipe.test-support :as sup])) 5 | 6 | (sup/def-ns-check-tests com.yetanalytics.xapipe.filter.path 7 | ;; TODO: These tests are very slow. Investigate pathetic's gens 8 | {:default {sup/stc-opts {:num-tests 1}}}) 9 | 10 | (deftest path-filter-pred-test 11 | (let [statements (read-string (slurp "dev-resources/statements/calibration_50.edn")) 12 | 13 | ;; Records to pass through the predicate 14 | records (map #(hash-map :statement %) 15 | statements)] 16 | (testing "excludes missing" 17 | (let [;; Generated statements have no stored 18 | pred (path-filter-pred {:ensure-paths [ 19 | [["stored"]] 20 | ]})] 21 | (is (= [] (filter pred records))))) 22 | (testing "includes present" 23 | (let [pred (path-filter-pred {:ensure-paths [ 24 | [["id"]] 25 | ]})] 26 | (is (= records (filter pred records))))) 27 | (testing "include intersection" 28 | (let [pred (path-filter-pred {:ensure-paths [ 29 | [["id"]] 30 | [["timestamp"]] 31 | ]})] 32 | (is (= records (filter pred records))))) 33 | (testing "can match" 34 | (let [pred (path-filter-pred 35 | {:match-paths [ 36 | [[["id"]] 37 | "66ff4283-a6ca-439c-b8fe-38ca398271e5"] 38 | ]})] 39 | (is (= 1 (count (filter pred records)))))) 40 | (testing "can match intersection" 41 | (let [pred (path-filter-pred 42 | {:match-paths [ 43 | [[["id"]] 44 | "66ff4283-a6ca-439c-b8fe-38ca398271e5"] 45 | [[["verb"]["id"]] 46 | "https://xapinet.org/xapi/yet/calibration/v1/concepts#didnt"] 47 | ]})] 48 | (is (= 1 (count (filter pred records)))))) 49 | 50 | (testing "can match union" 51 | (let [pred (path-filter-pred 52 | {:match-paths [ 53 | [[["id"]] 54 | "66ff4283-a6ca-439c-b8fe-38ca398271e5"] 55 | [[["id"]] 56 | "ee309867-15af-4c76-87b4-5fb49d687ee9"] 57 | ]})] 58 | (is (= 2 (count (filter pred records)))))))) 59 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/job/config_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.config-test 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.test :refer :all] 4 | [com.yetanalytics.xapipe.job.config :refer :all] 5 | [com.yetanalytics.xapipe.test-support :as sup])) 6 | 7 | (def minimal-config 8 | {:source 9 | {:request-config {:url-base "http://0.0.0.0:8080" 10 | :xapi-prefix "/xapi"}} 11 | :target 12 | {:request-config {:url-base "http://0.0.0.0:8081" 13 | :xapi-prefix "/xapi"}}}) 14 | 15 | (deftest minimal-config-test 16 | (testing "produces valid config" 17 | (is (s/valid? config-spec (ensure-defaults minimal-config)))) 18 | (testing "idempotent" 19 | (is (= (ensure-defaults 20 | minimal-config) 21 | (ensure-defaults 22 | (ensure-defaults 23 | minimal-config)))))) 24 | 25 | (sup/def-ns-check-tests 26 | com.yetanalytics.xapipe.job.config 27 | {:default {sup/stc-opts {:num-tests 10}}}) 28 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/job/json_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.json-test 2 | (:require [com.yetanalytics.xapipe.job.json :refer :all] 3 | [com.yetanalytics.xapipe.job :as job] 4 | [clojure.test :refer :all] 5 | [com.yetanalytics.xapipe.test-support :as sup] 6 | [clojure.spec.alpha :as s])) 7 | 8 | (sup/def-ns-check-tests 9 | com.yetanalytics.xapipe.job.json 10 | {:default {sup/stc-opts {:num-tests 25}}}) 11 | 12 | (deftest transit-roundtrip-test 13 | (is 14 | (= {:foo "bar" 15 | [{}] :sure} 16 | (read-transit-str 17 | (write-transit-str 18 | {:foo "bar" 19 | [{}] :sure}))))) 20 | 21 | (def complex-job 22 | {:id "foo", 23 | :config 24 | {:get-buffer-size 10, 25 | :statement-buffer-size 500, 26 | :batch-buffer-size 10, 27 | :batch-timeout 200, 28 | :cleanup-buffer-size 50, 29 | :source 30 | {:request-config 31 | {:url-base "http://0.0.0.0:8080", :xapi-prefix "/xapi"}, 32 | :batch-size 50, 33 | :backoff-opts {:budget 10000, :max-attempt 10}, 34 | :poll-interval 1000, 35 | :get-params {:limit 50}}, 36 | :target 37 | {:request-config 38 | {:url-base "http://0.0.0.0:8081", :xapi-prefix "/xapi"}, 39 | :batch-size 50, 40 | :backoff-opts {:budget 10000, :max-attempt 10}}, 41 | :filter 42 | {:pattern 43 | {:profile-urls ["dev-resources/profiles/calibration_strict_pattern.jsonld"] 44 | :pattern-ids []}}}, 45 | :state 46 | {:status :init, 47 | :cursor "1970-01-01T00:00:00.000000000Z", 48 | :source {:errors []}, 49 | :target {:errors []}, 50 | :errors [], 51 | :filter {:pattern 52 | {:accepts [], 53 | :rejects [], 54 | :states-map 55 | {"d7acfddb-f4c2-49f4-a081-ad1fb8490448" 56 | {"https://xapinet.org/xapi/yet/calibration_strict_pattern/v1/patterns#pattern-1" 57 | #{{:state 0, :accepted? false}}}}}}}}) 58 | 59 | (deftest complex-job-test 60 | (testing "Packs + unpacks a complex job" 61 | (testing "input sanity" 62 | (is (nil? (s/explain-data job/job-spec complex-job)))) 63 | (testing "Roundtrip" 64 | (let [roundtripped (-> complex-job job->json json->job)] 65 | (is (= complex-job 66 | roundtripped)))))) 67 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/job/state_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job.state-test 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.test :refer :all] 4 | [com.yetanalytics.xapipe.job.state :refer :all] 5 | [com.yetanalytics.xapipe.test-support :as sup])) 6 | 7 | (sup/def-ns-check-tests 8 | com.yetanalytics.xapipe.job.state 9 | {:default {sup/stc-opts {:num-tests 10}}}) 10 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/job_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.job-test 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.test :refer :all] 4 | [com.yetanalytics.xapipe.job :refer :all] 5 | [com.yetanalytics.xapipe.test-support :as sup])) 6 | 7 | (def minimal-config 8 | {:source 9 | {:request-config {:url-base "http://0.0.0.0:8080" 10 | :xapi-prefix "/xapi"}} 11 | :target 12 | {:request-config {:url-base "http://0.0.0.0:8081" 13 | :xapi-prefix "/xapi"}}}) 14 | 15 | (deftest minimal-config-test 16 | (testing "sanity check: minimal input config is valid" 17 | (is (s/valid? :com.yetanalytics.xapipe.job/config 18 | minimal-config)))) 19 | 20 | (deftest init-job-defaults-test 21 | (testing "given an id and minimal config it constructs a valid job" 22 | (is (s/valid? job-spec 23 | (init-job 24 | "foo" 25 | minimal-config)))) 26 | (testing "applies defaults" 27 | (is (= {:id "foo", 28 | :version 0, 29 | :config 30 | {:get-buffer-size 10, 31 | :statement-buffer-size 500, 32 | :batch-buffer-size 10, 33 | :batch-timeout 200, 34 | :cleanup-buffer-size 50, 35 | :source 36 | {:request-config 37 | {:url-base "http://0.0.0.0:8080", :xapi-prefix "/xapi"}, 38 | :batch-size 50, 39 | :backoff-opts {:budget 10000, :max-attempt 10}, 40 | :poll-interval 1000, 41 | :get-params {:limit 50}}, 42 | :target 43 | {:request-config 44 | {:url-base "http://0.0.0.0:8081", :xapi-prefix "/xapi"}, 45 | :batch-size 50, 46 | :backoff-opts {:budget 10000, :max-attempt 10}}, 47 | :filter {}}, 48 | :state 49 | {:status :init, 50 | :cursor "1970-01-01T00:00:00.000000000Z", 51 | :source {:errors []}, 52 | :target {:errors []}, 53 | :errors [], 54 | :filter {}}} 55 | 56 | (init-job 57 | "foo" 58 | minimal-config))))) 59 | 60 | (sup/def-ns-check-tests 61 | com.yetanalytics.xapipe.job 62 | {:default {sup/stc-opts {:num-tests 10}}}) 63 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/test_support/lrs.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.test-support.lrs 2 | "LRS and LRS Facade facilities" 3 | (:require [com.yetanalytics.lrs.protocol :as lrsp] 4 | [com.yetanalytics.lrs.xapi.statements :as ss] 5 | [com.yetanalytics.lrs.xapi.statements.timestamp :as timestamp] 6 | [cheshire.core :as json] 7 | [clojure.java.io :as io] 8 | [com.yetanalytics.lrs.util :as lrsu])) 9 | 10 | ;; An LRS that accepts statements but does not retain them 11 | (deftype SinkLRS [] 12 | lrsp/StatementsResource 13 | (-store-statements [_ _ statements attachments] 14 | {:statement-ids 15 | (into [] 16 | (map #(ss/normalize-id (get % "id")) 17 | (map 18 | (fn [s stamp] 19 | (ss/prepare-statement 20 | (assoc s "stored" stamp))) 21 | statements 22 | (timestamp/stamp-seq))))}) 23 | (-consistent-through [_ _ _] 24 | (ss/now-stamp)) 25 | lrsp/LRSAuth 26 | (-authenticate [_ _] 27 | {:result 28 | {:scopes #{:scope/all} 29 | :prefix "" 30 | :auth {:no-op {}}}}) 31 | (-authorize [_ _ _] 32 | {:result true})) 33 | 34 | (defn- make-more-url 35 | [xapi-path-prefix params last-id agent] 36 | (str xapi-path-prefix 37 | "/statements?" 38 | (lrsu/form-encode 39 | (cond-> params 40 | true (assoc :from last-id) 41 | ;; Re-encode the agent if present 42 | agent (assoc :agent (lrsu/json-string agent)))))) 43 | 44 | (defn stream-lrs 45 | "Create a read-only LRS that ignores all params except limit and streams 46 | statements from a json file" 47 | [path & {:keys [xapi-prefix] 48 | :or {xapi-prefix "/xapi"}}] 49 | (let [ss-atom (atom (json/parsed-seq (io/reader path)))] 50 | (reify 51 | lrsp/StatementsResource 52 | (-get-statements [_ 53 | _ 54 | {:keys [limit] 55 | :or {limit 50}} 56 | _] 57 | (let [[batch rest-ss] (split-at limit @ss-atom)] 58 | (reset! ss-atom rest-ss) 59 | {:attachments [] 60 | :statement-result 61 | (cond-> {:statements []} 62 | (not-empty batch) 63 | (update :statements into batch) 64 | 65 | (not-empty rest-ss) 66 | (assoc :more 67 | (str xapi-prefix 68 | "/statements?" 69 | (lrsu/form-encode 70 | {:limit limit 71 | :attachments true}))))})) 72 | 73 | (-consistent-through [_ _ _] 74 | (ss/now-stamp)) 75 | lrsp/LRSAuth 76 | (-authenticate [_ _] 77 | {:result 78 | {:scopes #{:scope/all} 79 | :prefix "" 80 | :auth {:no-op {}}}}) 81 | (-authorize [_ _ _] 82 | {:result true})))) 83 | 84 | (defn get-stream-lrs-range 85 | "Get the stored range of a large JSON file" 86 | [path] 87 | (let [ss (json/parsed-seq (io/reader path))] 88 | [(-> ss first (get "stored")) 89 | (-> ss last (get "stored"))])) 90 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/xapipe/util/async_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.xapipe.util.async-test 2 | (:require [clojure.test :refer :all] 3 | [com.yetanalytics.xapipe.util.async :refer :all] 4 | [clojure.core.async :as a] 5 | [com.yetanalytics.xapipe.test-support :as sup])) 6 | 7 | (use-fixtures :once (sup/instrument-fixture)) 8 | 9 | (deftest batch-filter-test 10 | (testing "with no predicates" 11 | (let [a-chan (a/chan 1000) 12 | b-chan (a/chan 20) 13 | cleanup-chan (a/chan 1000) 14 | _ (batch-filter 15 | a-chan 16 | b-chan 17 | 50 18 | 500 19 | :cleanup-chan cleanup-chan)] 20 | 21 | (a/onto-chan! a-chan (range 1000)) 22 | 23 | (let [batches (a/= 50) count :batch) 33 | batches))) 34 | (testing "empty states" 35 | (is 36 | (every? 37 | (fn [{:keys [filter-state]}] 38 | (empty? filter-state)) 39 | batches))) 40 | (testing "no dropped" 41 | (is (= [] 42 | dropped)) 43 | (is (= 0 44 | (count dropped))))))) 45 | (testing "With stateless predicates" 46 | (let [a-chan (a/chan 1000) 47 | b-chan (a/chan 20) 48 | cleanup-chan (a/chan 1000) 49 | _ (batch-filter 50 | a-chan 51 | b-chan 52 | 50 53 | 500 54 | :stateless-predicates 55 | {:odd? odd?} 56 | :cleanup-chan cleanup-chan)] 57 | 58 | (a/onto-chan! a-chan (range 1000)) 59 | 60 | (let [batches (a/statements-test 14 | (testing "deletes extra attachments" 15 | (let [attachment 16 | (sgen/generate 17 | (s/gen :com.yetanalytics.xapipe.client.multipart-mixed/attachment))] 18 | (response->statements {:body 19 | {:statement-result {:statements []} 20 | :attachments [attachment]}}) 21 | (is (false? (.exists (:tempfile attachment))))))) 22 | -------------------------------------------------------------------------------- /src/test/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | --------------------------------------------------------------------------------