├── .dir-locals.el ├── .github └── workflows │ ├── build.yml │ ├── nvd.yml │ └── test.yml ├── .gitignore ├── .nvd ├── config.json └── suppression.xml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── deps.edn ├── dev-resources ├── bench │ ├── actors.json │ ├── calibration.jsonld │ ├── models.json │ └── params.json ├── input │ ├── gift128.json │ ├── gift64.json │ ├── mom128.json │ ├── mom64.json │ └── simple.json ├── models │ ├── empty.json │ ├── simple.json │ ├── simple_with_overrides.json │ ├── simple_with_repeat_max.json │ ├── simple_with_temporal.json │ ├── simple_with_weights.json │ ├── tccc_dev.json │ └── temporal │ │ ├── 10a_restart_current_pattern.json │ │ ├── 10b_restart_parent_pattern.json │ │ ├── 10c_restart_all_pattern.json │ │ ├── 10d_restart_child_pattern.json │ │ ├── 10e_restart_child_template.json │ │ ├── 11a_end_current_pattern.json │ │ ├── 11b_end_parent_pattern.json │ │ ├── 11c_end_all_pattern.json │ │ ├── 11d_end_child_pattern.json │ │ ├── 11e_end_child_template.json │ │ ├── 1a_year_2023.json │ │ ├── 1b_year_2023-2024.json │ │ ├── 1c_year_0-2023.json │ │ ├── 1d_year_0-2022.json │ │ ├── 2a_dow_Tues-Thurs_hour_8-12.json │ │ ├── 2b_dow_Tues-Thurs_hour_8-12-18.json │ │ ├── 3a_dom_28.json │ │ ├── 3b_dom_29.json │ │ ├── 3c_dom_30.json │ │ ├── 3d_dom_31.json │ │ ├── 4a_Dec_25.json │ │ ├── 4b_Dec_25_Mon.json │ │ ├── 5a_Jan_1_sec_10-30.json │ │ ├── 5b_Jan_1_min_even_sec_10-30.json │ │ ├── 5c_Jan_1_hour_even_sec_10-30.json │ │ ├── 5d_Jan_1_min_hour_even_sec_10-30.json │ │ ├── 6a_hours_period_every_second_hour.json │ │ ├── 6b_hours_period_every_start_hour.json │ │ ├── 6c_hours_fixed_period_every_second_hour.json │ │ ├── 6d_hours_fixed_period_every_start_hour.json │ │ ├── 7a_millis_period.json │ │ ├── 7b_seconds_period.json │ │ ├── 7c_minutes_period.json │ │ ├── 7d_hours_period.json │ │ ├── 7e_days_period.json │ │ ├── 7f_weeks_period.json │ │ ├── 8a_millis_fixed_period.json │ │ ├── 8b_seconds_fixed_period.json │ │ ├── 8c_minutes_fixed_period.json │ │ ├── 8d_hours_fixed_period.json │ │ ├── 8e_days_fixed_period.json │ │ ├── 8f_weeks_fixed_period.json │ │ ├── 9a_outer_bound_larger.json │ │ └── 9b_outer_bound_smaller.json ├── parameters │ ├── ds_102.json │ ├── simple.json │ ├── tccc_dev.json │ └── temporal.json ├── personae │ ├── array │ │ └── simple_array.json │ ├── bench.json │ ├── ds_102_128.json │ ├── ds_102_64.json │ ├── simple.json │ ├── tccc_dev.json │ └── temporal.json ├── profiles │ ├── acrossx │ │ ├── fixed.jsonld │ │ └── profile.jsonld │ ├── activity.jsonld │ ├── activity_streams │ │ ├── fixed.jsonld │ │ └── profile.jsonld │ ├── calibration.jsonld │ ├── cmi5 │ │ └── fixed.json │ ├── gift │ │ └── gift.jsonld │ ├── minimal.jsonld │ ├── no_concept.jsonld │ ├── referential.jsonld │ ├── tccc │ │ └── cuf_hc_video_and_asm_student_survey_profile.jsonld │ ├── temporal.jsonld │ ├── tincan │ │ └── profile.jsonld │ ├── tla │ │ └── mom.jsonld │ └── video │ │ └── profile.jsonld └── xapi │ └── statements │ ├── array │ └── tccc_dev.json │ ├── long.json │ └── simple.json ├── doc ├── intro.md └── temporal_structure.md ├── onyx-resources └── onyx_config.edn ├── scripts ├── gc.sh ├── gc_checkpoints.sh ├── onyx.sh ├── peer.sh ├── peer_driver.sh ├── repl.sh ├── run.sh ├── server.sh └── submit_job.sh ├── src ├── cli │ └── com │ │ └── yetanalytics │ │ └── datasim │ │ ├── cli.clj │ │ └── cli │ │ ├── generate.clj │ │ ├── input.clj │ │ └── util.clj ├── dev │ ├── bench.clj │ └── user.clj ├── dev_onyx │ └── com │ │ └── yetanalytics │ │ └── datasim │ │ └── onyx │ │ ├── dev_peer.clj │ │ └── scratch.clj ├── main │ └── com │ │ └── yetanalytics │ │ ├── datasim.clj │ │ └── datasim │ │ ├── client.clj │ │ ├── input.clj │ │ ├── input │ │ ├── model.clj │ │ ├── model │ │ │ ├── alignments.clj │ │ │ └── personae.clj │ │ ├── parameters.clj │ │ ├── personae.clj │ │ └── profile.clj │ │ ├── model.clj │ │ ├── model │ │ ├── bounds.clj │ │ └── periods.clj │ │ ├── sim.clj │ │ ├── util │ │ ├── async.clj │ │ ├── errors.clj │ │ ├── io.clj │ │ ├── random.clj │ │ └── sequence.clj │ │ └── xapi │ │ ├── actor.clj │ │ ├── path.clj │ │ ├── profile.clj │ │ ├── profile │ │ ├── activity.clj │ │ ├── extension.clj │ │ ├── pattern.clj │ │ ├── template.clj │ │ └── verb.clj │ │ ├── rule.clj │ │ ├── statement.clj │ │ └── statement │ │ └── healing.clj ├── onyx │ └── com │ │ └── yetanalytics │ │ └── datasim │ │ └── onyx │ │ ├── aeron_media_driver.clj │ │ ├── config.clj │ │ ├── http.clj │ │ ├── job.clj │ │ ├── main.clj │ │ ├── peer.clj │ │ ├── repl.clj │ │ ├── scheduling │ │ └── semi_colocated_task_scheduler.clj │ │ ├── sim.clj │ │ └── util.clj ├── server │ └── com │ │ └── yetanalytics │ │ └── datasim │ │ └── server.clj ├── test │ └── com │ │ └── yetanalytics │ │ ├── cli_test.clj │ │ ├── datasim │ │ ├── input │ │ │ ├── model_test.clj │ │ │ ├── personae_test.clj │ │ │ └── profile_test.clj │ │ ├── input_test.clj │ │ ├── model │ │ │ ├── bounds_test.clj │ │ │ └── periods_test.clj │ │ ├── test_constants.clj │ │ ├── test_containers.clj │ │ ├── util │ │ │ ├── io_test.clj │ │ │ └── random_test.clj │ │ └── xapi │ │ │ ├── path_test.clj │ │ │ ├── profile_test.clj │ │ │ ├── rule_test.clj │ │ │ └── statement_test.clj │ │ ├── datasim_test.clj │ │ ├── datasim_test_temporal.clj │ │ └── server_test.clj └── test_onyx │ └── com │ └── yetanalytics │ └── datasim │ └── onyx │ ├── job_test.clj │ └── scheduling │ └── semi_colocated_task_scheduler_test.clj └── template ├── 0_vpc.yml ├── 1_hose.yml ├── 1_zk.yml └── 2_cluster.yml /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;;; Directory Local Variables 2 | ;;; For more information see (info "(emacs) Directory Variables") 3 | 4 | ((nil . 5 | ((cider-clojure-cli-global-options . "-A:cli:dev:test:onyx:onyx-dev")))) 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - '*' 7 | tags: 8 | - 'v*.*.*' # Enforce Semantic Versioning 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout Repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Setup CI environment 19 | uses: yetanalytics/actions/setup-env@v0.0.4 20 | 21 | - name: Build Bundle 22 | run: make bundle 23 | 24 | # Build docker for PR, tags 25 | - name: Extract metadata (tags, labels) for Docker 26 | id: meta 27 | uses: docker/metadata-action@v4 28 | with: 29 | images: yetanalytics/datasim 30 | tags: | 31 | type=ref,event=branch 32 | type=ref,event=pr 33 | type=semver,pattern={{version}} 34 | type=semver,pattern={{major}}.{{minor}} 35 | 36 | - name: Log in to Docker Hub 37 | uses: docker/login-action@v2 38 | with: 39 | username: ${{ secrets.DOCKERHUB_USERNAME }} 40 | password: ${{ secrets.DOCKERHUB_CI_TOKEN }} 41 | 42 | - name: Build and push Docker image 43 | uses: docker/build-push-action@v3 44 | with: 45 | context: . 46 | push: true 47 | tags: ${{ steps.meta.outputs.tags }} 48 | labels: ${{ steps.meta.outputs.labels }} 49 | 50 | - name: Compress Bundle 51 | run: | # Need to cd so that the zip file doesn't contain the parent dirs 52 | cd target/bundle 53 | zip -r ../../datasim-bundle.zip ./ 54 | 55 | - name: Archive Bundle 56 | uses: actions/upload-artifact@v4 57 | with: 58 | name: datasim-bundle-artifact-${{ github.sha }} 59 | path: datasim-bundle.zip 60 | 61 | deploy: 62 | runs-on: ubuntu-latest 63 | 64 | if: ${{ github.ref_type == 'tag' }} 65 | 66 | steps: 67 | - name: Checkout repository 68 | uses: actions/checkout@v4 69 | 70 | - name: Setup CD environment 71 | uses: yetanalytics/action-setup-env@v1 72 | 73 | - name: Extract version 74 | id: version 75 | run: echo version=${GITHUB_REF#refs\/tags\/v} >> $GITHUB_OUTPUT 76 | 77 | - name: Build and deploy to Clojars 78 | uses: yetanalytics/action-deploy-clojars@v1 79 | with: 80 | artifact-id: 'datasim' 81 | version: ${{ steps.version.outputs.version }} 82 | resource-dirs: '[]' 83 | clojars-username: ${{ secrets.CLOJARS_USERNAME }} 84 | clojars-deploy-token: ${{ secrets.CLOJARS_PASSWORD }} 85 | 86 | release: 87 | runs-on: ubuntu-latest 88 | needs: build 89 | if: ${{ github.ref_type == 'tag' }} 90 | steps: 91 | - name: Checkout repository 92 | uses: actions/checkout@v4 93 | 94 | - name: Download Bundle Artifact 95 | uses: actions/download-artifact@v4 96 | with: 97 | name: datasim-bundle-artifact-${{ github.sha }} 98 | 99 | - name: Craft Draft Release 100 | uses: softprops/action-gh-release@v2 101 | with: 102 | # Defaults: 103 | # name: [tag name] 104 | # tag_name: github.ref 105 | draft: true 106 | files: datasim-bundle.zip 107 | -------------------------------------------------------------------------------- /.github/workflows/nvd.yml: -------------------------------------------------------------------------------- 1 | name: Periodic NVD Scan 2 | 3 | on: 4 | schedule: 5 | - cron: '0 8 * * 1-5' # Every weekday at 8:00 AM 6 | 7 | jobs: 8 | nvd-scan: 9 | uses: yetanalytics/workflow-nvd/.github/workflows/nvd-scan.yml@v1 10 | with: 11 | nvd-clojure-version: '3.2.0' 12 | classpath-command: 'clojure -Spath -A:cli:server' 13 | nvd-config-filename: '.nvd/config.json' 14 | notify-slack: true 15 | secrets: 16 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 17 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: push 4 | 5 | jobs: 6 | nvd-scan: 7 | uses: yetanalytics/workflow-nvd/.github/workflows/nvd-scan.yml@v1 8 | with: 9 | nvd-clojure-version: '3.2.0' 10 | # onyx dep is outdated and abandoned so don't bother scanning 11 | classpath-command: 'clojure -Spath -A:cli:server' 12 | nvd-config-filename: '.nvd/config.json' 13 | 14 | test: 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | target: [test-unit, test-unit-onyx, test-cli] 20 | 21 | steps: 22 | - name: Checkout Repository 23 | uses: actions/checkout@v3 24 | 25 | - name: Setup CI environment 26 | uses: yetanalytics/actions/setup-env@v0.0.4 27 | 28 | - name: Log into DockerHub 29 | if: ${{ matrix.target == 'test-unit' }} 30 | uses: docker/login-action@v3 31 | with: 32 | username: ${{ secrets.DOCKERHUB_USERNAME }} 33 | password: ${{ secrets.DOCKERHUB_TOKEN }} 34 | 35 | - name: Run Makefile Target ${{ matrix.target }} 36 | run: make ${{ matrix.target }} 37 | 38 | validate-template: 39 | runs-on: ubuntu-latest 40 | 41 | # These permissions are needed by configure-aws-credentials in order 42 | # to interact with GitHub's OIDC Token endpoint. 43 | permissions: 44 | id-token: write # required to use OIDC authentication 45 | contents: read # required to checkout the code from the repo 46 | 47 | steps: 48 | - name: Checkout Repository 49 | uses: actions/checkout@v3 50 | 51 | - name: Setup CI environment 52 | uses: yetanalytics/actions/setup-env@v0.0.4 53 | 54 | - name: Configure AWS Credentials 55 | uses: aws-actions/configure-aws-credentials@v1-node16 56 | with: 57 | role-to-assume: ${{ secrets.CF_VALIDATE_TEMPLATE_ROLE_ARN }} 58 | role-duration-seconds: 900 # 15 min; minimal duration possible 59 | aws-region: us-east-1 60 | 61 | - name: Run Makefile Target validate-template 62 | run: make validate-template 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /target 3 | /classes 4 | /checkouts 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | /.cpcache 10 | /.lein-* 11 | /.nrepl-history 12 | /.nrepl-port 13 | .hgignore 14 | .hg/ 15 | *.md.tmp 16 | *.log 17 | .rebel_readline_history 18 | 19 | # VSCode 20 | .calva/ 21 | .clj-kondo/ 22 | .lsp/ 23 | .vscode/ 24 | 25 | -------------------------------------------------------------------------------- /.nvd/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "nvd": {"suppression-file": ".nvd/suppression.xml"} 3 | } 4 | -------------------------------------------------------------------------------- /.nvd/suppression.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | ^pkg:maven\/(?!org\.clojure\/clojure).*$ 8 | cpe:/a:clojure:clojure 9 | CVE-2017-20189 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.16 2 | MAINTAINER Milt Reder 3 | 4 | ADD target/bundle /bundle 5 | 6 | RUN apk update \ 7 | && apk upgrade \ 8 | && apk add ca-certificates \ 9 | && update-ca-certificates \ 10 | && apk add --no-cache openjdk11-jre \ 11 | && rm -rf /var/cache/apk/* 12 | 13 | WORKDIR /bundle 14 | EXPOSE 9090 15 | ENTRYPOINT ["bin/run.sh"] 16 | CMD ["-h"] 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean bundle test-cli test-cli-comprehensive test-cli-output test-unit test-unit-onyx ci server test-bundle-output validate-template 2 | 3 | GROUP_ID ?= com.yetanalytics 4 | ARTIFACT_ID ?= datasim 5 | VERSION ?= 0.4.4 6 | 7 | clean: 8 | rm -rf target 9 | 10 | target/bundle/datasim_cli.jar: 11 | mkdir -p target/bundle 12 | rm -f pom.xml 13 | clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:cli]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-cli :version '"$(VERSION)"' :jar target/bundle/datasim_cli.jar :main-class com.yetanalytics.datasim.cli 14 | rm -f pom.xml 15 | 16 | target/bundle/datasim_server.jar: # no AOT for this one 17 | mkdir -p target/bundle 18 | rm -f pom.xml 19 | clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:server]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-server :version '"$(VERSION)"' :jar target/bundle/datasim_server.jar :main-class com.yetanalytics.datasim.server 20 | rm -f pom.xml 21 | 22 | target/bundle/datasim_onyx.jar: 23 | mkdir -p target/bundle 24 | rm -f pom.xml 25 | TIMBRE_LOG_LEVEL=:info clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:onyx,:cli]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-onyx :version '"$(VERSION)"' :jar target/bundle/datasim_onyx.jar :main-class com.yetanalytics.datasim.onyx.main 26 | rm -f pom.xml 27 | 28 | target/bundle/bin: 29 | mkdir -p target/bundle/bin 30 | cp -r scripts/*.sh target/bundle/bin 31 | chmod +x target/bundle/bin 32 | 33 | target/bundle: target/bundle/bin target/bundle/datasim_cli.jar target/bundle/datasim_server.jar target/bundle/datasim_onyx.jar 34 | 35 | bundle: target/bundle 36 | 37 | # Tests 38 | 39 | test-unit: 40 | clojure -Adev:cli:server:test:run-test 41 | 42 | test-unit-onyx: 43 | clojure -Adev:cli:onyx:onyx-test:run-onyx-test 44 | 45 | test-cli: 46 | clojure -A:cli:run-cli validate-input -p dev-resources/profiles/cmi5/fixed.json -a dev-resources/personae/simple.json -m dev-resources/models/simple.json -o dev-resources/parameters/simple.json -v dev-resources/input/simple.json 47 | 48 | test-cli-comprehensive: 49 | clojure -A:cli:run-cli validate-input -i dev-resources/input/simple.json -v dev-resources/input/simple.json 50 | 51 | test-cli-output: 52 | clojure -A:cli:run-cli generate -i dev-resources/input/simple.json 53 | 54 | test-bundle-output: bundle 55 | cd target/bundle; bin/run.sh generate -i ../../dev-resources/input/simple.json 56 | 57 | validate-template: 58 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/0_vpc.yml 59 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/1_hose.yml 60 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/1_zk.yml 61 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/2_cluster.yml 62 | 63 | ci: test-unit test-unit-onyx test-cli validate-template 64 | 65 | server: 66 | clojure -A:server:run-server 67 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["resources" "src/main"] 2 | :deps {org.clojure/clojure {:mvn/version "1.11.2"} 3 | org.clojure/core.async {:mvn/version "1.6.673"} 4 | org.clojure/core.memoize {:mvn/version "1.0.257"} 5 | com.yetanalytics/xapi-schema 6 | {:mvn/version "1.2.0" 7 | :exclusions [org.clojure/clojurescript]} 8 | com.yetanalytics/project-pan 9 | {:mvn/version "0.4.3" 10 | :exclusions [org.clojure/clojurescript 11 | com.yetanalytics./xapi-schema]} 12 | com.yetanalytics/pathetic {:mvn/version "0.5.0"} 13 | com.yetanalytics/schemer {:mvn/version "0.1.0"} 14 | clojure.java-time/clojure.java-time {:mvn/version "1.2.0"} 15 | org.clojure/test.check {:mvn/version "1.1.1"} 16 | http-kit/http-kit {:mvn/version "2.7.0"} 17 | cheshire/cheshire {:mvn/version "5.12.0"}} 18 | :aliases 19 | {:dev {:extra-paths ["dev-resources" "src/dev"] 20 | :extra-deps {incanter/incanter-core {:mvn/version "1.9.3"} 21 | incanter/incanter-charts {:mvn/version "1.9.3"} 22 | criterium/criterium {:mvn/version "0.4.5"}}} 23 | :test {:extra-paths ["src/test" "src/test_onyx"] 24 | :extra-deps {same/ish {:mvn/version "0.1.6"} 25 | clj-test-containers/clj-test-containers {:mvn/version "0.7.4"}}} 26 | :run-test {:extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" 27 | :git/sha "dfb30dd"}} 28 | :main-opts ["-m" "cognitect.test-runner" 29 | "-d" "src/test"]} 30 | :cli {:extra-paths ["src/cli"] 31 | :extra-deps {org.clojure/tools.cli {:mvn/version "1.0.219"}}} 32 | :run-cli {:main-opts ["-m" "com.yetanalytics.datasim.cli"]} 33 | :server {:extra-paths ["src/server"] 34 | :extra-deps 35 | {io.pedestal/pedestal.jetty {:mvn/version "0.6.3" 36 | :exclusions [org.eclipse.jetty.http2/http2-server]} 37 | io.pedestal/pedestal.service {:mvn/version "0.6.3"} 38 | org.eclipse.jetty.http2/http2-server {:mvn/version "9.4.54.v20240208"} 39 | org.slf4j/slf4j-simple {:mvn/version "1.7.28"} 40 | clj-http/clj-http {:mvn/version "3.12.3"} 41 | environ/environ {:mvn/version "1.1.0"} 42 | ;; Buddy/BouncyCastle deps 43 | buddy/buddy-auth {:mvn/version "3.0.323" 44 | :exclusions [cheshire/cheshire 45 | buddy/buddy-sign]} 46 | buddy/buddy-sign {:mvn/version "3.5.346" 47 | :exclusions [org.bouncycastle/bcprov-jdk18on]} 48 | org.bouncycastle/bcprov-jdk18on {:mvn/version "1.78"}}} 49 | :run-server {:main-opts ["-m" "com.yetanalytics.datasim.server"]} 50 | :onyx {:extra-paths ["onyx-resources" "src/onyx"] 51 | :extra-deps {com.amazonaws/aws-java-sdk-s3 {:mvn/version "1.11.899"} 52 | com.amazonaws/aws-java-sdk-core {:mvn/version "1.11.899"} 53 | org.onyxplatform/onyx {:mvn/version "0.14.6" 54 | :exclusions 55 | ;; TODO: More exclusions probably 56 | [org.clojure/clojure 57 | org.clojure/core.async 58 | org.slf4j/slf4j-nop 59 | com.amazonaws/aws-java-sdk-s3]} 60 | aleph/aleph {:mvn/version "0.4.7-alpha7"} 61 | com.fzakaria/slf4j-timbre {:mvn/version "0.3.20"} 62 | org.onyxplatform/lib-onyx {:mvn/version "0.14.1.0"} 63 | org.onyxplatform/onyx-http {:mvn/version "0.14.5.0" 64 | :exclusions [aleph/aleph 65 | io.netty/netty-all]} 66 | ;; for local repl 67 | com.bhauman/rebel-readline {:mvn/version "0.1.4"} 68 | ;; for remote repl 69 | nrepl/nrepl {:mvn/version "0.8.3"} 70 | cider/cider-nrepl {:mvn/version "0.25.6"} 71 | org.onyxplatform/onyx-peer-http-query {:mvn/version "0.14.5.1-SNAPSHOT"} 72 | org.onyxplatform/onyx-amazon-s3 {:mvn/version "0.14.5.0" 73 | :exclusions [org.clojure/clojure 74 | org.onyxplatform/onyx 75 | com.amazonaws/aws-java-sdk-s3]}}} 76 | :onyx-dev {:extra-paths ["src/onyx_dev"]} 77 | :onyx-test {:extra-paths ["src/test_onyx"]} 78 | :run-onyx-test {:extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" 79 | :git/sha "dfb30dd"}} 80 | :main-opts ["-m" "cognitect.test-runner" 81 | "-d" "src/test_onyx"]} 82 | :depstar {:replace-deps ; tool usage is new in 2.x 83 | {seancorfield/depstar {:mvn/version "2.0.165"}} 84 | :ns-default hf.depstar 85 | :exec-args {:no-pom true}}}} 86 | -------------------------------------------------------------------------------- /dev-resources/bench/actors.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "trainees", 3 | "objectType": "Group", 4 | "member": [ 5 | { 6 | "name": "Dionne Knox", 7 | "mbox": "mailto:0@yetanalytics.com" 8 | }, 9 | { 10 | "name": "Penny Mayer", 11 | "mbox": "mailto:1@yetanalytics.com" 12 | }, 13 | { 14 | "name": "Stanley Ayers", 15 | "mbox": "mailto:2@yetanalytics.com" 16 | }, 17 | { 18 | "name": "Reid Joyner", 19 | "mbox": "mailto:3@yetanalytics.com" 20 | }, 21 | { 22 | "name": "Loretta Elliott", 23 | "mbox": "mailto:4@yetanalytics.com" 24 | }, 25 | { 26 | "name": "Hawkins Landry", 27 | "mbox": "mailto:5@yetanalytics.com" 28 | }, 29 | { 30 | "name": "Zimmerman Short", 31 | "mbox": "mailto:6@yetanalytics.com" 32 | }, 33 | { 34 | "name": "Jeannette Castaneda", 35 | "mbox": "mailto:7@yetanalytics.com" 36 | }, 37 | { 38 | "name": "Ramos Boyer", 39 | "mbox": "mailto:8@yetanalytics.com" 40 | }, 41 | { 42 | "name": "Montgomery Dale", 43 | "mbox": "mailto:9@yetanalytics.com" 44 | }, 45 | { 46 | "name": "Jerry Rose", 47 | "mbox": "mailto:10@yetanalytics.com" 48 | }, 49 | { 50 | "name": "Katina Armstrong", 51 | "mbox": "mailto:11@yetanalytics.com" 52 | }, 53 | { 54 | "name": "Dickson Gaines", 55 | "mbox": "mailto:12@yetanalytics.com" 56 | }, 57 | { 58 | "name": "Patricia Forbes", 59 | "mbox": "mailto:13@yetanalytics.com" 60 | }, 61 | { 62 | "name": "Petty Huber", 63 | "mbox": "mailto:14@yetanalytics.com" 64 | }, 65 | { 66 | "name": "Sexton Cooley", 67 | "mbox": "mailto:15@yetanalytics.com" 68 | }, 69 | { 70 | "name": "Nola Burns", 71 | "mbox": "mailto:16@yetanalytics.com" 72 | }, 73 | { 74 | "name": "Caitlin Lancaster", 75 | "mbox": "mailto:17@yetanalytics.com" 76 | }, 77 | { 78 | "name": "Dean Roth", 79 | "mbox": "mailto:18@yetanalytics.com" 80 | }, 81 | { 82 | "name": "Wilson Chambers", 83 | "mbox": "mailto:19@yetanalytics.com" 84 | }, 85 | { 86 | "name": "Helga Hutchinson", 87 | "mbox": "mailto:20@yetanalytics.com" 88 | }, 89 | { 90 | "name": "Mary Flores", 91 | "mbox": "mailto:21@yetanalytics.com" 92 | }, 93 | { 94 | "name": "Baker Orr", 95 | "mbox": "mailto:22@yetanalytics.com" 96 | }, 97 | { 98 | "name": "Randall Fernandez", 99 | "mbox": "mailto:23@yetanalytics.com" 100 | }, 101 | { 102 | "name": "Margie Pratt", 103 | "mbox": "mailto:24@yetanalytics.com" 104 | }, 105 | { 106 | "name": "Christian Baxter", 107 | "mbox": "mailto:25@yetanalytics.com" 108 | }, 109 | { 110 | "name": "Guerrero Glass", 111 | "mbox": "mailto:26@yetanalytics.com" 112 | }, 113 | { 114 | "name": "Marian Fox", 115 | "mbox": "mailto:27@yetanalytics.com" 116 | }, 117 | { 118 | "name": "Ayers Farley", 119 | "mbox": "mailto:28@yetanalytics.com" 120 | }, 121 | { 122 | "name": "Milagros Hudson", 123 | "mbox": "mailto:29@yetanalytics.com" 124 | }, 125 | { 126 | "name": "Brittany French", 127 | "mbox": "mailto:30@yetanalytics.com" 128 | }, 129 | { 130 | "name": "Mueller Ward", 131 | "mbox": "mailto:31@yetanalytics.com" 132 | }, 133 | { 134 | "name": "Emily Lucas", 135 | "mbox": "mailto:32@yetanalytics.com" 136 | }, 137 | { 138 | "name": "Cohen Shepherd", 139 | "mbox": "mailto:33@yetanalytics.com" 140 | } 141 | ] 142 | } 143 | -------------------------------------------------------------------------------- /dev-resources/bench/models.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bob@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "activities": [ 10 | { 11 | "id": "https://example.org/activity/a", 12 | "weight": 0.5 13 | }, 14 | { 15 | "id": "https://example.org/activity/c", 16 | "weight": 0.2 17 | } 18 | ] 19 | }, 20 | { 21 | "personae": [ 22 | { 23 | "id": "mbox::mailto:alice@example.org", 24 | "type": "Agent" 25 | } 26 | ], 27 | "activities": [ 28 | { 29 | "id": "https://example.org/activity/c", 30 | "weight": 0.7 31 | }, 32 | { 33 | "id": "https://example.org/activity/d", 34 | "weight": 0.02 35 | } 36 | ] 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /dev-resources/bench/params.json: -------------------------------------------------------------------------------- 1 | {"start": "2019-11-18T11:38:39.219768Z", 2 | "end": "2021-01-18T11:38:39.219768Z", 3 | "timezone": "America/New_York", 4 | "seed": 44} 5 | -------------------------------------------------------------------------------- /dev-resources/models/empty.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /dev-resources/models/simple.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bobfake@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "activities": [ 10 | { 11 | "id": "https://example.org/activity/a", 12 | "weight": 0.5 13 | }, 14 | { 15 | "id": "https://example.org/activity/c", 16 | "weight": 0.2 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/simple_with_overrides.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bobfake@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "activities": [ 10 | { 11 | "id": "https://example.org/activity/a", 12 | "weight": 0.5 13 | }, 14 | { 15 | "id": "https://example.org/activity/c", 16 | "weight": 0.2 17 | } 18 | ], 19 | "objectOverrides": [ 20 | { 21 | "weight": 0.7, 22 | "object": { 23 | "objectType": "Activity", 24 | "id": "https://www.whatever.com/activities#course2", 25 | "definition": { 26 | "name": { 27 | "en-US": "Course 2" 28 | }, 29 | "description": { 30 | "en-US": "Course Description 2" 31 | }, 32 | "type": "http://adlnet.gov/expapi/activities/course" 33 | } 34 | } 35 | 36 | }, 37 | { 38 | "weight": 0.3, 39 | "object": { 40 | "objectType": "Agent", 41 | "name": "Owen Overrider", 42 | "mbox": "mailto:owoverrider@example.com" 43 | } 44 | 45 | } 46 | ] 47 | } 48 | ] 49 | -------------------------------------------------------------------------------- /dev-resources/models/simple_with_repeat_max.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bobfake@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "patterns": [ 10 | { 11 | "id": "https://w3id.org/xapi/cmi5#satisfieds", 12 | "repeatMax": 1 13 | }, 14 | { 15 | "id": "https://w3id.org/xapi/cmi5#typicalsessions", 16 | "repeatMax": 1 17 | } 18 | ] 19 | }, 20 | { 21 | "personae": [ 22 | { 23 | "id": "mbox::mailto:frederstaz@example.org", 24 | "type": "Agent" 25 | } 26 | ], 27 | "patterns": [ 28 | { 29 | "id": "https://w3id.org/xapi/cmi5#satisfieds", 30 | "repeatMax": 100 31 | }, 32 | { 33 | "id": "https://w3id.org/xapi/cmi5#typicalsessions", 34 | "repeatMax": 100 35 | } 36 | ] 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /dev-resources/models/simple_with_temporal.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bobfake@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "patterns": [ 10 | { 11 | "id": "https://w3id.org/xapi/cmi5#satisfieds", 12 | "periods": [ 13 | { 14 | "min": 1, 15 | "mean": 1.0, 16 | "unit": "hours" 17 | } 18 | ] 19 | } 20 | ] 21 | }, 22 | { 23 | "personae": [ 24 | { 25 | "id": "mbox::mailto:frederstaz@example.org", 26 | "type": "Agent" 27 | } 28 | ], 29 | "patterns": [ 30 | { 31 | "id": "https://w3id.org/xapi/cmi5#typicalsessions", 32 | "bounds": [ 33 | { 34 | "months": [["January", "October"], "December"] 35 | } 36 | ] 37 | } 38 | ], 39 | "templates": [ 40 | { 41 | "id": "https://w3id.org/xapi/cmi5#launched", 42 | "bounds": [ 43 | { 44 | "minutes": [[0, 10]] 45 | } 46 | ], 47 | "boundRestarts": [ 48 | "https://w3id.org/xapi/cmi5#typicalsessions" 49 | ] 50 | }, 51 | { 52 | "id": "https://w3id.org/xapi/cmi5#initialized", 53 | "bounds": [ 54 | { 55 | "minutes": [[10, 59, 5]] 56 | } 57 | ] 58 | } 59 | ] 60 | } 61 | ] 62 | -------------------------------------------------------------------------------- /dev-resources/models/simple_with_weights.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "personae": [ 4 | { 5 | "id": "mbox::mailto:bobfake@example.org", 6 | "type": "Agent" 7 | } 8 | ], 9 | "activityTypes": [ 10 | { 11 | "id": "https://w3id.org/xapi/cmi5/activities/block", 12 | "weight": 1.0 13 | }, 14 | { 15 | "id": "https://w3id.org/xapi/cmi5/activitytype/block", 16 | "weight": 1.0 17 | }, 18 | { 19 | "id": "https://w3id.org/xapi/cmi5/activities/course", 20 | "weight": 0.0 21 | }, 22 | { 23 | "id": "https://w3id.org/xapi/cmi5/activitytype/course", 24 | "weight": 0.0 25 | } 26 | ], 27 | "patterns": [ 28 | { 29 | "id": "https://w3id.org/xapi/cmi5#satisfieds", 30 | "repeatMax": 0 31 | } 32 | ] 33 | }, 34 | { 35 | "personae": [ 36 | { 37 | "id": "mbox::mailto:frederstaz@example.org", 38 | "type": "Agent" 39 | } 40 | ], 41 | "activityTypes": [ 42 | { 43 | "id": "https://w3id.org/xapi/cmi5/activities/block", 44 | "weight": 0.0 45 | }, 46 | { 47 | "id": "https://w3id.org/xapi/cmi5/activitytype/block", 48 | "weight": 0.0 49 | }, 50 | { 51 | "id": "https://w3id.org/xapi/cmi5/activities/course", 52 | "weight": 1.0 53 | }, 54 | { 55 | "id": "https://w3id.org/xapi/cmi5/activitytype/course", 56 | "weight": 1.0 57 | } 58 | ], 59 | "patterns": [ 60 | { 61 | "id": "https://w3id.org/xapi/cmi5#satisfieds", 62 | "repeatMax": 0 63 | } 64 | ] 65 | } 66 | ] 67 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/10a_restart_current_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]] 12 | } 13 | ], 14 | "boundRestarts": [ 15 | "http://yetanalytics.com/temporal/pattern-B2" 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/10b_restart_parent_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]] 12 | } 13 | ], 14 | "boundRestarts": [ 15 | "http://yetanalytics.com/temporal/pattern-A" 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/10c_restart_all_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]] 12 | } 13 | ], 14 | "boundRestarts": [ 15 | "http://yetanalytics.com/temporal/pattern-A", 16 | "http://yetanalytics.com/temporal/pattern-B2", 17 | "http://yetanalytics.com/temporal/pattern-C3", 18 | "http://yetanalytics.com/temporal/template-5" 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/10d_restart_child_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]] 12 | } 13 | ], 14 | "boundRestarts": [ 15 | "http://yetanalytics.com/temporal/pattern-C3", 16 | "http://yetanalytics.com/temporal/pattern-C4" 17 | ] 18 | } 19 | ] 20 | } 21 | ] 22 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/10e_restart_child_template.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]] 12 | } 13 | ], 14 | "boundRestarts": [ 15 | "http://yetanalytics.com/temporal/template-5", 16 | "http://yetanalytics.com/temporal/template-6", 17 | "http://yetanalytics.com/temporal/template-7", 18 | "http://yetanalytics.com/temporal/template-8" 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/11a_end_current_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ], 17 | "boundRestarts": [ 18 | "http://yetanalytics.com/temporal/pattern-B2" 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/11b_end_parent_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ], 17 | "boundRestarts": [ 18 | "http://yetanalytics.com/temporal/pattern-A" 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/11c_end_all_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ], 17 | "boundRestarts": [ 18 | "http://yetanalytics.com/temporal/pattern-A", 19 | "http://yetanalytics.com/temporal/pattern-B2", 20 | "http://yetanalytics.com/temporal/pattern-C3", 21 | "http://yetanalytics.com/temporal/template-5" 22 | ] 23 | } 24 | ] 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/11d_end_child_pattern.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ], 17 | "boundRestarts": [ 18 | "http://yetanalytics.com/temporal/pattern-C3", 19 | "http://yetanalytics.com/temporal/pattern-C4" 20 | ] 21 | } 22 | ] 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/11e_end_child_template.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-B2", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ], 17 | "boundRestarts": [ 18 | "http://yetanalytics.com/temporal/template-5", 19 | "http://yetanalytics.com/temporal/template-6", 20 | "http://yetanalytics.com/temporal/template-7", 21 | "http://yetanalytics.com/temporal/template-8" 22 | ] 23 | } 24 | ] 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/1a_year_2023.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "days" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/1b_year_2023-2024.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [[2023, 2024]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "days" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/1c_year_0-2023.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [[0, 2023]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "days" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/1d_year_0-2022.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [[0, 2022]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "days" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/2a_dow_Tues-Thurs_hour_8-12.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfWeek": ["Tuesday", "Thursday"], 9 | "hours": [[8, 12]] 10 | } 11 | ], 12 | "periods": [ 13 | { 14 | "mean": 10, 15 | "unit": "minutes" 16 | } 17 | ] 18 | } 19 | ] 20 | } 21 | ] 22 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/2b_dow_Tues-Thurs_hour_8-12-18.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfWeek": ["Tuesday"], 9 | "hours": [[8, 12]] 10 | }, 11 | { 12 | "daysOfWeek": ["Thursday"], 13 | "hours": [[13, 18]] 14 | } 15 | ], 16 | "periods": [ 17 | { 18 | "mean": 10, 19 | "unit": "minutes" 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | ] 26 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/3a_dom_28.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfMonth": [28] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 10, 14 | "unit": "minutes" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/3b_dom_29.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfMonth": [29] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 10, 14 | "unit": "minutes" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/3c_dom_30.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfMonth": [30] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 10, 14 | "unit": "minutes" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/3d_dom_31.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "daysOfMonth": [31] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 10, 14 | "unit": "minutes" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/4a_Dec_25.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "months": [12], 9 | "daysOfMonth": [25] 10 | } 11 | ] 12 | } 13 | ] 14 | } 15 | ] 16 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/4b_Dec_25_Mon.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "months": [12], 9 | "daysOfMonth": [25], 10 | "daysOfWeek": [1] 11 | } 12 | ] 13 | } 14 | ] 15 | } 16 | ] 17 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/5a_Jan_1_sec_10-30.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "seconds": [[10, 30]] 12 | } 13 | ] 14 | } 15 | ] 16 | } 17 | ] 18 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/5b_Jan_1_min_even_sec_10-30.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "minutes": [[0, 59, 2]], 12 | "seconds": [[10, 30]] 13 | } 14 | ] 15 | } 16 | ] 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/5c_Jan_1_hour_even_sec_10-30.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]], 12 | "seconds": [[10, 30]] 13 | } 14 | ] 15 | } 16 | ] 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/5d_Jan_1_min_hour_even_sec_10-30.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 23, 2]], 12 | "minutes": [[0, 59, 2]], 13 | "seconds": [[10, 30]] 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | ] 20 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/6a_hours_period_every_second_hour.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [[0, 23, 2]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/6b_hours_period_every_start_hour.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/6c_hours_fixed_period_every_second_hour.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [[0, 23, 2]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "fixed": 1, 14 | "unit": "hours" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/6d_hours_fixed_period_every_start_hour.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [0] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "fixed": 1, 14 | "unit": "hours" 15 | } 16 | ] 17 | } 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7a_millis_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [0], 12 | "minutes": [0], 13 | "seconds": [[0, 7]] 14 | } 15 | ], 16 | "periods": [ 17 | { 18 | "bounds": [ 19 | { 20 | "seconds": [0] 21 | } 22 | ], 23 | "min": 0, 24 | "mean": 10, 25 | "unit": "millis" 26 | }, 27 | { 28 | "bounds": [ 29 | { 30 | "seconds": [1] 31 | } 32 | ], 33 | "min": 0, 34 | "mean": 20, 35 | "unit": "millis" 36 | }, 37 | { 38 | "bounds": [ 39 | { 40 | "seconds": [2] 41 | } 42 | ], 43 | "min": 10, 44 | "mean": 10, 45 | "unit": "millis" 46 | }, 47 | { 48 | "bounds": [ 49 | { 50 | "seconds": [3] 51 | } 52 | ], 53 | "min": 10, 54 | "mean": 20, 55 | "unit": "millis" 56 | }, 57 | { 58 | "bounds": [ 59 | { 60 | "seconds": [4] 61 | } 62 | ], 63 | "min": 20, 64 | "mean": 10, 65 | "unit": "millis" 66 | }, 67 | { 68 | "bounds": [ 69 | { 70 | "seconds": [5] 71 | } 72 | ], 73 | "min": 20, 74 | "mean": 20, 75 | "unit": "millis" 76 | }, 77 | { 78 | "bounds": [ 79 | { 80 | "seconds": [6] 81 | } 82 | ], 83 | "min": 30, 84 | "mean": 10, 85 | "unit": "millis" 86 | }, 87 | { 88 | "bounds": [ 89 | { 90 | "seconds": [7] 91 | } 92 | ], 93 | "min": 30, 94 | "mean": 20, 95 | "unit": "millis" 96 | }, 97 | { 98 | "min": 0, 99 | "mean": 10, 100 | "unit": "millis" 101 | } 102 | ] 103 | } 104 | ] 105 | } 106 | ] 107 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7b_seconds_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [0], 12 | "minutes": [[0, 7]] 13 | } 14 | ], 15 | "periods": [ 16 | { 17 | "bounds": [ 18 | { 19 | "minutes": [0] 20 | } 21 | ], 22 | "min": 0, 23 | "mean": 1, 24 | "unit": "seconds" 25 | }, 26 | { 27 | "bounds": [ 28 | { 29 | "minutes": [1] 30 | } 31 | ], 32 | "min": 0, 33 | "mean": 2, 34 | "unit": "seconds" 35 | }, 36 | { 37 | "bounds": [ 38 | { 39 | "minutes": [2] 40 | } 41 | ], 42 | "min": 1, 43 | "mean": 1, 44 | "unit": "seconds" 45 | }, 46 | { 47 | "bounds": [ 48 | { 49 | "minutes": [3] 50 | } 51 | ], 52 | "min": 1, 53 | "mean": 2, 54 | "unit": "seconds" 55 | }, 56 | { 57 | "bounds": [ 58 | { 59 | "minutes": [4] 60 | } 61 | ], 62 | "min": 2, 63 | "mean": 1, 64 | "unit": "seconds" 65 | }, 66 | { 67 | "bounds": [ 68 | { 69 | "minutes": [5] 70 | } 71 | ], 72 | "min": 2, 73 | "mean": 2, 74 | "unit": "seconds" 75 | }, 76 | { 77 | "bounds": [ 78 | { 79 | "minutes": [6] 80 | } 81 | ], 82 | "min": 3, 83 | "mean": 1, 84 | "unit": "seconds" 85 | }, 86 | { 87 | "bounds": [ 88 | { 89 | "minutes": [7] 90 | } 91 | ], 92 | "min": 3, 93 | "mean": 2, 94 | "unit": "seconds" 95 | }, 96 | { 97 | "min": 0, 98 | "mean": 1, 99 | "unit": "seconds" 100 | } 101 | ] 102 | } 103 | ] 104 | } 105 | ] 106 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7c_minutes_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 7]] 12 | } 13 | ], 14 | "periods": [ 15 | { 16 | "bounds": [ 17 | { 18 | "hours": [0] 19 | } 20 | ], 21 | "min": 0, 22 | "mean": 1, 23 | "unit": "minutes" 24 | }, 25 | { 26 | "bounds": [ 27 | { 28 | "hours": [1] 29 | } 30 | ], 31 | "min": 0, 32 | "mean": 2, 33 | "unit": "minutes" 34 | }, 35 | { 36 | "bounds": [ 37 | { 38 | "hours": [2] 39 | } 40 | ], 41 | "min": 1, 42 | "mean": 1, 43 | "unit": "minutes" 44 | }, 45 | { 46 | "bounds": [ 47 | { 48 | "hours": [3] 49 | } 50 | ], 51 | "min": 1, 52 | "mean": 2, 53 | "unit": "minutes" 54 | }, 55 | { 56 | "bounds": [ 57 | { 58 | "hours": [4] 59 | } 60 | ], 61 | "min": 2, 62 | "mean": 1, 63 | "unit": "minutes" 64 | }, 65 | { 66 | "bounds": [ 67 | { 68 | "hours": [5] 69 | } 70 | ], 71 | "min": 2, 72 | "mean": 2, 73 | "unit": "minutes" 74 | }, 75 | { 76 | "bounds": [ 77 | { 78 | "hours": [6] 79 | } 80 | ], 81 | "min": 3, 82 | "mean": 1, 83 | "unit": "minutes" 84 | }, 85 | { 86 | "bounds": [ 87 | { 88 | "hours": [7] 89 | } 90 | ], 91 | "min": 3, 92 | "mean": 2, 93 | "unit": "minutes" 94 | }, 95 | { 96 | "min": 0, 97 | "mean": 1, 98 | "unit": "minutes" 99 | } 100 | ] 101 | } 102 | ] 103 | } 104 | ] 105 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7d_hours_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [[1, 8]] 11 | } 12 | ], 13 | "periods": [ 14 | { 15 | "bounds": [ 16 | { 17 | "daysOfMonth": [1] 18 | } 19 | ], 20 | "min": 0, 21 | "mean": 1, 22 | "unit": "hours" 23 | }, 24 | { 25 | "bounds": [ 26 | { 27 | "daysOfMonth": [2] 28 | } 29 | ], 30 | "min": 0, 31 | "mean": 2, 32 | "unit": "hours" 33 | }, 34 | { 35 | "bounds": [ 36 | { 37 | "daysOfMonth": [3] 38 | } 39 | ], 40 | "min": 1, 41 | "mean": 1, 42 | "unit": "hours" 43 | }, 44 | { 45 | "bounds": [ 46 | { 47 | "daysOfMonth": [4] 48 | } 49 | ], 50 | "min": 1, 51 | "mean": 2, 52 | "unit": "hours" 53 | }, 54 | { 55 | "bounds": [ 56 | { 57 | "daysOfMonth": [5] 58 | } 59 | ], 60 | "min": 2, 61 | "mean": 1, 62 | "unit": "hours" 63 | }, 64 | { 65 | "bounds": [ 66 | { 67 | "daysOfMonth": [6] 68 | } 69 | ], 70 | "min": 2, 71 | "mean": 2, 72 | "unit": "hours" 73 | }, 74 | { 75 | "bounds": [ 76 | { 77 | "daysOfMonth": [7] 78 | } 79 | ], 80 | "min": 3, 81 | "mean": 1, 82 | "unit": "hours" 83 | }, 84 | { 85 | "bounds": [ 86 | { 87 | "daysOfMonth": [8] 88 | } 89 | ], 90 | "min": 3, 91 | "mean": 2, 92 | "unit": "hours" 93 | }, 94 | { 95 | "min": 0, 96 | "mean": 1, 97 | "unit": "hours" 98 | } 99 | ] 100 | } 101 | ] 102 | } 103 | ] 104 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7e_days_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": [[1, 8]] 10 | } 11 | ], 12 | "periods": [ 13 | { 14 | "bounds": [ 15 | { 16 | "months": [1] 17 | } 18 | ], 19 | "min": 0, 20 | "mean": 1, 21 | "unit": "days" 22 | }, 23 | { 24 | "bounds": [ 25 | { 26 | "months": [2] 27 | } 28 | ], 29 | "min": 0, 30 | "mean": 2, 31 | "unit": "days" 32 | }, 33 | { 34 | "bounds": [ 35 | { 36 | "months": [3] 37 | } 38 | ], 39 | "min": 1, 40 | "mean": 1, 41 | "unit": "days" 42 | }, 43 | { 44 | "bounds": [ 45 | { 46 | "months": [4] 47 | } 48 | ], 49 | "min": 1, 50 | "mean": 2, 51 | "unit": "days" 52 | }, 53 | { 54 | "bounds": [ 55 | { 56 | "months": [5] 57 | } 58 | ], 59 | "min": 2, 60 | "mean": 1, 61 | "unit": "days" 62 | }, 63 | { 64 | "bounds": [ 65 | { 66 | "months": [6] 67 | } 68 | ], 69 | "min": 2, 70 | "mean": 2, 71 | "unit": "days" 72 | }, 73 | { 74 | "bounds": [ 75 | { 76 | "months": [7] 77 | } 78 | ], 79 | "min": 3, 80 | "mean": 1, 81 | "unit": "days" 82 | }, 83 | { 84 | "bounds": [ 85 | { 86 | "months": [8] 87 | } 88 | ], 89 | "min": 3, 90 | "mean": 2, 91 | "unit": "days" 92 | }, 93 | { 94 | "min": 0, 95 | "mean": 1, 96 | "unit": "days" 97 | } 98 | ] 99 | } 100 | ] 101 | } 102 | ] 103 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/7f_weeks_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": [[1, 4]] 10 | } 11 | ], 12 | "periods": [ 13 | { 14 | "bounds": [ 15 | { 16 | "months": [1] 17 | } 18 | ], 19 | "min": 0, 20 | "mean": 1, 21 | "unit": "weeks" 22 | }, 23 | { 24 | "bounds": [ 25 | { 26 | "months": [2] 27 | } 28 | ], 29 | "min": 0, 30 | "mean": 2, 31 | "unit": "weeks" 32 | }, 33 | { 34 | "bounds": [ 35 | { 36 | "months": [3] 37 | } 38 | ], 39 | "min": 1, 40 | "mean": 1, 41 | "unit": "weeks" 42 | }, 43 | { 44 | "bounds": [ 45 | { 46 | "months": [4] 47 | } 48 | ], 49 | "min": 1, 50 | "mean": 2, 51 | "unit": "weeks" 52 | }, 53 | { 54 | "min": 0, 55 | "mean": 1, 56 | "unit": "weeks" 57 | } 58 | ] 59 | } 60 | ] 61 | } 62 | ] 63 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8a_millis_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [0], 12 | "minutes": [0], 13 | "seconds": [[0, 7]] 14 | } 15 | ], 16 | "periods": [ 17 | { 18 | "bounds": [ 19 | { 20 | "seconds": [0] 21 | } 22 | ], 23 | "fixed": 10, 24 | "unit": "millis" 25 | }, 26 | { 27 | "bounds": [ 28 | { 29 | "seconds": [1] 30 | } 31 | ], 32 | "fixed": 20, 33 | "unit": "millis" 34 | }, 35 | { 36 | "bounds": [ 37 | { 38 | "seconds": [2] 39 | } 40 | ], 41 | "fixed": 30, 42 | "unit": "millis" 43 | }, 44 | { 45 | "bounds": [ 46 | { 47 | "seconds": [3] 48 | } 49 | ], 50 | "fixed": 40, 51 | "unit": "millis" 52 | }, 53 | { 54 | "bounds": [ 55 | { 56 | "seconds": [4] 57 | } 58 | ], 59 | "fixed": 50, 60 | "unit": "millis" 61 | }, 62 | { 63 | "bounds": [ 64 | { 65 | "seconds": [5] 66 | } 67 | ], 68 | "fixed": 60, 69 | "unit": "millis" 70 | }, 71 | { 72 | "bounds": [ 73 | { 74 | "seconds": [6] 75 | } 76 | ], 77 | "fixed": 70, 78 | "unit": "millis" 79 | }, 80 | { 81 | "bounds": [ 82 | { 83 | "seconds": [7] 84 | } 85 | ], 86 | "fixed": 80, 87 | "unit": "millis" 88 | } 89 | ] 90 | } 91 | ] 92 | } 93 | ] 94 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8b_seconds_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [0], 12 | "minutes": [[0, 7]] 13 | } 14 | ], 15 | "periods": [ 16 | { 17 | "bounds": [ 18 | { 19 | "minutes": [0] 20 | } 21 | ], 22 | "fixed": 1, 23 | "unit": "seconds" 24 | }, 25 | { 26 | "bounds": [ 27 | { 28 | "minutes": [1] 29 | } 30 | ], 31 | "fixed": 2, 32 | "unit": "seconds" 33 | }, 34 | { 35 | "bounds": [ 36 | { 37 | "minutes": [2] 38 | } 39 | ], 40 | "fixed": 3, 41 | "unit": "seconds" 42 | }, 43 | { 44 | "bounds": [ 45 | { 46 | "minutes": [3] 47 | } 48 | ], 49 | "fixed": 4, 50 | "unit": "seconds" 51 | }, 52 | { 53 | "bounds": [ 54 | { 55 | "minutes": [4] 56 | } 57 | ], 58 | "fixed": 5, 59 | "unit": "seconds" 60 | }, 61 | { 62 | "bounds": [ 63 | { 64 | "minutes": [5] 65 | } 66 | ], 67 | "fixed": 6, 68 | "unit": "seconds" 69 | }, 70 | { 71 | "bounds": [ 72 | { 73 | "minutes": [6] 74 | } 75 | ], 76 | "fixed": 7, 77 | "unit": "seconds" 78 | }, 79 | { 80 | "bounds": [ 81 | { 82 | "minutes": [7] 83 | } 84 | ], 85 | "fixed": 8, 86 | "unit": "seconds" 87 | }, 88 | { 89 | "fixed": 1, 90 | "unit": "seconds" 91 | } 92 | ] 93 | } 94 | ] 95 | } 96 | ] 97 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8c_minutes_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [1], 11 | "hours": [[0, 7]] 12 | } 13 | ], 14 | "periods": [ 15 | { 16 | "bounds": [ 17 | { 18 | "hours": [0] 19 | } 20 | ], 21 | "fixed": 1, 22 | "unit": "minutes" 23 | }, 24 | { 25 | "bounds": [ 26 | { 27 | "hours": [1] 28 | } 29 | ], 30 | "fixed": 2, 31 | "unit": "minutes" 32 | }, 33 | { 34 | "bounds": [ 35 | { 36 | "hours": [2] 37 | } 38 | ], 39 | "fixed": 3, 40 | "unit": "minutes" 41 | }, 42 | { 43 | "bounds": [ 44 | { 45 | "hours": [3] 46 | } 47 | ], 48 | "fixed": 4, 49 | "unit": "minutes" 50 | }, 51 | { 52 | "bounds": [ 53 | { 54 | "hours": [4] 55 | } 56 | ], 57 | "fixed": 5, 58 | "unit": "minutes" 59 | }, 60 | { 61 | "bounds": [ 62 | { 63 | "hours": [5] 64 | } 65 | ], 66 | "fixed": 6, 67 | "unit": "minutes" 68 | }, 69 | { 70 | "bounds": [ 71 | { 72 | "hours": [6] 73 | } 74 | ], 75 | "fixed": 7, 76 | "unit": "minutes" 77 | }, 78 | { 79 | "bounds": [ 80 | { 81 | "hours": [7] 82 | } 83 | ], 84 | "fixed": 8, 85 | "unit": "minutes" 86 | }, 87 | { 88 | "fixed": 1, 89 | "unit": "minutes" 90 | } 91 | ] 92 | } 93 | ] 94 | } 95 | ] 96 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8d_hours_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": ["January"], 10 | "daysOfMonth": [[1, 8]] 11 | } 12 | ], 13 | "periods": [ 14 | { 15 | "bounds": [ 16 | { 17 | "daysOfMonth": [1] 18 | } 19 | ], 20 | "fixed": 1, 21 | "unit": "hours" 22 | }, 23 | { 24 | "bounds": [ 25 | { 26 | "daysOfMonth": [2] 27 | } 28 | ], 29 | "fixed": 2, 30 | "unit": "hours" 31 | }, 32 | { 33 | "bounds": [ 34 | { 35 | "daysOfMonth": [3] 36 | } 37 | ], 38 | "fixed": 3, 39 | "unit": "hours" 40 | }, 41 | { 42 | "bounds": [ 43 | { 44 | "daysOfMonth": [4] 45 | } 46 | ], 47 | "fixed": 4, 48 | "unit": "hours" 49 | }, 50 | { 51 | "bounds": [ 52 | { 53 | "daysOfMonth": [5] 54 | } 55 | ], 56 | "fixed": 5, 57 | "unit": "hours" 58 | }, 59 | { 60 | "bounds": [ 61 | { 62 | "daysOfMonth": [6] 63 | } 64 | ], 65 | "fixed": 6, 66 | "unit": "hours" 67 | }, 68 | { 69 | "bounds": [ 70 | { 71 | "daysOfMonth": [7] 72 | } 73 | ], 74 | "fixed": 7, 75 | "unit": "hours" 76 | }, 77 | { 78 | "bounds": [ 79 | { 80 | "daysOfMonth": [8] 81 | } 82 | ], 83 | "fixed": 8, 84 | "unit": "hours" 85 | }, 86 | { 87 | "fixed": 1, 88 | "unit": "hours" 89 | } 90 | ] 91 | } 92 | ] 93 | } 94 | ] 95 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8e_days_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": [[1, 8]] 10 | } 11 | ], 12 | "periods": [ 13 | { 14 | "bounds": [ 15 | { 16 | "months": [1] 17 | } 18 | ], 19 | "fixed": 1, 20 | "unit": "days" 21 | }, 22 | { 23 | "bounds": [ 24 | { 25 | "months": [2] 26 | } 27 | ], 28 | "fixed": 2, 29 | "unit": "days" 30 | }, 31 | { 32 | "bounds": [ 33 | { 34 | "months": [3] 35 | } 36 | ], 37 | "fixed": 3, 38 | "unit": "days" 39 | }, 40 | { 41 | "bounds": [ 42 | { 43 | "months": [4] 44 | } 45 | ], 46 | "fixed": 4, 47 | "unit": "days" 48 | }, 49 | { 50 | "bounds": [ 51 | { 52 | "months": [5] 53 | } 54 | ], 55 | "fixed": 5, 56 | "unit": "days" 57 | }, 58 | { 59 | "bounds": [ 60 | { 61 | "months": [6] 62 | } 63 | ], 64 | "fixed": 6, 65 | "unit": "days" 66 | }, 67 | { 68 | "bounds": [ 69 | { 70 | "months": [7] 71 | } 72 | ], 73 | "fixed": 7, 74 | "unit": "days" 75 | }, 76 | { 77 | "bounds": [ 78 | { 79 | "months": [8] 80 | } 81 | ], 82 | "fixed": 8, 83 | "unit": "days" 84 | }, 85 | { 86 | "fixed": 1, 87 | "unit": "days" 88 | } 89 | ] 90 | } 91 | ] 92 | } 93 | ] 94 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/8f_weeks_fixed_period.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "years": [2023], 9 | "months": [[1, 4]] 10 | } 11 | ], 12 | "periods": [ 13 | { 14 | "bounds": [ 15 | { 16 | "months": [1] 17 | } 18 | ], 19 | "fixed": 1, 20 | "unit": "weeks" 21 | }, 22 | { 23 | "bounds": [ 24 | { 25 | "months": [2] 26 | } 27 | ], 28 | "fixed": 2, 29 | "unit": "weeks" 30 | }, 31 | { 32 | "bounds": [ 33 | { 34 | "months": [3] 35 | } 36 | ], 37 | "fixed": 3, 38 | "unit": "weeks" 39 | }, 40 | { 41 | "bounds": [ 42 | { 43 | "months": [4] 44 | } 45 | ], 46 | "fixed": 4, 47 | "unit": "weeks" 48 | }, 49 | { 50 | "fixed": 5, 51 | "unit": "weeks" 52 | } 53 | ] 54 | } 55 | ] 56 | } 57 | ] 58 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/9a_outer_bound_larger.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [[0, 18]] 9 | } 10 | ] 11 | }, 12 | { 13 | "id": "http://yetanalytics.com/temporal/pattern-B1", 14 | "bounds": [ 15 | { 16 | "hours": [[6, 12]] 17 | } 18 | ], 19 | "periods": [ 20 | { 21 | "mean": 1, 22 | "unit": "hours" 23 | } 24 | ] 25 | }, 26 | { 27 | "id": "http://yetanalytics.com/temporal/pattern-B2", 28 | "bounds": [ 29 | { 30 | "hours": [[6, 12]] 31 | } 32 | ], 33 | "periods": [ 34 | { 35 | "mean": 1, 36 | "unit": "hours" 37 | } 38 | ] 39 | } 40 | ] 41 | } 42 | ] 43 | -------------------------------------------------------------------------------- /dev-resources/models/temporal/9b_outer_bound_smaller.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "patterns": [ 4 | { 5 | "id": "http://yetanalytics.com/temporal/pattern-A", 6 | "bounds": [ 7 | { 8 | "hours": [[6, 12]] 9 | } 10 | ], 11 | "periods": [ 12 | { 13 | "mean": 1, 14 | "unit": "hours" 15 | } 16 | ] 17 | }, 18 | { 19 | "id": "http://yetanalytics.com/temporal/pattern-B1", 20 | "bounds": [ 21 | { 22 | "hours": [[0, 18]] 23 | } 24 | ] 25 | }, 26 | { 27 | "id": "http://yetanalytics.com/temporal/pattern-B1", 28 | "bounds": [ 29 | { 30 | "hours": [[0, 18]] 31 | } 32 | ] 33 | } 34 | ] 35 | } 36 | ] 37 | -------------------------------------------------------------------------------- /dev-resources/parameters/ds_102.json: -------------------------------------------------------------------------------- 1 | { 2 | "start": "2021-03-18T16:00:00.000000Z", 3 | "timezone": "America/New_York", 4 | "seed": 42, 5 | "max": 1000000000 6 | } 7 | -------------------------------------------------------------------------------- /dev-resources/parameters/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "start": "2019-11-18T11:38:39.219768Z", 3 | "end": "2019-11-19T11:38:39.219768Z", 4 | "timezone": "America/New_York", 5 | "seed": 42 6 | } 7 | -------------------------------------------------------------------------------- /dev-resources/parameters/tccc_dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "start": "2020-02-10T11:38:39.219768Z", 3 | "end": "2020-02-25T17:38:39.219768Z", 4 | "timezone": "America/New_York", 5 | "seed": 40 6 | } 7 | -------------------------------------------------------------------------------- /dev-resources/parameters/temporal.json: -------------------------------------------------------------------------------- 1 | { 2 | "start": "2023-01-01T00:00:00Z", 3 | "end": "2026-01-01T00:00:00Z", 4 | "timezone": "UTC", 5 | "seed": 116 6 | } 7 | -------------------------------------------------------------------------------- /dev-resources/personae/array/simple_array.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "trainees", 4 | "objectType": "Group", 5 | "member": [ 6 | { 7 | "name": "Bob Fakename", 8 | "mbox": "mailto:bobfake@example.org", 9 | "role": "Lead Developer" 10 | }, 11 | { 12 | "name": "Alice Faux", 13 | "mbox": "mailto:alicefaux@example.org", 14 | "role": "Data Engineer" 15 | }, 16 | { 17 | "name": "Fred Ersatz", 18 | "mbox": "mailto:frederstaz@example.org" 19 | } 20 | ] 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /dev-resources/personae/bench.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "trainees", 3 | "objectType": "Group", 4 | "member": [ 5 | { 6 | "name": "Dionne Knox", 7 | "mbox": "mailto:0@yetanalytics.com" 8 | }, 9 | { 10 | "name": "Penny Mayer", 11 | "mbox": "mailto:1@yetanalytics.com" 12 | }, 13 | { 14 | "name": "Stanley Ayers", 15 | "mbox": "mailto:2@yetanalytics.com" 16 | }, 17 | { 18 | "name": "Reid Joyner", 19 | "mbox": "mailto:3@yetanalytics.com" 20 | }, 21 | { 22 | "name": "Loretta Elliott", 23 | "mbox": "mailto:4@yetanalytics.com" 24 | }, 25 | { 26 | "name": "Hawkins Landry", 27 | "mbox": "mailto:5@yetanalytics.com" 28 | }, 29 | { 30 | "name": "Zimmerman Short", 31 | "mbox": "mailto:6@yetanalytics.com" 32 | }, 33 | { 34 | "name": "Jeannette Castaneda", 35 | "mbox": "mailto:7@yetanalytics.com" 36 | }, 37 | { 38 | "name": "Ramos Boyer", 39 | "mbox": "mailto:8@yetanalytics.com" 40 | }, 41 | { 42 | "name": "Montgomery Dale", 43 | "mbox": "mailto:9@yetanalytics.com" 44 | }, 45 | { 46 | "name": "Jerry Rose", 47 | "mbox": "mailto:10@yetanalytics.com" 48 | }, 49 | { 50 | "name": "Katina Armstrong", 51 | "mbox": "mailto:11@yetanalytics.com" 52 | }, 53 | { 54 | "name": "Dickson Gaines", 55 | "mbox": "mailto:12@yetanalytics.com" 56 | }, 57 | { 58 | "name": "Patricia Forbes", 59 | "mbox": "mailto:13@yetanalytics.com" 60 | }, 61 | { 62 | "name": "Petty Huber", 63 | "mbox": "mailto:14@yetanalytics.com" 64 | }, 65 | { 66 | "name": "Sexton Cooley", 67 | "mbox": "mailto:15@yetanalytics.com" 68 | }, 69 | { 70 | "name": "Nola Burns", 71 | "mbox": "mailto:16@yetanalytics.com" 72 | }, 73 | { 74 | "name": "Caitlin Lancaster", 75 | "mbox": "mailto:17@yetanalytics.com" 76 | }, 77 | { 78 | "name": "Dean Roth", 79 | "mbox": "mailto:18@yetanalytics.com" 80 | }, 81 | { 82 | "name": "Wilson Chambers", 83 | "mbox": "mailto:19@yetanalytics.com" 84 | }, 85 | { 86 | "name": "Helga Hutchinson", 87 | "mbox": "mailto:20@yetanalytics.com" 88 | }, 89 | { 90 | "name": "Mary Flores", 91 | "mbox": "mailto:21@yetanalytics.com" 92 | }, 93 | { 94 | "name": "Baker Orr", 95 | "mbox": "mailto:22@yetanalytics.com" 96 | }, 97 | { 98 | "name": "Randall Fernandez", 99 | "mbox": "mailto:23@yetanalytics.com" 100 | }, 101 | { 102 | "name": "Margie Pratt", 103 | "mbox": "mailto:24@yetanalytics.com" 104 | }, 105 | { 106 | "name": "Christian Baxter", 107 | "mbox": "mailto:25@yetanalytics.com" 108 | }, 109 | { 110 | "name": "Guerrero Glass", 111 | "mbox": "mailto:26@yetanalytics.com" 112 | }, 113 | { 114 | "name": "Marian Fox", 115 | "mbox": "mailto:27@yetanalytics.com" 116 | }, 117 | { 118 | "name": "Ayers Farley", 119 | "mbox": "mailto:28@yetanalytics.com" 120 | }, 121 | { 122 | "name": "Milagros Hudson", 123 | "mbox": "mailto:29@yetanalytics.com" 124 | }, 125 | { 126 | "name": "Brittany French", 127 | "mbox": "mailto:30@yetanalytics.com" 128 | }, 129 | { 130 | "name": "Mueller Ward", 131 | "mbox": "mailto:31@yetanalytics.com" 132 | }, 133 | { 134 | "name": "Emily Lucas", 135 | "mbox": "mailto:32@yetanalytics.com" 136 | }, 137 | { 138 | "name": "Cohen Shepherd", 139 | "mbox": "mailto:33@yetanalytics.com" 140 | } 141 | ] 142 | } 143 | -------------------------------------------------------------------------------- /dev-resources/personae/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "trainees", 3 | "objectType": "Group", 4 | "member": [ 5 | { 6 | "name": "Bob Fakename", 7 | "mbox": "mailto:bobfake@example.org", 8 | "role": "Lead Developer" 9 | }, 10 | { 11 | "name": "Alice Faux", 12 | "mbox": "mailto:alicefaux@example.org", 13 | "role": "Data Engineer" 14 | }, 15 | { 16 | "name": "Fred Ersatz", 17 | "mbox": "mailto:frederstaz@example.org" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /dev-resources/personae/tccc_dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "trainees", 3 | "objectType" : "Group", 4 | "member" : [ 5 | { 6 | "name" : "Bob Nelson", 7 | "mbox" : "mailto:bob@example.org" 8 | }, 9 | { 10 | "name" : "Phil Walker", 11 | "mbox" : "mailto:phil@example.org" 12 | }, 13 | { 14 | "name" : "Sally Davis", 15 | "mbox" : "mailto:sally@example.org" 16 | }, 17 | { 18 | "name" : "Steve Stewart", 19 | "mbox" : "mailto:steve@example.org" 20 | }, 21 | { 22 | "name" : "Fred Evans", 23 | "mbox" : "mailto:fred@example.org" 24 | }, 25 | { 26 | "name" : "Alice Edwards", 27 | "mbox" : "mailto:alice@example.org" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /dev-resources/personae/temporal.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Singleton Group", 3 | "objectType": "Group", 4 | "member": [ 5 | { 6 | "name": "SCROM", 7 | "mbox": "mailto:scrom@yetanalytics.org", 8 | "role": "Merciful God of Learning Data" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /dev-resources/profiles/minimal.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "https://xapinet.org/xapi/yet/minimal", 3 | "definition" : { 4 | "en" : "This xAPI Profile demonstrates the minimal required properties of an xAPI profile" 5 | }, 6 | "@context" : "https://w3id.org/xapi/profiles/context", 7 | "prefLabel" : { 8 | "en" : "Minimal - Experimental xAPI Profile" 9 | }, 10 | "type" : "Profile", 11 | "author" : { 12 | "url" : "https://www.yetanalytics.com/", 13 | "name" : "Yet Analytics", 14 | "type" : "Organization" 15 | }, 16 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0", 17 | "versions" : [ 18 | { 19 | "id" : "https://xapinet.org/xapi/yet/minimal/v1", 20 | "generatedAtTime" : "2020-03-25T15:45:31.907Z" 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /dev-resources/profiles/no_concept.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id": "https://www.example.com/xapi/profile", 3 | "conformsTo": "https://w3id.org/xapi/profiles#1.0", 4 | "versions": [ 5 | { 6 | "id": "https://example.com/xapi/profile/v1.0.0", 7 | "generatedAtTime": "2022-11-15T16:35:26.240428Z" 8 | } 9 | ], 10 | "@context": "https://w3id.org/xapi/profiles/context", 11 | "definition": { 12 | "en": "Test Profile with no Verbs, Activities, or other Concepts" 13 | }, 14 | "prefLabel": { 15 | "en": "Test Profile" 16 | }, 17 | "type": "Profile", 18 | "author": { 19 | "type": "Organization", 20 | "name": "Yet Analytics" 21 | }, 22 | "templates": [ 23 | { 24 | "definition": { 25 | "en": "Test Template" 26 | }, 27 | "prefLabel": { 28 | "en": "Test Template" 29 | }, 30 | "type": "StatementTemplate", 31 | "rules": [ 32 | { 33 | "location": "$.context", 34 | "presence": "included" 35 | } 36 | ], 37 | "inScheme": "https://example.com/xapi/profile/v1.0.0", 38 | "id": "https://example.com/xapi/profile/template" 39 | } 40 | ], 41 | "patterns": [ 42 | { 43 | "definition": { 44 | "en": "Test Pattern" 45 | }, 46 | "primary": true, 47 | "prefLabel": { 48 | "en": "Test Pattern" 49 | }, 50 | "type": "Pattern", 51 | "inScheme": "https://example.com/xapi/profile/v1.0.0", 52 | "id": "https://example.com/xapi/profile/pattern", 53 | "sequence": [ 54 | "https://example.com/xapi/profile/template" 55 | ] 56 | } 57 | ] 58 | } 59 | -------------------------------------------------------------------------------- /dev-resources/profiles/referential.jsonld: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "https://xapinet.org/xapi/yet/referential", 3 | "definition" : { 4 | "en" : "Test Profile that refers to another" 5 | }, 6 | "@context" : "https://w3id.org/xapi/profiles/context", 7 | "prefLabel" : { 8 | "en" : "REFERENTIAL xAPI Profile" 9 | }, 10 | "type" : "Profile", 11 | "author" : { 12 | "url" : "https://www.yetanalytics.com/", 13 | "name" : "Yet Analytics", 14 | "type" : "Organization" 15 | }, 16 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0", 17 | "versions" : [ 18 | { 19 | "id" : "https://xapinet.org/xapi/yet/referential/v0.0.0", 20 | "generatedAtTime" : "2022-06-27T20:34:27.823870Z" 21 | } 22 | ], 23 | "patterns" : [ 24 | { 25 | "id": "https://xapinet.org/xapi/yet/referential#completed_session", 26 | "type": "Pattern", 27 | "primary": true, 28 | "inScheme": "https://xapinet.org/xapi/yet/referential/v0.0.0", 29 | "prefLabel": { 30 | "en": "completed session" 31 | }, 32 | "definition": { 33 | "en": "This pattern describes the sequence of statements sent over the an entire course registration." 34 | }, 35 | "sequence": [ 36 | "https://w3id.org/xapi/tla#launched", 37 | "https://w3id.org/xapi/tla#initialized", 38 | "https://w3id.org/xapi/tla#completed", 39 | "https://w3id.org/xapi/tla#terminated" 40 | ] 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /dev-resources/xapi/statements/long.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "6690e6c9-3ef0-4ed3-8b37-7f3964730bee", 3 | "actor": { 4 | "name": "Team PB", 5 | "mbox": "mailto:teampb@example.com", 6 | "member": [ 7 | { 8 | "name": "Andrew Downes", 9 | "account": { 10 | "homePage": "http://www.example.com", 11 | "name": "13936749" 12 | }, 13 | "objectType": "Agent" 14 | }, 15 | { 16 | "name": "Toby Nichols", 17 | "openid": "http://toby.openid.example.org/", 18 | "objectType": "Agent" 19 | }, 20 | { 21 | "name": "Ena Hills", 22 | "mbox_sha1sum": "ebd31e95054c018b10727ccffd2ef2ec3a016ee9", 23 | "objectType": "Agent" 24 | } 25 | ], 26 | "objectType": "Group" 27 | }, 28 | "verb": { 29 | "id": "http://adlnet.gov/expapi/verbs/attended", 30 | "display": { 31 | "en-GB": "attended", 32 | "en-US": "attended" 33 | } 34 | }, 35 | "result": { 36 | "extensions": { 37 | "http://example.com/profiles/meetings/resultextensions/minuteslocation": "X:\\meetings\\minutes\\examplemeeting.one" 38 | }, 39 | "success": true, 40 | "completion": true, 41 | "response": "We agreed on some example actions.", 42 | "duration": "PT1H0M0S" 43 | }, 44 | "context": { 45 | "registration": "ec531277-b57b-4c15-8d91-d292c5b2b8f7", 46 | "contextActivities": { 47 | "parent": [ 48 | { 49 | "id": "http://www.example.com/meetings/series/267", 50 | "objectType": "Activity" 51 | } 52 | ], 53 | "category": [ 54 | { 55 | "id": "http://www.example.com/meetings/categories/teammeeting", 56 | "objectType": "Activity", 57 | "definition": { 58 | "name": { 59 | "en": "team meeting" 60 | }, 61 | "description": { 62 | "en": "A category of meeting used for regular team meetings." 63 | }, 64 | "type": "http://example.com/expapi/activities/meetingcategory" 65 | } 66 | } 67 | ], 68 | "other": [ 69 | { 70 | "id": "http://www.example.com/meetings/occurances/34257", 71 | "objectType": "Activity" 72 | }, 73 | { 74 | "id": "http://www.example.com/meetings/occurances/3425567", 75 | "objectType": "Activity" 76 | } 77 | ] 78 | }, 79 | "instructor" : 80 | { 81 | "name": "Andrew Downes", 82 | "account": { 83 | "homePage": "http://www.example.com", 84 | "name": "13936749" 85 | }, 86 | "objectType": "Agent" 87 | }, 88 | "team": 89 | { 90 | "name": "Team PB", 91 | "mbox": "mailto:teampb@example.com", 92 | "objectType": "Group" 93 | }, 94 | "platform" : "Example virtual meeting software", 95 | "language" : "tlh", 96 | "statement" : { 97 | "objectType":"StatementRef", 98 | "id" :"6690e6c9-3ef0-4ed3-8b37-7f3964730bee" 99 | } 100 | 101 | }, 102 | "timestamp": "2013-05-18T05:32:34.804Z", 103 | "stored": "2013-05-18T05:32:34.804Z", 104 | "authority": { 105 | "account": { 106 | "homePage": "http://cloud.scorm.com/", 107 | "name": "anonymous" 108 | }, 109 | "objectType": "Agent" 110 | }, 111 | "version": "1.0.0", 112 | "object": { 113 | "id": "http://www.example.com/meetings/occurances/34534", 114 | "definition": { 115 | "extensions": { 116 | "http://example.com/profiles/meetings/activitydefinitionextensions/room": {"name": "Kilby", "id" : "http://example.com/rooms/342"} 117 | }, 118 | "name": { 119 | "en-GB": "example meeting", 120 | "en-US": "example meeting" 121 | }, 122 | "description": { 123 | "en-GB": "An example meeting that happened on a specific occasion with certain people present.", 124 | "en-US": "An example meeting that happened on a specific occasion with certain people present." 125 | }, 126 | "type": "http://adlnet.gov/expapi/activities/meeting", 127 | "moreInfo": "http://virtualmeeting.example.com/345256" 128 | }, 129 | "objectType": "Activity" 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /dev-resources/xapi/statements/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "id":"fd41c918-b88b-4b20-a0a5-a4c32391aaa0", 3 | "timestamp": "2015-11-18T12:17:00+00:00", 4 | "actor":{ 5 | "objectType": "Agent", 6 | "name":"Project Tin Can API", 7 | "mbox":"mailto:user@example.com" 8 | }, 9 | "verb":{ 10 | "id":"http://example.com/xapi/verbs#sent-a-statement", 11 | "display":{ 12 | "en-US":"sent" 13 | } 14 | }, 15 | "object":{ 16 | "id":"http://example.com/xapi/activity/simplestatement", 17 | "definition":{ 18 | "name":{ 19 | "en-US":"simple statement" 20 | }, 21 | "description":{ 22 | "en-US":"A simple Experience API statement. Note that the LRS does not need to have any prior information about the Actor (learner), the verb, or the Activity/object." 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to datasim 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /doc/temporal_structure.md: -------------------------------------------------------------------------------- 1 | # Temporal Structure 2 | 3 | In order to deterministically generate events over time, we need a model that does a few things: 4 | 5 | * generates from a seed and keeps things deterministic all the way down 6 | * generates discrete events over time in realistic patterns by incorporating: 7 | * Day/Night 8 | * Daily/Weekly Work Hours 9 | * Independent factor (ARMA) 10 | * dependent factor (group dynamic, etc) 11 | * generates unique activity for each actor 12 | * that is still beholden to dependent factors 13 | 14 | At the event level, we'll need the following: 15 | 16 | * when did the event occur? 17 | * who is the actor for this event? 18 | * a long representing the seed for generating this event. 19 | * should be a function of 20 | * the simulation (base) seed 21 | * the time 22 | * the actor 23 | * ~~a double representing the coarse "effectiveness" of the actor when the event occurs~~ 24 | * ~~harder to reason about this one, as no value/desirability is available~~ 25 | -------------------------------------------------------------------------------- /onyx-resources/onyx_config.edn: -------------------------------------------------------------------------------- 1 | {;; :default profile is dev operation 2 | :env-config 3 | {:onyx/tenancy-id #profile {:default #env TENANCY_ID} 4 | :zookeeper/address #profile {:default #or [#env ZK_ADDRESS 5 | "127.0.0.1:2188"] 6 | :prod #env ZK_ADDRESS} 7 | :zookeeper/server? #profile {:default true 8 | :prod false} 9 | :zookeeper.server/port #profile {:default #long #or [#env ZK_SERVER_PORT 10 | 2188]} 11 | :onyx.log/config #profile {:default nil 12 | :prod {:level :info}}} 13 | :peer-config 14 | {:onyx/tenancy-id #ref [:env-config :onyx/tenancy-id] 15 | :zookeeper/address #ref [:env-config :zookeeper/address] 16 | :onyx.peer/job-scheduler :onyx.job-scheduler/balanced 17 | :onyx.peer/zookeeper-timeout #long #or [#env ZK_TIMEOUT 18 | 1000] 19 | :onyx.messaging/impl :aeron 20 | :onyx.messaging/bind-addr #or [#env BIND_ADDR 21 | "0.0.0.0"] 22 | :onyx.messaging/peer-port #long #or [#env PEER_PORT 23 | 40200] 24 | :onyx.messaging.aeron/embedded-driver? #profile {:default true 25 | :prod false} 26 | :onyx.log/config #profile {:default nil 27 | :prod {:level :info}} 28 | :onyx.query/server? true 29 | :onyx.query.server/ip #or [#env SERVER_IP 30 | "0.0.0.0"] 31 | :onyx.query.server/port #long #or [#env SERVER_PORT 32 | 8888] ;; port change so it isn't the same as dev lrs 33 | ;; do allow short circuit locally, so the default colo works 34 | :onyx.messaging/allow-short-circuit? #profile {:default true 35 | :prod true} 36 | :onyx.peer/storage #keyword #or [#env PEER_STORAGE 37 | :zookeeper] 38 | :onyx.peer/storage.s3.bucket #or [#env PEER_STORAGE_BUCKET 39 | ""] 40 | :onyx.peer/storage.s3.region #or [#env AWS_REGION 41 | "us-east-1"] 42 | :onyx.peer/storage.s3.prefix #or [#env PEER_STORAGE_PREFIX 43 | #profile {:default "onyx_dev/"}] 44 | ;; Aeron max term buffer size 45 | :onyx.messaging/term-buffer-size.segment 16777216 46 | 47 | ;; :onyx.peer/coordinator-barrier-period-ms 5000 48 | 49 | :onyx.peer/storage.timeout #long #or [#env PEER_STORAGE_TIMEOUT 50 | 120000] 51 | ;; need this or constant timeout 52 | :onyx.peer/storage.s3.accelerate? #boolean #or [#env PEER_STORAGE_ACCELERATION 53 | "false"] 54 | 55 | #_#_:onyx.peer/tags #profile {:default [:gen :out] 56 | :prod [#keyword #env PEER_TAG]} 57 | ;; :onyx.peer/publisher-liveness-timeout-ms 120000 58 | ;; :onyx.peer/subscriber-liveness-timeout-ms 120000 59 | ;; :onyx.peer/coordinator-max-sleep-ms 10 60 | ;; :onyx.peer/coordinator-barrier-period-ms 2000 61 | ;; :onyx.peer/heartbeat-ms 1000 62 | } 63 | :launch-config 64 | {:n-vpeers #long #or [#env N_VPEERS 65 | 4]}} 66 | -------------------------------------------------------------------------------- /scripts/gc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh gc $@ 3 | -------------------------------------------------------------------------------- /scripts/gc_checkpoints.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh gc-checkpoints $@ 3 | -------------------------------------------------------------------------------- /scripts/onyx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -server $JAVA_OPTS -jar ./datasim_onyx.jar $@ 3 | -------------------------------------------------------------------------------- /scripts/peer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh start-peer $@ 3 | -------------------------------------------------------------------------------- /scripts/peer_driver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh start-driver $@ 3 | -------------------------------------------------------------------------------- /scripts/repl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh repl 3 | -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -jar ./datasim_cli.jar $@ 3 | -------------------------------------------------------------------------------- /scripts/server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -server -jar ./datasim_server.jar $@ 3 | -------------------------------------------------------------------------------- /scripts/submit_job.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sh ./bin/onyx.sh submit-job $@ 3 | -------------------------------------------------------------------------------- /src/cli/com/yetanalytics/datasim/cli.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.cli 2 | (:require [clojure.tools.cli :as cli] 3 | [com.yetanalytics.datasim.cli.input :as cli-input] 4 | [com.yetanalytics.datasim.cli.generate :as cli-gen] 5 | [com.yetanalytics.datasim.cli.util :as u]) 6 | (:gen-class)) 7 | 8 | (def top-level-options 9 | [["-h" "--help" "Display the top-level help guide."]]) 10 | 11 | (def top-level-summary 12 | (str "Usage: 'datasim ' or 'datasim [-h|--help]'.\n" 13 | "\n" 14 | "where the subcommand can be one of the following:\n" 15 | " validate-input: Validate the input and create an input JSON file.\n" 16 | " generate: Generate statements from input and print to stdout.\n" 17 | " generate-post: Generate statements from input and POST them to an LRS.\n" 18 | "\n" 19 | "Run 'datasim --help' for more info on each subcommand.")) 20 | 21 | (defn -main [& args] 22 | (let [{:keys [options arguments summary errors]} 23 | (cli/parse-opts args top-level-options 24 | :in-order true 25 | :summary-fn (fn [_] top-level-summary)) 26 | [subcommand & rest-args] 27 | arguments] 28 | (cond 29 | (:help options) 30 | (println summary) 31 | (not subcommand) 32 | (print "No subcommand entered.\n\n" summary) 33 | :else 34 | (let [results (case subcommand 35 | "validate-input" (cli-input/validate-input! rest-args) 36 | "generate" (cli-gen/generate! rest-args) 37 | "generate-post" (cli-gen/generate-post! rest-args) 38 | (u/bail! errors))] 39 | (when-some [errors (:errors results)] 40 | (u/bail! errors)))))) 41 | -------------------------------------------------------------------------------- /src/cli/com/yetanalytics/datasim/cli/util.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.cli.util 2 | (:require [clojure.tools.cli :as cli] 3 | [com.yetanalytics.datasim.util.io :as dio])) 4 | 5 | (defn conj-input 6 | "Conj the input (either a Profile or a personae) to return a 7 | vector of inputs, e.g. `-p profile-1 -p profile-2` becomes 8 | `[profile-1 profile-2]`." 9 | [opt-map id v] 10 | (update opt-map id (fnil conj []) v)) 11 | 12 | (defn conj-param-input 13 | "Add a parameter named by id." 14 | [opt-map id v] 15 | (update-in opt-map [:parameters id] (fnil conj []) v)) 16 | 17 | (defn bail! 18 | "Print error messages to standard error and exit." 19 | [errors & {:keys [status] 20 | :or {status 1}}] 21 | (dio/println-err-coll errors) 22 | (System/exit status)) 23 | 24 | (defn exec-subcommand 25 | "Execute `exec-fn` for a subcommand with arguments `args`, where the 26 | valid options are `cli-options`." 27 | [cli-options exec-fn args] 28 | (let [{:keys [options summary errors]} 29 | (cli/parse-opts args cli-options) 30 | {:keys [help]} 31 | options 32 | errors* (not-empty errors)] 33 | (cond 34 | help (println summary) 35 | errors* (bail! errors*) 36 | :else (exec-fn options)))) 37 | -------------------------------------------------------------------------------- /src/dev/bench.clj: -------------------------------------------------------------------------------- 1 | (ns bench 2 | (:require [criterium.core :as c] 3 | [com.yetanalytics.datasim.sim :as sim] 4 | [com.yetanalytics.datasim.input :as input])) 5 | 6 | (defn statements-per-second 7 | "How many statements per second will datasim generate?" 8 | [input] 9 | (let [sim-seq (sim/sim-seq input) 10 | t-start (System/currentTimeMillis) 11 | t-end (+ t-start 1000)] 12 | (loop [[_ & rest-s] sim-seq 13 | s-count 0] 14 | (if (> t-end (System/currentTimeMillis)) 15 | (recur rest-s (inc s-count)) 16 | s-count)))) 17 | 18 | (comment 19 | (def input 20 | {:profiles [(input/from-location :profile :json "dev-resources/bench/calibration.jsonld")] 21 | :personae-array [(input/from-location :personae :json "dev-resources/bench/actors.json")] 22 | :models (input/from-location :models :json "dev-resources/bench/models.json") 23 | :parameters (input/from-location :parameters :json "dev-resources/bench/params.json")}) 24 | 25 | (c/with-progress-reporting 26 | (c/quick-bench (do (doall (take 1000 (sim/sim-seq input))) 27 | nil))) 28 | 29 | ;; Evaluation count : 6 in 6 samples of 1 calls. 30 | ;; Execution time mean : 201.061588 ms 31 | ;; Execution time std-deviation : 6.698801 ms 32 | ;; Execution time lower quantile : 196.857258 ms ( 2.5%) 33 | ;; Execution time upper quantile : 212.503809 ms (97.5%) 34 | ;; Overhead used : 7.430081 ns 35 | 36 | (statements-per-second input) ;; => 4938 37 | 38 | (quot (reduce + (repeatedly 60 #(statements-per-second input))) 39 | 60) ;; => 4462 40 | 41 | ) 42 | -------------------------------------------------------------------------------- /src/dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user 2 | (:require [clojure.repl :refer [source doc apropos]] 3 | [clojure.pprint :refer [pprint]])) 4 | 5 | (set! *warn-on-reflection* true) 6 | -------------------------------------------------------------------------------- /src/dev_onyx/com/yetanalytics/datasim/onyx/dev_peer.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.dev-peer 2 | (:gen-class) 3 | (:require onyx.plugin.http-output 4 | onyx.plugin.seq 5 | onyx.plugin.s3-output 6 | onyx.api 7 | com.yetanalytics.datasim.onyx.sim 8 | com.yetanalytics.datasim.onyx.http 9 | [com.yetanalytics.datasim.onyx.config :as config])) 10 | 11 | ;; Trying just the peer 12 | 13 | (defn -main 14 | [& args] 15 | ;; VERY MUCH JUST DEV RIGHT NOW 16 | (let [;; DEV ENV config, should go away for prod things 17 | id (random-uuid) 18 | {:keys [env-config peer-config] 19 | {:keys [n-vpeers]} :launch-config} (-> (config/get-config) 20 | (assoc-in [:env-config :onyx/tenancy-id] id) 21 | (assoc-in [:peer-config :onyx/tenancy-id] id)) 22 | env (onyx.api/start-env env-config) 23 | 24 | ;; start peer group 25 | peer-group (onyx.api/start-peer-group peer-config) 26 | 27 | v-peers (onyx.api/start-peers n-vpeers peer-group)] 28 | 29 | 30 | (.addShutdownHook (Runtime/getRuntime) 31 | (Thread. ^Runnable #(do 32 | (onyx.api/shutdown-peers v-peers) 33 | 34 | (onyx.api/shutdown-peer-group peer-group) 35 | 36 | (onyx.api/shutdown-env env)))))) 37 | -------------------------------------------------------------------------------- /src/dev_onyx/com/yetanalytics/datasim/onyx/scratch.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.scratch 2 | "Scratch ns for playing with onyx" 3 | (:require 4 | [clojure.core.async :as a] 5 | com.yetanalytics.datasim.onyx.main ;; get all boot-up ns stuff 6 | [onyx.plugin.http-output :as http] 7 | [onyx.plugin.core-async :as ap] 8 | [onyx.plugin.seq] 9 | [onyx.api] 10 | [onyx.test-helper :as th] 11 | [com.yetanalytics.datasim.onyx.sim :as dsim] 12 | [com.yetanalytics.datasim.onyx.job :as job] 13 | [com.yetanalytics.datasim.onyx.config :as config])) 14 | 15 | (comment 16 | ;; Run things in an enclosed environment 17 | (let [id (random-uuid) 18 | {:keys [env-config peer-config]} (-> (config/get-config) 19 | (assoc-in [:env-config :onyx/tenancy-id] id) 20 | (assoc-in [:peer-config :onyx/tenancy-id] id))] 21 | (th/with-test-env 22 | [{:keys [n-peers 23 | env 24 | peer-group 25 | peers] 26 | :as test-env} [;; n-peers 27 | 3 ;; 28 | env-config 29 | peer-config 30 | ]] 31 | (let [;; Submit the job 32 | submission (onyx.api/submit-job 33 | peer-config 34 | (job/config 35 | {:input-loc "dev-resources/input/mom64.json" 36 | :gen-concurrency 1 37 | :gen-batch-size 10 38 | :post-concurrency 1 39 | :override-max 100 40 | :out-ratio 1 41 | :out-mode :lrs 42 | :lrs {:endpoint "http://localhost:8080/xapi"}}))] 43 | 44 | ;; Wait for jorb to finish if you like 45 | (println 'started submission) 46 | (onyx.api/await-job-completion peer-config (:job-id submission)) 47 | (println 'done submission) 48 | ))) 49 | 50 | ) 51 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.datasim.input :as input] 4 | [com.yetanalytics.datasim.sim :as sim])) 5 | 6 | (s/fdef read-input 7 | :args (s/cat :location string?) 8 | :ret ::input) 9 | 10 | (defn read-input 11 | "Read input at `location` and return DATASIM input for use with 12 | other functions." 13 | [location] 14 | (input/from-location :input :json location)) 15 | 16 | ;; TODO: Add fdefs for generation functions 17 | ;; TODO: Extract more implementation code from `sim` namespace 18 | 19 | #_{:clj-kondo/ignore [:unused-binding]} 20 | (defn generate-seq 21 | "Given `input`, produce a lazy sequence of statements in a synchronous 22 | fashion." 23 | [input & {:keys [select-agents] :as kwargs}] 24 | (sim/sim-seq input kwargs)) 25 | 26 | (defn generate-map 27 | "Given `input`, produce a map from actor IFIs to lazy sequences of statements 28 | that use those actors, all in a synchronous fashion." 29 | [input] 30 | (sim/build-skeleton input)) 31 | 32 | #_{:clj-kondo/ignore [:unused-binding]} 33 | (defn generate-seq-async 34 | "Given `input`, produce a `core.async` channels that contains a generated 35 | sequence of simulated statements; this is for parallel generation." 36 | [input & {:keys [select-agents pad-chan-max] :as kwargs}] 37 | (sim/sim-chan input kwargs)) 38 | 39 | #_{:clj-kondo/ignore [:unused-binding]} 40 | (defn generate-map-async 41 | "Given `input`, produce a map from actor IFIs to `core.async` channels, 42 | each with their own generated sequence of simulated statements; this 43 | is for parallel generation." 44 | [input & {:keys [select-agents pad-chan-max sort buffer-size] :as kwargs}] 45 | (sim/sim-chans input kwargs)) 46 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/client.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.client 2 | "Simple xAPI LRS client functions." 3 | (:require [clojure.core.async :as a] 4 | [clojure.java.io :as io] 5 | [cheshire.core :as json] 6 | [org.httpkit.client :as http] 7 | [com.yetanalytics.datasim.util.io :as dio])) 8 | 9 | (defn post-error-message 10 | "Error message for when POSTing to an LRS fails." 11 | [status error] 12 | (format "POST Request FAILED with STATUS: %d, MESSAGE: %s" 13 | status 14 | (or (some-> error ex-message) ""))) 15 | 16 | (def default-http-options 17 | {:headers {"X-Experience-Api-Version" "1.0.3" 18 | "Content-Type" "application/json"}}) 19 | 20 | (defn- decode-body [body] 21 | (with-open [rdr (io/reader body)] 22 | (json/decode-stream rdr))) 23 | 24 | (defn- format-url [endpoint] 25 | (format "%s/statements" endpoint)) 26 | 27 | (defn- post-options [http-options batch] 28 | (merge default-http-options 29 | http-options 30 | {:body (json/encode batch) 31 | :as :stream})) 32 | 33 | ;; `http/post` cannot be resolved since it's defined using `http/defreq` 34 | #_{:clj-kondo/ignore [:unresolved-var]} 35 | (defn- post-batch 36 | ([endpoint http-options batch] 37 | (http/post (format-url endpoint) 38 | (post-options http-options batch))) 39 | ([endpoint http-options batch callback-fn] 40 | (http/post (format-url endpoint) 41 | (post-options http-options batch) 42 | callback-fn))) 43 | 44 | (defn post-statements 45 | "Given LRS options and a `statement-seq`, send them to an LRS in synchronous 46 | batches. If `print-ids?` is `true`, returned statement IDs will be printed 47 | to stdout. `username` and `password` in the options map are the Basic Auth 48 | credentials of the LRS." 49 | [{:keys [endpoint 50 | batch-size 51 | username 52 | password] 53 | :or {batch-size 25}} 54 | statement-seq 55 | & {:keys [print-ids?] 56 | :or {print-ids? true}}] 57 | ;; TODO: Exponential backoff, etc 58 | (let [http-options {:basic-auth [username password]}] 59 | (loop [batches (partition-all batch-size statement-seq) 60 | success 0 61 | fail []] 62 | (if-let [batch (first batches)] 63 | (let [{:keys [status body] :as response} 64 | @(post-batch endpoint http-options batch)] 65 | (if (<= 200 status 299) 66 | ;; Success! 67 | (let [statement-ids (decode-body body)] 68 | (when print-ids? 69 | (dio/println-coll statement-ids)) 70 | (recur (rest batches) 71 | (+ success (count statement-ids)) 72 | fail)) 73 | ;; Failure 74 | (let [response* (cond-> response 75 | body (assoc :body (decode-body body)))] 76 | {:success success 77 | :fail (conj fail response*)}))) 78 | ;; Batch finished POSTing 79 | {:success success 80 | :fail fail})))) 81 | 82 | (defn post-statements-async 83 | "Given LRS options and a channel with statements, send them to an LRS in 84 | asynchronous batches. `username` and `password` in the options map are the 85 | Basic Auth credentials of the LRS. 86 | 87 | Returns a channel that will reciveve `[:success ]` 88 | for each batch or `[:fail ]`. Will stop sending on failure." 89 | [{:keys [endpoint 90 | batch-size 91 | username 92 | password] 93 | :or {batch-size 25}} 94 | statement-chan 95 | & {:keys [concurrency 96 | buffer-in 97 | buffer-out] 98 | :or {concurrency 4 99 | buffer-in 100 ; 10x default batch size 100 | buffer-out 100}}] 101 | (let [http-opts {:basic-auth [username password]} 102 | run? (atom true) 103 | in-chan (a/chan buffer-in (partition-all batch-size)) 104 | out-chan (a/chan buffer-out) ; is this.. backpressure? 105 | callback (fn [port {:keys [status body error] :as ret}] 106 | (if (or (not (<= 200 status 299)) 107 | error) 108 | ;; Error: Stop further processing 109 | (do 110 | (swap! run? not) 111 | (a/put! port [:fail ret])) 112 | ;; Success: Continue 113 | (a/put! port 114 | [:success (decode-body body)])) 115 | ;; Close the return channel 116 | (a/close! port)) 117 | async-fn (fn [batch port] 118 | (let [callback (partial callback port)] 119 | (if @run? 120 | (post-batch endpoint http-opts batch callback) 121 | (a/close! port))))] 122 | (a/pipeline-async concurrency out-chan async-fn in-chan) 123 | ;; Pipe to in-chan 124 | (a/pipe statement-chan in-chan) 125 | ;; Return the out chan 126 | out-chan)) 127 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/input/model.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.model 2 | "Model input specs and parsing." 3 | (:require [clojure.spec.alpha :as s] 4 | [com.yetanalytics.datasim.util.errors :as errs] 5 | [com.yetanalytics.datasim.input.model.alignments :as alignments] 6 | [com.yetanalytics.datasim.input.model.personae :as personae])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (s/def ::personae 13 | (s/every personae/persona-spec :kind vector? :min-count 1)) 14 | 15 | (s/def ::verbs 16 | (s/every alignments/verb-spec :kind vector?)) 17 | 18 | (s/def ::activities 19 | (s/every alignments/activity-spec :kind vector?)) 20 | 21 | (s/def ::activityTypes 22 | (s/every alignments/activity-type-spec :kind vector?)) 23 | 24 | (s/def ::patterns 25 | (s/every alignments/pattern-spec :kind vector?)) 26 | 27 | (s/def ::templates 28 | (s/every alignments/template-spec :kind vector?)) 29 | 30 | (s/def ::objectOverrides 31 | (s/every alignments/object-override-spec :kind vector?)) 32 | 33 | (def model-spec 34 | (s/keys :opt-un [::personae 35 | ::verbs 36 | ::activities 37 | ::activityTypes 38 | ::patterns 39 | ::templates 40 | ::objectOverrides])) 41 | 42 | (s/def ::models 43 | (s/and (s/every model-spec) 44 | personae/distinct-personae?)) 45 | 46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 47 | ;; Validation 48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 49 | 50 | (defn validate-models 51 | [models] 52 | (some->> (s/explain-data ::models models) 53 | (errs/explain-to-map-coll ::models))) 54 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/input/model/personae.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.model.personae 2 | (:require [clojure.spec.alpha :as s] 3 | [com.yetanalytics.datasim.xapi.actor :as actor] 4 | [com.yetanalytics.datasim.input.model.personae.agent :as-alias agent] 5 | [com.yetanalytics.datasim.input.model.personae.group :as-alias group] 6 | [com.yetanalytics.datasim.input.model.personae.role :as-alias role])) 7 | 8 | (defmulti persona-spec* :type) 9 | 10 | (s/def ::agent/id ::actor/actor-ifi) 11 | (s/def ::agent/type #{"Agent"}) 12 | 13 | (defmethod persona-spec* "Agent" [_] 14 | (s/keys :req-un [::agent/id ::agent/type])) 15 | 16 | (s/def ::group/id ::actor/actor-ifi) 17 | (s/def ::group/type #{"Group"}) 18 | 19 | (defmethod persona-spec* "Group" [_] 20 | (s/keys :req-un [::group/id ::group/type])) 21 | 22 | (s/def ::role/id (s/and string? not-empty)) 23 | (s/def ::role/type #{"Role"}) 24 | 25 | (defmethod persona-spec* "Role" [_] 26 | (s/keys :req-un [::role/id ::role/type])) 27 | 28 | (def persona-spec 29 | (s/multi-spec persona-spec* :type)) 30 | 31 | (defn distinct-personae? 32 | "Are each of the `:personae` in `map-coll` distinct from each other?" 33 | [map-coll] 34 | (let [personaes (map :personae map-coll)] 35 | (= (-> personaes count) 36 | (-> personaes distinct count)))) 37 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/input/parameters.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.parameters 2 | "Parameter input specs and parsing." 3 | (:require [clojure.spec.alpha :as s] 4 | [java-time.api :as t] 5 | [xapi-schema.spec :as xs] 6 | [com.yetanalytics.pan.objects.profile :as prof] 7 | [com.yetanalytics.pan.objects.pattern :as pat] 8 | [com.yetanalytics.datasim.util.random :as random] 9 | [com.yetanalytics.datasim.util.errors :as errs]) 10 | (:import [clojure.lang ExceptionInfo] 11 | [java.time.zone ZoneRulesException] 12 | [java.time Instant])) 13 | 14 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 15 | ;; Specs 16 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 17 | 18 | ;; All options are optional, but everything except `end` will get defaults 19 | 20 | ;; (optional) start of the simulation (inclusive), 8601 stamp 21 | (s/def ::start 22 | ::xs/timestamp) 23 | 24 | ;; (optional) start of the returned statements (if after ::start). 25 | ;; This lets us page through sims to later times. Defaults to ::start 26 | (s/def ::from 27 | ::xs/timestamp) 28 | 29 | ;; (optional) end of the simulation (exclusive), 8601 stamp 30 | (s/def ::end 31 | (s/nilable ::xs/timestamp)) 32 | 33 | (defn- timezone-string? [s] 34 | (try (t/zone-id s) 35 | (catch ExceptionInfo exi 36 | (if (= ZoneRulesException (type (ex-cause exi))) 37 | false 38 | (throw exi))))) 39 | 40 | ;; (optional) timezone, defaults to UTC 41 | (s/def ::timezone 42 | (s/and string? 43 | not-empty 44 | timezone-string?)) 45 | 46 | ;; Seed is required, but will be generated if not present 47 | (s/def ::seed 48 | int?) 49 | 50 | ;; Max number of statements returned 51 | (s/def ::max 52 | pos-int?) 53 | 54 | ;; Max number of bound restarts before giving up 55 | (s/def ::maxRestarts 56 | pos-int?) 57 | 58 | ;; Restrict Generation to these profile IDs 59 | (s/def ::genProfiles 60 | (s/every ::prof/id)) 61 | 62 | ;; Restrict Generation to these pattern IDs 63 | (s/def ::genPatterns 64 | (s/every ::pat/id)) 65 | 66 | (defn- ordered-timestamps? 67 | "Are the `start`, `from`, and `end` timestamps ordered properly?" 68 | [{:keys [start from end]}] 69 | (let [start-t (t/instant start) 70 | ?from-t (some->> from t/instant) 71 | ?end-t (some->> end t/instant)] 72 | (and (or (not ?end-t) 73 | (t/before? start-t ?end-t)) 74 | (or (not ?end-t) 75 | (not ?from-t) 76 | (t/before? ?from-t ?end-t)) 77 | (or (not ?from-t) 78 | (= ?from-t start-t) 79 | (t/before? start-t ?from-t))))) 80 | 81 | (s/def ::parameters 82 | (s/and 83 | (s/keys :req-un [::start 84 | ::timezone 85 | ::seed] 86 | :opt-un [::end 87 | ::from 88 | ::max 89 | ::maxRestarts 90 | ::genProfiles 91 | ::genPatterns]) 92 | ordered-timestamps?)) 93 | 94 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 95 | ;; Validation 96 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 97 | 98 | (defn validate-parameters 99 | [parameters] 100 | (some->> (s/explain-data ::parameters parameters) 101 | (errs/explain-to-map-coll ::parameters))) 102 | 103 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 104 | ;; Defaults 105 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 106 | 107 | (def utc-timezone "UTC") 108 | (def default-max-restarts 5) 109 | 110 | (defn apply-defaults 111 | "Apply defaults to `params` with the current time and a random seed. 112 | If `params` is not provided simply return the default parameters." 113 | ([] 114 | (apply-defaults {})) 115 | ([{:keys [start from timezone seed maxRestarts] :as params}] 116 | (merge 117 | params 118 | (let [start (or start (.toString (Instant/now)))] 119 | {:start start 120 | :from (or from start) 121 | :timezone (or timezone utc-timezone) 122 | :seed (or seed (random/rand-unbound-int (random/rng))) 123 | :maxRestarts (or maxRestarts default-max-restarts)})))) 124 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/input/personae.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.personae 2 | "Personae input specs and parsing." 3 | (:require [clojure.spec.alpha :as s] 4 | [clojure.walk :as w] 5 | [com.yetanalytics.datasim.xapi.actor :as agent] 6 | [com.yetanalytics.datasim.util.errors :as errs])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | ;; We model the input personae as an xAPI group. 13 | ;; It can be anonymous, but the name may be used in some way. 14 | 15 | ;; If functionality is added to express further groupings we'll have to revise 16 | ;; this strategy. 17 | 18 | ;; Note: We cannot apply xapi-schema specs directly, as xapi-schema restrict 19 | ;; which properties can be in the Group, including the `role` property. 20 | ;; We still use `agent` and `group` spec namespaces from xapi-schema. 21 | 22 | (s/def ::role string?) 23 | 24 | (s/def ::agent 25 | (s/keys :req-un [(or :agent/mbox 26 | :agent/mbox_sha1sum 27 | :agent/openid 28 | :agent/account)] 29 | :opt-un [:agent/name 30 | :agent/objectType 31 | ::role])) 32 | 33 | (s/def ::member 34 | (s/coll-of ::agent :kind vector? :min-count 1 :gen-max 3)) 35 | 36 | (s/def ::group 37 | (s/or :anonymous (s/keys :req-un [:group/objectType 38 | (or :group/mbox 39 | :group/mbox_sha1sum 40 | :group/openid 41 | :group/account)] 42 | :opt-un [:group/name ::member]) 43 | :identified (s/keys :req-un [:group/objectType ::member] 44 | :opt-un [:group/name]))) 45 | 46 | (defn- remove-nil-vals 47 | "Remove nil values from an associative structure. Does not recurse." 48 | [m] 49 | (reduce-kv 50 | (fn [m* k v] (cond-> m* (some? v) (assoc k v))) 51 | {} 52 | m)) 53 | 54 | ;; An open-validating group spec, ignores extra nils 55 | (s/def ::personae 56 | (s/and (s/conformer remove-nil-vals) 57 | (s/conformer w/keywordize-keys w/stringify-keys) 58 | ::group)) 59 | 60 | (defn- distinct-member-ids? 61 | [personaes] 62 | (let [member-ids (->> personaes 63 | (map :member) 64 | (apply concat) 65 | (map agent/actor-ifi))] 66 | (= (-> member-ids count) 67 | (-> member-ids distinct count)))) 68 | 69 | (s/def ::personae-array 70 | (s/and (s/every ::personae :min-count 1 :into []) 71 | distinct-member-ids?)) 72 | 73 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 74 | ;; Validation 75 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 76 | 77 | (defn validate-personae 78 | [personae] 79 | (some->> (s/explain-data ::personae personae) 80 | (errs/explain-to-map-coll ::personae))) 81 | 82 | (defn validate-personae-array 83 | [personae-array] 84 | (some->> (s/explain-data ::personae-array personae-array) 85 | (errs/explain-to-map-coll ::personae-array))) 86 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/input/profile.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.profile 2 | "Profile input parsing." 3 | (:require [clojure.spec.alpha :as s] 4 | [clojure.string :as cstr] 5 | [com.yetanalytics.pan :as pan] 6 | [com.yetanalytics.pan.objects.profile :as pan-profile] 7 | [com.yetanalytics.datasim.util.errors :as errs])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Specs 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (s/def ::profiles 14 | (s/every ::pan-profile/profile :min-count 1)) 15 | 16 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 17 | ;; Validation 18 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 19 | 20 | (defn validate-profile 21 | [{:keys [id] :as profile}] 22 | (some->> (pan/validate-profile profile 23 | :syntax? true 24 | :result :type-path-string) 25 | (errs/type-path-string-m->map-coll id))) 26 | 27 | (defn validate-profiles 28 | [profiles] 29 | (if (vector? profiles) 30 | (let [prof-errs (pan/validate-profile-coll profiles 31 | :syntax? true 32 | :pattern-rels? true 33 | :result :type-path-string)] 34 | (errs/type-path-string-ms->map-coll (map :id profiles) 35 | prof-errs)) 36 | ;; TODO: Something more solid/less hacky, particularly in the Pan lib itself 37 | [{:path [::profiles] 38 | :text "Profiles must be a vector!" 39 | :id [::profiles]}])) 40 | 41 | ;; Helpers to `input/validate-pattern-filters` 42 | 43 | (defn- validate-pattern-filters-emsg 44 | [pattern-id pattern-id-set] 45 | (format 46 | "Pattern ID %s is not among primary patterns in provided profiles: %s" 47 | pattern-id 48 | (cstr/join \, pattern-id-set))) 49 | 50 | (defn- validate-profile-fitlers-emsg 51 | [profile-id profile-id-set] 52 | (format "Profile ID %s is not one of provided profiles: %s" 53 | profile-id 54 | (cstr/join \, profile-id-set))) 55 | 56 | (defn validate-pattern-filters 57 | [profiles gen-patterns] 58 | (let [pattern-id-set (->> profiles 59 | (mapcat :patterns) 60 | (keep (fn [{:keys [id primary]}] (when primary id))) 61 | (into #{}))] 62 | (for [[idx pattern-id] (map-indexed vector gen-patterns) 63 | :when (not (contains? pattern-id-set pattern-id))] 64 | {:id (str "parameters-genPatterns-" idx) 65 | :path [:parameters :genPatterns idx] 66 | :text (validate-pattern-filters-emsg pattern-id pattern-id-set)}))) 67 | 68 | (defn validate-profile-filters 69 | [profiles gen-profiles] 70 | (let [profile-id-set (->> profiles (map :id) (into #{}))] 71 | (for [[idx profile-id] (map-indexed vector gen-profiles) 72 | :when (not (contains? profile-id-set profile-id))] 73 | {:id (str "parameters-genProfiles-" idx) 74 | :path [:parameters :genProfiles idx] 75 | :text (validate-profile-fitlers-emsg profile-id profile-id-set)}))) 76 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/model/periods.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.model.periods 2 | (:require [clojure.spec.alpha :as s] 3 | [java-time.api :as t] 4 | [com.yetanalytics.datasim.input.model.alignments :as align] 5 | [com.yetanalytics.datasim.util.random :as random] 6 | [com.yetanalytics.datasim.model.bounds :as bounds])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (s/def ::min nat-int?) 13 | 14 | (s/def ::mean pos-int?) 15 | 16 | (s/def ::fixed pos-int?) 17 | 18 | (s/def ::period 19 | (s/or :variable (s/keys :req-un [::min ::mean] 20 | :opt-un [::bounds/bounds]) 21 | :fixed (s/keys :req-un [::fixed] 22 | :opt-un [::bounds/bounds]))) 23 | 24 | (s/def ::periods 25 | (s/every ::period)) 26 | 27 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 28 | ;; Constants 29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | 31 | (def ms-per-second 32 | 1000) 33 | 34 | (def ms-per-minute 35 | 60000) 36 | 37 | (def ms-per-hour 38 | 3600000) 39 | 40 | (def ms-per-day 41 | 86400000) 42 | 43 | (def ms-per-week 44 | 604800000) 45 | 46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 47 | ;; Compilation 48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 49 | 50 | (defn- convert-time 51 | "Convert time `t` into milliseconds based on the time `unit`. Coerces 52 | any doubles into integers." 53 | [t unit] 54 | (long (case unit 55 | :millis t 56 | :seconds (* t ms-per-second) 57 | :minutes (* t ms-per-minute) 58 | :hours (* t ms-per-hour) 59 | :days (* t ms-per-day) 60 | :weeks (* t ms-per-week)))) 61 | 62 | (defn- convert-period 63 | [{:keys [min mean fixed unit bounds]}] 64 | (let [unit* (or (some-> unit keyword) :minutes) 65 | mean* (or (some-> mean (convert-time unit*)) ms-per-minute) 66 | min* (or (some-> min (convert-time unit*)) 0) 67 | fixed* (some-> fixed (convert-time unit*)) 68 | bounds* (some-> bounds bounds/convert-bounds)] 69 | (cond-> {} 70 | fixed* (assoc :fixed fixed*) 71 | (not fixed*) (assoc :min min* 72 | :mean mean*) 73 | bounds* (assoc :bounds bounds*)))) 74 | 75 | (s/fdef convert-periods 76 | :args (s/cat :periods ::align/periods) 77 | :ret ::periods) 78 | 79 | (defn convert-periods 80 | [periods] 81 | (mapv convert-period periods)) 82 | 83 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 84 | ;; Runtime 85 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 86 | 87 | (defn- generate-period 88 | [rng {:keys [mean min fixed]}] 89 | (or fixed 90 | (let [rate (/ 1.0 mean) 91 | t-diff (long (random/rand-exp rng rate))] 92 | (+ min t-diff)))) 93 | 94 | (defn- add-period 95 | [date-time rng period] 96 | (t/plus date-time (t/millis (generate-period rng period)))) 97 | 98 | (s/fdef add-periods 99 | :args (s/cat :date-time t/local-date-time? 100 | :rng ::rng 101 | :periods ::periods) 102 | :ret t/local-date-time?) 103 | 104 | (defn add-periods 105 | "Add a random amount of milliseonds to `date-time` based on the first map of 106 | valid parameter map in `periods`. A valid map either has no time bounds or 107 | bounds that satisfy `date-time`. The millis amount is an exponentially 108 | distributed random variable with `period` parameters `:mean` and `:min`. 109 | The generated sequence that uses these periodic date-times will thus occur 110 | as a Poisson random process." 111 | [date-time rng periods] 112 | (let [periods (or periods 113 | [{:min 0 114 | :mean ms-per-minute}]) 115 | some-bound (fn [{:keys [bounds] :as period}] 116 | (when (bounds/bounded-time? bounds date-time) period))] 117 | (if-some [period (some some-bound periods)] 118 | (add-period date-time rng period) 119 | (throw (ex-info "Timestamp does not satisfy any period `bounds`; no default period present." 120 | {:type ::outside-period-bound 121 | :periods periods 122 | :date-time date-time}))))) 123 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/util/async.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.async 2 | "`core.async` utilities, for parallel generation." 3 | (:require [clojure.core.async :as a] 4 | [clojure.core.async.impl.protocols :as ap])) 5 | 6 | (defn chan? 7 | "Is `x` a proper `core.async` channel?" 8 | [x] 9 | (satisfies? ap/Channel x)) 10 | 11 | (defn heads-chans 12 | "Get a sorted map of `[head-val ]`, where `channel` is open." 13 | ([chans] 14 | (heads-chans compare chans)) 15 | ([compfn chans] 16 | (a/go-loop [chan-set (set chans) 17 | head-chan-m (sorted-map-by compfn)] 18 | (if-let [chan-seq (seq chan-set)] 19 | (let [[head chan] (a/alts! chan-seq)] 20 | (recur (disj chan-set chan) 21 | (cond-> head-chan-m 22 | head (assoc head chan)))) 23 | head-chan-m)))) 24 | 25 | (defn sequence-messages 26 | "Given an output channel `out-chan`, comparator `compfn`, and one or more 27 | channels `chans` containing ordered values, returns a channel that will 28 | receive an ordered sequence of messages, closing when the last channel 29 | closes." 30 | ([chans] 31 | (sequence-messages (a/chan) compare chans)) 32 | ([out-chan compfn chans] 33 | (a/go-loop [head-chan-m (a/! out-chan min-head) 36 | (let [?next-h (a/ head-chan-m 38 | (dissoc min-head) 39 | (cond-> 40 | ?next-h (assoc ?next-h chan)))))) 41 | (a/close! out-chan))) 42 | out-chan)) 43 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/util/errors.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.errors 2 | (:require [clojure.spec.alpha :as s])) 3 | 4 | (defn explain-to-map-coll 5 | "Convert spec error data for personae, models, and parameters 6 | into a map coll acceptable to the Datasim UI." 7 | [spec-kw spec-ed] 8 | (->> (::s/problems spec-ed) 9 | (map-indexed 10 | (fn [idx problem] 11 | (let [epath (into [spec-kw] (:in problem)) 12 | estr (-> problem 13 | (assoc ::s/problems [problem]) 14 | s/explain-printer 15 | with-out-str) 16 | eid (str (name spec-kw) "-" idx)] 17 | {:path epath 18 | :text estr 19 | :id eid}))) 20 | vec)) 21 | 22 | (defn type-path-string-m->map-coll 23 | "Convert Pan's `type-path-string-map`, using `profile-id`, into a 24 | map coll acceptable to the Datasim UI. Returns `nil` if there are 25 | no errors." 26 | [profile-id type-path-string-map] 27 | (->> type-path-string-map 28 | (reduce-kv (fn [acc etype epath-estr-m] 29 | (reduce-kv (fn [acc* epath estr] 30 | (conj acc* {:path (into [profile-id etype] 31 | epath) 32 | :text estr})) 33 | acc 34 | epath-estr-m)) 35 | []) 36 | (map-indexed (fn [idx emap] 37 | (assoc emap :id (str "profile-" idx)))) 38 | vec 39 | not-empty)) 40 | 41 | (defn type-path-string-ms->map-coll 42 | "Convert Pan's `type-path-string-maps`, using `profile-ids`, into a 43 | map coll acceptable to the Datasim UI. Returns `nil` if there are 44 | not errors." 45 | [profile-ids type-path-string-maps] 46 | (->> type-path-string-maps 47 | (map (fn [profile-id etype-epath-estr-m] 48 | [profile-id etype-epath-estr-m]) 49 | profile-ids) 50 | (reduce (fn [acc [prof-id etype-epath-estr-m]] 51 | (reduce-kv (fn [acc* etype epath-estr-m] 52 | (reduce-kv (fn [acc** epath estr] 53 | (let [epath* (into [prof-id etype] 54 | epath)] 55 | (conj acc** 56 | {:path epath* 57 | :text estr}))) 58 | acc* 59 | epath-estr-m)) 60 | acc 61 | etype-epath-estr-m)) 62 | []) 63 | (map-indexed (fn [idx emap] 64 | (assoc emap :id (str "profiles-" idx)))) 65 | vec 66 | not-empty)) 67 | 68 | (def bar 69 | (apply str (repeat 80 \=))) 70 | 71 | (defn map-coll->strs 72 | "Form a list of CLI error strings from an error map coll." 73 | [map-coll] 74 | (for [{:keys [id path text]} map-coll] 75 | (format 76 | "%s\nINPUT ERROR: %s\n%s\npath: %s\n\n%s" 77 | bar 78 | id 79 | bar 80 | (pr-str path) 81 | text))) 82 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/util/io.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.io 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as cstr] 4 | [cheshire.core :as json]) 5 | (:import [java.io IOException])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Exceptions 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (defn- throw-parse-error [cause-exn] 12 | (throw (ex-info "Parse Error" 13 | {:type ::parse-error} 14 | cause-exn))) 15 | 16 | (defn- throw-unparse-error [cause-exn] 17 | (throw (ex-info "Unparse Error" 18 | {:type ::unparse-error} 19 | cause-exn))) 20 | 21 | (defn- throw-io-error [location cause-exn] 22 | (throw (ex-info "I/O Error" 23 | {:type ::io-error 24 | :location location} 25 | cause-exn))) 26 | 27 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 28 | ;; Keyword Key Functions 29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | 31 | (defn- keywordize-json-key 32 | [k] 33 | (keyword (cond-> k 34 | (= \@ (first k)) 35 | (cstr/replace-first \@ \_)))) 36 | 37 | (defn- stringify-edn-key 38 | [k] 39 | (let [named-key (name k)] 40 | (cond-> named-key 41 | (= \_ (first named-key)) 42 | (cstr/replace-first \_ \@)))) 43 | 44 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 45 | ;; JSON I/O Functions 46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 47 | 48 | ;; TODO: Also have a `key-fn?` kwarg here? 49 | (defn read-json-location 50 | "Reads in a file from the given `location` and parses it into EDN. If 51 | the data being read is an array or sequence, return all the data 52 | as a seq." 53 | [location] 54 | (try (with-open [r (io/reader location)] 55 | (try (doall (json/parse-stream r keywordize-json-key)) 56 | (catch Exception e 57 | (throw-parse-error e)))) 58 | (catch IOException e 59 | (throw-io-error location e)))) 60 | 61 | (defn write-json 62 | "Write the contents of `data` to `writer`, which can be returned by 63 | `clojure.java.io/writer`. Applies `stringify-edn-key` if `key-fn?` is 64 | `true`." 65 | [data writer & {:keys [key-fn?] :or {key-fn? true}}] 66 | (try (if key-fn? 67 | (json/generate-stream data writer {:key-fn stringify-edn-key}) 68 | (json/generate-stream data writer)) 69 | (catch Exception e 70 | (throw-unparse-error e)))) 71 | 72 | ;; Avoid `with-open` to prevent stdout/stderr writers from being closed 73 | 74 | (defn write-json-stdout 75 | "Write the contents of `data` to standard output. Applies `stringify-edn-key` 76 | if `key-fn?` is `true`." 77 | [data & {:keys [key-fn?] :or {key-fn? true}}] 78 | (let [w (io/writer *out*)] 79 | (write-json data w :key-fn? key-fn?) 80 | (.write w "\n") 81 | (.flush w))) 82 | 83 | (defn write-json-stderr 84 | "Write the contents of `data` to standard error. Applies `stringify-edn-key` 85 | if `key-fn?` is `true`." 86 | [data & {:keys [key-fn?] :or {key-fn? true}}] 87 | (let [w (io/writer *err*)] 88 | (write-json data w :key-fn? key-fn?) 89 | (.write w "\n") 90 | (.flush w))) 91 | 92 | ;; Use `with-open` to ensure that the file writer gets closed 93 | 94 | (defn write-json-file 95 | "Write the contents of `data` to the file `location`; a file will be 96 | created if it does not exist. Stringifies keyword keys. Applies 97 | `stringify-edn-key` if `key-fn?` is `true`." 98 | [data location & {:keys [key-fn?] :or {key-fn? true}}] 99 | (let [file (io/file location)] 100 | (io/make-parents file) 101 | (with-open [w (io/writer location)] 102 | (write-json data w :key-fn? key-fn?)))) 103 | 104 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 105 | ;; String I/O Functions 106 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 107 | 108 | (defn println-coll 109 | "Print each string in `str-coll` followed by a newline to stdout, 110 | flushing the output buffer after each line." 111 | [str-coll] 112 | (doseq [s str-coll] 113 | (println s) 114 | (flush))) 115 | 116 | (defn println-err-coll 117 | "Print each string in `err-str-coll` followed by a newline to stderr, 118 | flushing the output buffer after each line." 119 | [err-str-coll] 120 | (binding [*out* *err*] 121 | (println-coll err-str-coll))) 122 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/util/sequence.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.sequence) 2 | 3 | (defn seq-sort 4 | "Given a key-fn (that must return a number) and a vector of seqs, return the 5 | items from all seqs ordered by that, ascending. 6 | If incoming seqs are monotonic, the output is as well." 7 | [key-fn seqs] 8 | (lazy-seq 9 | (when-let [seqs' (not-empty 10 | (into [] 11 | (keep not-empty 12 | seqs)))] 13 | (let [[idx [head & rest-seq]] 14 | (apply min-key (comp key-fn 15 | first 16 | second) 17 | (map-indexed vector 18 | seqs'))] 19 | (cons head 20 | (seq-sort 21 | key-fn 22 | (assoc seqs' idx rest-seq))))))) 23 | 24 | ;; https://clojuredocs.org/clojure.core/chunk#example-5c9cebc3e4b0ca44402ef6ec 25 | (defn re-chunk 26 | "takes a sequence (already chunked or not) 27 | and produces another sequence with different chunking size." 28 | [n xs] 29 | (lazy-seq 30 | (when-let [s (seq (take n xs))] 31 | (let [cb (chunk-buffer n)] 32 | (doseq [x s] (chunk-append cb x)) 33 | (chunk-cons (chunk cb) (re-chunk n (drop n xs))))))) 34 | 35 | (comment 36 | (require '[com.yetanalytics.datasim.util.random :as r]) 37 | 38 | (defn rand-monotonic-seq 39 | [seed & [start]] 40 | (let [rng (r/seed-rng seed)] 41 | (rest (iterate #(+ % (r/rand rng)) (or start 0))))) 42 | 43 | (let [ss (seq-sort 44 | identity 45 | (into [] 46 | (for [n (range 20)] 47 | (take 10 (rand-monotonic-seq n)))))] 48 | (= (sort ss) ss)) ;; => true 49 | ) 50 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/xapi/actor.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.xapi.actor 2 | "Utilities for Agents and Groups (collectively known as Actors)." 3 | (:require [clojure.string :as cstr] 4 | [clojure.spec.alpha :as s] 5 | [clojure.spec.gen.alpha :as sgen] 6 | [xapi-schema.spec :as xs])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (def ^:private ifi-prefix-gen 13 | (sgen/elements ["mbox::" "mbox_sha1sum::" "account::" "openid::"])) 14 | 15 | (def ^:private ifi-body-gen 16 | (sgen/such-that not-empty (sgen/string-ascii))) 17 | 18 | (s/def ::actor-ifi 19 | (s/with-gen 20 | (s/and string? 21 | not-empty 22 | (fn [s] 23 | (or (cstr/starts-with? s "mbox::") 24 | (cstr/starts-with? s "account::") 25 | (cstr/starts-with? s "mbox_sha1sum::") 26 | (cstr/starts-with? s "openid::")))) 27 | ;; Monads are fun! 28 | ;; This composes a the two IFI generators together to create a 29 | ;; (str ifi-prefix ifi-body) generator 30 | #(sgen/bind ifi-prefix-gen 31 | (fn [ifi-prefix] 32 | (sgen/bind ifi-body-gen 33 | (fn [ifi-body] 34 | (sgen/return (str ifi-prefix ifi-body)))))))) 35 | 36 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 37 | ;; Functions 38 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 39 | 40 | (s/fdef actor-ifi 41 | :args (s/cat :actor ::xs/actor) 42 | :ret ::actor-ifi) 43 | 44 | (defn actor-ifi 45 | "Return a string representing the IFI of an Agent or Group. Will be prefixed 46 | with the IFI property and two colons `::`. 47 | 48 | For accounts, the homepage will precede the name, and they will be delimited 49 | by a comma as so: `account::https://foo.bar,baz`. 50 | 51 | If an IFI cannot be found, returns `nil`." 52 | [{:keys [mbox 53 | mbox_sha1sum 54 | openid] 55 | {:keys [name homePage]} :account}] 56 | (or (and mbox 57 | (format "mbox::%s" mbox)) 58 | (and name homePage 59 | (format "account::%s,%s" 60 | homePage name)) 61 | (and mbox_sha1sum 62 | (format "mbox_sha1sum::%s" mbox_sha1sum)) 63 | (and openid 64 | (format "openid::%s" openid)))) 65 | 66 | (s/fdef groups->agent-group-ifi-map 67 | :args (s/cat :group-coll (s/every ::xs/identified-group)) 68 | :ret (s/map-of ::actor-ifi ::actor-ifi)) 69 | 70 | (defn groups->agent-group-ifi-map 71 | "Convert `group-coll` into a map from member agent IFIs to group IFIs." 72 | [group-coll] 73 | (reduce 74 | (fn [m {agents :member :as personae}] 75 | (let [group-ifi (actor-ifi personae)] 76 | (reduce 77 | (fn [m* actor] 78 | (let [agent-ifi (actor-ifi actor)] 79 | (assoc m* agent-ifi group-ifi))) 80 | m 81 | agents))) 82 | {} 83 | group-coll)) 84 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/xapi/profile/extension.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.xapi.profile.extension 2 | "Creation of `extension-spec-map` for Profile compilation." 3 | (:require [clojure.spec.alpha :as s] 4 | [com.yetanalytics.schemer :as schemer] 5 | [com.yetanalytics.pan.axioms :as ax] 6 | [com.yetanalytics.datasim.xapi.profile :as-alias profile])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (def json-schema-spec 13 | (s/or :keyword (s/and keyword? s/get-spec) 14 | :spec s/spec?)) 15 | 16 | (s/def ::activity (s/map-of ::ax/iri (s/nilable json-schema-spec))) 17 | (s/def ::context (s/map-of ::ax/iri (s/nilable json-schema-spec))) 18 | (s/def ::result (s/map-of ::ax/iri (s/nilable json-schema-spec))) 19 | 20 | (s/def ::extension-spec-map 21 | (s/keys :req-un [::activity ::context ::result])) 22 | 23 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 24 | ;; Functions 25 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 26 | 27 | (s/fdef create-extension-spec-map 28 | :args (s/cat :type-iri-map ::profile/type-iri-map) 29 | :ret ::extension-spec-map) 30 | 31 | (defn create-extension-spec-map 32 | [type-iri-map] 33 | (let [ext->spec (fn [ext] 34 | (some->> ext :inlineSchema (schemer/schema->spec nil))) 35 | reduce-exts (fn [ext-map] 36 | (update-vals ext-map ext->spec)) 37 | act-iri-map (get type-iri-map "ActivityExtension") 38 | ctx-iri-map (get type-iri-map "ContextExtension") 39 | res-iri-map (get type-iri-map "ResultExtension")] 40 | {:activity (reduce-exts act-iri-map) 41 | :context (reduce-exts ctx-iri-map) 42 | :result (reduce-exts res-iri-map)})) 43 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/xapi/profile/template.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.xapi.profile.template 2 | "Creation of `statement-base-map` and `parsed-rules-map` for Profile 3 | compilation." 4 | (:require [clojure.spec.alpha :as s] 5 | [com.yetanalytics.pan.objects.template :as template] 6 | [com.yetanalytics.datasim.xapi.rule :as rule] 7 | [com.yetanalytics.datasim.xapi.profile :as-alias profile])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Specs 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | (s/def ::template ::template/template) 14 | 15 | (s/def ::statement-base-map 16 | (s/map-of ::template/id map?)) 17 | 18 | (s/def ::parsed-rules-map 19 | (s/map-of ::template/id (s/every ::rule/parsed-rule))) 20 | 21 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 22 | ;; Statement Base 23 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 24 | 25 | (defn- activity-type->activity-base 26 | [activity-type] 27 | {"definition" {"type" activity-type}}) 28 | 29 | (defn- usage-type->attachment-base 30 | [attachment-usage-type] 31 | {"usageType" attachment-usage-type}) 32 | 33 | (s/fdef template->statement-base 34 | :args (s/cat :template ::template/template) 35 | :ret map?) 36 | 37 | (defn template->statement-base 38 | "Form the base of a statement from the Determining Properties of 39 | the Template. Elements of array-valued properties (the context 40 | activity types and the attachment usage types) are added in order." 41 | [{profile-version-id :inScheme 42 | verb-id :verb 43 | object-activity-type :objectActivityType 44 | category-activity-types :contextCategoryActivityType 45 | grouping-activity-types :contextGroupingActivityType 46 | parent-activity-types :contextParentActivityType 47 | other-activity-types :contextOtherActivityType 48 | attachment-usage-types :attachmentUsageType 49 | ;; TODO: StatementRef properties 50 | ;; object-statement-ref :objectStatementRefTemplate 51 | ;; context-statement-ref :contextStatementRefTemplate 52 | }] 53 | (cond-> {} 54 | verb-id 55 | (assoc-in ["verb" "id"] verb-id) 56 | object-activity-type 57 | (assoc-in ["object" "definition" "type"] object-activity-type) 58 | category-activity-types 59 | (assoc-in ["context" "contextActivities" "category"] 60 | (mapv activity-type->activity-base category-activity-types)) 61 | grouping-activity-types 62 | (assoc-in ["context" "contextActivities" "grouping"] 63 | (mapv activity-type->activity-base grouping-activity-types)) 64 | parent-activity-types 65 | (assoc-in ["context" "contextActivities" "parent"] 66 | (mapv activity-type->activity-base parent-activity-types)) 67 | other-activity-types 68 | (assoc-in ["context" "contextActivities" "other"] 69 | (mapv activity-type->activity-base other-activity-types)) 70 | attachment-usage-types 71 | (assoc-in ["attachments"] 72 | (mapv usage-type->attachment-base attachment-usage-types)) 73 | profile-version-id ; always true 74 | (update-in ["context" "contextActivities" "category"] 75 | (fnil conj []) 76 | {"id" profile-version-id}))) 77 | 78 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 79 | ;; Statement Rule Application 80 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 81 | 82 | (s/fdef template->parsed-rules 83 | :args (s/cat :type-iri-map ::type-iri-map 84 | :activity-map ::activity-map 85 | :template ::template/template) 86 | :ret (s/every ::rule/parsed-rule)) 87 | 88 | (defn template->parsed-rules 89 | "Return a collection of parsed rules derived from the template `rules`. 90 | Uses the object Determining Properties to assist with rule parsing." 91 | [{object-activity-type :objectActivityType 92 | object-statement-ref :objectStatementRefTemplate 93 | rules :rules}] 94 | (cond 95 | object-activity-type 96 | (rule/parse-rules :activity-type rules) 97 | object-statement-ref 98 | (rule/parse-rules :statement-ref rules) 99 | :else 100 | (rule/parse-rules rules))) 101 | 102 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 103 | ;; Profile Templates -> Statement Base + Parsed Rules 104 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 105 | 106 | (s/fdef create-statement-base-map 107 | :args (s/cat :type-iri-map ::profile/type-iri-map) 108 | :ret ::statement-base-map) 109 | 110 | (defn create-statement-base-map 111 | "Given Statement Templates in `type-iri-map`, return a map from those 112 | templates' IDs to the base xAPI Statements they form from their 113 | Determining Properties and inScheme." 114 | [type-iri-map] 115 | (-> type-iri-map 116 | (get "StatementTemplate") 117 | (update-vals template->statement-base))) 118 | 119 | (s/fdef create-parsed-rules-map 120 | :args (s/cat :type-iri-map? ::profile/type-iri-map) 121 | :ret ::parsed-rules-map) 122 | 123 | (defn create-parsed-rules-map 124 | "Given Statement Templates in `type-iri-map`, return a map from those 125 | templates' IDs to those their parsed rules" 126 | [type-iri-map] 127 | (-> type-iri-map 128 | (get "StatementTemplate") 129 | (update-vals template->parsed-rules))) 130 | 131 | (s/fdef update-parsed-rules-map 132 | :args (s/cat :profile-map ::profile/profile-map 133 | :parsed-rules-map ::parsed-rules-map) 134 | :ret ::parsed-rules-map) 135 | 136 | (defn update-parsed-rules-map 137 | "Use information from `profile-map` to complete the rules in 138 | `parsed-rules-map` by adding additional valuesets or spec generators." 139 | [profile-map parsed-rules-map] 140 | (update-vals parsed-rules-map 141 | (partial rule/add-rules-valuegen profile-map))) 142 | -------------------------------------------------------------------------------- /src/main/com/yetanalytics/datasim/xapi/profile/verb.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.xapi.profile.verb 2 | "Creation of `verb-map` for Profile compilation." 3 | (:require [clojure.spec.alpha :as s] 4 | [clojure.walk :as w] 5 | [xapi-schema.spec :as xs] 6 | [com.yetanalytics.datasim.xapi.profile :as-alias profile])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Specs 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (s/def ::verb-map 13 | (s/map-of ::xs/iri ::xs/verb)) 14 | 15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | ;; Functions 17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 18 | 19 | (defn- profile->statement-verb 20 | [{:keys [id prefLabel]}] 21 | {"id" id 22 | "display" (w/stringify-keys prefLabel)}) 23 | 24 | (s/fdef create-verb-map 25 | :args (s/cat :type-iri-map ::profile/type-iri-map) 26 | :ret ::verb-map) 27 | 28 | (defn create-verb-map 29 | "Create a map of verb IDs to Statement verbs out of Profile Verbs from 30 | `type-iri-map`." 31 | [type-iri-map] 32 | (-> type-iri-map (get "Verb") (update-vals profile->statement-verb))) 33 | -------------------------------------------------------------------------------- /src/onyx/com/yetanalytics/datasim/onyx/aeron_media_driver.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.aeron-media-driver 2 | (:require [clojure.core.async :refer [chan (System/getenv "ONYX_PROFILE") keyword)] 7 | (aero/read-config (io/resource "onyx_config.edn") 8 | {:profile (or profile :default)}))) 9 | -------------------------------------------------------------------------------- /src/onyx/com/yetanalytics/datasim/onyx/http.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.http 2 | (:require [cheshire.core :as json])) 3 | 4 | (defn after-read-batch 5 | "Encode statements and add request" 6 | [event {lrs-request ::lrs-request 7 | :as lifecycle}] 8 | (update event 9 | :onyx.core/batch 10 | (partial map 11 | (fn [{:keys [statements] 12 | :as segment}] 13 | (merge segment 14 | (assoc-in lrs-request 15 | [:args :body] 16 | ;; TODO: Buffr 17 | (json/generate-string statements))))))) 18 | 19 | (defn after-batch 20 | "Tear things down" 21 | [event lifecycle] 22 | (update event 23 | :onyx.core/batch 24 | (partial map 25 | (fn [{:keys [statements] 26 | :as segment}] 27 | (-> segment 28 | (dissoc :statements) 29 | (update :args dissoc :body :basic-auth) 30 | (assoc :statement-count (count statements))))))) 31 | 32 | (def out-calls 33 | {:lifecycle/after-read-batch after-read-batch 34 | :lifecycle/after-batch after-batch}) 35 | 36 | (defn post-success? 37 | [{:keys [status error]}] 38 | (and (= 200 status) 39 | (not error))) 40 | -------------------------------------------------------------------------------- /src/onyx/com/yetanalytics/datasim/onyx/peer.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.peer 2 | (:require onyx.plugin.http-output 3 | onyx.api 4 | com.yetanalytics.datasim.onyx.sim 5 | com.yetanalytics.datasim.onyx.http 6 | com.yetanalytics.datasim.onyx.job 7 | [com.yetanalytics.datasim.onyx.config :as config])) 8 | 9 | (defn start-peer! 10 | "Launch a peer, optionally pass config" 11 | [& [config-override]] 12 | (let [{:keys [env-config 13 | peer-config 14 | launch-config] 15 | {:keys [n-vpeers]} :launch-config} (or config-override 16 | (config/get-config)) 17 | 18 | env (onyx.api/start-env env-config) 19 | ;; start peer group 20 | peer-group (onyx.api/start-peer-group peer-config) 21 | 22 | v-peers (onyx.api/start-peers n-vpeers peer-group)] 23 | 24 | 25 | (.addShutdownHook (Runtime/getRuntime) 26 | (Thread. ^Runnable #(do 27 | (onyx.api/shutdown-peers v-peers) 28 | (onyx.api/shutdown-peer-group peer-group) 29 | (onyx.api/shutdown-env env)))))) 30 | -------------------------------------------------------------------------------- /src/onyx/com/yetanalytics/datasim/onyx/repl.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.repl 2 | "Cluster repl" 3 | (:require clojure.main 4 | rebel-readline.core 5 | rebel-readline.clojure.line-reader 6 | rebel-readline.clojure.service.local 7 | rebel-readline.clojure.main)) 8 | 9 | (defn repl! 10 | [] 11 | (rebel-readline.core/with-line-reader 12 | (rebel-readline.clojure.line-reader/create 13 | (rebel-readline.clojure.service.local/create)) 14 | (clojure.main/repl 15 | :prompt (fn []) ;; prompt is handled by line-reader 16 | :read (rebel-readline.clojure.main/create-repl-read)))) 17 | -------------------------------------------------------------------------------- /src/onyx/com/yetanalytics/datasim/onyx/util.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.onyx.util 2 | (:require [com.yetanalytics.datasim.input :as input] 3 | [cheshire.core :as json] 4 | [byte-streams :as bs])) 5 | 6 | 7 | (defn round-robin 8 | "Given a number of partitions/buckets and a sequence, return a set of sets 9 | dividing up xs" 10 | [num-parts 11 | xs] 12 | (set 13 | (remove empty? 14 | (reduce 15 | (fn [vs [x idx]] 16 | (update vs idx conj x)) 17 | (into [] 18 | (repeat num-parts #{})) 19 | (map 20 | vector 21 | xs 22 | (cycle (range num-parts))))))) 23 | 24 | (defn batch->smile 25 | "Convert a batch of segments to SMILE data" 26 | ^bytes [segments] 27 | (json/generate-smile 28 | (mapcat :statements 29 | segments))) 30 | 31 | (defn batch->json 32 | "Convert a batch of segments to JSON bytes" 33 | ^bytes [segments] 34 | (bs/to-byte-array 35 | (json/generate-string 36 | (mapcat :statements 37 | segments)))) 38 | 39 | (defn override-max! 40 | [input mo] 41 | (assoc-in input [:parameters :max] mo)) 42 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/cli_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.cli-test 2 | "Integration tests for the DATASIM CLI." 3 | (:require [clojure.test :refer [deftest testing is]] 4 | [clojure.string :as cstr] 5 | [org.httpkit.client :as http] 6 | [clj-test-containers.core :as tc] 7 | [com.yetanalytics.datasim.input :as input] 8 | [com.yetanalytics.datasim.cli.input :as cli-input] 9 | [com.yetanalytics.datasim.cli.generate :as cli-gen] 10 | [com.yetanalytics.datasim.test-containers :as ds-tc])) 11 | 12 | (def json-post-header 13 | {"X-Experience-Api-Version" "1.0.3" 14 | "Content-Type" "application/json"}) 15 | 16 | (deftest validate-input-test 17 | (testing "validate-input subcommand" 18 | (cli-input/validate-input! 19 | ["-p" "dev-resources/profiles/cmi5/fixed.json" 20 | "-a" "dev-resources/personae/simple.json" 21 | "-m" "dev-resources/models/simple.json" 22 | "-o" "dev-resources/parameters/simple.json" 23 | "-v" "dev-resources/input/simple2.json"]) 24 | (let [input* (input/from-location 25 | :input :json 26 | "dev-resources/input/simple2.json")] 27 | (is (nil? (input/validate :input input*)))))) 28 | 29 | (deftest generate-test 30 | (let [cont (tc/start! ds-tc/validate-server-container) 31 | host (:host cont) 32 | port (get (:mapped-ports cont) 8080)] 33 | (testing "generate subcommand" 34 | (let [results 35 | (with-out-str 36 | (cli-gen/generate! 37 | ["-i" "dev-resources/input/simple.json"]))] 38 | (is (string? results)) 39 | (is (every? 40 | (fn [stmt-str] 41 | (let [validate-res 42 | #_{:clj-kondo/ignore [:unresolved-var]} 43 | @(http/post 44 | (format "http://%s:%d/statements" host port) 45 | {:headers json-post-header 46 | :body stmt-str 47 | :as :stream})] 48 | (= 204 (:status validate-res)))) 49 | (take 25 (cstr/split-lines results)))))) 50 | (testing "generate-post subcommand - sync" 51 | (let [results 52 | (cli-gen/generate-post! 53 | ["-i" "dev-resources/input/simple.json" 54 | "-E" (format "http://%s:%d" host port) 55 | "-B" "1" 56 | "-L" "1" 57 | "--no-async"])] 58 | ;; Errors would indicate 4xx response from Persephone server 59 | (is (nil? (:errors results))))) 60 | (testing "generate-post subcommand - async" 61 | (let [cont (tc/start! ds-tc/validate-server-container) 62 | host (:host cont) 63 | port (get (:mapped-ports cont) 8080) 64 | res (cli-gen/generate-post! 65 | ["-i" "dev-resources/input/simple.json" 66 | "-E" (format "http://%s:%d" host port) 67 | "-B" "1" 68 | "-L" "1" 69 | "--async"])] 70 | (is (nil? (:errors res))))) 71 | (tc/stop! cont))) 72 | 73 | (deftest generate-test-2 74 | (testing "generate-post subcommand on match serever" 75 | (let [cont (tc/start! ds-tc/match-server-container) 76 | host (:host cont) 77 | port (get (:mapped-ports cont) 8080) 78 | res (cli-gen/generate-post! 79 | ["-i" "dev-resources/input/simple.json" 80 | "-E" (format "http://%s:%d" host port) 81 | "-B" "25" 82 | "-L" "25" 83 | "--no-async"])] 84 | (is (nil? (:errors res))) 85 | (tc/stop! cont)))) 86 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/input/personae_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.personae-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [com.yetanalytics.datasim.util.io :as dio] 4 | [com.yetanalytics.datasim.input.personae :as personae] 5 | [com.yetanalytics.datasim.test-constants :as const])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Constants 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (def simple-personae 12 | (dio/read-json-location const/simple-personae-filepath)) 13 | 14 | (def tc3-personae 15 | (dio/read-json-location const/tc3-personae-filepath)) 16 | 17 | (def temporal-personae 18 | (dio/read-json-location const/temporal-personae-filepath)) 19 | 20 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 21 | ;; Tests 22 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 23 | 24 | (deftest personae-test 25 | (testing "personae without roles" 26 | (is (nil? (personae/validate-personae simple-personae))) 27 | (is (nil? (personae/validate-personae tc3-personae))) 28 | (is (nil? (personae/validate-personae temporal-personae)))) 29 | (testing "personae with roles" 30 | (is (nil? (-> simple-personae 31 | (assoc-in [:member 0 :role] "Lead Developer") 32 | (assoc-in [:member 1 :role] "Data Engineer") 33 | (assoc-in [:member 2 :role] "CEO") 34 | personae/validate-personae))) 35 | (is (nil? (-> tc3-personae 36 | (assoc-in [:member 0 :role] "xAPI God") 37 | (assoc-in [:member 1 :role] "Company Chief") 38 | (assoc-in [:member 2 :role] "Questions Queen") 39 | (assoc-in [:member 3 :role] "Learning Lord") 40 | (assoc-in [:member 4 :role] "Simulations Nomand") 41 | (assoc-in [:member 5 :role] "Statement Merchant") 42 | personae/validate-personae))))) 43 | 44 | (deftest personae-array-validation-test 45 | (testing "personae-array spec" 46 | (is (nil? (personae/validate-personae-array 47 | [const/simple-personae const/tc3-personae]))) 48 | (is (some? (personae/validate-personae-array 49 | [(-> const/simple-personae 50 | (assoc-in [:member 0 :mbox] "not-an-email")) 51 | const/tc3-personae]))) 52 | (is (some? (personae/validate-personae-array [])))) 53 | (testing "duplicate member ids across different groups" 54 | (is (some? (personae/validate-personae-array 55 | [(-> const/simple-personae 56 | (assoc-in [:member 0 :mbox] "mailto:bob@example.org") 57 | (assoc-in [:member 1 :mbox] "mailto:alice@example.org") 58 | (assoc-in [:member 2 :mbox] "mailto:fred@example.org")) 59 | const/tc3-personae]))))) 60 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/input/profile_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input.profile-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [clojure.data.json :as json] 4 | [com.yetanalytics.pan :as pan] 5 | [com.yetanalytics.datasim.input.profile :as profile] 6 | [com.yetanalytics.datasim.util.io :as dio] 7 | [com.yetanalytics.datasim.test-constants :as const]) 8 | (:import [java.io File])) 9 | 10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | ;; Constants 12 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 13 | 14 | (def minimal-profile 15 | (dio/read-json-location const/minimal-profile-filepath)) 16 | 17 | (def minimal-profile-map 18 | {:id "https://xapinet.org/xapi/yet/minimal" 19 | :type "Profile" 20 | :_context "https://w3id.org/xapi/profiles/context" 21 | :conformsTo "https://w3id.org/xapi/profiles#1.0" 22 | :prefLabel {:en "Minimal - Experimental xAPI Profile"} 23 | :definition {:en "This xAPI Profile demonstrates the minimal required properties of an xAPI profile"} 24 | :versions [{:id "https://xapinet.org/xapi/yet/minimal/v1" 25 | :generatedAtTime "2020-03-25T15:45:31.907Z"}] 26 | :author {:url "https://www.yetanalytics.com/" 27 | :name "Yet Analytics" 28 | :type "Organization"}}) 29 | 30 | (def cmi5-satisfied-bad-1 "https://w3id.org/xapi/cmi5#satisfiedbad1") 31 | (def cmi5-satisfied-bad-2 "https://w3id.org/xapi/cmi5#satisfiedbad2") 32 | (def cmi5-satisfied-bad-3 "https://w3id.org/xapi/cmi5#satisfiedbad3") 33 | (def cmi5-initialized "https://w3id.org/xapi/cmi5#initialized") 34 | (def cmi5-terminated "https://w3id.org/xapi/cmi5#terminated") 35 | (def cmi5-completed "https://w3id.org/xapi/cmi5#completed") 36 | 37 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 38 | ;; Tests 39 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 40 | 41 | (defn- profile-key [k] 42 | (let [kname (name k)] 43 | (if (= "@context" kname) 44 | :_context 45 | (keyword kname)))) 46 | 47 | (deftest minimal-profile-test 48 | (testing "produces the correct profile" 49 | ;; Coerce `minimal-profile` back into non-record map 50 | (is (= minimal-profile-map 51 | (into {} minimal-profile)))) 52 | (testing "is valid" 53 | (is (nil? (profile/validate-profile minimal-profile)))) 54 | (testing "is valid when written" 55 | (let [^File tf (File/createTempFile "profiletest" nil)] 56 | (try 57 | (dio/write-json-file minimal-profile tf) 58 | (is (nil? (pan/validate-profile 59 | (json/read-str (slurp tf) :key-fn profile-key)))) 60 | (finally 61 | (.delete tf)))))) 62 | 63 | (deftest profile-cosmos-validation-test 64 | (testing "input is valid if all template refs are valid" 65 | (is (nil? (profile/validate-profiles [const/cmi5-profile])))) 66 | (testing "input is invalid if invalid template ref iri exists" 67 | (is (= 1 (->> [(-> const/cmi5-profile 68 | (assoc-in [:patterns 0 :zeroOrMore] 69 | "https://w3id.org/xapi/cmi5#bad-template"))] 70 | profile/validate-profiles 71 | count))) 72 | ;; XXX: If we replaced satisfiedbad3 with satisfiedbad2, we only get a count 73 | ;; of 2 errors, not 3. 74 | (is (= 3 75 | (->> [(-> const/cmi5-profile 76 | (assoc-in [:patterns 0 :zeroOrMore] cmi5-satisfied-bad-1) 77 | (assoc-in [:patterns 1 :sequence 0] cmi5-satisfied-bad-2) 78 | (assoc-in [:patterns 1 :sequence 2] cmi5-satisfied-bad-3))] 79 | profile/validate-profiles 80 | count)))) 81 | (testing "validation works for multi-profile cosmos" 82 | (is (nil? (profile/validate-profiles 83 | [const/cmi5-profile const/video-profile]))) 84 | ;; Add connections between Profiles 85 | (is (nil? (profile/validate-profiles 86 | [const/cmi5-profile 87 | (-> const/video-profile 88 | (assoc-in [:patterns 0 :sequence 0] cmi5-initialized) 89 | (assoc-in [:patterns 0 :sequence 2] cmi5-terminated) 90 | (assoc-in [:patterns 1 :alternates 6] cmi5-completed))]))) 91 | (is (= 1 (->> [(-> const/cmi5-profile 92 | (assoc-in [:patterns 0 :zeroOrMore] 93 | "https://w3id.org/xapi/cmi5#bad-template")) 94 | const/video-profile] 95 | profile/validate-profiles 96 | count)))) 97 | (testing "fixed / valid profiles" 98 | (is (nil? (profile/validate-profiles [const/acrossx-profile]))) 99 | (is (nil? (profile/validate-profiles [const/activity-streams-profile]))) 100 | (is (nil? (profile/validate-profiles [const/tc3-profile])))) 101 | ;; Following tests exist to point out flaws in Profiles 102 | (testing "invalid profiles" 103 | ;; AcrossX and ActivityStreams violate spec: 104 | ;; "related MUST only be used on deprecated Concepts" 105 | (is (some? (profile/validate-profiles [const/acrossx-profile*]))) 106 | (is (some? (profile/validate-profiles [const/activity-streams-profile*]))))) 107 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/input_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.input-test 2 | (:require [clojure.test :refer [deftest testing is are]] 3 | [com.yetanalytics.datasim.input 4 | :refer [from-location validate validate-throw] 5 | :as input] 6 | [com.yetanalytics.datasim.test-constants :as const]) 7 | (:import [clojure.lang ExceptionInfo])) 8 | 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | ;; Input Read Tests 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | 13 | ;; We need to test that the process of reading itself is valid, rather than 14 | ;; entirely rely on already-read fixtures 15 | 16 | (deftest valid-invalid-read-test 17 | (are [test-name 18 | invalid-reason 19 | input-key 20 | file-loc 21 | invalidator-fn] 22 | (testing (format "Reading %s" test-name) 23 | (let [x (from-location input-key :json file-loc)] 24 | (testing "is valid" 25 | (is (nil? (validate input-key x)))) 26 | (testing (format "is invalid due to %s" invalid-reason) 27 | (is (some? (validate input-key (invalidator-fn x)))) 28 | (is (thrown? ExceptionInfo 29 | (validate-throw input-key (invalidator-fn x))))))) 30 | "xAPI Profile" "non-IRI ID" 31 | :profile const/cmi5-profile-filepath 32 | #(assoc % :id "foo") 33 | 34 | "Actor Personae" "empty `:members` coll" 35 | :personae const/simple-personae-filepath 36 | #(assoc % :member []) 37 | 38 | "Actor Models" "invalid due to invalid alignments" 39 | :models const/simple-models-filepath 40 | #(conj % {:personae [{:id "notanid" 41 | :type "notatype"}] 42 | :verbs [{:component "notaniri" 43 | :weight "bar"}]}) 44 | 45 | "Actor Models, Long" "invalid alignments" 46 | :models const/tc3-models-filepath 47 | #(conj % {:personae [{:id "notanid" 48 | :type "notatype"}] 49 | :verbs [{:component "notaniri" 50 | :weight "bar"}]}) 51 | 52 | "Actor Models w/ Overrides" "invalid alignments" 53 | :models const/simple-overrides-models-filepath 54 | #(conj % {:personae [{:id "notanid" 55 | :type "notatype"}] 56 | :objectOverrides [{:component "notaniri" 57 | :weight "bar"}]}) 58 | 59 | "Simulation Parameters" "non-numeric seed" 60 | :parameters const/simple-parameters-filepath 61 | #(assoc % :seed "hey") 62 | 63 | "Combined Input Spec" "`:profiles` not being a vector" 64 | :input const/simple-input-filepath 65 | #(update % :profiles first))) 66 | 67 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 68 | ;; Input Validation Tests 69 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 70 | 71 | (deftest subobject-validation-test 72 | (testing "input is valid with a minimal profile" 73 | (is (nil? (input/validate 74 | :input 75 | (assoc-in const/simple-input 76 | [:profiles 0] 77 | const/minimal-profile)))))) 78 | 79 | (deftest combined-input-validation-test 80 | (testing "combined input is valid" 81 | (is (nil? (validate :input const/simple-input))) 82 | (is (try (validate-throw :input const/simple-input) 83 | true 84 | (catch Exception _ false)))) 85 | (testing "combined input is invalid" 86 | (testing "with invalid genProfiles" 87 | (is (try 88 | (validate-throw 89 | :input 90 | (assoc-in const/simple-input 91 | [:parameters :genProfiles] 92 | ["http://example.com/nonexistent.jsonld"])) 93 | false 94 | (catch Exception _ true)))) 95 | (testing "with invalid genPatterns" 96 | (is (try 97 | (validate-throw 98 | :input 99 | (assoc-in const/simple-input 100 | [:parameters :genPatterns] 101 | ["http://example.com/nonexistent#pattern"])) 102 | false 103 | (catch Exception _ true)))))) 104 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/model/periods_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.model.periods-test 2 | (:require [clojure.test :refer [deftest testing is are]] 3 | [java-time.api :as t] 4 | [com.yetanalytics.datasim.util.random :as random] 5 | [com.yetanalytics.datasim.model.periods :as periods])) 6 | 7 | (def new-years-date-time 8 | "The first moment of 2023, 2023-01-01T00:00:00, as a LocalDateTime" 9 | (t/local-date-time 2023 1 1 0 0 0)) 10 | 11 | (deftest time-period-test 12 | (testing "`convert-periods?` function" 13 | (is (= [{:min 2 14 | :mean 2} 15 | {:min 2000 16 | :mean 2000} 17 | {:min 120000 18 | :mean 120000} 19 | {:min 7200000 20 | :mean 7200000} 21 | {:min 172800000 22 | :mean 172800000} 23 | {:min 1209600000 24 | :mean 1209600000}] 25 | (periods/convert-periods 26 | [{:min 2 27 | :mean 2 28 | :unit "millis"} 29 | {:min 2 30 | :mean 2 31 | :unit "seconds"} 32 | {:min 2 33 | :mean 2 34 | :unit "minutes"} 35 | {:min 2 36 | :mean 2 37 | :unit "hours"} 38 | {:min 2 39 | :mean 2 40 | :unit "days"} 41 | {:min 2 42 | :mean 2 43 | :unit "weeks"}]))) 44 | (is (= [{:fixed 2} 45 | {:fixed 2000} 46 | {:fixed 120000} 47 | {:fixed 7200000} 48 | {:fixed 172800000} 49 | {:fixed 1209600000}] 50 | (periods/convert-periods 51 | [{:fixed 2 52 | :unit "millis"} 53 | {:fixed 2 54 | :unit "seconds"} 55 | {:fixed 2 56 | :unit "minutes"} 57 | {:fixed 2 58 | :unit "hours"} 59 | {:fixed 2 60 | :unit "days"} 61 | {:fixed 2 62 | :unit "weeks"}]))) 63 | (is (= [{:min 60000 ; minute default 64 | :mean 60000} 65 | {:fixed 60000}] 66 | (periods/convert-periods 67 | [{:min 1 68 | :mean 1} 69 | {:min 1 70 | :mean 1 71 | :fixed 1}])))) 72 | (testing "`add-periods` function" 73 | (testing "(fixed times)" 74 | (are [expected periods] 75 | (= (t/local-date-time expected) 76 | (periods/add-periods 77 | new-years-date-time 78 | (random/seed-rng 100) ; rng doesn't matter here 79 | (periods/convert-periods periods))) 80 | "2023-01-01T00:01:00" [{:fixed 1}] 81 | "2023-01-01T00:30:00" [{:fixed 1 82 | :bounds [{:years [2024]}]} 83 | {:fixed 30 84 | :bounds [{:years [2023]}]}])) 85 | (testing "(minimum times)" 86 | (are [expected-min periods] 87 | (t/before? 88 | (t/local-date-time expected-min) 89 | (periods/add-periods 90 | new-years-date-time 91 | (random/rng) 92 | (periods/convert-periods periods))) 93 | "2023-01-01T00:00:30" [{:min 30 94 | :unit "seconds"}] 95 | "2023-01-01T00:30:00" [{:min 30 96 | :unit "minutes"}] 97 | "2023-01-01T12:00:00" [{:min 12 98 | :unit "hours"}] 99 | "2023-01-10T00:00:00" [{:min 10 100 | :unit "days"}] 101 | "2023-01-14T00:00:00" [{:min 2 102 | :unit "weeks"}])) 103 | (testing "(mean times)" 104 | ;; We use 6 * sd = 6 * mean (b/c exponential distribution) 105 | (are [expected-max periods] 106 | (t/before? 107 | new-years-date-time 108 | (periods/add-periods 109 | new-years-date-time 110 | (random/rng) 111 | (periods/convert-periods periods)) 112 | (t/local-date-time expected-max)) 113 | "2023-01-01T00:03:00" [{:mean 30 114 | :unit "seconds"}] 115 | "2023-01-01T00:30:00" [{:mean 5 116 | :unit "minutes"}] 117 | "2023-01-01T12:00:00" [{:mean 2 118 | :unit "hours"}] 119 | "2023-01-06T00:00:00" [{:mean 1 120 | :unit "days"}] 121 | "2023-01-21T00:00:00" [{:mean 0.5 122 | :unit "weeks"}])))) 123 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/test_constants.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.test-constants 2 | "Constants for input items, i.e. profiles, personae, models, and 3 | parameters." 4 | (:require [clojure.java.io :as io] 5 | [com.yetanalytics.datasim.input :as input])) 6 | 7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8 | ;; Filepath Names 9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | ;; Profiles 12 | 13 | (def minimal-profile-filepath 14 | "dev-resources/profiles/minimal.jsonld") 15 | (def no-concept-profile-filepath 16 | "dev-resources/profiles/no_concept.jsonld") 17 | (def cmi5-profile-filepath 18 | "dev-resources/profiles/cmi5/fixed.json") 19 | (def video-profile-filepath 20 | "dev-resources/profiles/video/profile.jsonld") 21 | (def acrossx-profile-filepath* 22 | "dev-resources/profiles/acrossx/profile.jsonld") 23 | (def acrossx-profile-filepath 24 | "dev-resources/profiles/acrossx/fixed.jsonld") 25 | (def activity-streams-profile-filepath* 26 | "dev-resources/profiles/activity_streams/profile.jsonld") 27 | (def activity-streams-profile-filepath 28 | "dev-resources/profiles/activity_streams/fixed.jsonld") 29 | (def activity-profile-filepath 30 | "dev-resources/profiles/activity.jsonld") 31 | (def mom-profile-filepath 32 | "dev-resources/profiles/tla/mom.jsonld") 33 | (def referential-profile-filepath 34 | "dev-resources/profiles/referential.jsonld") 35 | (def tc3-profile-filepath 36 | "dev-resources/profiles/tccc/cuf_hc_video_and_asm_student_survey_profile.jsonld") 37 | (def temporal-profile-filepath 38 | "dev-resources/profiles/temporal.jsonld") 39 | 40 | ;; Personae 41 | 42 | (def simple-personae-filepath 43 | "dev-resources/personae/simple.json") 44 | (def tc3-personae-filepath 45 | "dev-resources/personae/tccc_dev.json") 46 | (def temporal-personae-filepath 47 | "dev-resources/personae/temporal.json") 48 | 49 | ;; Models 50 | 51 | (def simple-models-filepath 52 | "dev-resources/models/simple.json") 53 | (def simple-overrides-models-filepath 54 | "dev-resources/models/simple_with_overrides.json") 55 | (def simple-temporal-models-filepath 56 | "dev-resources/models/simple_with_temporal.json") 57 | (def simple-repeat-max-models-filepath 58 | "dev-resources/models/simple_with_repeat_max.json") 59 | (def tc3-models-filepath 60 | "dev-resources/models/tccc_dev.json") 61 | 62 | (def temporal-models-filepath 63 | "dev-resources/models/temporal/") 64 | (def temporal-models-filepath-coll* 65 | (-> temporal-models-filepath io/as-file .list seq)) 66 | (def temporal-models-filepath-coll 67 | (map (partial str temporal-models-filepath) 68 | temporal-models-filepath-coll*)) 69 | 70 | ;; Parameters 71 | 72 | (def simple-parameters-filepath 73 | "dev-resources/parameters/simple.json") 74 | (def temporal-parameters-filepath 75 | "dev-resources/parameters/temporal.json") 76 | 77 | ;; Combined Input 78 | 79 | (def simple-input-filepath 80 | "dev-resources/input/simple.json") 81 | 82 | ;; Miscellaneous 83 | 84 | (def simple-statement-filepath 85 | "dev-resources/xapi/statements/simple.json") 86 | (def long-statement-filepath 87 | "dev-resources/xapi/statements/long.json") 88 | 89 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 90 | ;; JSON Datas 91 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 92 | 93 | ;; Profiles 94 | 95 | (def minimal-profile 96 | (input/from-location :profile :json minimal-profile-filepath)) 97 | 98 | (def no-concept-profile 99 | (input/from-location :profile :json no-concept-profile-filepath)) 100 | 101 | (def cmi5-profile 102 | (input/from-location :profile :json cmi5-profile-filepath)) 103 | 104 | (def video-profile 105 | (input/from-location :profile :json video-profile-filepath)) 106 | 107 | (def acrossx-profile* 108 | (input/from-location :profile :json acrossx-profile-filepath*)) 109 | 110 | (def acrossx-profile 111 | (input/from-location :profile :json acrossx-profile-filepath)) 112 | 113 | (def activity-streams-profile* 114 | (input/from-location :profile :json activity-streams-profile-filepath*)) 115 | 116 | (def activity-streams-profile 117 | (input/from-location :profile :json activity-streams-profile-filepath)) 118 | 119 | (def activity-profile 120 | (input/from-location :profile :json activity-profile-filepath)) 121 | 122 | (def mom-profile 123 | (input/from-location :profile :json mom-profile-filepath)) 124 | 125 | (def referential-profile 126 | (input/from-location :profile :json referential-profile-filepath)) 127 | 128 | (def tc3-profile 129 | (input/from-location :profile :json tc3-profile-filepath)) 130 | 131 | (def temporal-profile 132 | (input/from-location :profile :json temporal-profile-filepath)) 133 | 134 | ;; Personae 135 | 136 | (def simple-personae 137 | (input/from-location :personae :json simple-personae-filepath)) 138 | 139 | (def tc3-personae 140 | (input/from-location :personae :json tc3-personae-filepath)) 141 | 142 | (def temporal-personae 143 | (input/from-location :personae :json temporal-personae-filepath)) 144 | 145 | ;; Models 146 | 147 | (def simple-overrides-models 148 | (input/from-location :models :json simple-overrides-models-filepath)) 149 | 150 | (def simple-temporal-models 151 | (input/from-location :models :json simple-temporal-models-filepath)) 152 | 153 | (def simple-repeat-max-models 154 | (input/from-location :models :json simple-repeat-max-models-filepath)) 155 | 156 | (def temporal-models-coll 157 | (map (partial input/from-location :models :json) 158 | temporal-models-filepath-coll)) 159 | 160 | ;; Parameters 161 | 162 | (def temporal-parameters 163 | (input/from-location :parameters :json temporal-parameters-filepath)) 164 | 165 | ;; Combined Input 166 | 167 | (def simple-input 168 | (input/from-location :input :json simple-input-filepath)) 169 | 170 | (def temporal-input-map 171 | (zipmap temporal-models-filepath-coll* 172 | (map (fn [temporal-model] 173 | {:profiles [temporal-profile] 174 | :personae-array [temporal-personae] 175 | :parameters temporal-parameters 176 | :models temporal-model}) 177 | temporal-models-coll))) 178 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/test_containers.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.test-containers 2 | (:require [clj-test-containers.core :as tc])) 3 | 4 | (def server-validate-command 5 | ["/persephone/bin/server.sh" "validate" 6 | "-p" "dev-resources/profiles/cmi5/fixed.json"]) 7 | 8 | (def server-match-command 9 | ["/persephone/bin/server.sh" "match" 10 | "-p" "dev-resources/profiles/cmi5/fixed.json"]) 11 | 12 | (def container-map 13 | {:image-name "yetanalytics/persephone:v0.9.1" 14 | :exposed-ports [8080]}) 15 | 16 | (def filesystem-map 17 | {:host-path "dev-resources/" 18 | :container-path "/persephone/dev-resources/" 19 | :mode :read-only}) 20 | 21 | (def validate-server-container 22 | (-> (assoc container-map :command server-validate-command) 23 | tc/create 24 | (tc/bind-filesystem! filesystem-map))) 25 | 26 | (def match-server-container 27 | (-> (assoc container-map :command server-match-command) 28 | tc/create 29 | (tc/bind-filesystem! filesystem-map))) 30 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/util/io_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.io-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [com.yetanalytics.datasim.util.io :as io] 4 | [com.yetanalytics.datasim.test-constants :as const])) 5 | 6 | (deftest read-json-location-test 7 | (testing "Reads files as json" 8 | (is (map? (io/read-json-location const/cmi5-profile-filepath)))) 9 | (testing "Throws wrapped I/O errors when it can't read something" 10 | (is (= ::io/io-error 11 | (try (io/read-json-location "notthere.json") 12 | (catch clojure.lang.ExceptionInfo exi 13 | (:type (ex-data exi)))))) 14 | (is (= ::io/io-error 15 | (try (io/read-json-location "https://notarealhost329083") 16 | (catch clojure.lang.ExceptionInfo exi 17 | (:type (ex-data exi)))))) 18 | (is (= ::io/io-error 19 | (try (io/read-json-location "https://www.google.com/bingisgreat") 20 | (catch clojure.lang.ExceptionInfo exi 21 | (:type (ex-data exi))))))) 22 | (testing "Throws wrapped parser errors when it can't read a format" 23 | (is (= ::io/parse-error 24 | (try (io/read-json-location "deps.edn") 25 | (catch clojure.lang.ExceptionInfo exi 26 | (:type (ex-data exi)))))))) 27 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/datasim/util/random_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.datasim.util.random-test 2 | (:require [clojure.test :refer [deftest testing is]] 3 | [clojure.math :as math] 4 | [clojure.spec.test.alpha :as stest] 5 | [com.yetanalytics.datasim.util.random :as r])) 6 | 7 | (deftest random-functions-test 8 | (testing "Generative tests" 9 | (let [results (stest/check 10 | `#{r/rng 11 | r/seed-rng 12 | r/rand 13 | r/rand-int 14 | r/rand-unbound-int 15 | r/rand-gaussian 16 | r/rand-boolean 17 | r/rand-exp 18 | r/rand-uuid 19 | r/rand-nth 20 | r/shuffle 21 | r/random-sample 22 | r/choose 23 | r/choose-map}) 24 | {:keys [total 25 | check-passed]} (stest/summarize-results results)] 26 | (is (= total check-passed))))) 27 | 28 | (def choose-total 4000) 29 | 30 | (def choose-coll [:foo :bar :baz :qux]) 31 | 32 | (def choose-map {:foo :FOO :bar :BAR :baz :BAZ :qux :QUX}) 33 | 34 | ;; 4 standard deviations means that the bounds will only be exceeded in 35 | ;; 1 in 15,787 runs. 36 | ;; See: https://en.wikipedia.org/wiki/68–95–99.7_rule##Table_of_numerical_values 37 | (def sd-limit 4) 38 | 39 | ;; The means and standard deviations are derived from interpreting the 40 | ;; event of choosing a particular kewyord as a binomial distribution. 41 | ;; See: https://en.wikipedia.org/wiki/Binomial_distribution 42 | (defmacro test-equal-choose [weights] 43 | (let [prob# 0.25 44 | mean# (* choose-total prob#) 45 | sd# (math/sqrt (* choose-total prob# (- 1 prob#))) 46 | lo-bound# (- mean# (* sd-limit sd#)) 47 | hi-bound# (+ mean# (* sd-limit sd#))] 48 | `(let [~'the-rng (r/rng) 49 | ~'choose-fn (fn [] (r/choose ~'the-rng ~weights ~choose-coll)) 50 | ~'results (repeatedly ~choose-total ~'choose-fn) 51 | ~'freq-map (frequencies ~'results)] 52 | (is (< ~lo-bound# (:foo ~'freq-map) ~hi-bound#)) 53 | (is (< ~lo-bound# (:bar ~'freq-map) ~hi-bound#)) 54 | (is (< ~lo-bound# (:baz ~'freq-map) ~hi-bound#)) 55 | (is (< ~lo-bound# (:qux ~'freq-map) ~hi-bound#))))) 56 | 57 | (deftest choose-test 58 | (testing "choose function: equal weights" 59 | (test-equal-choose {}) 60 | (test-equal-choose {:unrelated-key-1 0.2 :unrelated-key 0.8}) 61 | (test-equal-choose {:foo 0.5 :baz 0.5}) 62 | (test-equal-choose {:foo 0.0 :bar 0.0 :baz 0.0 :qux 0.0}) 63 | (test-equal-choose {:foo 0.2 :bar 0.2 :baz 0.2 :qux 0.2}) 64 | (test-equal-choose {:foo 0.8 :bar 0.8 :baz 0.8 :qux 0.8}) 65 | (test-equal-choose {:foo 1.0 :bar 1.0 :baz 1.0 :qux 1.0})) 66 | (testing "choose function: only one non-zero weight" 67 | (let [weights {:foo 0 :bar 0 :baz 1 :qux 0} 68 | the-rng (r/rng) 69 | choose-fn (fn [] (r/choose the-rng weights choose-coll)) 70 | results (repeatedly choose-total choose-fn) 71 | freq-map (frequencies results)] 72 | (is (= choose-total (:baz freq-map))) 73 | (is (nil? (:foo freq-map))) 74 | (is (nil? (:bar freq-map))) 75 | (is (nil? (:qux freq-map))))) 76 | ;; The probabilities are derived from integrating over the area defined 77 | ;; by the joint uniform distributions where (< x2 (max x1 t2)), where 78 | ;; x1 and x2 are the independent (but not identical!) random variables 79 | ;; and t1 and t2 are the upper bounds of the uniform distributions. 80 | ;; In Clojure pseudocode: 81 | ;; (-> (/ 1 (* t1 t2)) 82 | ;; (integrate 0 (max x1 t2) dx2) 83 | ;; (integrate 0 t1 dx1)) 84 | ;; = (- 1 (/ t2 (* 2 t1)) 85 | (testing "choose function: two different non-zero weights" 86 | (let [weights {:foo 0 :bar 0 :baz 1 :qux 0.4} 87 | prob-1 0.8 88 | prob-2 0.2 89 | mean-1 (* choose-total prob-1) 90 | mean-2 (* choose-total prob-2) 91 | sd (math/sqrt (* choose-total prob-1 prob-2)) 92 | lo-1 (- mean-1 (* sd-limit sd)) 93 | lo-2 (- mean-2 (* sd-limit sd)) 94 | hi-1 (+ mean-1 (* sd-limit sd)) 95 | hi-2 (+ mean-2 (* sd-limit sd)) 96 | the-rng (r/rng) 97 | choose-fn (fn [] (r/choose the-rng weights choose-coll)) 98 | results (repeatedly choose-total choose-fn) 99 | freq-map (frequencies results)] 100 | (is (< lo-1 (:baz freq-map) hi-1)) 101 | (is (< lo-2 (:qux freq-map) hi-2)) 102 | (is (nil? (:foo freq-map))) 103 | (is (nil? (:bar freq-map))))) 104 | (testing "choose-map function works just like choose" 105 | (let [weights {:foo 0.2 :bar 0.4 :baz 0.6 :qux 0.8} 106 | seed (r/rand-unbound-int (r/rng)) 107 | rng-1 (r/seed-rng seed) 108 | rng-2 (r/seed-rng seed) 109 | choose-fn (fn [] (r/choose rng-1 weights choose-coll)) 110 | choose-map-fn (fn [] (r/choose-map rng-2 weights choose-map)) 111 | results-1 (repeatedly choose-total choose-fn) 112 | results-2 (repeatedly choose-total choose-map-fn) 113 | freq-map-1 (frequencies results-1) 114 | freq-map-2 (frequencies results-2)] 115 | (is (= (:foo freq-map-1) 116 | (:FOO freq-map-2))) 117 | (is (= (:bar freq-map-1) 118 | (:BAR freq-map-2))) 119 | (is (= (:baz freq-map-1) 120 | (:BAZ freq-map-2))) 121 | (is (= (:qux freq-map-1) 122 | (:QUX freq-map-2)))))) 123 | -------------------------------------------------------------------------------- /src/test/com/yetanalytics/server_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.yetanalytics.server-test 2 | "Integration tests for the DATASIM server." 3 | (:require [clojure.test :refer [deftest testing is]] 4 | [clojure.string :as cstr] 5 | [io.pedestal.http :as http] 6 | [org.httpkit.client :as httpkit] 7 | [clj-test-containers.core :as tc] 8 | [com.yetanalytics.datasim.test-constants :as const] 9 | [com.yetanalytics.datasim.test-containers :as ds-tc] 10 | [com.yetanalytics.datasim.server :as server])) 11 | 12 | (def server 13 | (server/create-server)) 14 | 15 | (def profile-string 16 | (format "[%s]" (slurp const/cmi5-profile-filepath))) 17 | 18 | (def personaes-string 19 | (format "[%s]" (slurp const/simple-personae-filepath))) 20 | 21 | (def models-string 22 | (slurp const/simple-models-filepath)) 23 | 24 | (def parameters-string 25 | (slurp const/simple-parameters-filepath)) 26 | 27 | (def post-header 28 | {"X-Experience-Api-Version" "1.0.3" 29 | "Content-Type" "multipart/form-data"}) 30 | 31 | (def json-post-header 32 | {"X-Experience-Api-Version" "1.0.3" 33 | "Content-Type" "application/json"}) 34 | 35 | (def multipart-content 36 | [{:name "profiles" 37 | :content profile-string} 38 | {:name "personae-array" 39 | :content personaes-string} 40 | {:name "models" 41 | :content models-string} 42 | {:name "parameters" 43 | :content parameters-string}]) 44 | 45 | (defn- multipart-post-content [lrs-endpoint api-key api-secret] 46 | (into multipart-content 47 | [{:name "lrs-endpoint" 48 | :content lrs-endpoint} 49 | {:name "api-key" 50 | :content api-key} 51 | {:name "api-secret-key" 52 | :content api-secret} 53 | {:name "send-to-lrs" 54 | :content "true"}])) 55 | 56 | (deftest server-test 57 | (testing "server" 58 | (let [_ (http/start server) 59 | cont (tc/start! ds-tc/validate-server-container) 60 | host (:host cont) 61 | post (get (:mapped-ports cont) 8080) 62 | lrs-url (format "http://%s:%d/statements" host post)] 63 | (testing "GET /health endpoint" 64 | (let [{:keys [status]} 65 | #_{:clj-kondo/ignore [:unresolved-var]} 66 | @(httpkit/get 67 | "http://0.0.0.0:9090/health")] 68 | (is (= 200 status)))) 69 | (testing "POST /api/v1/generate endpoint" 70 | (let [{:keys [status body]} 71 | #_{:clj-kondo/ignore [:unresolved-var]} 72 | @(httpkit/post 73 | "http://0.0.0.0:9090/api/v1/generate" 74 | {:headers post-header 75 | :basic-auth ["username" "password"] 76 | :multipart multipart-content})] 77 | (is (= 200 status)) 78 | (is (every? 79 | (fn [stmt-str] 80 | (let [validate-res 81 | #_{:clj-kondo/ignore [:unresolved-var]} 82 | @(httpkit/post 83 | lrs-url 84 | {:headers json-post-header 85 | :body stmt-str 86 | :as :stream})] 87 | (= 204 (:status validate-res)))) 88 | (take 25 (rest (cstr/split-lines body))))))) 89 | (testing "POST /api/v1/generate endpoint w/ LRS" 90 | (let [endpoint (format "http://%s:%d" host post) 91 | {:keys [status]} 92 | #_{:clj-kondo/ignore [:unresolved-var]} 93 | @(httpkit/post 94 | "http://0.0.0.0:9090/api/v1/generate" 95 | {:headers post-header 96 | :basic-auth ["username" "password"] 97 | :multipart (multipart-post-content endpoint "foo" "bar")})] 98 | (is (= 200 status)))) 99 | (tc/stop! cont) 100 | (http/stop server)))) 101 | --------------------------------------------------------------------------------