├── .dir-locals.el
├── .github
└── workflows
│ ├── build.yml
│ ├── nvd.yml
│ └── test.yml
├── .gitignore
├── .nvd
├── config.json
└── suppression.xml
├── CHANGELOG.md
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── deps.edn
├── dev-resources
├── bench
│ ├── actors.json
│ ├── calibration.jsonld
│ ├── models.json
│ └── params.json
├── input
│ ├── gift128.json
│ ├── gift64.json
│ ├── mom128.json
│ ├── mom64.json
│ └── simple.json
├── models
│ ├── empty.json
│ ├── simple.json
│ ├── simple_with_overrides.json
│ ├── simple_with_repeat_max.json
│ ├── simple_with_temporal.json
│ ├── simple_with_weights.json
│ ├── tccc_dev.json
│ └── temporal
│ │ ├── 10a_restart_current_pattern.json
│ │ ├── 10b_restart_parent_pattern.json
│ │ ├── 10c_restart_all_pattern.json
│ │ ├── 10d_restart_child_pattern.json
│ │ ├── 10e_restart_child_template.json
│ │ ├── 11a_end_current_pattern.json
│ │ ├── 11b_end_parent_pattern.json
│ │ ├── 11c_end_all_pattern.json
│ │ ├── 11d_end_child_pattern.json
│ │ ├── 11e_end_child_template.json
│ │ ├── 1a_year_2023.json
│ │ ├── 1b_year_2023-2024.json
│ │ ├── 1c_year_0-2023.json
│ │ ├── 1d_year_0-2022.json
│ │ ├── 2a_dow_Tues-Thurs_hour_8-12.json
│ │ ├── 2b_dow_Tues-Thurs_hour_8-12-18.json
│ │ ├── 3a_dom_28.json
│ │ ├── 3b_dom_29.json
│ │ ├── 3c_dom_30.json
│ │ ├── 3d_dom_31.json
│ │ ├── 4a_Dec_25.json
│ │ ├── 4b_Dec_25_Mon.json
│ │ ├── 5a_Jan_1_sec_10-30.json
│ │ ├── 5b_Jan_1_min_even_sec_10-30.json
│ │ ├── 5c_Jan_1_hour_even_sec_10-30.json
│ │ ├── 5d_Jan_1_min_hour_even_sec_10-30.json
│ │ ├── 6a_hours_period_every_second_hour.json
│ │ ├── 6b_hours_period_every_start_hour.json
│ │ ├── 6c_hours_fixed_period_every_second_hour.json
│ │ ├── 6d_hours_fixed_period_every_start_hour.json
│ │ ├── 7a_millis_period.json
│ │ ├── 7b_seconds_period.json
│ │ ├── 7c_minutes_period.json
│ │ ├── 7d_hours_period.json
│ │ ├── 7e_days_period.json
│ │ ├── 7f_weeks_period.json
│ │ ├── 8a_millis_fixed_period.json
│ │ ├── 8b_seconds_fixed_period.json
│ │ ├── 8c_minutes_fixed_period.json
│ │ ├── 8d_hours_fixed_period.json
│ │ ├── 8e_days_fixed_period.json
│ │ ├── 8f_weeks_fixed_period.json
│ │ ├── 9a_outer_bound_larger.json
│ │ └── 9b_outer_bound_smaller.json
├── parameters
│ ├── ds_102.json
│ ├── simple.json
│ ├── tccc_dev.json
│ └── temporal.json
├── personae
│ ├── array
│ │ └── simple_array.json
│ ├── bench.json
│ ├── ds_102_128.json
│ ├── ds_102_64.json
│ ├── simple.json
│ ├── tccc_dev.json
│ └── temporal.json
├── profiles
│ ├── acrossx
│ │ ├── fixed.jsonld
│ │ └── profile.jsonld
│ ├── activity.jsonld
│ ├── activity_streams
│ │ ├── fixed.jsonld
│ │ └── profile.jsonld
│ ├── calibration.jsonld
│ ├── cmi5
│ │ └── fixed.json
│ ├── gift
│ │ └── gift.jsonld
│ ├── minimal.jsonld
│ ├── no_concept.jsonld
│ ├── referential.jsonld
│ ├── tccc
│ │ └── cuf_hc_video_and_asm_student_survey_profile.jsonld
│ ├── temporal.jsonld
│ ├── tincan
│ │ └── profile.jsonld
│ ├── tla
│ │ └── mom.jsonld
│ └── video
│ │ └── profile.jsonld
└── xapi
│ └── statements
│ ├── array
│ └── tccc_dev.json
│ ├── long.json
│ └── simple.json
├── doc
├── intro.md
└── temporal_structure.md
├── onyx-resources
└── onyx_config.edn
├── scripts
├── gc.sh
├── gc_checkpoints.sh
├── onyx.sh
├── peer.sh
├── peer_driver.sh
├── repl.sh
├── run.sh
├── server.sh
└── submit_job.sh
├── src
├── cli
│ └── com
│ │ └── yetanalytics
│ │ └── datasim
│ │ ├── cli.clj
│ │ └── cli
│ │ ├── generate.clj
│ │ ├── input.clj
│ │ └── util.clj
├── dev
│ ├── bench.clj
│ └── user.clj
├── dev_onyx
│ └── com
│ │ └── yetanalytics
│ │ └── datasim
│ │ └── onyx
│ │ ├── dev_peer.clj
│ │ └── scratch.clj
├── main
│ └── com
│ │ └── yetanalytics
│ │ ├── datasim.clj
│ │ └── datasim
│ │ ├── client.clj
│ │ ├── input.clj
│ │ ├── input
│ │ ├── model.clj
│ │ ├── model
│ │ │ ├── alignments.clj
│ │ │ └── personae.clj
│ │ ├── parameters.clj
│ │ ├── personae.clj
│ │ └── profile.clj
│ │ ├── model.clj
│ │ ├── model
│ │ ├── bounds.clj
│ │ └── periods.clj
│ │ ├── sim.clj
│ │ ├── util
│ │ ├── async.clj
│ │ ├── errors.clj
│ │ ├── io.clj
│ │ ├── random.clj
│ │ └── sequence.clj
│ │ └── xapi
│ │ ├── actor.clj
│ │ ├── path.clj
│ │ ├── profile.clj
│ │ ├── profile
│ │ ├── activity.clj
│ │ ├── extension.clj
│ │ ├── pattern.clj
│ │ ├── template.clj
│ │ └── verb.clj
│ │ ├── rule.clj
│ │ ├── statement.clj
│ │ └── statement
│ │ └── healing.clj
├── onyx
│ └── com
│ │ └── yetanalytics
│ │ └── datasim
│ │ └── onyx
│ │ ├── aeron_media_driver.clj
│ │ ├── config.clj
│ │ ├── http.clj
│ │ ├── job.clj
│ │ ├── main.clj
│ │ ├── peer.clj
│ │ ├── repl.clj
│ │ ├── scheduling
│ │ └── semi_colocated_task_scheduler.clj
│ │ ├── sim.clj
│ │ └── util.clj
├── server
│ └── com
│ │ └── yetanalytics
│ │ └── datasim
│ │ └── server.clj
├── test
│ └── com
│ │ └── yetanalytics
│ │ ├── cli_test.clj
│ │ ├── datasim
│ │ ├── input
│ │ │ ├── model_test.clj
│ │ │ ├── personae_test.clj
│ │ │ └── profile_test.clj
│ │ ├── input_test.clj
│ │ ├── model
│ │ │ ├── bounds_test.clj
│ │ │ └── periods_test.clj
│ │ ├── test_constants.clj
│ │ ├── test_containers.clj
│ │ ├── util
│ │ │ ├── io_test.clj
│ │ │ └── random_test.clj
│ │ └── xapi
│ │ │ ├── path_test.clj
│ │ │ ├── profile_test.clj
│ │ │ ├── rule_test.clj
│ │ │ └── statement_test.clj
│ │ ├── datasim_test.clj
│ │ ├── datasim_test_temporal.clj
│ │ └── server_test.clj
└── test_onyx
│ └── com
│ └── yetanalytics
│ └── datasim
│ └── onyx
│ ├── job_test.clj
│ └── scheduling
│ └── semi_colocated_task_scheduler_test.clj
└── template
├── 0_vpc.yml
├── 1_hose.yml
├── 1_zk.yml
└── 2_cluster.yml
/.dir-locals.el:
--------------------------------------------------------------------------------
1 | ;;; Directory Local Variables
2 | ;;; For more information see (info "(emacs) Directory Variables")
3 |
4 | ((nil .
5 | ((cider-clojure-cli-global-options . "-A:cli:dev:test:onyx:onyx-dev"))))
6 |
--------------------------------------------------------------------------------
/.github/workflows/build.yml:
--------------------------------------------------------------------------------
1 | name: CD
2 |
3 | on:
4 | push:
5 | branches:
6 | - '*'
7 | tags:
8 | - 'v*.*.*' # Enforce Semantic Versioning
9 |
10 | jobs:
11 | build:
12 | runs-on: ubuntu-latest
13 |
14 | steps:
15 | - name: Checkout Repository
16 | uses: actions/checkout@v4
17 |
18 | - name: Setup CI environment
19 | uses: yetanalytics/actions/setup-env@v0.0.4
20 |
21 | - name: Build Bundle
22 | run: make bundle
23 |
24 | # Build docker for PR, tags
25 | - name: Extract metadata (tags, labels) for Docker
26 | id: meta
27 | uses: docker/metadata-action@v4
28 | with:
29 | images: yetanalytics/datasim
30 | tags: |
31 | type=ref,event=branch
32 | type=ref,event=pr
33 | type=semver,pattern={{version}}
34 | type=semver,pattern={{major}}.{{minor}}
35 |
36 | - name: Log in to Docker Hub
37 | uses: docker/login-action@v2
38 | with:
39 | username: ${{ secrets.DOCKERHUB_USERNAME }}
40 | password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
41 |
42 | - name: Build and push Docker image
43 | uses: docker/build-push-action@v3
44 | with:
45 | context: .
46 | push: true
47 | tags: ${{ steps.meta.outputs.tags }}
48 | labels: ${{ steps.meta.outputs.labels }}
49 |
50 | - name: Compress Bundle
51 | run: | # Need to cd so that the zip file doesn't contain the parent dirs
52 | cd target/bundle
53 | zip -r ../../datasim-bundle.zip ./
54 |
55 | - name: Archive Bundle
56 | uses: actions/upload-artifact@v4
57 | with:
58 | name: datasim-bundle-artifact-${{ github.sha }}
59 | path: datasim-bundle.zip
60 |
61 | deploy:
62 | runs-on: ubuntu-latest
63 |
64 | if: ${{ github.ref_type == 'tag' }}
65 |
66 | steps:
67 | - name: Checkout repository
68 | uses: actions/checkout@v4
69 |
70 | - name: Setup CD environment
71 | uses: yetanalytics/action-setup-env@v1
72 |
73 | - name: Extract version
74 | id: version
75 | run: echo version=${GITHUB_REF#refs\/tags\/v} >> $GITHUB_OUTPUT
76 |
77 | - name: Build and deploy to Clojars
78 | uses: yetanalytics/action-deploy-clojars@v1
79 | with:
80 | artifact-id: 'datasim'
81 | version: ${{ steps.version.outputs.version }}
82 | resource-dirs: '[]'
83 | clojars-username: ${{ secrets.CLOJARS_USERNAME }}
84 | clojars-deploy-token: ${{ secrets.CLOJARS_PASSWORD }}
85 |
86 | release:
87 | runs-on: ubuntu-latest
88 | needs: build
89 | if: ${{ github.ref_type == 'tag' }}
90 | steps:
91 | - name: Checkout repository
92 | uses: actions/checkout@v4
93 |
94 | - name: Download Bundle Artifact
95 | uses: actions/download-artifact@v4
96 | with:
97 | name: datasim-bundle-artifact-${{ github.sha }}
98 |
99 | - name: Craft Draft Release
100 | uses: softprops/action-gh-release@v2
101 | with:
102 | # Defaults:
103 | # name: [tag name]
104 | # tag_name: github.ref
105 | draft: true
106 | files: datasim-bundle.zip
107 |
--------------------------------------------------------------------------------
/.github/workflows/nvd.yml:
--------------------------------------------------------------------------------
1 | name: Periodic NVD Scan
2 |
3 | on:
4 | schedule:
5 | - cron: '0 8 * * 1-5' # Every weekday at 8:00 AM
6 |
7 | jobs:
8 | nvd-scan:
9 | uses: yetanalytics/workflow-nvd/.github/workflows/nvd-scan.yml@v1
10 | with:
11 | nvd-clojure-version: '3.2.0'
12 | classpath-command: 'clojure -Spath -A:cli:server'
13 | nvd-config-filename: '.nvd/config.json'
14 | notify-slack: true
15 | secrets:
16 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
17 |
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on: push
4 |
5 | jobs:
6 | nvd-scan:
7 | uses: yetanalytics/workflow-nvd/.github/workflows/nvd-scan.yml@v1
8 | with:
9 | nvd-clojure-version: '3.2.0'
10 | # onyx dep is outdated and abandoned so don't bother scanning
11 | classpath-command: 'clojure -Spath -A:cli:server'
12 | nvd-config-filename: '.nvd/config.json'
13 |
14 | test:
15 | runs-on: ubuntu-latest
16 |
17 | strategy:
18 | matrix:
19 | target: [test-unit, test-unit-onyx, test-cli]
20 |
21 | steps:
22 | - name: Checkout Repository
23 | uses: actions/checkout@v3
24 |
25 | - name: Setup CI environment
26 | uses: yetanalytics/actions/setup-env@v0.0.4
27 |
28 | - name: Log into DockerHub
29 | if: ${{ matrix.target == 'test-unit' }}
30 | uses: docker/login-action@v3
31 | with:
32 | username: ${{ secrets.DOCKERHUB_USERNAME }}
33 | password: ${{ secrets.DOCKERHUB_TOKEN }}
34 |
35 | - name: Run Makefile Target ${{ matrix.target }}
36 | run: make ${{ matrix.target }}
37 |
38 | validate-template:
39 | runs-on: ubuntu-latest
40 |
41 | # These permissions are needed by configure-aws-credentials in order
42 | # to interact with GitHub's OIDC Token endpoint.
43 | permissions:
44 | id-token: write # required to use OIDC authentication
45 | contents: read # required to checkout the code from the repo
46 |
47 | steps:
48 | - name: Checkout Repository
49 | uses: actions/checkout@v3
50 |
51 | - name: Setup CI environment
52 | uses: yetanalytics/actions/setup-env@v0.0.4
53 |
54 | - name: Configure AWS Credentials
55 | uses: aws-actions/configure-aws-credentials@v1-node16
56 | with:
57 | role-to-assume: ${{ secrets.CF_VALIDATE_TEMPLATE_ROLE_ARN }}
58 | role-duration-seconds: 900 # 15 min; minimal duration possible
59 | aws-region: us-east-1
60 |
61 | - name: Run Makefile Target validate-template
62 | run: make validate-template
63 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | /target
3 | /classes
4 | /checkouts
5 | pom.xml
6 | pom.xml.asc
7 | *.jar
8 | *.class
9 | /.cpcache
10 | /.lein-*
11 | /.nrepl-history
12 | /.nrepl-port
13 | .hgignore
14 | .hg/
15 | *.md.tmp
16 | *.log
17 | .rebel_readline_history
18 |
19 | # VSCode
20 | .calva/
21 | .clj-kondo/
22 | .lsp/
23 | .vscode/
24 |
25 |
--------------------------------------------------------------------------------
/.nvd/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "nvd": {"suppression-file": ".nvd/suppression.xml"}
3 | }
4 |
--------------------------------------------------------------------------------
/.nvd/suppression.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
7 | ^pkg:maven\/(?!org\.clojure\/clojure).*$
8 | cpe:/a:clojure:clojure
9 | CVE-2017-20189
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM alpine:3.16
2 | MAINTAINER Milt Reder
3 |
4 | ADD target/bundle /bundle
5 |
6 | RUN apk update \
7 | && apk upgrade \
8 | && apk add ca-certificates \
9 | && update-ca-certificates \
10 | && apk add --no-cache openjdk11-jre \
11 | && rm -rf /var/cache/apk/*
12 |
13 | WORKDIR /bundle
14 | EXPOSE 9090
15 | ENTRYPOINT ["bin/run.sh"]
16 | CMD ["-h"]
17 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: clean bundle test-cli test-cli-comprehensive test-cli-output test-unit test-unit-onyx ci server test-bundle-output validate-template
2 |
3 | GROUP_ID ?= com.yetanalytics
4 | ARTIFACT_ID ?= datasim
5 | VERSION ?= 0.4.4
6 |
7 | clean:
8 | rm -rf target
9 |
10 | target/bundle/datasim_cli.jar:
11 | mkdir -p target/bundle
12 | rm -f pom.xml
13 | clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:cli]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-cli :version '"$(VERSION)"' :jar target/bundle/datasim_cli.jar :main-class com.yetanalytics.datasim.cli
14 | rm -f pom.xml
15 |
16 | target/bundle/datasim_server.jar: # no AOT for this one
17 | mkdir -p target/bundle
18 | rm -f pom.xml
19 | clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:server]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-server :version '"$(VERSION)"' :jar target/bundle/datasim_server.jar :main-class com.yetanalytics.datasim.server
20 | rm -f pom.xml
21 |
22 | target/bundle/datasim_onyx.jar:
23 | mkdir -p target/bundle
24 | rm -f pom.xml
25 | TIMBRE_LOG_LEVEL=:info clojure -X:depstar uberjar :no-pom false :sync-pom true :aliases '[:onyx,:cli]' :aot true :group-id $(GROUP_ID) :artifact-id $(ARTIFACT_ID)-onyx :version '"$(VERSION)"' :jar target/bundle/datasim_onyx.jar :main-class com.yetanalytics.datasim.onyx.main
26 | rm -f pom.xml
27 |
28 | target/bundle/bin:
29 | mkdir -p target/bundle/bin
30 | cp -r scripts/*.sh target/bundle/bin
31 | chmod +x target/bundle/bin
32 |
33 | target/bundle: target/bundle/bin target/bundle/datasim_cli.jar target/bundle/datasim_server.jar target/bundle/datasim_onyx.jar
34 |
35 | bundle: target/bundle
36 |
37 | # Tests
38 |
39 | test-unit:
40 | clojure -Adev:cli:server:test:run-test
41 |
42 | test-unit-onyx:
43 | clojure -Adev:cli:onyx:onyx-test:run-onyx-test
44 |
45 | test-cli:
46 | clojure -A:cli:run-cli validate-input -p dev-resources/profiles/cmi5/fixed.json -a dev-resources/personae/simple.json -m dev-resources/models/simple.json -o dev-resources/parameters/simple.json -v dev-resources/input/simple.json
47 |
48 | test-cli-comprehensive:
49 | clojure -A:cli:run-cli validate-input -i dev-resources/input/simple.json -v dev-resources/input/simple.json
50 |
51 | test-cli-output:
52 | clojure -A:cli:run-cli generate -i dev-resources/input/simple.json
53 |
54 | test-bundle-output: bundle
55 | cd target/bundle; bin/run.sh generate -i ../../dev-resources/input/simple.json
56 |
57 | validate-template:
58 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/0_vpc.yml
59 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/1_hose.yml
60 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/1_zk.yml
61 | AWS_PAGER="" aws cloudformation validate-template --template-body file://template/2_cluster.yml
62 |
63 | ci: test-unit test-unit-onyx test-cli validate-template
64 |
65 | server:
66 | clojure -A:server:run-server
67 |
--------------------------------------------------------------------------------
/deps.edn:
--------------------------------------------------------------------------------
1 | {:paths ["resources" "src/main"]
2 | :deps {org.clojure/clojure {:mvn/version "1.11.2"}
3 | org.clojure/core.async {:mvn/version "1.6.673"}
4 | org.clojure/core.memoize {:mvn/version "1.0.257"}
5 | com.yetanalytics/xapi-schema
6 | {:mvn/version "1.2.0"
7 | :exclusions [org.clojure/clojurescript]}
8 | com.yetanalytics/project-pan
9 | {:mvn/version "0.4.3"
10 | :exclusions [org.clojure/clojurescript
11 | com.yetanalytics./xapi-schema]}
12 | com.yetanalytics/pathetic {:mvn/version "0.5.0"}
13 | com.yetanalytics/schemer {:mvn/version "0.1.0"}
14 | clojure.java-time/clojure.java-time {:mvn/version "1.2.0"}
15 | org.clojure/test.check {:mvn/version "1.1.1"}
16 | http-kit/http-kit {:mvn/version "2.7.0"}
17 | cheshire/cheshire {:mvn/version "5.12.0"}}
18 | :aliases
19 | {:dev {:extra-paths ["dev-resources" "src/dev"]
20 | :extra-deps {incanter/incanter-core {:mvn/version "1.9.3"}
21 | incanter/incanter-charts {:mvn/version "1.9.3"}
22 | criterium/criterium {:mvn/version "0.4.5"}}}
23 | :test {:extra-paths ["src/test" "src/test_onyx"]
24 | :extra-deps {same/ish {:mvn/version "0.1.6"}
25 | clj-test-containers/clj-test-containers {:mvn/version "0.7.4"}}}
26 | :run-test {:extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1"
27 | :git/sha "dfb30dd"}}
28 | :main-opts ["-m" "cognitect.test-runner"
29 | "-d" "src/test"]}
30 | :cli {:extra-paths ["src/cli"]
31 | :extra-deps {org.clojure/tools.cli {:mvn/version "1.0.219"}}}
32 | :run-cli {:main-opts ["-m" "com.yetanalytics.datasim.cli"]}
33 | :server {:extra-paths ["src/server"]
34 | :extra-deps
35 | {io.pedestal/pedestal.jetty {:mvn/version "0.6.3"
36 | :exclusions [org.eclipse.jetty.http2/http2-server]}
37 | io.pedestal/pedestal.service {:mvn/version "0.6.3"}
38 | org.eclipse.jetty.http2/http2-server {:mvn/version "9.4.54.v20240208"}
39 | org.slf4j/slf4j-simple {:mvn/version "1.7.28"}
40 | clj-http/clj-http {:mvn/version "3.12.3"}
41 | environ/environ {:mvn/version "1.1.0"}
42 | ;; Buddy/BouncyCastle deps
43 | buddy/buddy-auth {:mvn/version "3.0.323"
44 | :exclusions [cheshire/cheshire
45 | buddy/buddy-sign]}
46 | buddy/buddy-sign {:mvn/version "3.5.346"
47 | :exclusions [org.bouncycastle/bcprov-jdk18on]}
48 | org.bouncycastle/bcprov-jdk18on {:mvn/version "1.78"}}}
49 | :run-server {:main-opts ["-m" "com.yetanalytics.datasim.server"]}
50 | :onyx {:extra-paths ["onyx-resources" "src/onyx"]
51 | :extra-deps {com.amazonaws/aws-java-sdk-s3 {:mvn/version "1.11.899"}
52 | com.amazonaws/aws-java-sdk-core {:mvn/version "1.11.899"}
53 | org.onyxplatform/onyx {:mvn/version "0.14.6"
54 | :exclusions
55 | ;; TODO: More exclusions probably
56 | [org.clojure/clojure
57 | org.clojure/core.async
58 | org.slf4j/slf4j-nop
59 | com.amazonaws/aws-java-sdk-s3]}
60 | aleph/aleph {:mvn/version "0.4.7-alpha7"}
61 | com.fzakaria/slf4j-timbre {:mvn/version "0.3.20"}
62 | org.onyxplatform/lib-onyx {:mvn/version "0.14.1.0"}
63 | org.onyxplatform/onyx-http {:mvn/version "0.14.5.0"
64 | :exclusions [aleph/aleph
65 | io.netty/netty-all]}
66 | ;; for local repl
67 | com.bhauman/rebel-readline {:mvn/version "0.1.4"}
68 | ;; for remote repl
69 | nrepl/nrepl {:mvn/version "0.8.3"}
70 | cider/cider-nrepl {:mvn/version "0.25.6"}
71 | org.onyxplatform/onyx-peer-http-query {:mvn/version "0.14.5.1-SNAPSHOT"}
72 | org.onyxplatform/onyx-amazon-s3 {:mvn/version "0.14.5.0"
73 | :exclusions [org.clojure/clojure
74 | org.onyxplatform/onyx
75 | com.amazonaws/aws-java-sdk-s3]}}}
76 | :onyx-dev {:extra-paths ["src/onyx_dev"]}
77 | :onyx-test {:extra-paths ["src/test_onyx"]}
78 | :run-onyx-test {:extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1"
79 | :git/sha "dfb30dd"}}
80 | :main-opts ["-m" "cognitect.test-runner"
81 | "-d" "src/test_onyx"]}
82 | :depstar {:replace-deps ; tool usage is new in 2.x
83 | {seancorfield/depstar {:mvn/version "2.0.165"}}
84 | :ns-default hf.depstar
85 | :exec-args {:no-pom true}}}}
86 |
--------------------------------------------------------------------------------
/dev-resources/bench/actors.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "trainees",
3 | "objectType": "Group",
4 | "member": [
5 | {
6 | "name": "Dionne Knox",
7 | "mbox": "mailto:0@yetanalytics.com"
8 | },
9 | {
10 | "name": "Penny Mayer",
11 | "mbox": "mailto:1@yetanalytics.com"
12 | },
13 | {
14 | "name": "Stanley Ayers",
15 | "mbox": "mailto:2@yetanalytics.com"
16 | },
17 | {
18 | "name": "Reid Joyner",
19 | "mbox": "mailto:3@yetanalytics.com"
20 | },
21 | {
22 | "name": "Loretta Elliott",
23 | "mbox": "mailto:4@yetanalytics.com"
24 | },
25 | {
26 | "name": "Hawkins Landry",
27 | "mbox": "mailto:5@yetanalytics.com"
28 | },
29 | {
30 | "name": "Zimmerman Short",
31 | "mbox": "mailto:6@yetanalytics.com"
32 | },
33 | {
34 | "name": "Jeannette Castaneda",
35 | "mbox": "mailto:7@yetanalytics.com"
36 | },
37 | {
38 | "name": "Ramos Boyer",
39 | "mbox": "mailto:8@yetanalytics.com"
40 | },
41 | {
42 | "name": "Montgomery Dale",
43 | "mbox": "mailto:9@yetanalytics.com"
44 | },
45 | {
46 | "name": "Jerry Rose",
47 | "mbox": "mailto:10@yetanalytics.com"
48 | },
49 | {
50 | "name": "Katina Armstrong",
51 | "mbox": "mailto:11@yetanalytics.com"
52 | },
53 | {
54 | "name": "Dickson Gaines",
55 | "mbox": "mailto:12@yetanalytics.com"
56 | },
57 | {
58 | "name": "Patricia Forbes",
59 | "mbox": "mailto:13@yetanalytics.com"
60 | },
61 | {
62 | "name": "Petty Huber",
63 | "mbox": "mailto:14@yetanalytics.com"
64 | },
65 | {
66 | "name": "Sexton Cooley",
67 | "mbox": "mailto:15@yetanalytics.com"
68 | },
69 | {
70 | "name": "Nola Burns",
71 | "mbox": "mailto:16@yetanalytics.com"
72 | },
73 | {
74 | "name": "Caitlin Lancaster",
75 | "mbox": "mailto:17@yetanalytics.com"
76 | },
77 | {
78 | "name": "Dean Roth",
79 | "mbox": "mailto:18@yetanalytics.com"
80 | },
81 | {
82 | "name": "Wilson Chambers",
83 | "mbox": "mailto:19@yetanalytics.com"
84 | },
85 | {
86 | "name": "Helga Hutchinson",
87 | "mbox": "mailto:20@yetanalytics.com"
88 | },
89 | {
90 | "name": "Mary Flores",
91 | "mbox": "mailto:21@yetanalytics.com"
92 | },
93 | {
94 | "name": "Baker Orr",
95 | "mbox": "mailto:22@yetanalytics.com"
96 | },
97 | {
98 | "name": "Randall Fernandez",
99 | "mbox": "mailto:23@yetanalytics.com"
100 | },
101 | {
102 | "name": "Margie Pratt",
103 | "mbox": "mailto:24@yetanalytics.com"
104 | },
105 | {
106 | "name": "Christian Baxter",
107 | "mbox": "mailto:25@yetanalytics.com"
108 | },
109 | {
110 | "name": "Guerrero Glass",
111 | "mbox": "mailto:26@yetanalytics.com"
112 | },
113 | {
114 | "name": "Marian Fox",
115 | "mbox": "mailto:27@yetanalytics.com"
116 | },
117 | {
118 | "name": "Ayers Farley",
119 | "mbox": "mailto:28@yetanalytics.com"
120 | },
121 | {
122 | "name": "Milagros Hudson",
123 | "mbox": "mailto:29@yetanalytics.com"
124 | },
125 | {
126 | "name": "Brittany French",
127 | "mbox": "mailto:30@yetanalytics.com"
128 | },
129 | {
130 | "name": "Mueller Ward",
131 | "mbox": "mailto:31@yetanalytics.com"
132 | },
133 | {
134 | "name": "Emily Lucas",
135 | "mbox": "mailto:32@yetanalytics.com"
136 | },
137 | {
138 | "name": "Cohen Shepherd",
139 | "mbox": "mailto:33@yetanalytics.com"
140 | }
141 | ]
142 | }
143 |
--------------------------------------------------------------------------------
/dev-resources/bench/models.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bob@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "activities": [
10 | {
11 | "id": "https://example.org/activity/a",
12 | "weight": 0.5
13 | },
14 | {
15 | "id": "https://example.org/activity/c",
16 | "weight": 0.2
17 | }
18 | ]
19 | },
20 | {
21 | "personae": [
22 | {
23 | "id": "mbox::mailto:alice@example.org",
24 | "type": "Agent"
25 | }
26 | ],
27 | "activities": [
28 | {
29 | "id": "https://example.org/activity/c",
30 | "weight": 0.7
31 | },
32 | {
33 | "id": "https://example.org/activity/d",
34 | "weight": 0.02
35 | }
36 | ]
37 | }
38 | ]
39 |
--------------------------------------------------------------------------------
/dev-resources/bench/params.json:
--------------------------------------------------------------------------------
1 | {"start": "2019-11-18T11:38:39.219768Z",
2 | "end": "2021-01-18T11:38:39.219768Z",
3 | "timezone": "America/New_York",
4 | "seed": 44}
5 |
--------------------------------------------------------------------------------
/dev-resources/models/empty.json:
--------------------------------------------------------------------------------
1 | []
2 |
--------------------------------------------------------------------------------
/dev-resources/models/simple.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bobfake@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "activities": [
10 | {
11 | "id": "https://example.org/activity/a",
12 | "weight": 0.5
13 | },
14 | {
15 | "id": "https://example.org/activity/c",
16 | "weight": 0.2
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/simple_with_overrides.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bobfake@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "activities": [
10 | {
11 | "id": "https://example.org/activity/a",
12 | "weight": 0.5
13 | },
14 | {
15 | "id": "https://example.org/activity/c",
16 | "weight": 0.2
17 | }
18 | ],
19 | "objectOverrides": [
20 | {
21 | "weight": 0.7,
22 | "object": {
23 | "objectType": "Activity",
24 | "id": "https://www.whatever.com/activities#course2",
25 | "definition": {
26 | "name": {
27 | "en-US": "Course 2"
28 | },
29 | "description": {
30 | "en-US": "Course Description 2"
31 | },
32 | "type": "http://adlnet.gov/expapi/activities/course"
33 | }
34 | }
35 |
36 | },
37 | {
38 | "weight": 0.3,
39 | "object": {
40 | "objectType": "Agent",
41 | "name": "Owen Overrider",
42 | "mbox": "mailto:owoverrider@example.com"
43 | }
44 |
45 | }
46 | ]
47 | }
48 | ]
49 |
--------------------------------------------------------------------------------
/dev-resources/models/simple_with_repeat_max.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bobfake@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "patterns": [
10 | {
11 | "id": "https://w3id.org/xapi/cmi5#satisfieds",
12 | "repeatMax": 1
13 | },
14 | {
15 | "id": "https://w3id.org/xapi/cmi5#typicalsessions",
16 | "repeatMax": 1
17 | }
18 | ]
19 | },
20 | {
21 | "personae": [
22 | {
23 | "id": "mbox::mailto:frederstaz@example.org",
24 | "type": "Agent"
25 | }
26 | ],
27 | "patterns": [
28 | {
29 | "id": "https://w3id.org/xapi/cmi5#satisfieds",
30 | "repeatMax": 100
31 | },
32 | {
33 | "id": "https://w3id.org/xapi/cmi5#typicalsessions",
34 | "repeatMax": 100
35 | }
36 | ]
37 | }
38 | ]
39 |
--------------------------------------------------------------------------------
/dev-resources/models/simple_with_temporal.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bobfake@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "patterns": [
10 | {
11 | "id": "https://w3id.org/xapi/cmi5#satisfieds",
12 | "periods": [
13 | {
14 | "min": 1,
15 | "mean": 1.0,
16 | "unit": "hours"
17 | }
18 | ]
19 | }
20 | ]
21 | },
22 | {
23 | "personae": [
24 | {
25 | "id": "mbox::mailto:frederstaz@example.org",
26 | "type": "Agent"
27 | }
28 | ],
29 | "patterns": [
30 | {
31 | "id": "https://w3id.org/xapi/cmi5#typicalsessions",
32 | "bounds": [
33 | {
34 | "months": [["January", "October"], "December"]
35 | }
36 | ]
37 | }
38 | ],
39 | "templates": [
40 | {
41 | "id": "https://w3id.org/xapi/cmi5#launched",
42 | "bounds": [
43 | {
44 | "minutes": [[0, 10]]
45 | }
46 | ],
47 | "boundRestarts": [
48 | "https://w3id.org/xapi/cmi5#typicalsessions"
49 | ]
50 | },
51 | {
52 | "id": "https://w3id.org/xapi/cmi5#initialized",
53 | "bounds": [
54 | {
55 | "minutes": [[10, 59, 5]]
56 | }
57 | ]
58 | }
59 | ]
60 | }
61 | ]
62 |
--------------------------------------------------------------------------------
/dev-resources/models/simple_with_weights.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "personae": [
4 | {
5 | "id": "mbox::mailto:bobfake@example.org",
6 | "type": "Agent"
7 | }
8 | ],
9 | "activityTypes": [
10 | {
11 | "id": "https://w3id.org/xapi/cmi5/activities/block",
12 | "weight": 1.0
13 | },
14 | {
15 | "id": "https://w3id.org/xapi/cmi5/activitytype/block",
16 | "weight": 1.0
17 | },
18 | {
19 | "id": "https://w3id.org/xapi/cmi5/activities/course",
20 | "weight": 0.0
21 | },
22 | {
23 | "id": "https://w3id.org/xapi/cmi5/activitytype/course",
24 | "weight": 0.0
25 | }
26 | ],
27 | "patterns": [
28 | {
29 | "id": "https://w3id.org/xapi/cmi5#satisfieds",
30 | "repeatMax": 0
31 | }
32 | ]
33 | },
34 | {
35 | "personae": [
36 | {
37 | "id": "mbox::mailto:frederstaz@example.org",
38 | "type": "Agent"
39 | }
40 | ],
41 | "activityTypes": [
42 | {
43 | "id": "https://w3id.org/xapi/cmi5/activities/block",
44 | "weight": 0.0
45 | },
46 | {
47 | "id": "https://w3id.org/xapi/cmi5/activitytype/block",
48 | "weight": 0.0
49 | },
50 | {
51 | "id": "https://w3id.org/xapi/cmi5/activities/course",
52 | "weight": 1.0
53 | },
54 | {
55 | "id": "https://w3id.org/xapi/cmi5/activitytype/course",
56 | "weight": 1.0
57 | }
58 | ],
59 | "patterns": [
60 | {
61 | "id": "https://w3id.org/xapi/cmi5#satisfieds",
62 | "repeatMax": 0
63 | }
64 | ]
65 | }
66 | ]
67 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/10a_restart_current_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]]
12 | }
13 | ],
14 | "boundRestarts": [
15 | "http://yetanalytics.com/temporal/pattern-B2"
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/10b_restart_parent_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]]
12 | }
13 | ],
14 | "boundRestarts": [
15 | "http://yetanalytics.com/temporal/pattern-A"
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/10c_restart_all_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]]
12 | }
13 | ],
14 | "boundRestarts": [
15 | "http://yetanalytics.com/temporal/pattern-A",
16 | "http://yetanalytics.com/temporal/pattern-B2",
17 | "http://yetanalytics.com/temporal/pattern-C3",
18 | "http://yetanalytics.com/temporal/template-5"
19 | ]
20 | }
21 | ]
22 | }
23 | ]
24 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/10d_restart_child_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]]
12 | }
13 | ],
14 | "boundRestarts": [
15 | "http://yetanalytics.com/temporal/pattern-C3",
16 | "http://yetanalytics.com/temporal/pattern-C4"
17 | ]
18 | }
19 | ]
20 | }
21 | ]
22 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/10e_restart_child_template.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]]
12 | }
13 | ],
14 | "boundRestarts": [
15 | "http://yetanalytics.com/temporal/template-5",
16 | "http://yetanalytics.com/temporal/template-6",
17 | "http://yetanalytics.com/temporal/template-7",
18 | "http://yetanalytics.com/temporal/template-8"
19 | ]
20 | }
21 | ]
22 | }
23 | ]
24 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/11a_end_current_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ],
17 | "boundRestarts": [
18 | "http://yetanalytics.com/temporal/pattern-B2"
19 | ]
20 | }
21 | ]
22 | }
23 | ]
24 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/11b_end_parent_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ],
17 | "boundRestarts": [
18 | "http://yetanalytics.com/temporal/pattern-A"
19 | ]
20 | }
21 | ]
22 | }
23 | ]
24 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/11c_end_all_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ],
17 | "boundRestarts": [
18 | "http://yetanalytics.com/temporal/pattern-A",
19 | "http://yetanalytics.com/temporal/pattern-B2",
20 | "http://yetanalytics.com/temporal/pattern-C3",
21 | "http://yetanalytics.com/temporal/template-5"
22 | ]
23 | }
24 | ]
25 | }
26 | ]
27 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/11d_end_child_pattern.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ],
17 | "boundRestarts": [
18 | "http://yetanalytics.com/temporal/pattern-C3",
19 | "http://yetanalytics.com/temporal/pattern-C4"
20 | ]
21 | }
22 | ]
23 | }
24 | ]
25 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/11e_end_child_template.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-B2",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ],
17 | "boundRestarts": [
18 | "http://yetanalytics.com/temporal/template-5",
19 | "http://yetanalytics.com/temporal/template-6",
20 | "http://yetanalytics.com/temporal/template-7",
21 | "http://yetanalytics.com/temporal/template-8"
22 | ]
23 | }
24 | ]
25 | }
26 | ]
27 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/1a_year_2023.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "days"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/1b_year_2023-2024.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [[2023, 2024]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "days"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/1c_year_0-2023.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [[0, 2023]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "days"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/1d_year_0-2022.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [[0, 2022]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "days"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/2a_dow_Tues-Thurs_hour_8-12.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfWeek": ["Tuesday", "Thursday"],
9 | "hours": [[8, 12]]
10 | }
11 | ],
12 | "periods": [
13 | {
14 | "mean": 10,
15 | "unit": "minutes"
16 | }
17 | ]
18 | }
19 | ]
20 | }
21 | ]
22 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/2b_dow_Tues-Thurs_hour_8-12-18.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfWeek": ["Tuesday"],
9 | "hours": [[8, 12]]
10 | },
11 | {
12 | "daysOfWeek": ["Thursday"],
13 | "hours": [[13, 18]]
14 | }
15 | ],
16 | "periods": [
17 | {
18 | "mean": 10,
19 | "unit": "minutes"
20 | }
21 | ]
22 | }
23 | ]
24 | }
25 | ]
26 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/3a_dom_28.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfMonth": [28]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 10,
14 | "unit": "minutes"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/3b_dom_29.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfMonth": [29]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 10,
14 | "unit": "minutes"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/3c_dom_30.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfMonth": [30]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 10,
14 | "unit": "minutes"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/3d_dom_31.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "daysOfMonth": [31]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 10,
14 | "unit": "minutes"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/4a_Dec_25.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "months": [12],
9 | "daysOfMonth": [25]
10 | }
11 | ]
12 | }
13 | ]
14 | }
15 | ]
16 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/4b_Dec_25_Mon.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "months": [12],
9 | "daysOfMonth": [25],
10 | "daysOfWeek": [1]
11 | }
12 | ]
13 | }
14 | ]
15 | }
16 | ]
17 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/5a_Jan_1_sec_10-30.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "seconds": [[10, 30]]
12 | }
13 | ]
14 | }
15 | ]
16 | }
17 | ]
18 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/5b_Jan_1_min_even_sec_10-30.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "minutes": [[0, 59, 2]],
12 | "seconds": [[10, 30]]
13 | }
14 | ]
15 | }
16 | ]
17 | }
18 | ]
19 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/5c_Jan_1_hour_even_sec_10-30.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]],
12 | "seconds": [[10, 30]]
13 | }
14 | ]
15 | }
16 | ]
17 | }
18 | ]
19 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/5d_Jan_1_min_hour_even_sec_10-30.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 23, 2]],
12 | "minutes": [[0, 59, 2]],
13 | "seconds": [[10, 30]]
14 | }
15 | ]
16 | }
17 | ]
18 | }
19 | ]
20 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/6a_hours_period_every_second_hour.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [[0, 23, 2]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/6b_hours_period_every_start_hour.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/6c_hours_fixed_period_every_second_hour.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [[0, 23, 2]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "fixed": 1,
14 | "unit": "hours"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/6d_hours_fixed_period_every_start_hour.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [0]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "fixed": 1,
14 | "unit": "hours"
15 | }
16 | ]
17 | }
18 | ]
19 | }
20 | ]
21 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7a_millis_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [0],
12 | "minutes": [0],
13 | "seconds": [[0, 7]]
14 | }
15 | ],
16 | "periods": [
17 | {
18 | "bounds": [
19 | {
20 | "seconds": [0]
21 | }
22 | ],
23 | "min": 0,
24 | "mean": 10,
25 | "unit": "millis"
26 | },
27 | {
28 | "bounds": [
29 | {
30 | "seconds": [1]
31 | }
32 | ],
33 | "min": 0,
34 | "mean": 20,
35 | "unit": "millis"
36 | },
37 | {
38 | "bounds": [
39 | {
40 | "seconds": [2]
41 | }
42 | ],
43 | "min": 10,
44 | "mean": 10,
45 | "unit": "millis"
46 | },
47 | {
48 | "bounds": [
49 | {
50 | "seconds": [3]
51 | }
52 | ],
53 | "min": 10,
54 | "mean": 20,
55 | "unit": "millis"
56 | },
57 | {
58 | "bounds": [
59 | {
60 | "seconds": [4]
61 | }
62 | ],
63 | "min": 20,
64 | "mean": 10,
65 | "unit": "millis"
66 | },
67 | {
68 | "bounds": [
69 | {
70 | "seconds": [5]
71 | }
72 | ],
73 | "min": 20,
74 | "mean": 20,
75 | "unit": "millis"
76 | },
77 | {
78 | "bounds": [
79 | {
80 | "seconds": [6]
81 | }
82 | ],
83 | "min": 30,
84 | "mean": 10,
85 | "unit": "millis"
86 | },
87 | {
88 | "bounds": [
89 | {
90 | "seconds": [7]
91 | }
92 | ],
93 | "min": 30,
94 | "mean": 20,
95 | "unit": "millis"
96 | },
97 | {
98 | "min": 0,
99 | "mean": 10,
100 | "unit": "millis"
101 | }
102 | ]
103 | }
104 | ]
105 | }
106 | ]
107 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7b_seconds_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [0],
12 | "minutes": [[0, 7]]
13 | }
14 | ],
15 | "periods": [
16 | {
17 | "bounds": [
18 | {
19 | "minutes": [0]
20 | }
21 | ],
22 | "min": 0,
23 | "mean": 1,
24 | "unit": "seconds"
25 | },
26 | {
27 | "bounds": [
28 | {
29 | "minutes": [1]
30 | }
31 | ],
32 | "min": 0,
33 | "mean": 2,
34 | "unit": "seconds"
35 | },
36 | {
37 | "bounds": [
38 | {
39 | "minutes": [2]
40 | }
41 | ],
42 | "min": 1,
43 | "mean": 1,
44 | "unit": "seconds"
45 | },
46 | {
47 | "bounds": [
48 | {
49 | "minutes": [3]
50 | }
51 | ],
52 | "min": 1,
53 | "mean": 2,
54 | "unit": "seconds"
55 | },
56 | {
57 | "bounds": [
58 | {
59 | "minutes": [4]
60 | }
61 | ],
62 | "min": 2,
63 | "mean": 1,
64 | "unit": "seconds"
65 | },
66 | {
67 | "bounds": [
68 | {
69 | "minutes": [5]
70 | }
71 | ],
72 | "min": 2,
73 | "mean": 2,
74 | "unit": "seconds"
75 | },
76 | {
77 | "bounds": [
78 | {
79 | "minutes": [6]
80 | }
81 | ],
82 | "min": 3,
83 | "mean": 1,
84 | "unit": "seconds"
85 | },
86 | {
87 | "bounds": [
88 | {
89 | "minutes": [7]
90 | }
91 | ],
92 | "min": 3,
93 | "mean": 2,
94 | "unit": "seconds"
95 | },
96 | {
97 | "min": 0,
98 | "mean": 1,
99 | "unit": "seconds"
100 | }
101 | ]
102 | }
103 | ]
104 | }
105 | ]
106 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7c_minutes_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 7]]
12 | }
13 | ],
14 | "periods": [
15 | {
16 | "bounds": [
17 | {
18 | "hours": [0]
19 | }
20 | ],
21 | "min": 0,
22 | "mean": 1,
23 | "unit": "minutes"
24 | },
25 | {
26 | "bounds": [
27 | {
28 | "hours": [1]
29 | }
30 | ],
31 | "min": 0,
32 | "mean": 2,
33 | "unit": "minutes"
34 | },
35 | {
36 | "bounds": [
37 | {
38 | "hours": [2]
39 | }
40 | ],
41 | "min": 1,
42 | "mean": 1,
43 | "unit": "minutes"
44 | },
45 | {
46 | "bounds": [
47 | {
48 | "hours": [3]
49 | }
50 | ],
51 | "min": 1,
52 | "mean": 2,
53 | "unit": "minutes"
54 | },
55 | {
56 | "bounds": [
57 | {
58 | "hours": [4]
59 | }
60 | ],
61 | "min": 2,
62 | "mean": 1,
63 | "unit": "minutes"
64 | },
65 | {
66 | "bounds": [
67 | {
68 | "hours": [5]
69 | }
70 | ],
71 | "min": 2,
72 | "mean": 2,
73 | "unit": "minutes"
74 | },
75 | {
76 | "bounds": [
77 | {
78 | "hours": [6]
79 | }
80 | ],
81 | "min": 3,
82 | "mean": 1,
83 | "unit": "minutes"
84 | },
85 | {
86 | "bounds": [
87 | {
88 | "hours": [7]
89 | }
90 | ],
91 | "min": 3,
92 | "mean": 2,
93 | "unit": "minutes"
94 | },
95 | {
96 | "min": 0,
97 | "mean": 1,
98 | "unit": "minutes"
99 | }
100 | ]
101 | }
102 | ]
103 | }
104 | ]
105 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7d_hours_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [[1, 8]]
11 | }
12 | ],
13 | "periods": [
14 | {
15 | "bounds": [
16 | {
17 | "daysOfMonth": [1]
18 | }
19 | ],
20 | "min": 0,
21 | "mean": 1,
22 | "unit": "hours"
23 | },
24 | {
25 | "bounds": [
26 | {
27 | "daysOfMonth": [2]
28 | }
29 | ],
30 | "min": 0,
31 | "mean": 2,
32 | "unit": "hours"
33 | },
34 | {
35 | "bounds": [
36 | {
37 | "daysOfMonth": [3]
38 | }
39 | ],
40 | "min": 1,
41 | "mean": 1,
42 | "unit": "hours"
43 | },
44 | {
45 | "bounds": [
46 | {
47 | "daysOfMonth": [4]
48 | }
49 | ],
50 | "min": 1,
51 | "mean": 2,
52 | "unit": "hours"
53 | },
54 | {
55 | "bounds": [
56 | {
57 | "daysOfMonth": [5]
58 | }
59 | ],
60 | "min": 2,
61 | "mean": 1,
62 | "unit": "hours"
63 | },
64 | {
65 | "bounds": [
66 | {
67 | "daysOfMonth": [6]
68 | }
69 | ],
70 | "min": 2,
71 | "mean": 2,
72 | "unit": "hours"
73 | },
74 | {
75 | "bounds": [
76 | {
77 | "daysOfMonth": [7]
78 | }
79 | ],
80 | "min": 3,
81 | "mean": 1,
82 | "unit": "hours"
83 | },
84 | {
85 | "bounds": [
86 | {
87 | "daysOfMonth": [8]
88 | }
89 | ],
90 | "min": 3,
91 | "mean": 2,
92 | "unit": "hours"
93 | },
94 | {
95 | "min": 0,
96 | "mean": 1,
97 | "unit": "hours"
98 | }
99 | ]
100 | }
101 | ]
102 | }
103 | ]
104 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7e_days_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": [[1, 8]]
10 | }
11 | ],
12 | "periods": [
13 | {
14 | "bounds": [
15 | {
16 | "months": [1]
17 | }
18 | ],
19 | "min": 0,
20 | "mean": 1,
21 | "unit": "days"
22 | },
23 | {
24 | "bounds": [
25 | {
26 | "months": [2]
27 | }
28 | ],
29 | "min": 0,
30 | "mean": 2,
31 | "unit": "days"
32 | },
33 | {
34 | "bounds": [
35 | {
36 | "months": [3]
37 | }
38 | ],
39 | "min": 1,
40 | "mean": 1,
41 | "unit": "days"
42 | },
43 | {
44 | "bounds": [
45 | {
46 | "months": [4]
47 | }
48 | ],
49 | "min": 1,
50 | "mean": 2,
51 | "unit": "days"
52 | },
53 | {
54 | "bounds": [
55 | {
56 | "months": [5]
57 | }
58 | ],
59 | "min": 2,
60 | "mean": 1,
61 | "unit": "days"
62 | },
63 | {
64 | "bounds": [
65 | {
66 | "months": [6]
67 | }
68 | ],
69 | "min": 2,
70 | "mean": 2,
71 | "unit": "days"
72 | },
73 | {
74 | "bounds": [
75 | {
76 | "months": [7]
77 | }
78 | ],
79 | "min": 3,
80 | "mean": 1,
81 | "unit": "days"
82 | },
83 | {
84 | "bounds": [
85 | {
86 | "months": [8]
87 | }
88 | ],
89 | "min": 3,
90 | "mean": 2,
91 | "unit": "days"
92 | },
93 | {
94 | "min": 0,
95 | "mean": 1,
96 | "unit": "days"
97 | }
98 | ]
99 | }
100 | ]
101 | }
102 | ]
103 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/7f_weeks_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": [[1, 4]]
10 | }
11 | ],
12 | "periods": [
13 | {
14 | "bounds": [
15 | {
16 | "months": [1]
17 | }
18 | ],
19 | "min": 0,
20 | "mean": 1,
21 | "unit": "weeks"
22 | },
23 | {
24 | "bounds": [
25 | {
26 | "months": [2]
27 | }
28 | ],
29 | "min": 0,
30 | "mean": 2,
31 | "unit": "weeks"
32 | },
33 | {
34 | "bounds": [
35 | {
36 | "months": [3]
37 | }
38 | ],
39 | "min": 1,
40 | "mean": 1,
41 | "unit": "weeks"
42 | },
43 | {
44 | "bounds": [
45 | {
46 | "months": [4]
47 | }
48 | ],
49 | "min": 1,
50 | "mean": 2,
51 | "unit": "weeks"
52 | },
53 | {
54 | "min": 0,
55 | "mean": 1,
56 | "unit": "weeks"
57 | }
58 | ]
59 | }
60 | ]
61 | }
62 | ]
63 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8a_millis_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [0],
12 | "minutes": [0],
13 | "seconds": [[0, 7]]
14 | }
15 | ],
16 | "periods": [
17 | {
18 | "bounds": [
19 | {
20 | "seconds": [0]
21 | }
22 | ],
23 | "fixed": 10,
24 | "unit": "millis"
25 | },
26 | {
27 | "bounds": [
28 | {
29 | "seconds": [1]
30 | }
31 | ],
32 | "fixed": 20,
33 | "unit": "millis"
34 | },
35 | {
36 | "bounds": [
37 | {
38 | "seconds": [2]
39 | }
40 | ],
41 | "fixed": 30,
42 | "unit": "millis"
43 | },
44 | {
45 | "bounds": [
46 | {
47 | "seconds": [3]
48 | }
49 | ],
50 | "fixed": 40,
51 | "unit": "millis"
52 | },
53 | {
54 | "bounds": [
55 | {
56 | "seconds": [4]
57 | }
58 | ],
59 | "fixed": 50,
60 | "unit": "millis"
61 | },
62 | {
63 | "bounds": [
64 | {
65 | "seconds": [5]
66 | }
67 | ],
68 | "fixed": 60,
69 | "unit": "millis"
70 | },
71 | {
72 | "bounds": [
73 | {
74 | "seconds": [6]
75 | }
76 | ],
77 | "fixed": 70,
78 | "unit": "millis"
79 | },
80 | {
81 | "bounds": [
82 | {
83 | "seconds": [7]
84 | }
85 | ],
86 | "fixed": 80,
87 | "unit": "millis"
88 | }
89 | ]
90 | }
91 | ]
92 | }
93 | ]
94 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8b_seconds_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [0],
12 | "minutes": [[0, 7]]
13 | }
14 | ],
15 | "periods": [
16 | {
17 | "bounds": [
18 | {
19 | "minutes": [0]
20 | }
21 | ],
22 | "fixed": 1,
23 | "unit": "seconds"
24 | },
25 | {
26 | "bounds": [
27 | {
28 | "minutes": [1]
29 | }
30 | ],
31 | "fixed": 2,
32 | "unit": "seconds"
33 | },
34 | {
35 | "bounds": [
36 | {
37 | "minutes": [2]
38 | }
39 | ],
40 | "fixed": 3,
41 | "unit": "seconds"
42 | },
43 | {
44 | "bounds": [
45 | {
46 | "minutes": [3]
47 | }
48 | ],
49 | "fixed": 4,
50 | "unit": "seconds"
51 | },
52 | {
53 | "bounds": [
54 | {
55 | "minutes": [4]
56 | }
57 | ],
58 | "fixed": 5,
59 | "unit": "seconds"
60 | },
61 | {
62 | "bounds": [
63 | {
64 | "minutes": [5]
65 | }
66 | ],
67 | "fixed": 6,
68 | "unit": "seconds"
69 | },
70 | {
71 | "bounds": [
72 | {
73 | "minutes": [6]
74 | }
75 | ],
76 | "fixed": 7,
77 | "unit": "seconds"
78 | },
79 | {
80 | "bounds": [
81 | {
82 | "minutes": [7]
83 | }
84 | ],
85 | "fixed": 8,
86 | "unit": "seconds"
87 | },
88 | {
89 | "fixed": 1,
90 | "unit": "seconds"
91 | }
92 | ]
93 | }
94 | ]
95 | }
96 | ]
97 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8c_minutes_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [1],
11 | "hours": [[0, 7]]
12 | }
13 | ],
14 | "periods": [
15 | {
16 | "bounds": [
17 | {
18 | "hours": [0]
19 | }
20 | ],
21 | "fixed": 1,
22 | "unit": "minutes"
23 | },
24 | {
25 | "bounds": [
26 | {
27 | "hours": [1]
28 | }
29 | ],
30 | "fixed": 2,
31 | "unit": "minutes"
32 | },
33 | {
34 | "bounds": [
35 | {
36 | "hours": [2]
37 | }
38 | ],
39 | "fixed": 3,
40 | "unit": "minutes"
41 | },
42 | {
43 | "bounds": [
44 | {
45 | "hours": [3]
46 | }
47 | ],
48 | "fixed": 4,
49 | "unit": "minutes"
50 | },
51 | {
52 | "bounds": [
53 | {
54 | "hours": [4]
55 | }
56 | ],
57 | "fixed": 5,
58 | "unit": "minutes"
59 | },
60 | {
61 | "bounds": [
62 | {
63 | "hours": [5]
64 | }
65 | ],
66 | "fixed": 6,
67 | "unit": "minutes"
68 | },
69 | {
70 | "bounds": [
71 | {
72 | "hours": [6]
73 | }
74 | ],
75 | "fixed": 7,
76 | "unit": "minutes"
77 | },
78 | {
79 | "bounds": [
80 | {
81 | "hours": [7]
82 | }
83 | ],
84 | "fixed": 8,
85 | "unit": "minutes"
86 | },
87 | {
88 | "fixed": 1,
89 | "unit": "minutes"
90 | }
91 | ]
92 | }
93 | ]
94 | }
95 | ]
96 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8d_hours_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": ["January"],
10 | "daysOfMonth": [[1, 8]]
11 | }
12 | ],
13 | "periods": [
14 | {
15 | "bounds": [
16 | {
17 | "daysOfMonth": [1]
18 | }
19 | ],
20 | "fixed": 1,
21 | "unit": "hours"
22 | },
23 | {
24 | "bounds": [
25 | {
26 | "daysOfMonth": [2]
27 | }
28 | ],
29 | "fixed": 2,
30 | "unit": "hours"
31 | },
32 | {
33 | "bounds": [
34 | {
35 | "daysOfMonth": [3]
36 | }
37 | ],
38 | "fixed": 3,
39 | "unit": "hours"
40 | },
41 | {
42 | "bounds": [
43 | {
44 | "daysOfMonth": [4]
45 | }
46 | ],
47 | "fixed": 4,
48 | "unit": "hours"
49 | },
50 | {
51 | "bounds": [
52 | {
53 | "daysOfMonth": [5]
54 | }
55 | ],
56 | "fixed": 5,
57 | "unit": "hours"
58 | },
59 | {
60 | "bounds": [
61 | {
62 | "daysOfMonth": [6]
63 | }
64 | ],
65 | "fixed": 6,
66 | "unit": "hours"
67 | },
68 | {
69 | "bounds": [
70 | {
71 | "daysOfMonth": [7]
72 | }
73 | ],
74 | "fixed": 7,
75 | "unit": "hours"
76 | },
77 | {
78 | "bounds": [
79 | {
80 | "daysOfMonth": [8]
81 | }
82 | ],
83 | "fixed": 8,
84 | "unit": "hours"
85 | },
86 | {
87 | "fixed": 1,
88 | "unit": "hours"
89 | }
90 | ]
91 | }
92 | ]
93 | }
94 | ]
95 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8e_days_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": [[1, 8]]
10 | }
11 | ],
12 | "periods": [
13 | {
14 | "bounds": [
15 | {
16 | "months": [1]
17 | }
18 | ],
19 | "fixed": 1,
20 | "unit": "days"
21 | },
22 | {
23 | "bounds": [
24 | {
25 | "months": [2]
26 | }
27 | ],
28 | "fixed": 2,
29 | "unit": "days"
30 | },
31 | {
32 | "bounds": [
33 | {
34 | "months": [3]
35 | }
36 | ],
37 | "fixed": 3,
38 | "unit": "days"
39 | },
40 | {
41 | "bounds": [
42 | {
43 | "months": [4]
44 | }
45 | ],
46 | "fixed": 4,
47 | "unit": "days"
48 | },
49 | {
50 | "bounds": [
51 | {
52 | "months": [5]
53 | }
54 | ],
55 | "fixed": 5,
56 | "unit": "days"
57 | },
58 | {
59 | "bounds": [
60 | {
61 | "months": [6]
62 | }
63 | ],
64 | "fixed": 6,
65 | "unit": "days"
66 | },
67 | {
68 | "bounds": [
69 | {
70 | "months": [7]
71 | }
72 | ],
73 | "fixed": 7,
74 | "unit": "days"
75 | },
76 | {
77 | "bounds": [
78 | {
79 | "months": [8]
80 | }
81 | ],
82 | "fixed": 8,
83 | "unit": "days"
84 | },
85 | {
86 | "fixed": 1,
87 | "unit": "days"
88 | }
89 | ]
90 | }
91 | ]
92 | }
93 | ]
94 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/8f_weeks_fixed_period.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "years": [2023],
9 | "months": [[1, 4]]
10 | }
11 | ],
12 | "periods": [
13 | {
14 | "bounds": [
15 | {
16 | "months": [1]
17 | }
18 | ],
19 | "fixed": 1,
20 | "unit": "weeks"
21 | },
22 | {
23 | "bounds": [
24 | {
25 | "months": [2]
26 | }
27 | ],
28 | "fixed": 2,
29 | "unit": "weeks"
30 | },
31 | {
32 | "bounds": [
33 | {
34 | "months": [3]
35 | }
36 | ],
37 | "fixed": 3,
38 | "unit": "weeks"
39 | },
40 | {
41 | "bounds": [
42 | {
43 | "months": [4]
44 | }
45 | ],
46 | "fixed": 4,
47 | "unit": "weeks"
48 | },
49 | {
50 | "fixed": 5,
51 | "unit": "weeks"
52 | }
53 | ]
54 | }
55 | ]
56 | }
57 | ]
58 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/9a_outer_bound_larger.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [[0, 18]]
9 | }
10 | ]
11 | },
12 | {
13 | "id": "http://yetanalytics.com/temporal/pattern-B1",
14 | "bounds": [
15 | {
16 | "hours": [[6, 12]]
17 | }
18 | ],
19 | "periods": [
20 | {
21 | "mean": 1,
22 | "unit": "hours"
23 | }
24 | ]
25 | },
26 | {
27 | "id": "http://yetanalytics.com/temporal/pattern-B2",
28 | "bounds": [
29 | {
30 | "hours": [[6, 12]]
31 | }
32 | ],
33 | "periods": [
34 | {
35 | "mean": 1,
36 | "unit": "hours"
37 | }
38 | ]
39 | }
40 | ]
41 | }
42 | ]
43 |
--------------------------------------------------------------------------------
/dev-resources/models/temporal/9b_outer_bound_smaller.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "patterns": [
4 | {
5 | "id": "http://yetanalytics.com/temporal/pattern-A",
6 | "bounds": [
7 | {
8 | "hours": [[6, 12]]
9 | }
10 | ],
11 | "periods": [
12 | {
13 | "mean": 1,
14 | "unit": "hours"
15 | }
16 | ]
17 | },
18 | {
19 | "id": "http://yetanalytics.com/temporal/pattern-B1",
20 | "bounds": [
21 | {
22 | "hours": [[0, 18]]
23 | }
24 | ]
25 | },
26 | {
27 | "id": "http://yetanalytics.com/temporal/pattern-B1",
28 | "bounds": [
29 | {
30 | "hours": [[0, 18]]
31 | }
32 | ]
33 | }
34 | ]
35 | }
36 | ]
37 |
--------------------------------------------------------------------------------
/dev-resources/parameters/ds_102.json:
--------------------------------------------------------------------------------
1 | {
2 | "start": "2021-03-18T16:00:00.000000Z",
3 | "timezone": "America/New_York",
4 | "seed": 42,
5 | "max": 1000000000
6 | }
7 |
--------------------------------------------------------------------------------
/dev-resources/parameters/simple.json:
--------------------------------------------------------------------------------
1 | {
2 | "start": "2019-11-18T11:38:39.219768Z",
3 | "end": "2019-11-19T11:38:39.219768Z",
4 | "timezone": "America/New_York",
5 | "seed": 42
6 | }
7 |
--------------------------------------------------------------------------------
/dev-resources/parameters/tccc_dev.json:
--------------------------------------------------------------------------------
1 | {
2 | "start": "2020-02-10T11:38:39.219768Z",
3 | "end": "2020-02-25T17:38:39.219768Z",
4 | "timezone": "America/New_York",
5 | "seed": 40
6 | }
7 |
--------------------------------------------------------------------------------
/dev-resources/parameters/temporal.json:
--------------------------------------------------------------------------------
1 | {
2 | "start": "2023-01-01T00:00:00Z",
3 | "end": "2026-01-01T00:00:00Z",
4 | "timezone": "UTC",
5 | "seed": 116
6 | }
7 |
--------------------------------------------------------------------------------
/dev-resources/personae/array/simple_array.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "name": "trainees",
4 | "objectType": "Group",
5 | "member": [
6 | {
7 | "name": "Bob Fakename",
8 | "mbox": "mailto:bobfake@example.org",
9 | "role": "Lead Developer"
10 | },
11 | {
12 | "name": "Alice Faux",
13 | "mbox": "mailto:alicefaux@example.org",
14 | "role": "Data Engineer"
15 | },
16 | {
17 | "name": "Fred Ersatz",
18 | "mbox": "mailto:frederstaz@example.org"
19 | }
20 | ]
21 | }
22 | ]
23 |
--------------------------------------------------------------------------------
/dev-resources/personae/bench.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "trainees",
3 | "objectType": "Group",
4 | "member": [
5 | {
6 | "name": "Dionne Knox",
7 | "mbox": "mailto:0@yetanalytics.com"
8 | },
9 | {
10 | "name": "Penny Mayer",
11 | "mbox": "mailto:1@yetanalytics.com"
12 | },
13 | {
14 | "name": "Stanley Ayers",
15 | "mbox": "mailto:2@yetanalytics.com"
16 | },
17 | {
18 | "name": "Reid Joyner",
19 | "mbox": "mailto:3@yetanalytics.com"
20 | },
21 | {
22 | "name": "Loretta Elliott",
23 | "mbox": "mailto:4@yetanalytics.com"
24 | },
25 | {
26 | "name": "Hawkins Landry",
27 | "mbox": "mailto:5@yetanalytics.com"
28 | },
29 | {
30 | "name": "Zimmerman Short",
31 | "mbox": "mailto:6@yetanalytics.com"
32 | },
33 | {
34 | "name": "Jeannette Castaneda",
35 | "mbox": "mailto:7@yetanalytics.com"
36 | },
37 | {
38 | "name": "Ramos Boyer",
39 | "mbox": "mailto:8@yetanalytics.com"
40 | },
41 | {
42 | "name": "Montgomery Dale",
43 | "mbox": "mailto:9@yetanalytics.com"
44 | },
45 | {
46 | "name": "Jerry Rose",
47 | "mbox": "mailto:10@yetanalytics.com"
48 | },
49 | {
50 | "name": "Katina Armstrong",
51 | "mbox": "mailto:11@yetanalytics.com"
52 | },
53 | {
54 | "name": "Dickson Gaines",
55 | "mbox": "mailto:12@yetanalytics.com"
56 | },
57 | {
58 | "name": "Patricia Forbes",
59 | "mbox": "mailto:13@yetanalytics.com"
60 | },
61 | {
62 | "name": "Petty Huber",
63 | "mbox": "mailto:14@yetanalytics.com"
64 | },
65 | {
66 | "name": "Sexton Cooley",
67 | "mbox": "mailto:15@yetanalytics.com"
68 | },
69 | {
70 | "name": "Nola Burns",
71 | "mbox": "mailto:16@yetanalytics.com"
72 | },
73 | {
74 | "name": "Caitlin Lancaster",
75 | "mbox": "mailto:17@yetanalytics.com"
76 | },
77 | {
78 | "name": "Dean Roth",
79 | "mbox": "mailto:18@yetanalytics.com"
80 | },
81 | {
82 | "name": "Wilson Chambers",
83 | "mbox": "mailto:19@yetanalytics.com"
84 | },
85 | {
86 | "name": "Helga Hutchinson",
87 | "mbox": "mailto:20@yetanalytics.com"
88 | },
89 | {
90 | "name": "Mary Flores",
91 | "mbox": "mailto:21@yetanalytics.com"
92 | },
93 | {
94 | "name": "Baker Orr",
95 | "mbox": "mailto:22@yetanalytics.com"
96 | },
97 | {
98 | "name": "Randall Fernandez",
99 | "mbox": "mailto:23@yetanalytics.com"
100 | },
101 | {
102 | "name": "Margie Pratt",
103 | "mbox": "mailto:24@yetanalytics.com"
104 | },
105 | {
106 | "name": "Christian Baxter",
107 | "mbox": "mailto:25@yetanalytics.com"
108 | },
109 | {
110 | "name": "Guerrero Glass",
111 | "mbox": "mailto:26@yetanalytics.com"
112 | },
113 | {
114 | "name": "Marian Fox",
115 | "mbox": "mailto:27@yetanalytics.com"
116 | },
117 | {
118 | "name": "Ayers Farley",
119 | "mbox": "mailto:28@yetanalytics.com"
120 | },
121 | {
122 | "name": "Milagros Hudson",
123 | "mbox": "mailto:29@yetanalytics.com"
124 | },
125 | {
126 | "name": "Brittany French",
127 | "mbox": "mailto:30@yetanalytics.com"
128 | },
129 | {
130 | "name": "Mueller Ward",
131 | "mbox": "mailto:31@yetanalytics.com"
132 | },
133 | {
134 | "name": "Emily Lucas",
135 | "mbox": "mailto:32@yetanalytics.com"
136 | },
137 | {
138 | "name": "Cohen Shepherd",
139 | "mbox": "mailto:33@yetanalytics.com"
140 | }
141 | ]
142 | }
143 |
--------------------------------------------------------------------------------
/dev-resources/personae/simple.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "trainees",
3 | "objectType": "Group",
4 | "member": [
5 | {
6 | "name": "Bob Fakename",
7 | "mbox": "mailto:bobfake@example.org",
8 | "role": "Lead Developer"
9 | },
10 | {
11 | "name": "Alice Faux",
12 | "mbox": "mailto:alicefaux@example.org",
13 | "role": "Data Engineer"
14 | },
15 | {
16 | "name": "Fred Ersatz",
17 | "mbox": "mailto:frederstaz@example.org"
18 | }
19 | ]
20 | }
21 |
--------------------------------------------------------------------------------
/dev-resources/personae/tccc_dev.json:
--------------------------------------------------------------------------------
1 | {
2 | "name" : "trainees",
3 | "objectType" : "Group",
4 | "member" : [
5 | {
6 | "name" : "Bob Nelson",
7 | "mbox" : "mailto:bob@example.org"
8 | },
9 | {
10 | "name" : "Phil Walker",
11 | "mbox" : "mailto:phil@example.org"
12 | },
13 | {
14 | "name" : "Sally Davis",
15 | "mbox" : "mailto:sally@example.org"
16 | },
17 | {
18 | "name" : "Steve Stewart",
19 | "mbox" : "mailto:steve@example.org"
20 | },
21 | {
22 | "name" : "Fred Evans",
23 | "mbox" : "mailto:fred@example.org"
24 | },
25 | {
26 | "name" : "Alice Edwards",
27 | "mbox" : "mailto:alice@example.org"
28 | }
29 | ]
30 | }
--------------------------------------------------------------------------------
/dev-resources/personae/temporal.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "Singleton Group",
3 | "objectType": "Group",
4 | "member": [
5 | {
6 | "name": "SCROM",
7 | "mbox": "mailto:scrom@yetanalytics.org",
8 | "role": "Merciful God of Learning Data"
9 | }
10 | ]
11 | }
12 |
--------------------------------------------------------------------------------
/dev-resources/profiles/minimal.jsonld:
--------------------------------------------------------------------------------
1 | {
2 | "id" : "https://xapinet.org/xapi/yet/minimal",
3 | "definition" : {
4 | "en" : "This xAPI Profile demonstrates the minimal required properties of an xAPI profile"
5 | },
6 | "@context" : "https://w3id.org/xapi/profiles/context",
7 | "prefLabel" : {
8 | "en" : "Minimal - Experimental xAPI Profile"
9 | },
10 | "type" : "Profile",
11 | "author" : {
12 | "url" : "https://www.yetanalytics.com/",
13 | "name" : "Yet Analytics",
14 | "type" : "Organization"
15 | },
16 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0",
17 | "versions" : [
18 | {
19 | "id" : "https://xapinet.org/xapi/yet/minimal/v1",
20 | "generatedAtTime" : "2020-03-25T15:45:31.907Z"
21 | }
22 | ]
23 | }
24 |
--------------------------------------------------------------------------------
/dev-resources/profiles/no_concept.jsonld:
--------------------------------------------------------------------------------
1 | {
2 | "id": "https://www.example.com/xapi/profile",
3 | "conformsTo": "https://w3id.org/xapi/profiles#1.0",
4 | "versions": [
5 | {
6 | "id": "https://example.com/xapi/profile/v1.0.0",
7 | "generatedAtTime": "2022-11-15T16:35:26.240428Z"
8 | }
9 | ],
10 | "@context": "https://w3id.org/xapi/profiles/context",
11 | "definition": {
12 | "en": "Test Profile with no Verbs, Activities, or other Concepts"
13 | },
14 | "prefLabel": {
15 | "en": "Test Profile"
16 | },
17 | "type": "Profile",
18 | "author": {
19 | "type": "Organization",
20 | "name": "Yet Analytics"
21 | },
22 | "templates": [
23 | {
24 | "definition": {
25 | "en": "Test Template"
26 | },
27 | "prefLabel": {
28 | "en": "Test Template"
29 | },
30 | "type": "StatementTemplate",
31 | "rules": [
32 | {
33 | "location": "$.context",
34 | "presence": "included"
35 | }
36 | ],
37 | "inScheme": "https://example.com/xapi/profile/v1.0.0",
38 | "id": "https://example.com/xapi/profile/template"
39 | }
40 | ],
41 | "patterns": [
42 | {
43 | "definition": {
44 | "en": "Test Pattern"
45 | },
46 | "primary": true,
47 | "prefLabel": {
48 | "en": "Test Pattern"
49 | },
50 | "type": "Pattern",
51 | "inScheme": "https://example.com/xapi/profile/v1.0.0",
52 | "id": "https://example.com/xapi/profile/pattern",
53 | "sequence": [
54 | "https://example.com/xapi/profile/template"
55 | ]
56 | }
57 | ]
58 | }
59 |
--------------------------------------------------------------------------------
/dev-resources/profiles/referential.jsonld:
--------------------------------------------------------------------------------
1 | {
2 | "id" : "https://xapinet.org/xapi/yet/referential",
3 | "definition" : {
4 | "en" : "Test Profile that refers to another"
5 | },
6 | "@context" : "https://w3id.org/xapi/profiles/context",
7 | "prefLabel" : {
8 | "en" : "REFERENTIAL xAPI Profile"
9 | },
10 | "type" : "Profile",
11 | "author" : {
12 | "url" : "https://www.yetanalytics.com/",
13 | "name" : "Yet Analytics",
14 | "type" : "Organization"
15 | },
16 | "conformsTo" : "https://w3id.org/xapi/profiles#1.0",
17 | "versions" : [
18 | {
19 | "id" : "https://xapinet.org/xapi/yet/referential/v0.0.0",
20 | "generatedAtTime" : "2022-06-27T20:34:27.823870Z"
21 | }
22 | ],
23 | "patterns" : [
24 | {
25 | "id": "https://xapinet.org/xapi/yet/referential#completed_session",
26 | "type": "Pattern",
27 | "primary": true,
28 | "inScheme": "https://xapinet.org/xapi/yet/referential/v0.0.0",
29 | "prefLabel": {
30 | "en": "completed session"
31 | },
32 | "definition": {
33 | "en": "This pattern describes the sequence of statements sent over the an entire course registration."
34 | },
35 | "sequence": [
36 | "https://w3id.org/xapi/tla#launched",
37 | "https://w3id.org/xapi/tla#initialized",
38 | "https://w3id.org/xapi/tla#completed",
39 | "https://w3id.org/xapi/tla#terminated"
40 | ]
41 | }
42 | ]
43 | }
44 |
--------------------------------------------------------------------------------
/dev-resources/xapi/statements/long.json:
--------------------------------------------------------------------------------
1 | {
2 | "id": "6690e6c9-3ef0-4ed3-8b37-7f3964730bee",
3 | "actor": {
4 | "name": "Team PB",
5 | "mbox": "mailto:teampb@example.com",
6 | "member": [
7 | {
8 | "name": "Andrew Downes",
9 | "account": {
10 | "homePage": "http://www.example.com",
11 | "name": "13936749"
12 | },
13 | "objectType": "Agent"
14 | },
15 | {
16 | "name": "Toby Nichols",
17 | "openid": "http://toby.openid.example.org/",
18 | "objectType": "Agent"
19 | },
20 | {
21 | "name": "Ena Hills",
22 | "mbox_sha1sum": "ebd31e95054c018b10727ccffd2ef2ec3a016ee9",
23 | "objectType": "Agent"
24 | }
25 | ],
26 | "objectType": "Group"
27 | },
28 | "verb": {
29 | "id": "http://adlnet.gov/expapi/verbs/attended",
30 | "display": {
31 | "en-GB": "attended",
32 | "en-US": "attended"
33 | }
34 | },
35 | "result": {
36 | "extensions": {
37 | "http://example.com/profiles/meetings/resultextensions/minuteslocation": "X:\\meetings\\minutes\\examplemeeting.one"
38 | },
39 | "success": true,
40 | "completion": true,
41 | "response": "We agreed on some example actions.",
42 | "duration": "PT1H0M0S"
43 | },
44 | "context": {
45 | "registration": "ec531277-b57b-4c15-8d91-d292c5b2b8f7",
46 | "contextActivities": {
47 | "parent": [
48 | {
49 | "id": "http://www.example.com/meetings/series/267",
50 | "objectType": "Activity"
51 | }
52 | ],
53 | "category": [
54 | {
55 | "id": "http://www.example.com/meetings/categories/teammeeting",
56 | "objectType": "Activity",
57 | "definition": {
58 | "name": {
59 | "en": "team meeting"
60 | },
61 | "description": {
62 | "en": "A category of meeting used for regular team meetings."
63 | },
64 | "type": "http://example.com/expapi/activities/meetingcategory"
65 | }
66 | }
67 | ],
68 | "other": [
69 | {
70 | "id": "http://www.example.com/meetings/occurances/34257",
71 | "objectType": "Activity"
72 | },
73 | {
74 | "id": "http://www.example.com/meetings/occurances/3425567",
75 | "objectType": "Activity"
76 | }
77 | ]
78 | },
79 | "instructor" :
80 | {
81 | "name": "Andrew Downes",
82 | "account": {
83 | "homePage": "http://www.example.com",
84 | "name": "13936749"
85 | },
86 | "objectType": "Agent"
87 | },
88 | "team":
89 | {
90 | "name": "Team PB",
91 | "mbox": "mailto:teampb@example.com",
92 | "objectType": "Group"
93 | },
94 | "platform" : "Example virtual meeting software",
95 | "language" : "tlh",
96 | "statement" : {
97 | "objectType":"StatementRef",
98 | "id" :"6690e6c9-3ef0-4ed3-8b37-7f3964730bee"
99 | }
100 |
101 | },
102 | "timestamp": "2013-05-18T05:32:34.804Z",
103 | "stored": "2013-05-18T05:32:34.804Z",
104 | "authority": {
105 | "account": {
106 | "homePage": "http://cloud.scorm.com/",
107 | "name": "anonymous"
108 | },
109 | "objectType": "Agent"
110 | },
111 | "version": "1.0.0",
112 | "object": {
113 | "id": "http://www.example.com/meetings/occurances/34534",
114 | "definition": {
115 | "extensions": {
116 | "http://example.com/profiles/meetings/activitydefinitionextensions/room": {"name": "Kilby", "id" : "http://example.com/rooms/342"}
117 | },
118 | "name": {
119 | "en-GB": "example meeting",
120 | "en-US": "example meeting"
121 | },
122 | "description": {
123 | "en-GB": "An example meeting that happened on a specific occasion with certain people present.",
124 | "en-US": "An example meeting that happened on a specific occasion with certain people present."
125 | },
126 | "type": "http://adlnet.gov/expapi/activities/meeting",
127 | "moreInfo": "http://virtualmeeting.example.com/345256"
128 | },
129 | "objectType": "Activity"
130 | }
131 | }
132 |
--------------------------------------------------------------------------------
/dev-resources/xapi/statements/simple.json:
--------------------------------------------------------------------------------
1 | {
2 | "id":"fd41c918-b88b-4b20-a0a5-a4c32391aaa0",
3 | "timestamp": "2015-11-18T12:17:00+00:00",
4 | "actor":{
5 | "objectType": "Agent",
6 | "name":"Project Tin Can API",
7 | "mbox":"mailto:user@example.com"
8 | },
9 | "verb":{
10 | "id":"http://example.com/xapi/verbs#sent-a-statement",
11 | "display":{
12 | "en-US":"sent"
13 | }
14 | },
15 | "object":{
16 | "id":"http://example.com/xapi/activity/simplestatement",
17 | "definition":{
18 | "name":{
19 | "en-US":"simple statement"
20 | },
21 | "description":{
22 | "en-US":"A simple Experience API statement. Note that the LRS does not need to have any prior information about the Actor (learner), the verb, or the Activity/object."
23 | }
24 | }
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/doc/intro.md:
--------------------------------------------------------------------------------
1 | # Introduction to datasim
2 |
3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/)
4 |
--------------------------------------------------------------------------------
/doc/temporal_structure.md:
--------------------------------------------------------------------------------
1 | # Temporal Structure
2 |
3 | In order to deterministically generate events over time, we need a model that does a few things:
4 |
5 | * generates from a seed and keeps things deterministic all the way down
6 | * generates discrete events over time in realistic patterns by incorporating:
7 | * Day/Night
8 | * Daily/Weekly Work Hours
9 | * Independent factor (ARMA)
10 | * dependent factor (group dynamic, etc)
11 | * generates unique activity for each actor
12 | * that is still beholden to dependent factors
13 |
14 | At the event level, we'll need the following:
15 |
16 | * when did the event occur?
17 | * who is the actor for this event?
18 | * a long representing the seed for generating this event.
19 | * should be a function of
20 | * the simulation (base) seed
21 | * the time
22 | * the actor
23 | * ~~a double representing the coarse "effectiveness" of the actor when the event occurs~~
24 | * ~~harder to reason about this one, as no value/desirability is available~~
25 |
--------------------------------------------------------------------------------
/onyx-resources/onyx_config.edn:
--------------------------------------------------------------------------------
1 | {;; :default profile is dev operation
2 | :env-config
3 | {:onyx/tenancy-id #profile {:default #env TENANCY_ID}
4 | :zookeeper/address #profile {:default #or [#env ZK_ADDRESS
5 | "127.0.0.1:2188"]
6 | :prod #env ZK_ADDRESS}
7 | :zookeeper/server? #profile {:default true
8 | :prod false}
9 | :zookeeper.server/port #profile {:default #long #or [#env ZK_SERVER_PORT
10 | 2188]}
11 | :onyx.log/config #profile {:default nil
12 | :prod {:level :info}}}
13 | :peer-config
14 | {:onyx/tenancy-id #ref [:env-config :onyx/tenancy-id]
15 | :zookeeper/address #ref [:env-config :zookeeper/address]
16 | :onyx.peer/job-scheduler :onyx.job-scheduler/balanced
17 | :onyx.peer/zookeeper-timeout #long #or [#env ZK_TIMEOUT
18 | 1000]
19 | :onyx.messaging/impl :aeron
20 | :onyx.messaging/bind-addr #or [#env BIND_ADDR
21 | "0.0.0.0"]
22 | :onyx.messaging/peer-port #long #or [#env PEER_PORT
23 | 40200]
24 | :onyx.messaging.aeron/embedded-driver? #profile {:default true
25 | :prod false}
26 | :onyx.log/config #profile {:default nil
27 | :prod {:level :info}}
28 | :onyx.query/server? true
29 | :onyx.query.server/ip #or [#env SERVER_IP
30 | "0.0.0.0"]
31 | :onyx.query.server/port #long #or [#env SERVER_PORT
32 | 8888] ;; port change so it isn't the same as dev lrs
33 | ;; do allow short circuit locally, so the default colo works
34 | :onyx.messaging/allow-short-circuit? #profile {:default true
35 | :prod true}
36 | :onyx.peer/storage #keyword #or [#env PEER_STORAGE
37 | :zookeeper]
38 | :onyx.peer/storage.s3.bucket #or [#env PEER_STORAGE_BUCKET
39 | ""]
40 | :onyx.peer/storage.s3.region #or [#env AWS_REGION
41 | "us-east-1"]
42 | :onyx.peer/storage.s3.prefix #or [#env PEER_STORAGE_PREFIX
43 | #profile {:default "onyx_dev/"}]
44 | ;; Aeron max term buffer size
45 | :onyx.messaging/term-buffer-size.segment 16777216
46 |
47 | ;; :onyx.peer/coordinator-barrier-period-ms 5000
48 |
49 | :onyx.peer/storage.timeout #long #or [#env PEER_STORAGE_TIMEOUT
50 | 120000]
51 | ;; need this or constant timeout
52 | :onyx.peer/storage.s3.accelerate? #boolean #or [#env PEER_STORAGE_ACCELERATION
53 | "false"]
54 |
55 | #_#_:onyx.peer/tags #profile {:default [:gen :out]
56 | :prod [#keyword #env PEER_TAG]}
57 | ;; :onyx.peer/publisher-liveness-timeout-ms 120000
58 | ;; :onyx.peer/subscriber-liveness-timeout-ms 120000
59 | ;; :onyx.peer/coordinator-max-sleep-ms 10
60 | ;; :onyx.peer/coordinator-barrier-period-ms 2000
61 | ;; :onyx.peer/heartbeat-ms 1000
62 | }
63 | :launch-config
64 | {:n-vpeers #long #or [#env N_VPEERS
65 | 4]}}
66 |
--------------------------------------------------------------------------------
/scripts/gc.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh gc $@
3 |
--------------------------------------------------------------------------------
/scripts/gc_checkpoints.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh gc-checkpoints $@
3 |
--------------------------------------------------------------------------------
/scripts/onyx.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | java -server $JAVA_OPTS -jar ./datasim_onyx.jar $@
3 |
--------------------------------------------------------------------------------
/scripts/peer.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh start-peer $@
3 |
--------------------------------------------------------------------------------
/scripts/peer_driver.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh start-driver $@
3 |
--------------------------------------------------------------------------------
/scripts/repl.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh repl
3 |
--------------------------------------------------------------------------------
/scripts/run.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | java -jar ./datasim_cli.jar $@
3 |
--------------------------------------------------------------------------------
/scripts/server.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | java -server -jar ./datasim_server.jar $@
3 |
--------------------------------------------------------------------------------
/scripts/submit_job.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | sh ./bin/onyx.sh submit-job $@
3 |
--------------------------------------------------------------------------------
/src/cli/com/yetanalytics/datasim/cli.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.cli
2 | (:require [clojure.tools.cli :as cli]
3 | [com.yetanalytics.datasim.cli.input :as cli-input]
4 | [com.yetanalytics.datasim.cli.generate :as cli-gen]
5 | [com.yetanalytics.datasim.cli.util :as u])
6 | (:gen-class))
7 |
8 | (def top-level-options
9 | [["-h" "--help" "Display the top-level help guide."]])
10 |
11 | (def top-level-summary
12 | (str "Usage: 'datasim ' or 'datasim [-h|--help]'.\n"
13 | "\n"
14 | "where the subcommand can be one of the following:\n"
15 | " validate-input: Validate the input and create an input JSON file.\n"
16 | " generate: Generate statements from input and print to stdout.\n"
17 | " generate-post: Generate statements from input and POST them to an LRS.\n"
18 | "\n"
19 | "Run 'datasim --help' for more info on each subcommand."))
20 |
21 | (defn -main [& args]
22 | (let [{:keys [options arguments summary errors]}
23 | (cli/parse-opts args top-level-options
24 | :in-order true
25 | :summary-fn (fn [_] top-level-summary))
26 | [subcommand & rest-args]
27 | arguments]
28 | (cond
29 | (:help options)
30 | (println summary)
31 | (not subcommand)
32 | (print "No subcommand entered.\n\n" summary)
33 | :else
34 | (let [results (case subcommand
35 | "validate-input" (cli-input/validate-input! rest-args)
36 | "generate" (cli-gen/generate! rest-args)
37 | "generate-post" (cli-gen/generate-post! rest-args)
38 | (u/bail! errors))]
39 | (when-some [errors (:errors results)]
40 | (u/bail! errors))))))
41 |
--------------------------------------------------------------------------------
/src/cli/com/yetanalytics/datasim/cli/util.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.cli.util
2 | (:require [clojure.tools.cli :as cli]
3 | [com.yetanalytics.datasim.util.io :as dio]))
4 |
5 | (defn conj-input
6 | "Conj the input (either a Profile or a personae) to return a
7 | vector of inputs, e.g. `-p profile-1 -p profile-2` becomes
8 | `[profile-1 profile-2]`."
9 | [opt-map id v]
10 | (update opt-map id (fnil conj []) v))
11 |
12 | (defn conj-param-input
13 | "Add a parameter named by id."
14 | [opt-map id v]
15 | (update-in opt-map [:parameters id] (fnil conj []) v))
16 |
17 | (defn bail!
18 | "Print error messages to standard error and exit."
19 | [errors & {:keys [status]
20 | :or {status 1}}]
21 | (dio/println-err-coll errors)
22 | (System/exit status))
23 |
24 | (defn exec-subcommand
25 | "Execute `exec-fn` for a subcommand with arguments `args`, where the
26 | valid options are `cli-options`."
27 | [cli-options exec-fn args]
28 | (let [{:keys [options summary errors]}
29 | (cli/parse-opts args cli-options)
30 | {:keys [help]}
31 | options
32 | errors* (not-empty errors)]
33 | (cond
34 | help (println summary)
35 | errors* (bail! errors*)
36 | :else (exec-fn options))))
37 |
--------------------------------------------------------------------------------
/src/dev/bench.clj:
--------------------------------------------------------------------------------
1 | (ns bench
2 | (:require [criterium.core :as c]
3 | [com.yetanalytics.datasim.sim :as sim]
4 | [com.yetanalytics.datasim.input :as input]))
5 |
6 | (defn statements-per-second
7 | "How many statements per second will datasim generate?"
8 | [input]
9 | (let [sim-seq (sim/sim-seq input)
10 | t-start (System/currentTimeMillis)
11 | t-end (+ t-start 1000)]
12 | (loop [[_ & rest-s] sim-seq
13 | s-count 0]
14 | (if (> t-end (System/currentTimeMillis))
15 | (recur rest-s (inc s-count))
16 | s-count))))
17 |
18 | (comment
19 | (def input
20 | {:profiles [(input/from-location :profile :json "dev-resources/bench/calibration.jsonld")]
21 | :personae-array [(input/from-location :personae :json "dev-resources/bench/actors.json")]
22 | :models (input/from-location :models :json "dev-resources/bench/models.json")
23 | :parameters (input/from-location :parameters :json "dev-resources/bench/params.json")})
24 |
25 | (c/with-progress-reporting
26 | (c/quick-bench (do (doall (take 1000 (sim/sim-seq input)))
27 | nil)))
28 |
29 | ;; Evaluation count : 6 in 6 samples of 1 calls.
30 | ;; Execution time mean : 201.061588 ms
31 | ;; Execution time std-deviation : 6.698801 ms
32 | ;; Execution time lower quantile : 196.857258 ms ( 2.5%)
33 | ;; Execution time upper quantile : 212.503809 ms (97.5%)
34 | ;; Overhead used : 7.430081 ns
35 |
36 | (statements-per-second input) ;; => 4938
37 |
38 | (quot (reduce + (repeatedly 60 #(statements-per-second input)))
39 | 60) ;; => 4462
40 |
41 | )
42 |
--------------------------------------------------------------------------------
/src/dev/user.clj:
--------------------------------------------------------------------------------
1 | (ns user
2 | (:require [clojure.repl :refer [source doc apropos]]
3 | [clojure.pprint :refer [pprint]]))
4 |
5 | (set! *warn-on-reflection* true)
6 |
--------------------------------------------------------------------------------
/src/dev_onyx/com/yetanalytics/datasim/onyx/dev_peer.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.dev-peer
2 | (:gen-class)
3 | (:require onyx.plugin.http-output
4 | onyx.plugin.seq
5 | onyx.plugin.s3-output
6 | onyx.api
7 | com.yetanalytics.datasim.onyx.sim
8 | com.yetanalytics.datasim.onyx.http
9 | [com.yetanalytics.datasim.onyx.config :as config]))
10 |
11 | ;; Trying just the peer
12 |
13 | (defn -main
14 | [& args]
15 | ;; VERY MUCH JUST DEV RIGHT NOW
16 | (let [;; DEV ENV config, should go away for prod things
17 | id (random-uuid)
18 | {:keys [env-config peer-config]
19 | {:keys [n-vpeers]} :launch-config} (-> (config/get-config)
20 | (assoc-in [:env-config :onyx/tenancy-id] id)
21 | (assoc-in [:peer-config :onyx/tenancy-id] id))
22 | env (onyx.api/start-env env-config)
23 |
24 | ;; start peer group
25 | peer-group (onyx.api/start-peer-group peer-config)
26 |
27 | v-peers (onyx.api/start-peers n-vpeers peer-group)]
28 |
29 |
30 | (.addShutdownHook (Runtime/getRuntime)
31 | (Thread. ^Runnable #(do
32 | (onyx.api/shutdown-peers v-peers)
33 |
34 | (onyx.api/shutdown-peer-group peer-group)
35 |
36 | (onyx.api/shutdown-env env))))))
37 |
--------------------------------------------------------------------------------
/src/dev_onyx/com/yetanalytics/datasim/onyx/scratch.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.scratch
2 | "Scratch ns for playing with onyx"
3 | (:require
4 | [clojure.core.async :as a]
5 | com.yetanalytics.datasim.onyx.main ;; get all boot-up ns stuff
6 | [onyx.plugin.http-output :as http]
7 | [onyx.plugin.core-async :as ap]
8 | [onyx.plugin.seq]
9 | [onyx.api]
10 | [onyx.test-helper :as th]
11 | [com.yetanalytics.datasim.onyx.sim :as dsim]
12 | [com.yetanalytics.datasim.onyx.job :as job]
13 | [com.yetanalytics.datasim.onyx.config :as config]))
14 |
15 | (comment
16 | ;; Run things in an enclosed environment
17 | (let [id (random-uuid)
18 | {:keys [env-config peer-config]} (-> (config/get-config)
19 | (assoc-in [:env-config :onyx/tenancy-id] id)
20 | (assoc-in [:peer-config :onyx/tenancy-id] id))]
21 | (th/with-test-env
22 | [{:keys [n-peers
23 | env
24 | peer-group
25 | peers]
26 | :as test-env} [;; n-peers
27 | 3 ;;
28 | env-config
29 | peer-config
30 | ]]
31 | (let [;; Submit the job
32 | submission (onyx.api/submit-job
33 | peer-config
34 | (job/config
35 | {:input-loc "dev-resources/input/mom64.json"
36 | :gen-concurrency 1
37 | :gen-batch-size 10
38 | :post-concurrency 1
39 | :override-max 100
40 | :out-ratio 1
41 | :out-mode :lrs
42 | :lrs {:endpoint "http://localhost:8080/xapi"}}))]
43 |
44 | ;; Wait for jorb to finish if you like
45 | (println 'started submission)
46 | (onyx.api/await-job-completion peer-config (:job-id submission))
47 | (println 'done submission)
48 | )))
49 |
50 | )
51 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim
2 | (:require [clojure.spec.alpha :as s]
3 | [com.yetanalytics.datasim.input :as input]
4 | [com.yetanalytics.datasim.sim :as sim]))
5 |
6 | (s/fdef read-input
7 | :args (s/cat :location string?)
8 | :ret ::input)
9 |
10 | (defn read-input
11 | "Read input at `location` and return DATASIM input for use with
12 | other functions."
13 | [location]
14 | (input/from-location :input :json location))
15 |
16 | ;; TODO: Add fdefs for generation functions
17 | ;; TODO: Extract more implementation code from `sim` namespace
18 |
19 | #_{:clj-kondo/ignore [:unused-binding]}
20 | (defn generate-seq
21 | "Given `input`, produce a lazy sequence of statements in a synchronous
22 | fashion."
23 | [input & {:keys [select-agents] :as kwargs}]
24 | (sim/sim-seq input kwargs))
25 |
26 | (defn generate-map
27 | "Given `input`, produce a map from actor IFIs to lazy sequences of statements
28 | that use those actors, all in a synchronous fashion."
29 | [input]
30 | (sim/build-skeleton input))
31 |
32 | #_{:clj-kondo/ignore [:unused-binding]}
33 | (defn generate-seq-async
34 | "Given `input`, produce a `core.async` channels that contains a generated
35 | sequence of simulated statements; this is for parallel generation."
36 | [input & {:keys [select-agents pad-chan-max] :as kwargs}]
37 | (sim/sim-chan input kwargs))
38 |
39 | #_{:clj-kondo/ignore [:unused-binding]}
40 | (defn generate-map-async
41 | "Given `input`, produce a map from actor IFIs to `core.async` channels,
42 | each with their own generated sequence of simulated statements; this
43 | is for parallel generation."
44 | [input & {:keys [select-agents pad-chan-max sort buffer-size] :as kwargs}]
45 | (sim/sim-chans input kwargs))
46 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/client.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.client
2 | "Simple xAPI LRS client functions."
3 | (:require [clojure.core.async :as a]
4 | [clojure.java.io :as io]
5 | [cheshire.core :as json]
6 | [org.httpkit.client :as http]
7 | [com.yetanalytics.datasim.util.io :as dio]))
8 |
9 | (defn post-error-message
10 | "Error message for when POSTing to an LRS fails."
11 | [status error]
12 | (format "POST Request FAILED with STATUS: %d, MESSAGE: %s"
13 | status
14 | (or (some-> error ex-message) "")))
15 |
16 | (def default-http-options
17 | {:headers {"X-Experience-Api-Version" "1.0.3"
18 | "Content-Type" "application/json"}})
19 |
20 | (defn- decode-body [body]
21 | (with-open [rdr (io/reader body)]
22 | (json/decode-stream rdr)))
23 |
24 | (defn- format-url [endpoint]
25 | (format "%s/statements" endpoint))
26 |
27 | (defn- post-options [http-options batch]
28 | (merge default-http-options
29 | http-options
30 | {:body (json/encode batch)
31 | :as :stream}))
32 |
33 | ;; `http/post` cannot be resolved since it's defined using `http/defreq`
34 | #_{:clj-kondo/ignore [:unresolved-var]}
35 | (defn- post-batch
36 | ([endpoint http-options batch]
37 | (http/post (format-url endpoint)
38 | (post-options http-options batch)))
39 | ([endpoint http-options batch callback-fn]
40 | (http/post (format-url endpoint)
41 | (post-options http-options batch)
42 | callback-fn)))
43 |
44 | (defn post-statements
45 | "Given LRS options and a `statement-seq`, send them to an LRS in synchronous
46 | batches. If `print-ids?` is `true`, returned statement IDs will be printed
47 | to stdout. `username` and `password` in the options map are the Basic Auth
48 | credentials of the LRS."
49 | [{:keys [endpoint
50 | batch-size
51 | username
52 | password]
53 | :or {batch-size 25}}
54 | statement-seq
55 | & {:keys [print-ids?]
56 | :or {print-ids? true}}]
57 | ;; TODO: Exponential backoff, etc
58 | (let [http-options {:basic-auth [username password]}]
59 | (loop [batches (partition-all batch-size statement-seq)
60 | success 0
61 | fail []]
62 | (if-let [batch (first batches)]
63 | (let [{:keys [status body] :as response}
64 | @(post-batch endpoint http-options batch)]
65 | (if (<= 200 status 299)
66 | ;; Success!
67 | (let [statement-ids (decode-body body)]
68 | (when print-ids?
69 | (dio/println-coll statement-ids))
70 | (recur (rest batches)
71 | (+ success (count statement-ids))
72 | fail))
73 | ;; Failure
74 | (let [response* (cond-> response
75 | body (assoc :body (decode-body body)))]
76 | {:success success
77 | :fail (conj fail response*)})))
78 | ;; Batch finished POSTing
79 | {:success success
80 | :fail fail}))))
81 |
82 | (defn post-statements-async
83 | "Given LRS options and a channel with statements, send them to an LRS in
84 | asynchronous batches. `username` and `password` in the options map are the
85 | Basic Auth credentials of the LRS.
86 |
87 | Returns a channel that will reciveve `[:success ]`
88 | for each batch or `[:fail ]`. Will stop sending on failure."
89 | [{:keys [endpoint
90 | batch-size
91 | username
92 | password]
93 | :or {batch-size 25}}
94 | statement-chan
95 | & {:keys [concurrency
96 | buffer-in
97 | buffer-out]
98 | :or {concurrency 4
99 | buffer-in 100 ; 10x default batch size
100 | buffer-out 100}}]
101 | (let [http-opts {:basic-auth [username password]}
102 | run? (atom true)
103 | in-chan (a/chan buffer-in (partition-all batch-size))
104 | out-chan (a/chan buffer-out) ; is this.. backpressure?
105 | callback (fn [port {:keys [status body error] :as ret}]
106 | (if (or (not (<= 200 status 299))
107 | error)
108 | ;; Error: Stop further processing
109 | (do
110 | (swap! run? not)
111 | (a/put! port [:fail ret]))
112 | ;; Success: Continue
113 | (a/put! port
114 | [:success (decode-body body)]))
115 | ;; Close the return channel
116 | (a/close! port))
117 | async-fn (fn [batch port]
118 | (let [callback (partial callback port)]
119 | (if @run?
120 | (post-batch endpoint http-opts batch callback)
121 | (a/close! port))))]
122 | (a/pipeline-async concurrency out-chan async-fn in-chan)
123 | ;; Pipe to in-chan
124 | (a/pipe statement-chan in-chan)
125 | ;; Return the out chan
126 | out-chan))
127 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/input/model.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.model
2 | "Model input specs and parsing."
3 | (:require [clojure.spec.alpha :as s]
4 | [com.yetanalytics.datasim.util.errors :as errs]
5 | [com.yetanalytics.datasim.input.model.alignments :as alignments]
6 | [com.yetanalytics.datasim.input.model.personae :as personae]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | (s/def ::personae
13 | (s/every personae/persona-spec :kind vector? :min-count 1))
14 |
15 | (s/def ::verbs
16 | (s/every alignments/verb-spec :kind vector?))
17 |
18 | (s/def ::activities
19 | (s/every alignments/activity-spec :kind vector?))
20 |
21 | (s/def ::activityTypes
22 | (s/every alignments/activity-type-spec :kind vector?))
23 |
24 | (s/def ::patterns
25 | (s/every alignments/pattern-spec :kind vector?))
26 |
27 | (s/def ::templates
28 | (s/every alignments/template-spec :kind vector?))
29 |
30 | (s/def ::objectOverrides
31 | (s/every alignments/object-override-spec :kind vector?))
32 |
33 | (def model-spec
34 | (s/keys :opt-un [::personae
35 | ::verbs
36 | ::activities
37 | ::activityTypes
38 | ::patterns
39 | ::templates
40 | ::objectOverrides]))
41 |
42 | (s/def ::models
43 | (s/and (s/every model-spec)
44 | personae/distinct-personae?))
45 |
46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
47 | ;; Validation
48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
49 |
50 | (defn validate-models
51 | [models]
52 | (some->> (s/explain-data ::models models)
53 | (errs/explain-to-map-coll ::models)))
54 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/input/model/personae.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.model.personae
2 | (:require [clojure.spec.alpha :as s]
3 | [com.yetanalytics.datasim.xapi.actor :as actor]
4 | [com.yetanalytics.datasim.input.model.personae.agent :as-alias agent]
5 | [com.yetanalytics.datasim.input.model.personae.group :as-alias group]
6 | [com.yetanalytics.datasim.input.model.personae.role :as-alias role]))
7 |
8 | (defmulti persona-spec* :type)
9 |
10 | (s/def ::agent/id ::actor/actor-ifi)
11 | (s/def ::agent/type #{"Agent"})
12 |
13 | (defmethod persona-spec* "Agent" [_]
14 | (s/keys :req-un [::agent/id ::agent/type]))
15 |
16 | (s/def ::group/id ::actor/actor-ifi)
17 | (s/def ::group/type #{"Group"})
18 |
19 | (defmethod persona-spec* "Group" [_]
20 | (s/keys :req-un [::group/id ::group/type]))
21 |
22 | (s/def ::role/id (s/and string? not-empty))
23 | (s/def ::role/type #{"Role"})
24 |
25 | (defmethod persona-spec* "Role" [_]
26 | (s/keys :req-un [::role/id ::role/type]))
27 |
28 | (def persona-spec
29 | (s/multi-spec persona-spec* :type))
30 |
31 | (defn distinct-personae?
32 | "Are each of the `:personae` in `map-coll` distinct from each other?"
33 | [map-coll]
34 | (let [personaes (map :personae map-coll)]
35 | (= (-> personaes count)
36 | (-> personaes distinct count))))
37 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/input/parameters.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.parameters
2 | "Parameter input specs and parsing."
3 | (:require [clojure.spec.alpha :as s]
4 | [java-time.api :as t]
5 | [xapi-schema.spec :as xs]
6 | [com.yetanalytics.pan.objects.profile :as prof]
7 | [com.yetanalytics.pan.objects.pattern :as pat]
8 | [com.yetanalytics.datasim.util.random :as random]
9 | [com.yetanalytics.datasim.util.errors :as errs])
10 | (:import [clojure.lang ExceptionInfo]
11 | [java.time.zone ZoneRulesException]
12 | [java.time Instant]))
13 |
14 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15 | ;; Specs
16 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
17 |
18 | ;; All options are optional, but everything except `end` will get defaults
19 |
20 | ;; (optional) start of the simulation (inclusive), 8601 stamp
21 | (s/def ::start
22 | ::xs/timestamp)
23 |
24 | ;; (optional) start of the returned statements (if after ::start).
25 | ;; This lets us page through sims to later times. Defaults to ::start
26 | (s/def ::from
27 | ::xs/timestamp)
28 |
29 | ;; (optional) end of the simulation (exclusive), 8601 stamp
30 | (s/def ::end
31 | (s/nilable ::xs/timestamp))
32 |
33 | (defn- timezone-string? [s]
34 | (try (t/zone-id s)
35 | (catch ExceptionInfo exi
36 | (if (= ZoneRulesException (type (ex-cause exi)))
37 | false
38 | (throw exi)))))
39 |
40 | ;; (optional) timezone, defaults to UTC
41 | (s/def ::timezone
42 | (s/and string?
43 | not-empty
44 | timezone-string?))
45 |
46 | ;; Seed is required, but will be generated if not present
47 | (s/def ::seed
48 | int?)
49 |
50 | ;; Max number of statements returned
51 | (s/def ::max
52 | pos-int?)
53 |
54 | ;; Max number of bound restarts before giving up
55 | (s/def ::maxRestarts
56 | pos-int?)
57 |
58 | ;; Restrict Generation to these profile IDs
59 | (s/def ::genProfiles
60 | (s/every ::prof/id))
61 |
62 | ;; Restrict Generation to these pattern IDs
63 | (s/def ::genPatterns
64 | (s/every ::pat/id))
65 |
66 | (defn- ordered-timestamps?
67 | "Are the `start`, `from`, and `end` timestamps ordered properly?"
68 | [{:keys [start from end]}]
69 | (let [start-t (t/instant start)
70 | ?from-t (some->> from t/instant)
71 | ?end-t (some->> end t/instant)]
72 | (and (or (not ?end-t)
73 | (t/before? start-t ?end-t))
74 | (or (not ?end-t)
75 | (not ?from-t)
76 | (t/before? ?from-t ?end-t))
77 | (or (not ?from-t)
78 | (= ?from-t start-t)
79 | (t/before? start-t ?from-t)))))
80 |
81 | (s/def ::parameters
82 | (s/and
83 | (s/keys :req-un [::start
84 | ::timezone
85 | ::seed]
86 | :opt-un [::end
87 | ::from
88 | ::max
89 | ::maxRestarts
90 | ::genProfiles
91 | ::genPatterns])
92 | ordered-timestamps?))
93 |
94 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
95 | ;; Validation
96 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
97 |
98 | (defn validate-parameters
99 | [parameters]
100 | (some->> (s/explain-data ::parameters parameters)
101 | (errs/explain-to-map-coll ::parameters)))
102 |
103 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
104 | ;; Defaults
105 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
106 |
107 | (def utc-timezone "UTC")
108 | (def default-max-restarts 5)
109 |
110 | (defn apply-defaults
111 | "Apply defaults to `params` with the current time and a random seed.
112 | If `params` is not provided simply return the default parameters."
113 | ([]
114 | (apply-defaults {}))
115 | ([{:keys [start from timezone seed maxRestarts] :as params}]
116 | (merge
117 | params
118 | (let [start (or start (.toString (Instant/now)))]
119 | {:start start
120 | :from (or from start)
121 | :timezone (or timezone utc-timezone)
122 | :seed (or seed (random/rand-unbound-int (random/rng)))
123 | :maxRestarts (or maxRestarts default-max-restarts)}))))
124 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/input/personae.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.personae
2 | "Personae input specs and parsing."
3 | (:require [clojure.spec.alpha :as s]
4 | [clojure.walk :as w]
5 | [com.yetanalytics.datasim.xapi.actor :as agent]
6 | [com.yetanalytics.datasim.util.errors :as errs]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | ;; We model the input personae as an xAPI group.
13 | ;; It can be anonymous, but the name may be used in some way.
14 |
15 | ;; If functionality is added to express further groupings we'll have to revise
16 | ;; this strategy.
17 |
18 | ;; Note: We cannot apply xapi-schema specs directly, as xapi-schema restrict
19 | ;; which properties can be in the Group, including the `role` property.
20 | ;; We still use `agent` and `group` spec namespaces from xapi-schema.
21 |
22 | (s/def ::role string?)
23 |
24 | (s/def ::agent
25 | (s/keys :req-un [(or :agent/mbox
26 | :agent/mbox_sha1sum
27 | :agent/openid
28 | :agent/account)]
29 | :opt-un [:agent/name
30 | :agent/objectType
31 | ::role]))
32 |
33 | (s/def ::member
34 | (s/coll-of ::agent :kind vector? :min-count 1 :gen-max 3))
35 |
36 | (s/def ::group
37 | (s/or :anonymous (s/keys :req-un [:group/objectType
38 | (or :group/mbox
39 | :group/mbox_sha1sum
40 | :group/openid
41 | :group/account)]
42 | :opt-un [:group/name ::member])
43 | :identified (s/keys :req-un [:group/objectType ::member]
44 | :opt-un [:group/name])))
45 |
46 | (defn- remove-nil-vals
47 | "Remove nil values from an associative structure. Does not recurse."
48 | [m]
49 | (reduce-kv
50 | (fn [m* k v] (cond-> m* (some? v) (assoc k v)))
51 | {}
52 | m))
53 |
54 | ;; An open-validating group spec, ignores extra nils
55 | (s/def ::personae
56 | (s/and (s/conformer remove-nil-vals)
57 | (s/conformer w/keywordize-keys w/stringify-keys)
58 | ::group))
59 |
60 | (defn- distinct-member-ids?
61 | [personaes]
62 | (let [member-ids (->> personaes
63 | (map :member)
64 | (apply concat)
65 | (map agent/actor-ifi))]
66 | (= (-> member-ids count)
67 | (-> member-ids distinct count))))
68 |
69 | (s/def ::personae-array
70 | (s/and (s/every ::personae :min-count 1 :into [])
71 | distinct-member-ids?))
72 |
73 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
74 | ;; Validation
75 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
76 |
77 | (defn validate-personae
78 | [personae]
79 | (some->> (s/explain-data ::personae personae)
80 | (errs/explain-to-map-coll ::personae)))
81 |
82 | (defn validate-personae-array
83 | [personae-array]
84 | (some->> (s/explain-data ::personae-array personae-array)
85 | (errs/explain-to-map-coll ::personae-array)))
86 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/input/profile.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.profile
2 | "Profile input parsing."
3 | (:require [clojure.spec.alpha :as s]
4 | [clojure.string :as cstr]
5 | [com.yetanalytics.pan :as pan]
6 | [com.yetanalytics.pan.objects.profile :as pan-profile]
7 | [com.yetanalytics.datasim.util.errors :as errs]))
8 |
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 | ;; Specs
11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12 |
13 | (s/def ::profiles
14 | (s/every ::pan-profile/profile :min-count 1))
15 |
16 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
17 | ;; Validation
18 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
19 |
20 | (defn validate-profile
21 | [{:keys [id] :as profile}]
22 | (some->> (pan/validate-profile profile
23 | :syntax? true
24 | :result :type-path-string)
25 | (errs/type-path-string-m->map-coll id)))
26 |
27 | (defn validate-profiles
28 | [profiles]
29 | (if (vector? profiles)
30 | (let [prof-errs (pan/validate-profile-coll profiles
31 | :syntax? true
32 | :pattern-rels? true
33 | :result :type-path-string)]
34 | (errs/type-path-string-ms->map-coll (map :id profiles)
35 | prof-errs))
36 | ;; TODO: Something more solid/less hacky, particularly in the Pan lib itself
37 | [{:path [::profiles]
38 | :text "Profiles must be a vector!"
39 | :id [::profiles]}]))
40 |
41 | ;; Helpers to `input/validate-pattern-filters`
42 |
43 | (defn- validate-pattern-filters-emsg
44 | [pattern-id pattern-id-set]
45 | (format
46 | "Pattern ID %s is not among primary patterns in provided profiles: %s"
47 | pattern-id
48 | (cstr/join \, pattern-id-set)))
49 |
50 | (defn- validate-profile-fitlers-emsg
51 | [profile-id profile-id-set]
52 | (format "Profile ID %s is not one of provided profiles: %s"
53 | profile-id
54 | (cstr/join \, profile-id-set)))
55 |
56 | (defn validate-pattern-filters
57 | [profiles gen-patterns]
58 | (let [pattern-id-set (->> profiles
59 | (mapcat :patterns)
60 | (keep (fn [{:keys [id primary]}] (when primary id)))
61 | (into #{}))]
62 | (for [[idx pattern-id] (map-indexed vector gen-patterns)
63 | :when (not (contains? pattern-id-set pattern-id))]
64 | {:id (str "parameters-genPatterns-" idx)
65 | :path [:parameters :genPatterns idx]
66 | :text (validate-pattern-filters-emsg pattern-id pattern-id-set)})))
67 |
68 | (defn validate-profile-filters
69 | [profiles gen-profiles]
70 | (let [profile-id-set (->> profiles (map :id) (into #{}))]
71 | (for [[idx profile-id] (map-indexed vector gen-profiles)
72 | :when (not (contains? profile-id-set profile-id))]
73 | {:id (str "parameters-genProfiles-" idx)
74 | :path [:parameters :genProfiles idx]
75 | :text (validate-profile-fitlers-emsg profile-id profile-id-set)})))
76 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/model/periods.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.model.periods
2 | (:require [clojure.spec.alpha :as s]
3 | [java-time.api :as t]
4 | [com.yetanalytics.datasim.input.model.alignments :as align]
5 | [com.yetanalytics.datasim.util.random :as random]
6 | [com.yetanalytics.datasim.model.bounds :as bounds]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | (s/def ::min nat-int?)
13 |
14 | (s/def ::mean pos-int?)
15 |
16 | (s/def ::fixed pos-int?)
17 |
18 | (s/def ::period
19 | (s/or :variable (s/keys :req-un [::min ::mean]
20 | :opt-un [::bounds/bounds])
21 | :fixed (s/keys :req-un [::fixed]
22 | :opt-un [::bounds/bounds])))
23 |
24 | (s/def ::periods
25 | (s/every ::period))
26 |
27 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
28 | ;; Constants
29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
30 |
31 | (def ms-per-second
32 | 1000)
33 |
34 | (def ms-per-minute
35 | 60000)
36 |
37 | (def ms-per-hour
38 | 3600000)
39 |
40 | (def ms-per-day
41 | 86400000)
42 |
43 | (def ms-per-week
44 | 604800000)
45 |
46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
47 | ;; Compilation
48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
49 |
50 | (defn- convert-time
51 | "Convert time `t` into milliseconds based on the time `unit`. Coerces
52 | any doubles into integers."
53 | [t unit]
54 | (long (case unit
55 | :millis t
56 | :seconds (* t ms-per-second)
57 | :minutes (* t ms-per-minute)
58 | :hours (* t ms-per-hour)
59 | :days (* t ms-per-day)
60 | :weeks (* t ms-per-week))))
61 |
62 | (defn- convert-period
63 | [{:keys [min mean fixed unit bounds]}]
64 | (let [unit* (or (some-> unit keyword) :minutes)
65 | mean* (or (some-> mean (convert-time unit*)) ms-per-minute)
66 | min* (or (some-> min (convert-time unit*)) 0)
67 | fixed* (some-> fixed (convert-time unit*))
68 | bounds* (some-> bounds bounds/convert-bounds)]
69 | (cond-> {}
70 | fixed* (assoc :fixed fixed*)
71 | (not fixed*) (assoc :min min*
72 | :mean mean*)
73 | bounds* (assoc :bounds bounds*))))
74 |
75 | (s/fdef convert-periods
76 | :args (s/cat :periods ::align/periods)
77 | :ret ::periods)
78 |
79 | (defn convert-periods
80 | [periods]
81 | (mapv convert-period periods))
82 |
83 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
84 | ;; Runtime
85 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
86 |
87 | (defn- generate-period
88 | [rng {:keys [mean min fixed]}]
89 | (or fixed
90 | (let [rate (/ 1.0 mean)
91 | t-diff (long (random/rand-exp rng rate))]
92 | (+ min t-diff))))
93 |
94 | (defn- add-period
95 | [date-time rng period]
96 | (t/plus date-time (t/millis (generate-period rng period))))
97 |
98 | (s/fdef add-periods
99 | :args (s/cat :date-time t/local-date-time?
100 | :rng ::rng
101 | :periods ::periods)
102 | :ret t/local-date-time?)
103 |
104 | (defn add-periods
105 | "Add a random amount of milliseonds to `date-time` based on the first map of
106 | valid parameter map in `periods`. A valid map either has no time bounds or
107 | bounds that satisfy `date-time`. The millis amount is an exponentially
108 | distributed random variable with `period` parameters `:mean` and `:min`.
109 | The generated sequence that uses these periodic date-times will thus occur
110 | as a Poisson random process."
111 | [date-time rng periods]
112 | (let [periods (or periods
113 | [{:min 0
114 | :mean ms-per-minute}])
115 | some-bound (fn [{:keys [bounds] :as period}]
116 | (when (bounds/bounded-time? bounds date-time) period))]
117 | (if-some [period (some some-bound periods)]
118 | (add-period date-time rng period)
119 | (throw (ex-info "Timestamp does not satisfy any period `bounds`; no default period present."
120 | {:type ::outside-period-bound
121 | :periods periods
122 | :date-time date-time})))))
123 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/util/async.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.async
2 | "`core.async` utilities, for parallel generation."
3 | (:require [clojure.core.async :as a]
4 | [clojure.core.async.impl.protocols :as ap]))
5 |
6 | (defn chan?
7 | "Is `x` a proper `core.async` channel?"
8 | [x]
9 | (satisfies? ap/Channel x))
10 |
11 | (defn heads-chans
12 | "Get a sorted map of `[head-val ]`, where `channel` is open."
13 | ([chans]
14 | (heads-chans compare chans))
15 | ([compfn chans]
16 | (a/go-loop [chan-set (set chans)
17 | head-chan-m (sorted-map-by compfn)]
18 | (if-let [chan-seq (seq chan-set)]
19 | (let [[head chan] (a/alts! chan-seq)]
20 | (recur (disj chan-set chan)
21 | (cond-> head-chan-m
22 | head (assoc head chan))))
23 | head-chan-m))))
24 |
25 | (defn sequence-messages
26 | "Given an output channel `out-chan`, comparator `compfn`, and one or more
27 | channels `chans` containing ordered values, returns a channel that will
28 | receive an ordered sequence of messages, closing when the last channel
29 | closes."
30 | ([chans]
31 | (sequence-messages (a/chan) compare chans))
32 | ([out-chan compfn chans]
33 | (a/go-loop [head-chan-m (a/! out-chan min-head)
36 | (let [?next-h (a/ head-chan-m
38 | (dissoc min-head)
39 | (cond->
40 | ?next-h (assoc ?next-h chan))))))
41 | (a/close! out-chan)))
42 | out-chan))
43 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/util/errors.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.errors
2 | (:require [clojure.spec.alpha :as s]))
3 |
4 | (defn explain-to-map-coll
5 | "Convert spec error data for personae, models, and parameters
6 | into a map coll acceptable to the Datasim UI."
7 | [spec-kw spec-ed]
8 | (->> (::s/problems spec-ed)
9 | (map-indexed
10 | (fn [idx problem]
11 | (let [epath (into [spec-kw] (:in problem))
12 | estr (-> problem
13 | (assoc ::s/problems [problem])
14 | s/explain-printer
15 | with-out-str)
16 | eid (str (name spec-kw) "-" idx)]
17 | {:path epath
18 | :text estr
19 | :id eid})))
20 | vec))
21 |
22 | (defn type-path-string-m->map-coll
23 | "Convert Pan's `type-path-string-map`, using `profile-id`, into a
24 | map coll acceptable to the Datasim UI. Returns `nil` if there are
25 | no errors."
26 | [profile-id type-path-string-map]
27 | (->> type-path-string-map
28 | (reduce-kv (fn [acc etype epath-estr-m]
29 | (reduce-kv (fn [acc* epath estr]
30 | (conj acc* {:path (into [profile-id etype]
31 | epath)
32 | :text estr}))
33 | acc
34 | epath-estr-m))
35 | [])
36 | (map-indexed (fn [idx emap]
37 | (assoc emap :id (str "profile-" idx))))
38 | vec
39 | not-empty))
40 |
41 | (defn type-path-string-ms->map-coll
42 | "Convert Pan's `type-path-string-maps`, using `profile-ids`, into a
43 | map coll acceptable to the Datasim UI. Returns `nil` if there are
44 | not errors."
45 | [profile-ids type-path-string-maps]
46 | (->> type-path-string-maps
47 | (map (fn [profile-id etype-epath-estr-m]
48 | [profile-id etype-epath-estr-m])
49 | profile-ids)
50 | (reduce (fn [acc [prof-id etype-epath-estr-m]]
51 | (reduce-kv (fn [acc* etype epath-estr-m]
52 | (reduce-kv (fn [acc** epath estr]
53 | (let [epath* (into [prof-id etype]
54 | epath)]
55 | (conj acc**
56 | {:path epath*
57 | :text estr})))
58 | acc*
59 | epath-estr-m))
60 | acc
61 | etype-epath-estr-m))
62 | [])
63 | (map-indexed (fn [idx emap]
64 | (assoc emap :id (str "profiles-" idx))))
65 | vec
66 | not-empty))
67 |
68 | (def bar
69 | (apply str (repeat 80 \=)))
70 |
71 | (defn map-coll->strs
72 | "Form a list of CLI error strings from an error map coll."
73 | [map-coll]
74 | (for [{:keys [id path text]} map-coll]
75 | (format
76 | "%s\nINPUT ERROR: %s\n%s\npath: %s\n\n%s"
77 | bar
78 | id
79 | bar
80 | (pr-str path)
81 | text)))
82 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/util/io.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.io
2 | (:require [clojure.java.io :as io]
3 | [clojure.string :as cstr]
4 | [cheshire.core :as json])
5 | (:import [java.io IOException]))
6 |
7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8 | ;; Exceptions
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 |
11 | (defn- throw-parse-error [cause-exn]
12 | (throw (ex-info "Parse Error"
13 | {:type ::parse-error}
14 | cause-exn)))
15 |
16 | (defn- throw-unparse-error [cause-exn]
17 | (throw (ex-info "Unparse Error"
18 | {:type ::unparse-error}
19 | cause-exn)))
20 |
21 | (defn- throw-io-error [location cause-exn]
22 | (throw (ex-info "I/O Error"
23 | {:type ::io-error
24 | :location location}
25 | cause-exn)))
26 |
27 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
28 | ;; Keyword Key Functions
29 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
30 |
31 | (defn- keywordize-json-key
32 | [k]
33 | (keyword (cond-> k
34 | (= \@ (first k))
35 | (cstr/replace-first \@ \_))))
36 |
37 | (defn- stringify-edn-key
38 | [k]
39 | (let [named-key (name k)]
40 | (cond-> named-key
41 | (= \_ (first named-key))
42 | (cstr/replace-first \_ \@))))
43 |
44 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
45 | ;; JSON I/O Functions
46 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
47 |
48 | ;; TODO: Also have a `key-fn?` kwarg here?
49 | (defn read-json-location
50 | "Reads in a file from the given `location` and parses it into EDN. If
51 | the data being read is an array or sequence, return all the data
52 | as a seq."
53 | [location]
54 | (try (with-open [r (io/reader location)]
55 | (try (doall (json/parse-stream r keywordize-json-key))
56 | (catch Exception e
57 | (throw-parse-error e))))
58 | (catch IOException e
59 | (throw-io-error location e))))
60 |
61 | (defn write-json
62 | "Write the contents of `data` to `writer`, which can be returned by
63 | `clojure.java.io/writer`. Applies `stringify-edn-key` if `key-fn?` is
64 | `true`."
65 | [data writer & {:keys [key-fn?] :or {key-fn? true}}]
66 | (try (if key-fn?
67 | (json/generate-stream data writer {:key-fn stringify-edn-key})
68 | (json/generate-stream data writer))
69 | (catch Exception e
70 | (throw-unparse-error e))))
71 |
72 | ;; Avoid `with-open` to prevent stdout/stderr writers from being closed
73 |
74 | (defn write-json-stdout
75 | "Write the contents of `data` to standard output. Applies `stringify-edn-key`
76 | if `key-fn?` is `true`."
77 | [data & {:keys [key-fn?] :or {key-fn? true}}]
78 | (let [w (io/writer *out*)]
79 | (write-json data w :key-fn? key-fn?)
80 | (.write w "\n")
81 | (.flush w)))
82 |
83 | (defn write-json-stderr
84 | "Write the contents of `data` to standard error. Applies `stringify-edn-key`
85 | if `key-fn?` is `true`."
86 | [data & {:keys [key-fn?] :or {key-fn? true}}]
87 | (let [w (io/writer *err*)]
88 | (write-json data w :key-fn? key-fn?)
89 | (.write w "\n")
90 | (.flush w)))
91 |
92 | ;; Use `with-open` to ensure that the file writer gets closed
93 |
94 | (defn write-json-file
95 | "Write the contents of `data` to the file `location`; a file will be
96 | created if it does not exist. Stringifies keyword keys. Applies
97 | `stringify-edn-key` if `key-fn?` is `true`."
98 | [data location & {:keys [key-fn?] :or {key-fn? true}}]
99 | (let [file (io/file location)]
100 | (io/make-parents file)
101 | (with-open [w (io/writer location)]
102 | (write-json data w :key-fn? key-fn?))))
103 |
104 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
105 | ;; String I/O Functions
106 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
107 |
108 | (defn println-coll
109 | "Print each string in `str-coll` followed by a newline to stdout,
110 | flushing the output buffer after each line."
111 | [str-coll]
112 | (doseq [s str-coll]
113 | (println s)
114 | (flush)))
115 |
116 | (defn println-err-coll
117 | "Print each string in `err-str-coll` followed by a newline to stderr,
118 | flushing the output buffer after each line."
119 | [err-str-coll]
120 | (binding [*out* *err*]
121 | (println-coll err-str-coll)))
122 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/util/sequence.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.sequence)
2 |
3 | (defn seq-sort
4 | "Given a key-fn (that must return a number) and a vector of seqs, return the
5 | items from all seqs ordered by that, ascending.
6 | If incoming seqs are monotonic, the output is as well."
7 | [key-fn seqs]
8 | (lazy-seq
9 | (when-let [seqs' (not-empty
10 | (into []
11 | (keep not-empty
12 | seqs)))]
13 | (let [[idx [head & rest-seq]]
14 | (apply min-key (comp key-fn
15 | first
16 | second)
17 | (map-indexed vector
18 | seqs'))]
19 | (cons head
20 | (seq-sort
21 | key-fn
22 | (assoc seqs' idx rest-seq)))))))
23 |
24 | ;; https://clojuredocs.org/clojure.core/chunk#example-5c9cebc3e4b0ca44402ef6ec
25 | (defn re-chunk
26 | "takes a sequence (already chunked or not)
27 | and produces another sequence with different chunking size."
28 | [n xs]
29 | (lazy-seq
30 | (when-let [s (seq (take n xs))]
31 | (let [cb (chunk-buffer n)]
32 | (doseq [x s] (chunk-append cb x))
33 | (chunk-cons (chunk cb) (re-chunk n (drop n xs)))))))
34 |
35 | (comment
36 | (require '[com.yetanalytics.datasim.util.random :as r])
37 |
38 | (defn rand-monotonic-seq
39 | [seed & [start]]
40 | (let [rng (r/seed-rng seed)]
41 | (rest (iterate #(+ % (r/rand rng)) (or start 0)))))
42 |
43 | (let [ss (seq-sort
44 | identity
45 | (into []
46 | (for [n (range 20)]
47 | (take 10 (rand-monotonic-seq n)))))]
48 | (= (sort ss) ss)) ;; => true
49 | )
50 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/xapi/actor.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.xapi.actor
2 | "Utilities for Agents and Groups (collectively known as Actors)."
3 | (:require [clojure.string :as cstr]
4 | [clojure.spec.alpha :as s]
5 | [clojure.spec.gen.alpha :as sgen]
6 | [xapi-schema.spec :as xs]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | (def ^:private ifi-prefix-gen
13 | (sgen/elements ["mbox::" "mbox_sha1sum::" "account::" "openid::"]))
14 |
15 | (def ^:private ifi-body-gen
16 | (sgen/such-that not-empty (sgen/string-ascii)))
17 |
18 | (s/def ::actor-ifi
19 | (s/with-gen
20 | (s/and string?
21 | not-empty
22 | (fn [s]
23 | (or (cstr/starts-with? s "mbox::")
24 | (cstr/starts-with? s "account::")
25 | (cstr/starts-with? s "mbox_sha1sum::")
26 | (cstr/starts-with? s "openid::"))))
27 | ;; Monads are fun!
28 | ;; This composes a the two IFI generators together to create a
29 | ;; (str ifi-prefix ifi-body) generator
30 | #(sgen/bind ifi-prefix-gen
31 | (fn [ifi-prefix]
32 | (sgen/bind ifi-body-gen
33 | (fn [ifi-body]
34 | (sgen/return (str ifi-prefix ifi-body))))))))
35 |
36 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
37 | ;; Functions
38 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
39 |
40 | (s/fdef actor-ifi
41 | :args (s/cat :actor ::xs/actor)
42 | :ret ::actor-ifi)
43 |
44 | (defn actor-ifi
45 | "Return a string representing the IFI of an Agent or Group. Will be prefixed
46 | with the IFI property and two colons `::`.
47 |
48 | For accounts, the homepage will precede the name, and they will be delimited
49 | by a comma as so: `account::https://foo.bar,baz`.
50 |
51 | If an IFI cannot be found, returns `nil`."
52 | [{:keys [mbox
53 | mbox_sha1sum
54 | openid]
55 | {:keys [name homePage]} :account}]
56 | (or (and mbox
57 | (format "mbox::%s" mbox))
58 | (and name homePage
59 | (format "account::%s,%s"
60 | homePage name))
61 | (and mbox_sha1sum
62 | (format "mbox_sha1sum::%s" mbox_sha1sum))
63 | (and openid
64 | (format "openid::%s" openid))))
65 |
66 | (s/fdef groups->agent-group-ifi-map
67 | :args (s/cat :group-coll (s/every ::xs/identified-group))
68 | :ret (s/map-of ::actor-ifi ::actor-ifi))
69 |
70 | (defn groups->agent-group-ifi-map
71 | "Convert `group-coll` into a map from member agent IFIs to group IFIs."
72 | [group-coll]
73 | (reduce
74 | (fn [m {agents :member :as personae}]
75 | (let [group-ifi (actor-ifi personae)]
76 | (reduce
77 | (fn [m* actor]
78 | (let [agent-ifi (actor-ifi actor)]
79 | (assoc m* agent-ifi group-ifi)))
80 | m
81 | agents)))
82 | {}
83 | group-coll))
84 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/xapi/profile/extension.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.xapi.profile.extension
2 | "Creation of `extension-spec-map` for Profile compilation."
3 | (:require [clojure.spec.alpha :as s]
4 | [com.yetanalytics.schemer :as schemer]
5 | [com.yetanalytics.pan.axioms :as ax]
6 | [com.yetanalytics.datasim.xapi.profile :as-alias profile]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | (def json-schema-spec
13 | (s/or :keyword (s/and keyword? s/get-spec)
14 | :spec s/spec?))
15 |
16 | (s/def ::activity (s/map-of ::ax/iri (s/nilable json-schema-spec)))
17 | (s/def ::context (s/map-of ::ax/iri (s/nilable json-schema-spec)))
18 | (s/def ::result (s/map-of ::ax/iri (s/nilable json-schema-spec)))
19 |
20 | (s/def ::extension-spec-map
21 | (s/keys :req-un [::activity ::context ::result]))
22 |
23 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
24 | ;; Functions
25 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
26 |
27 | (s/fdef create-extension-spec-map
28 | :args (s/cat :type-iri-map ::profile/type-iri-map)
29 | :ret ::extension-spec-map)
30 |
31 | (defn create-extension-spec-map
32 | [type-iri-map]
33 | (let [ext->spec (fn [ext]
34 | (some->> ext :inlineSchema (schemer/schema->spec nil)))
35 | reduce-exts (fn [ext-map]
36 | (update-vals ext-map ext->spec))
37 | act-iri-map (get type-iri-map "ActivityExtension")
38 | ctx-iri-map (get type-iri-map "ContextExtension")
39 | res-iri-map (get type-iri-map "ResultExtension")]
40 | {:activity (reduce-exts act-iri-map)
41 | :context (reduce-exts ctx-iri-map)
42 | :result (reduce-exts res-iri-map)}))
43 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/xapi/profile/template.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.xapi.profile.template
2 | "Creation of `statement-base-map` and `parsed-rules-map` for Profile
3 | compilation."
4 | (:require [clojure.spec.alpha :as s]
5 | [com.yetanalytics.pan.objects.template :as template]
6 | [com.yetanalytics.datasim.xapi.rule :as rule]
7 | [com.yetanalytics.datasim.xapi.profile :as-alias profile]))
8 |
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 | ;; Specs
11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12 |
13 | (s/def ::template ::template/template)
14 |
15 | (s/def ::statement-base-map
16 | (s/map-of ::template/id map?))
17 |
18 | (s/def ::parsed-rules-map
19 | (s/map-of ::template/id (s/every ::rule/parsed-rule)))
20 |
21 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
22 | ;; Statement Base
23 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
24 |
25 | (defn- activity-type->activity-base
26 | [activity-type]
27 | {"definition" {"type" activity-type}})
28 |
29 | (defn- usage-type->attachment-base
30 | [attachment-usage-type]
31 | {"usageType" attachment-usage-type})
32 |
33 | (s/fdef template->statement-base
34 | :args (s/cat :template ::template/template)
35 | :ret map?)
36 |
37 | (defn template->statement-base
38 | "Form the base of a statement from the Determining Properties of
39 | the Template. Elements of array-valued properties (the context
40 | activity types and the attachment usage types) are added in order."
41 | [{profile-version-id :inScheme
42 | verb-id :verb
43 | object-activity-type :objectActivityType
44 | category-activity-types :contextCategoryActivityType
45 | grouping-activity-types :contextGroupingActivityType
46 | parent-activity-types :contextParentActivityType
47 | other-activity-types :contextOtherActivityType
48 | attachment-usage-types :attachmentUsageType
49 | ;; TODO: StatementRef properties
50 | ;; object-statement-ref :objectStatementRefTemplate
51 | ;; context-statement-ref :contextStatementRefTemplate
52 | }]
53 | (cond-> {}
54 | verb-id
55 | (assoc-in ["verb" "id"] verb-id)
56 | object-activity-type
57 | (assoc-in ["object" "definition" "type"] object-activity-type)
58 | category-activity-types
59 | (assoc-in ["context" "contextActivities" "category"]
60 | (mapv activity-type->activity-base category-activity-types))
61 | grouping-activity-types
62 | (assoc-in ["context" "contextActivities" "grouping"]
63 | (mapv activity-type->activity-base grouping-activity-types))
64 | parent-activity-types
65 | (assoc-in ["context" "contextActivities" "parent"]
66 | (mapv activity-type->activity-base parent-activity-types))
67 | other-activity-types
68 | (assoc-in ["context" "contextActivities" "other"]
69 | (mapv activity-type->activity-base other-activity-types))
70 | attachment-usage-types
71 | (assoc-in ["attachments"]
72 | (mapv usage-type->attachment-base attachment-usage-types))
73 | profile-version-id ; always true
74 | (update-in ["context" "contextActivities" "category"]
75 | (fnil conj [])
76 | {"id" profile-version-id})))
77 |
78 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
79 | ;; Statement Rule Application
80 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
81 |
82 | (s/fdef template->parsed-rules
83 | :args (s/cat :type-iri-map ::type-iri-map
84 | :activity-map ::activity-map
85 | :template ::template/template)
86 | :ret (s/every ::rule/parsed-rule))
87 |
88 | (defn template->parsed-rules
89 | "Return a collection of parsed rules derived from the template `rules`.
90 | Uses the object Determining Properties to assist with rule parsing."
91 | [{object-activity-type :objectActivityType
92 | object-statement-ref :objectStatementRefTemplate
93 | rules :rules}]
94 | (cond
95 | object-activity-type
96 | (rule/parse-rules :activity-type rules)
97 | object-statement-ref
98 | (rule/parse-rules :statement-ref rules)
99 | :else
100 | (rule/parse-rules rules)))
101 |
102 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
103 | ;; Profile Templates -> Statement Base + Parsed Rules
104 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
105 |
106 | (s/fdef create-statement-base-map
107 | :args (s/cat :type-iri-map ::profile/type-iri-map)
108 | :ret ::statement-base-map)
109 |
110 | (defn create-statement-base-map
111 | "Given Statement Templates in `type-iri-map`, return a map from those
112 | templates' IDs to the base xAPI Statements they form from their
113 | Determining Properties and inScheme."
114 | [type-iri-map]
115 | (-> type-iri-map
116 | (get "StatementTemplate")
117 | (update-vals template->statement-base)))
118 |
119 | (s/fdef create-parsed-rules-map
120 | :args (s/cat :type-iri-map? ::profile/type-iri-map)
121 | :ret ::parsed-rules-map)
122 |
123 | (defn create-parsed-rules-map
124 | "Given Statement Templates in `type-iri-map`, return a map from those
125 | templates' IDs to those their parsed rules"
126 | [type-iri-map]
127 | (-> type-iri-map
128 | (get "StatementTemplate")
129 | (update-vals template->parsed-rules)))
130 |
131 | (s/fdef update-parsed-rules-map
132 | :args (s/cat :profile-map ::profile/profile-map
133 | :parsed-rules-map ::parsed-rules-map)
134 | :ret ::parsed-rules-map)
135 |
136 | (defn update-parsed-rules-map
137 | "Use information from `profile-map` to complete the rules in
138 | `parsed-rules-map` by adding additional valuesets or spec generators."
139 | [profile-map parsed-rules-map]
140 | (update-vals parsed-rules-map
141 | (partial rule/add-rules-valuegen profile-map)))
142 |
--------------------------------------------------------------------------------
/src/main/com/yetanalytics/datasim/xapi/profile/verb.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.xapi.profile.verb
2 | "Creation of `verb-map` for Profile compilation."
3 | (:require [clojure.spec.alpha :as s]
4 | [clojure.walk :as w]
5 | [xapi-schema.spec :as xs]
6 | [com.yetanalytics.datasim.xapi.profile :as-alias profile]))
7 |
8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9 | ;; Specs
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 |
12 | (s/def ::verb-map
13 | (s/map-of ::xs/iri ::xs/verb))
14 |
15 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
16 | ;; Functions
17 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
18 |
19 | (defn- profile->statement-verb
20 | [{:keys [id prefLabel]}]
21 | {"id" id
22 | "display" (w/stringify-keys prefLabel)})
23 |
24 | (s/fdef create-verb-map
25 | :args (s/cat :type-iri-map ::profile/type-iri-map)
26 | :ret ::verb-map)
27 |
28 | (defn create-verb-map
29 | "Create a map of verb IDs to Statement verbs out of Profile Verbs from
30 | `type-iri-map`."
31 | [type-iri-map]
32 | (-> type-iri-map (get "Verb") (update-vals profile->statement-verb)))
33 |
--------------------------------------------------------------------------------
/src/onyx/com/yetanalytics/datasim/onyx/aeron_media_driver.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.aeron-media-driver
2 | (:require [clojure.core.async :refer [chan (System/getenv "ONYX_PROFILE") keyword)]
7 | (aero/read-config (io/resource "onyx_config.edn")
8 | {:profile (or profile :default)})))
9 |
--------------------------------------------------------------------------------
/src/onyx/com/yetanalytics/datasim/onyx/http.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.http
2 | (:require [cheshire.core :as json]))
3 |
4 | (defn after-read-batch
5 | "Encode statements and add request"
6 | [event {lrs-request ::lrs-request
7 | :as lifecycle}]
8 | (update event
9 | :onyx.core/batch
10 | (partial map
11 | (fn [{:keys [statements]
12 | :as segment}]
13 | (merge segment
14 | (assoc-in lrs-request
15 | [:args :body]
16 | ;; TODO: Buffr
17 | (json/generate-string statements)))))))
18 |
19 | (defn after-batch
20 | "Tear things down"
21 | [event lifecycle]
22 | (update event
23 | :onyx.core/batch
24 | (partial map
25 | (fn [{:keys [statements]
26 | :as segment}]
27 | (-> segment
28 | (dissoc :statements)
29 | (update :args dissoc :body :basic-auth)
30 | (assoc :statement-count (count statements)))))))
31 |
32 | (def out-calls
33 | {:lifecycle/after-read-batch after-read-batch
34 | :lifecycle/after-batch after-batch})
35 |
36 | (defn post-success?
37 | [{:keys [status error]}]
38 | (and (= 200 status)
39 | (not error)))
40 |
--------------------------------------------------------------------------------
/src/onyx/com/yetanalytics/datasim/onyx/peer.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.peer
2 | (:require onyx.plugin.http-output
3 | onyx.api
4 | com.yetanalytics.datasim.onyx.sim
5 | com.yetanalytics.datasim.onyx.http
6 | com.yetanalytics.datasim.onyx.job
7 | [com.yetanalytics.datasim.onyx.config :as config]))
8 |
9 | (defn start-peer!
10 | "Launch a peer, optionally pass config"
11 | [& [config-override]]
12 | (let [{:keys [env-config
13 | peer-config
14 | launch-config]
15 | {:keys [n-vpeers]} :launch-config} (or config-override
16 | (config/get-config))
17 |
18 | env (onyx.api/start-env env-config)
19 | ;; start peer group
20 | peer-group (onyx.api/start-peer-group peer-config)
21 |
22 | v-peers (onyx.api/start-peers n-vpeers peer-group)]
23 |
24 |
25 | (.addShutdownHook (Runtime/getRuntime)
26 | (Thread. ^Runnable #(do
27 | (onyx.api/shutdown-peers v-peers)
28 | (onyx.api/shutdown-peer-group peer-group)
29 | (onyx.api/shutdown-env env))))))
30 |
--------------------------------------------------------------------------------
/src/onyx/com/yetanalytics/datasim/onyx/repl.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.repl
2 | "Cluster repl"
3 | (:require clojure.main
4 | rebel-readline.core
5 | rebel-readline.clojure.line-reader
6 | rebel-readline.clojure.service.local
7 | rebel-readline.clojure.main))
8 |
9 | (defn repl!
10 | []
11 | (rebel-readline.core/with-line-reader
12 | (rebel-readline.clojure.line-reader/create
13 | (rebel-readline.clojure.service.local/create))
14 | (clojure.main/repl
15 | :prompt (fn []) ;; prompt is handled by line-reader
16 | :read (rebel-readline.clojure.main/create-repl-read))))
17 |
--------------------------------------------------------------------------------
/src/onyx/com/yetanalytics/datasim/onyx/util.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.onyx.util
2 | (:require [com.yetanalytics.datasim.input :as input]
3 | [cheshire.core :as json]
4 | [byte-streams :as bs]))
5 |
6 |
7 | (defn round-robin
8 | "Given a number of partitions/buckets and a sequence, return a set of sets
9 | dividing up xs"
10 | [num-parts
11 | xs]
12 | (set
13 | (remove empty?
14 | (reduce
15 | (fn [vs [x idx]]
16 | (update vs idx conj x))
17 | (into []
18 | (repeat num-parts #{}))
19 | (map
20 | vector
21 | xs
22 | (cycle (range num-parts)))))))
23 |
24 | (defn batch->smile
25 | "Convert a batch of segments to SMILE data"
26 | ^bytes [segments]
27 | (json/generate-smile
28 | (mapcat :statements
29 | segments)))
30 |
31 | (defn batch->json
32 | "Convert a batch of segments to JSON bytes"
33 | ^bytes [segments]
34 | (bs/to-byte-array
35 | (json/generate-string
36 | (mapcat :statements
37 | segments))))
38 |
39 | (defn override-max!
40 | [input mo]
41 | (assoc-in input [:parameters :max] mo))
42 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/cli_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.cli-test
2 | "Integration tests for the DATASIM CLI."
3 | (:require [clojure.test :refer [deftest testing is]]
4 | [clojure.string :as cstr]
5 | [org.httpkit.client :as http]
6 | [clj-test-containers.core :as tc]
7 | [com.yetanalytics.datasim.input :as input]
8 | [com.yetanalytics.datasim.cli.input :as cli-input]
9 | [com.yetanalytics.datasim.cli.generate :as cli-gen]
10 | [com.yetanalytics.datasim.test-containers :as ds-tc]))
11 |
12 | (def json-post-header
13 | {"X-Experience-Api-Version" "1.0.3"
14 | "Content-Type" "application/json"})
15 |
16 | (deftest validate-input-test
17 | (testing "validate-input subcommand"
18 | (cli-input/validate-input!
19 | ["-p" "dev-resources/profiles/cmi5/fixed.json"
20 | "-a" "dev-resources/personae/simple.json"
21 | "-m" "dev-resources/models/simple.json"
22 | "-o" "dev-resources/parameters/simple.json"
23 | "-v" "dev-resources/input/simple2.json"])
24 | (let [input* (input/from-location
25 | :input :json
26 | "dev-resources/input/simple2.json")]
27 | (is (nil? (input/validate :input input*))))))
28 |
29 | (deftest generate-test
30 | (let [cont (tc/start! ds-tc/validate-server-container)
31 | host (:host cont)
32 | port (get (:mapped-ports cont) 8080)]
33 | (testing "generate subcommand"
34 | (let [results
35 | (with-out-str
36 | (cli-gen/generate!
37 | ["-i" "dev-resources/input/simple.json"]))]
38 | (is (string? results))
39 | (is (every?
40 | (fn [stmt-str]
41 | (let [validate-res
42 | #_{:clj-kondo/ignore [:unresolved-var]}
43 | @(http/post
44 | (format "http://%s:%d/statements" host port)
45 | {:headers json-post-header
46 | :body stmt-str
47 | :as :stream})]
48 | (= 204 (:status validate-res))))
49 | (take 25 (cstr/split-lines results))))))
50 | (testing "generate-post subcommand - sync"
51 | (let [results
52 | (cli-gen/generate-post!
53 | ["-i" "dev-resources/input/simple.json"
54 | "-E" (format "http://%s:%d" host port)
55 | "-B" "1"
56 | "-L" "1"
57 | "--no-async"])]
58 | ;; Errors would indicate 4xx response from Persephone server
59 | (is (nil? (:errors results)))))
60 | (testing "generate-post subcommand - async"
61 | (let [cont (tc/start! ds-tc/validate-server-container)
62 | host (:host cont)
63 | port (get (:mapped-ports cont) 8080)
64 | res (cli-gen/generate-post!
65 | ["-i" "dev-resources/input/simple.json"
66 | "-E" (format "http://%s:%d" host port)
67 | "-B" "1"
68 | "-L" "1"
69 | "--async"])]
70 | (is (nil? (:errors res)))))
71 | (tc/stop! cont)))
72 |
73 | (deftest generate-test-2
74 | (testing "generate-post subcommand on match serever"
75 | (let [cont (tc/start! ds-tc/match-server-container)
76 | host (:host cont)
77 | port (get (:mapped-ports cont) 8080)
78 | res (cli-gen/generate-post!
79 | ["-i" "dev-resources/input/simple.json"
80 | "-E" (format "http://%s:%d" host port)
81 | "-B" "25"
82 | "-L" "25"
83 | "--no-async"])]
84 | (is (nil? (:errors res)))
85 | (tc/stop! cont))))
86 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/input/personae_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.personae-test
2 | (:require [clojure.test :refer [deftest testing is]]
3 | [com.yetanalytics.datasim.util.io :as dio]
4 | [com.yetanalytics.datasim.input.personae :as personae]
5 | [com.yetanalytics.datasim.test-constants :as const]))
6 |
7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8 | ;; Constants
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 |
11 | (def simple-personae
12 | (dio/read-json-location const/simple-personae-filepath))
13 |
14 | (def tc3-personae
15 | (dio/read-json-location const/tc3-personae-filepath))
16 |
17 | (def temporal-personae
18 | (dio/read-json-location const/temporal-personae-filepath))
19 |
20 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
21 | ;; Tests
22 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
23 |
24 | (deftest personae-test
25 | (testing "personae without roles"
26 | (is (nil? (personae/validate-personae simple-personae)))
27 | (is (nil? (personae/validate-personae tc3-personae)))
28 | (is (nil? (personae/validate-personae temporal-personae))))
29 | (testing "personae with roles"
30 | (is (nil? (-> simple-personae
31 | (assoc-in [:member 0 :role] "Lead Developer")
32 | (assoc-in [:member 1 :role] "Data Engineer")
33 | (assoc-in [:member 2 :role] "CEO")
34 | personae/validate-personae)))
35 | (is (nil? (-> tc3-personae
36 | (assoc-in [:member 0 :role] "xAPI God")
37 | (assoc-in [:member 1 :role] "Company Chief")
38 | (assoc-in [:member 2 :role] "Questions Queen")
39 | (assoc-in [:member 3 :role] "Learning Lord")
40 | (assoc-in [:member 4 :role] "Simulations Nomand")
41 | (assoc-in [:member 5 :role] "Statement Merchant")
42 | personae/validate-personae)))))
43 |
44 | (deftest personae-array-validation-test
45 | (testing "personae-array spec"
46 | (is (nil? (personae/validate-personae-array
47 | [const/simple-personae const/tc3-personae])))
48 | (is (some? (personae/validate-personae-array
49 | [(-> const/simple-personae
50 | (assoc-in [:member 0 :mbox] "not-an-email"))
51 | const/tc3-personae])))
52 | (is (some? (personae/validate-personae-array []))))
53 | (testing "duplicate member ids across different groups"
54 | (is (some? (personae/validate-personae-array
55 | [(-> const/simple-personae
56 | (assoc-in [:member 0 :mbox] "mailto:bob@example.org")
57 | (assoc-in [:member 1 :mbox] "mailto:alice@example.org")
58 | (assoc-in [:member 2 :mbox] "mailto:fred@example.org"))
59 | const/tc3-personae])))))
60 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/input/profile_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input.profile-test
2 | (:require [clojure.test :refer [deftest testing is]]
3 | [clojure.data.json :as json]
4 | [com.yetanalytics.pan :as pan]
5 | [com.yetanalytics.datasim.input.profile :as profile]
6 | [com.yetanalytics.datasim.util.io :as dio]
7 | [com.yetanalytics.datasim.test-constants :as const])
8 | (:import [java.io File]))
9 |
10 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 | ;; Constants
12 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13 |
14 | (def minimal-profile
15 | (dio/read-json-location const/minimal-profile-filepath))
16 |
17 | (def minimal-profile-map
18 | {:id "https://xapinet.org/xapi/yet/minimal"
19 | :type "Profile"
20 | :_context "https://w3id.org/xapi/profiles/context"
21 | :conformsTo "https://w3id.org/xapi/profiles#1.0"
22 | :prefLabel {:en "Minimal - Experimental xAPI Profile"}
23 | :definition {:en "This xAPI Profile demonstrates the minimal required properties of an xAPI profile"}
24 | :versions [{:id "https://xapinet.org/xapi/yet/minimal/v1"
25 | :generatedAtTime "2020-03-25T15:45:31.907Z"}]
26 | :author {:url "https://www.yetanalytics.com/"
27 | :name "Yet Analytics"
28 | :type "Organization"}})
29 |
30 | (def cmi5-satisfied-bad-1 "https://w3id.org/xapi/cmi5#satisfiedbad1")
31 | (def cmi5-satisfied-bad-2 "https://w3id.org/xapi/cmi5#satisfiedbad2")
32 | (def cmi5-satisfied-bad-3 "https://w3id.org/xapi/cmi5#satisfiedbad3")
33 | (def cmi5-initialized "https://w3id.org/xapi/cmi5#initialized")
34 | (def cmi5-terminated "https://w3id.org/xapi/cmi5#terminated")
35 | (def cmi5-completed "https://w3id.org/xapi/cmi5#completed")
36 |
37 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
38 | ;; Tests
39 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
40 |
41 | (defn- profile-key [k]
42 | (let [kname (name k)]
43 | (if (= "@context" kname)
44 | :_context
45 | (keyword kname))))
46 |
47 | (deftest minimal-profile-test
48 | (testing "produces the correct profile"
49 | ;; Coerce `minimal-profile` back into non-record map
50 | (is (= minimal-profile-map
51 | (into {} minimal-profile))))
52 | (testing "is valid"
53 | (is (nil? (profile/validate-profile minimal-profile))))
54 | (testing "is valid when written"
55 | (let [^File tf (File/createTempFile "profiletest" nil)]
56 | (try
57 | (dio/write-json-file minimal-profile tf)
58 | (is (nil? (pan/validate-profile
59 | (json/read-str (slurp tf) :key-fn profile-key))))
60 | (finally
61 | (.delete tf))))))
62 |
63 | (deftest profile-cosmos-validation-test
64 | (testing "input is valid if all template refs are valid"
65 | (is (nil? (profile/validate-profiles [const/cmi5-profile]))))
66 | (testing "input is invalid if invalid template ref iri exists"
67 | (is (= 1 (->> [(-> const/cmi5-profile
68 | (assoc-in [:patterns 0 :zeroOrMore]
69 | "https://w3id.org/xapi/cmi5#bad-template"))]
70 | profile/validate-profiles
71 | count)))
72 | ;; XXX: If we replaced satisfiedbad3 with satisfiedbad2, we only get a count
73 | ;; of 2 errors, not 3.
74 | (is (= 3
75 | (->> [(-> const/cmi5-profile
76 | (assoc-in [:patterns 0 :zeroOrMore] cmi5-satisfied-bad-1)
77 | (assoc-in [:patterns 1 :sequence 0] cmi5-satisfied-bad-2)
78 | (assoc-in [:patterns 1 :sequence 2] cmi5-satisfied-bad-3))]
79 | profile/validate-profiles
80 | count))))
81 | (testing "validation works for multi-profile cosmos"
82 | (is (nil? (profile/validate-profiles
83 | [const/cmi5-profile const/video-profile])))
84 | ;; Add connections between Profiles
85 | (is (nil? (profile/validate-profiles
86 | [const/cmi5-profile
87 | (-> const/video-profile
88 | (assoc-in [:patterns 0 :sequence 0] cmi5-initialized)
89 | (assoc-in [:patterns 0 :sequence 2] cmi5-terminated)
90 | (assoc-in [:patterns 1 :alternates 6] cmi5-completed))])))
91 | (is (= 1 (->> [(-> const/cmi5-profile
92 | (assoc-in [:patterns 0 :zeroOrMore]
93 | "https://w3id.org/xapi/cmi5#bad-template"))
94 | const/video-profile]
95 | profile/validate-profiles
96 | count))))
97 | (testing "fixed / valid profiles"
98 | (is (nil? (profile/validate-profiles [const/acrossx-profile])))
99 | (is (nil? (profile/validate-profiles [const/activity-streams-profile])))
100 | (is (nil? (profile/validate-profiles [const/tc3-profile]))))
101 | ;; Following tests exist to point out flaws in Profiles
102 | (testing "invalid profiles"
103 | ;; AcrossX and ActivityStreams violate spec:
104 | ;; "related MUST only be used on deprecated Concepts"
105 | (is (some? (profile/validate-profiles [const/acrossx-profile*])))
106 | (is (some? (profile/validate-profiles [const/activity-streams-profile*])))))
107 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/input_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.input-test
2 | (:require [clojure.test :refer [deftest testing is are]]
3 | [com.yetanalytics.datasim.input
4 | :refer [from-location validate validate-throw]
5 | :as input]
6 | [com.yetanalytics.datasim.test-constants :as const])
7 | (:import [clojure.lang ExceptionInfo]))
8 |
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 | ;; Input Read Tests
11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12 |
13 | ;; We need to test that the process of reading itself is valid, rather than
14 | ;; entirely rely on already-read fixtures
15 |
16 | (deftest valid-invalid-read-test
17 | (are [test-name
18 | invalid-reason
19 | input-key
20 | file-loc
21 | invalidator-fn]
22 | (testing (format "Reading %s" test-name)
23 | (let [x (from-location input-key :json file-loc)]
24 | (testing "is valid"
25 | (is (nil? (validate input-key x))))
26 | (testing (format "is invalid due to %s" invalid-reason)
27 | (is (some? (validate input-key (invalidator-fn x))))
28 | (is (thrown? ExceptionInfo
29 | (validate-throw input-key (invalidator-fn x)))))))
30 | "xAPI Profile" "non-IRI ID"
31 | :profile const/cmi5-profile-filepath
32 | #(assoc % :id "foo")
33 |
34 | "Actor Personae" "empty `:members` coll"
35 | :personae const/simple-personae-filepath
36 | #(assoc % :member [])
37 |
38 | "Actor Models" "invalid due to invalid alignments"
39 | :models const/simple-models-filepath
40 | #(conj % {:personae [{:id "notanid"
41 | :type "notatype"}]
42 | :verbs [{:component "notaniri"
43 | :weight "bar"}]})
44 |
45 | "Actor Models, Long" "invalid alignments"
46 | :models const/tc3-models-filepath
47 | #(conj % {:personae [{:id "notanid"
48 | :type "notatype"}]
49 | :verbs [{:component "notaniri"
50 | :weight "bar"}]})
51 |
52 | "Actor Models w/ Overrides" "invalid alignments"
53 | :models const/simple-overrides-models-filepath
54 | #(conj % {:personae [{:id "notanid"
55 | :type "notatype"}]
56 | :objectOverrides [{:component "notaniri"
57 | :weight "bar"}]})
58 |
59 | "Simulation Parameters" "non-numeric seed"
60 | :parameters const/simple-parameters-filepath
61 | #(assoc % :seed "hey")
62 |
63 | "Combined Input Spec" "`:profiles` not being a vector"
64 | :input const/simple-input-filepath
65 | #(update % :profiles first)))
66 |
67 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
68 | ;; Input Validation Tests
69 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
70 |
71 | (deftest subobject-validation-test
72 | (testing "input is valid with a minimal profile"
73 | (is (nil? (input/validate
74 | :input
75 | (assoc-in const/simple-input
76 | [:profiles 0]
77 | const/minimal-profile))))))
78 |
79 | (deftest combined-input-validation-test
80 | (testing "combined input is valid"
81 | (is (nil? (validate :input const/simple-input)))
82 | (is (try (validate-throw :input const/simple-input)
83 | true
84 | (catch Exception _ false))))
85 | (testing "combined input is invalid"
86 | (testing "with invalid genProfiles"
87 | (is (try
88 | (validate-throw
89 | :input
90 | (assoc-in const/simple-input
91 | [:parameters :genProfiles]
92 | ["http://example.com/nonexistent.jsonld"]))
93 | false
94 | (catch Exception _ true))))
95 | (testing "with invalid genPatterns"
96 | (is (try
97 | (validate-throw
98 | :input
99 | (assoc-in const/simple-input
100 | [:parameters :genPatterns]
101 | ["http://example.com/nonexistent#pattern"]))
102 | false
103 | (catch Exception _ true))))))
104 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/model/periods_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.model.periods-test
2 | (:require [clojure.test :refer [deftest testing is are]]
3 | [java-time.api :as t]
4 | [com.yetanalytics.datasim.util.random :as random]
5 | [com.yetanalytics.datasim.model.periods :as periods]))
6 |
7 | (def new-years-date-time
8 | "The first moment of 2023, 2023-01-01T00:00:00, as a LocalDateTime"
9 | (t/local-date-time 2023 1 1 0 0 0))
10 |
11 | (deftest time-period-test
12 | (testing "`convert-periods?` function"
13 | (is (= [{:min 2
14 | :mean 2}
15 | {:min 2000
16 | :mean 2000}
17 | {:min 120000
18 | :mean 120000}
19 | {:min 7200000
20 | :mean 7200000}
21 | {:min 172800000
22 | :mean 172800000}
23 | {:min 1209600000
24 | :mean 1209600000}]
25 | (periods/convert-periods
26 | [{:min 2
27 | :mean 2
28 | :unit "millis"}
29 | {:min 2
30 | :mean 2
31 | :unit "seconds"}
32 | {:min 2
33 | :mean 2
34 | :unit "minutes"}
35 | {:min 2
36 | :mean 2
37 | :unit "hours"}
38 | {:min 2
39 | :mean 2
40 | :unit "days"}
41 | {:min 2
42 | :mean 2
43 | :unit "weeks"}])))
44 | (is (= [{:fixed 2}
45 | {:fixed 2000}
46 | {:fixed 120000}
47 | {:fixed 7200000}
48 | {:fixed 172800000}
49 | {:fixed 1209600000}]
50 | (periods/convert-periods
51 | [{:fixed 2
52 | :unit "millis"}
53 | {:fixed 2
54 | :unit "seconds"}
55 | {:fixed 2
56 | :unit "minutes"}
57 | {:fixed 2
58 | :unit "hours"}
59 | {:fixed 2
60 | :unit "days"}
61 | {:fixed 2
62 | :unit "weeks"}])))
63 | (is (= [{:min 60000 ; minute default
64 | :mean 60000}
65 | {:fixed 60000}]
66 | (periods/convert-periods
67 | [{:min 1
68 | :mean 1}
69 | {:min 1
70 | :mean 1
71 | :fixed 1}]))))
72 | (testing "`add-periods` function"
73 | (testing "(fixed times)"
74 | (are [expected periods]
75 | (= (t/local-date-time expected)
76 | (periods/add-periods
77 | new-years-date-time
78 | (random/seed-rng 100) ; rng doesn't matter here
79 | (periods/convert-periods periods)))
80 | "2023-01-01T00:01:00" [{:fixed 1}]
81 | "2023-01-01T00:30:00" [{:fixed 1
82 | :bounds [{:years [2024]}]}
83 | {:fixed 30
84 | :bounds [{:years [2023]}]}]))
85 | (testing "(minimum times)"
86 | (are [expected-min periods]
87 | (t/before?
88 | (t/local-date-time expected-min)
89 | (periods/add-periods
90 | new-years-date-time
91 | (random/rng)
92 | (periods/convert-periods periods)))
93 | "2023-01-01T00:00:30" [{:min 30
94 | :unit "seconds"}]
95 | "2023-01-01T00:30:00" [{:min 30
96 | :unit "minutes"}]
97 | "2023-01-01T12:00:00" [{:min 12
98 | :unit "hours"}]
99 | "2023-01-10T00:00:00" [{:min 10
100 | :unit "days"}]
101 | "2023-01-14T00:00:00" [{:min 2
102 | :unit "weeks"}]))
103 | (testing "(mean times)"
104 | ;; We use 6 * sd = 6 * mean (b/c exponential distribution)
105 | (are [expected-max periods]
106 | (t/before?
107 | new-years-date-time
108 | (periods/add-periods
109 | new-years-date-time
110 | (random/rng)
111 | (periods/convert-periods periods))
112 | (t/local-date-time expected-max))
113 | "2023-01-01T00:03:00" [{:mean 30
114 | :unit "seconds"}]
115 | "2023-01-01T00:30:00" [{:mean 5
116 | :unit "minutes"}]
117 | "2023-01-01T12:00:00" [{:mean 2
118 | :unit "hours"}]
119 | "2023-01-06T00:00:00" [{:mean 1
120 | :unit "days"}]
121 | "2023-01-21T00:00:00" [{:mean 0.5
122 | :unit "weeks"}]))))
123 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/test_constants.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.test-constants
2 | "Constants for input items, i.e. profiles, personae, models, and
3 | parameters."
4 | (:require [clojure.java.io :as io]
5 | [com.yetanalytics.datasim.input :as input]))
6 |
7 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8 | ;; Filepath Names
9 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10 |
11 | ;; Profiles
12 |
13 | (def minimal-profile-filepath
14 | "dev-resources/profiles/minimal.jsonld")
15 | (def no-concept-profile-filepath
16 | "dev-resources/profiles/no_concept.jsonld")
17 | (def cmi5-profile-filepath
18 | "dev-resources/profiles/cmi5/fixed.json")
19 | (def video-profile-filepath
20 | "dev-resources/profiles/video/profile.jsonld")
21 | (def acrossx-profile-filepath*
22 | "dev-resources/profiles/acrossx/profile.jsonld")
23 | (def acrossx-profile-filepath
24 | "dev-resources/profiles/acrossx/fixed.jsonld")
25 | (def activity-streams-profile-filepath*
26 | "dev-resources/profiles/activity_streams/profile.jsonld")
27 | (def activity-streams-profile-filepath
28 | "dev-resources/profiles/activity_streams/fixed.jsonld")
29 | (def activity-profile-filepath
30 | "dev-resources/profiles/activity.jsonld")
31 | (def mom-profile-filepath
32 | "dev-resources/profiles/tla/mom.jsonld")
33 | (def referential-profile-filepath
34 | "dev-resources/profiles/referential.jsonld")
35 | (def tc3-profile-filepath
36 | "dev-resources/profiles/tccc/cuf_hc_video_and_asm_student_survey_profile.jsonld")
37 | (def temporal-profile-filepath
38 | "dev-resources/profiles/temporal.jsonld")
39 |
40 | ;; Personae
41 |
42 | (def simple-personae-filepath
43 | "dev-resources/personae/simple.json")
44 | (def tc3-personae-filepath
45 | "dev-resources/personae/tccc_dev.json")
46 | (def temporal-personae-filepath
47 | "dev-resources/personae/temporal.json")
48 |
49 | ;; Models
50 |
51 | (def simple-models-filepath
52 | "dev-resources/models/simple.json")
53 | (def simple-overrides-models-filepath
54 | "dev-resources/models/simple_with_overrides.json")
55 | (def simple-temporal-models-filepath
56 | "dev-resources/models/simple_with_temporal.json")
57 | (def simple-repeat-max-models-filepath
58 | "dev-resources/models/simple_with_repeat_max.json")
59 | (def tc3-models-filepath
60 | "dev-resources/models/tccc_dev.json")
61 |
62 | (def temporal-models-filepath
63 | "dev-resources/models/temporal/")
64 | (def temporal-models-filepath-coll*
65 | (-> temporal-models-filepath io/as-file .list seq))
66 | (def temporal-models-filepath-coll
67 | (map (partial str temporal-models-filepath)
68 | temporal-models-filepath-coll*))
69 |
70 | ;; Parameters
71 |
72 | (def simple-parameters-filepath
73 | "dev-resources/parameters/simple.json")
74 | (def temporal-parameters-filepath
75 | "dev-resources/parameters/temporal.json")
76 |
77 | ;; Combined Input
78 |
79 | (def simple-input-filepath
80 | "dev-resources/input/simple.json")
81 |
82 | ;; Miscellaneous
83 |
84 | (def simple-statement-filepath
85 | "dev-resources/xapi/statements/simple.json")
86 | (def long-statement-filepath
87 | "dev-resources/xapi/statements/long.json")
88 |
89 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
90 | ;; JSON Datas
91 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
92 |
93 | ;; Profiles
94 |
95 | (def minimal-profile
96 | (input/from-location :profile :json minimal-profile-filepath))
97 |
98 | (def no-concept-profile
99 | (input/from-location :profile :json no-concept-profile-filepath))
100 |
101 | (def cmi5-profile
102 | (input/from-location :profile :json cmi5-profile-filepath))
103 |
104 | (def video-profile
105 | (input/from-location :profile :json video-profile-filepath))
106 |
107 | (def acrossx-profile*
108 | (input/from-location :profile :json acrossx-profile-filepath*))
109 |
110 | (def acrossx-profile
111 | (input/from-location :profile :json acrossx-profile-filepath))
112 |
113 | (def activity-streams-profile*
114 | (input/from-location :profile :json activity-streams-profile-filepath*))
115 |
116 | (def activity-streams-profile
117 | (input/from-location :profile :json activity-streams-profile-filepath))
118 |
119 | (def activity-profile
120 | (input/from-location :profile :json activity-profile-filepath))
121 |
122 | (def mom-profile
123 | (input/from-location :profile :json mom-profile-filepath))
124 |
125 | (def referential-profile
126 | (input/from-location :profile :json referential-profile-filepath))
127 |
128 | (def tc3-profile
129 | (input/from-location :profile :json tc3-profile-filepath))
130 |
131 | (def temporal-profile
132 | (input/from-location :profile :json temporal-profile-filepath))
133 |
134 | ;; Personae
135 |
136 | (def simple-personae
137 | (input/from-location :personae :json simple-personae-filepath))
138 |
139 | (def tc3-personae
140 | (input/from-location :personae :json tc3-personae-filepath))
141 |
142 | (def temporal-personae
143 | (input/from-location :personae :json temporal-personae-filepath))
144 |
145 | ;; Models
146 |
147 | (def simple-overrides-models
148 | (input/from-location :models :json simple-overrides-models-filepath))
149 |
150 | (def simple-temporal-models
151 | (input/from-location :models :json simple-temporal-models-filepath))
152 |
153 | (def simple-repeat-max-models
154 | (input/from-location :models :json simple-repeat-max-models-filepath))
155 |
156 | (def temporal-models-coll
157 | (map (partial input/from-location :models :json)
158 | temporal-models-filepath-coll))
159 |
160 | ;; Parameters
161 |
162 | (def temporal-parameters
163 | (input/from-location :parameters :json temporal-parameters-filepath))
164 |
165 | ;; Combined Input
166 |
167 | (def simple-input
168 | (input/from-location :input :json simple-input-filepath))
169 |
170 | (def temporal-input-map
171 | (zipmap temporal-models-filepath-coll*
172 | (map (fn [temporal-model]
173 | {:profiles [temporal-profile]
174 | :personae-array [temporal-personae]
175 | :parameters temporal-parameters
176 | :models temporal-model})
177 | temporal-models-coll)))
178 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/test_containers.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.test-containers
2 | (:require [clj-test-containers.core :as tc]))
3 |
4 | (def server-validate-command
5 | ["/persephone/bin/server.sh" "validate"
6 | "-p" "dev-resources/profiles/cmi5/fixed.json"])
7 |
8 | (def server-match-command
9 | ["/persephone/bin/server.sh" "match"
10 | "-p" "dev-resources/profiles/cmi5/fixed.json"])
11 |
12 | (def container-map
13 | {:image-name "yetanalytics/persephone:v0.9.1"
14 | :exposed-ports [8080]})
15 |
16 | (def filesystem-map
17 | {:host-path "dev-resources/"
18 | :container-path "/persephone/dev-resources/"
19 | :mode :read-only})
20 |
21 | (def validate-server-container
22 | (-> (assoc container-map :command server-validate-command)
23 | tc/create
24 | (tc/bind-filesystem! filesystem-map)))
25 |
26 | (def match-server-container
27 | (-> (assoc container-map :command server-match-command)
28 | tc/create
29 | (tc/bind-filesystem! filesystem-map)))
30 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/util/io_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.io-test
2 | (:require [clojure.test :refer [deftest testing is]]
3 | [com.yetanalytics.datasim.util.io :as io]
4 | [com.yetanalytics.datasim.test-constants :as const]))
5 |
6 | (deftest read-json-location-test
7 | (testing "Reads files as json"
8 | (is (map? (io/read-json-location const/cmi5-profile-filepath))))
9 | (testing "Throws wrapped I/O errors when it can't read something"
10 | (is (= ::io/io-error
11 | (try (io/read-json-location "notthere.json")
12 | (catch clojure.lang.ExceptionInfo exi
13 | (:type (ex-data exi))))))
14 | (is (= ::io/io-error
15 | (try (io/read-json-location "https://notarealhost329083")
16 | (catch clojure.lang.ExceptionInfo exi
17 | (:type (ex-data exi))))))
18 | (is (= ::io/io-error
19 | (try (io/read-json-location "https://www.google.com/bingisgreat")
20 | (catch clojure.lang.ExceptionInfo exi
21 | (:type (ex-data exi)))))))
22 | (testing "Throws wrapped parser errors when it can't read a format"
23 | (is (= ::io/parse-error
24 | (try (io/read-json-location "deps.edn")
25 | (catch clojure.lang.ExceptionInfo exi
26 | (:type (ex-data exi))))))))
27 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/datasim/util/random_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.datasim.util.random-test
2 | (:require [clojure.test :refer [deftest testing is]]
3 | [clojure.math :as math]
4 | [clojure.spec.test.alpha :as stest]
5 | [com.yetanalytics.datasim.util.random :as r]))
6 |
7 | (deftest random-functions-test
8 | (testing "Generative tests"
9 | (let [results (stest/check
10 | `#{r/rng
11 | r/seed-rng
12 | r/rand
13 | r/rand-int
14 | r/rand-unbound-int
15 | r/rand-gaussian
16 | r/rand-boolean
17 | r/rand-exp
18 | r/rand-uuid
19 | r/rand-nth
20 | r/shuffle
21 | r/random-sample
22 | r/choose
23 | r/choose-map})
24 | {:keys [total
25 | check-passed]} (stest/summarize-results results)]
26 | (is (= total check-passed)))))
27 |
28 | (def choose-total 4000)
29 |
30 | (def choose-coll [:foo :bar :baz :qux])
31 |
32 | (def choose-map {:foo :FOO :bar :BAR :baz :BAZ :qux :QUX})
33 |
34 | ;; 4 standard deviations means that the bounds will only be exceeded in
35 | ;; 1 in 15,787 runs.
36 | ;; See: https://en.wikipedia.org/wiki/68–95–99.7_rule##Table_of_numerical_values
37 | (def sd-limit 4)
38 |
39 | ;; The means and standard deviations are derived from interpreting the
40 | ;; event of choosing a particular kewyord as a binomial distribution.
41 | ;; See: https://en.wikipedia.org/wiki/Binomial_distribution
42 | (defmacro test-equal-choose [weights]
43 | (let [prob# 0.25
44 | mean# (* choose-total prob#)
45 | sd# (math/sqrt (* choose-total prob# (- 1 prob#)))
46 | lo-bound# (- mean# (* sd-limit sd#))
47 | hi-bound# (+ mean# (* sd-limit sd#))]
48 | `(let [~'the-rng (r/rng)
49 | ~'choose-fn (fn [] (r/choose ~'the-rng ~weights ~choose-coll))
50 | ~'results (repeatedly ~choose-total ~'choose-fn)
51 | ~'freq-map (frequencies ~'results)]
52 | (is (< ~lo-bound# (:foo ~'freq-map) ~hi-bound#))
53 | (is (< ~lo-bound# (:bar ~'freq-map) ~hi-bound#))
54 | (is (< ~lo-bound# (:baz ~'freq-map) ~hi-bound#))
55 | (is (< ~lo-bound# (:qux ~'freq-map) ~hi-bound#)))))
56 |
57 | (deftest choose-test
58 | (testing "choose function: equal weights"
59 | (test-equal-choose {})
60 | (test-equal-choose {:unrelated-key-1 0.2 :unrelated-key 0.8})
61 | (test-equal-choose {:foo 0.5 :baz 0.5})
62 | (test-equal-choose {:foo 0.0 :bar 0.0 :baz 0.0 :qux 0.0})
63 | (test-equal-choose {:foo 0.2 :bar 0.2 :baz 0.2 :qux 0.2})
64 | (test-equal-choose {:foo 0.8 :bar 0.8 :baz 0.8 :qux 0.8})
65 | (test-equal-choose {:foo 1.0 :bar 1.0 :baz 1.0 :qux 1.0}))
66 | (testing "choose function: only one non-zero weight"
67 | (let [weights {:foo 0 :bar 0 :baz 1 :qux 0}
68 | the-rng (r/rng)
69 | choose-fn (fn [] (r/choose the-rng weights choose-coll))
70 | results (repeatedly choose-total choose-fn)
71 | freq-map (frequencies results)]
72 | (is (= choose-total (:baz freq-map)))
73 | (is (nil? (:foo freq-map)))
74 | (is (nil? (:bar freq-map)))
75 | (is (nil? (:qux freq-map)))))
76 | ;; The probabilities are derived from integrating over the area defined
77 | ;; by the joint uniform distributions where (< x2 (max x1 t2)), where
78 | ;; x1 and x2 are the independent (but not identical!) random variables
79 | ;; and t1 and t2 are the upper bounds of the uniform distributions.
80 | ;; In Clojure pseudocode:
81 | ;; (-> (/ 1 (* t1 t2))
82 | ;; (integrate 0 (max x1 t2) dx2)
83 | ;; (integrate 0 t1 dx1))
84 | ;; = (- 1 (/ t2 (* 2 t1))
85 | (testing "choose function: two different non-zero weights"
86 | (let [weights {:foo 0 :bar 0 :baz 1 :qux 0.4}
87 | prob-1 0.8
88 | prob-2 0.2
89 | mean-1 (* choose-total prob-1)
90 | mean-2 (* choose-total prob-2)
91 | sd (math/sqrt (* choose-total prob-1 prob-2))
92 | lo-1 (- mean-1 (* sd-limit sd))
93 | lo-2 (- mean-2 (* sd-limit sd))
94 | hi-1 (+ mean-1 (* sd-limit sd))
95 | hi-2 (+ mean-2 (* sd-limit sd))
96 | the-rng (r/rng)
97 | choose-fn (fn [] (r/choose the-rng weights choose-coll))
98 | results (repeatedly choose-total choose-fn)
99 | freq-map (frequencies results)]
100 | (is (< lo-1 (:baz freq-map) hi-1))
101 | (is (< lo-2 (:qux freq-map) hi-2))
102 | (is (nil? (:foo freq-map)))
103 | (is (nil? (:bar freq-map)))))
104 | (testing "choose-map function works just like choose"
105 | (let [weights {:foo 0.2 :bar 0.4 :baz 0.6 :qux 0.8}
106 | seed (r/rand-unbound-int (r/rng))
107 | rng-1 (r/seed-rng seed)
108 | rng-2 (r/seed-rng seed)
109 | choose-fn (fn [] (r/choose rng-1 weights choose-coll))
110 | choose-map-fn (fn [] (r/choose-map rng-2 weights choose-map))
111 | results-1 (repeatedly choose-total choose-fn)
112 | results-2 (repeatedly choose-total choose-map-fn)
113 | freq-map-1 (frequencies results-1)
114 | freq-map-2 (frequencies results-2)]
115 | (is (= (:foo freq-map-1)
116 | (:FOO freq-map-2)))
117 | (is (= (:bar freq-map-1)
118 | (:BAR freq-map-2)))
119 | (is (= (:baz freq-map-1)
120 | (:BAZ freq-map-2)))
121 | (is (= (:qux freq-map-1)
122 | (:QUX freq-map-2))))))
123 |
--------------------------------------------------------------------------------
/src/test/com/yetanalytics/server_test.clj:
--------------------------------------------------------------------------------
1 | (ns com.yetanalytics.server-test
2 | "Integration tests for the DATASIM server."
3 | (:require [clojure.test :refer [deftest testing is]]
4 | [clojure.string :as cstr]
5 | [io.pedestal.http :as http]
6 | [org.httpkit.client :as httpkit]
7 | [clj-test-containers.core :as tc]
8 | [com.yetanalytics.datasim.test-constants :as const]
9 | [com.yetanalytics.datasim.test-containers :as ds-tc]
10 | [com.yetanalytics.datasim.server :as server]))
11 |
12 | (def server
13 | (server/create-server))
14 |
15 | (def profile-string
16 | (format "[%s]" (slurp const/cmi5-profile-filepath)))
17 |
18 | (def personaes-string
19 | (format "[%s]" (slurp const/simple-personae-filepath)))
20 |
21 | (def models-string
22 | (slurp const/simple-models-filepath))
23 |
24 | (def parameters-string
25 | (slurp const/simple-parameters-filepath))
26 |
27 | (def post-header
28 | {"X-Experience-Api-Version" "1.0.3"
29 | "Content-Type" "multipart/form-data"})
30 |
31 | (def json-post-header
32 | {"X-Experience-Api-Version" "1.0.3"
33 | "Content-Type" "application/json"})
34 |
35 | (def multipart-content
36 | [{:name "profiles"
37 | :content profile-string}
38 | {:name "personae-array"
39 | :content personaes-string}
40 | {:name "models"
41 | :content models-string}
42 | {:name "parameters"
43 | :content parameters-string}])
44 |
45 | (defn- multipart-post-content [lrs-endpoint api-key api-secret]
46 | (into multipart-content
47 | [{:name "lrs-endpoint"
48 | :content lrs-endpoint}
49 | {:name "api-key"
50 | :content api-key}
51 | {:name "api-secret-key"
52 | :content api-secret}
53 | {:name "send-to-lrs"
54 | :content "true"}]))
55 |
56 | (deftest server-test
57 | (testing "server"
58 | (let [_ (http/start server)
59 | cont (tc/start! ds-tc/validate-server-container)
60 | host (:host cont)
61 | post (get (:mapped-ports cont) 8080)
62 | lrs-url (format "http://%s:%d/statements" host post)]
63 | (testing "GET /health endpoint"
64 | (let [{:keys [status]}
65 | #_{:clj-kondo/ignore [:unresolved-var]}
66 | @(httpkit/get
67 | "http://0.0.0.0:9090/health")]
68 | (is (= 200 status))))
69 | (testing "POST /api/v1/generate endpoint"
70 | (let [{:keys [status body]}
71 | #_{:clj-kondo/ignore [:unresolved-var]}
72 | @(httpkit/post
73 | "http://0.0.0.0:9090/api/v1/generate"
74 | {:headers post-header
75 | :basic-auth ["username" "password"]
76 | :multipart multipart-content})]
77 | (is (= 200 status))
78 | (is (every?
79 | (fn [stmt-str]
80 | (let [validate-res
81 | #_{:clj-kondo/ignore [:unresolved-var]}
82 | @(httpkit/post
83 | lrs-url
84 | {:headers json-post-header
85 | :body stmt-str
86 | :as :stream})]
87 | (= 204 (:status validate-res))))
88 | (take 25 (rest (cstr/split-lines body)))))))
89 | (testing "POST /api/v1/generate endpoint w/ LRS"
90 | (let [endpoint (format "http://%s:%d" host post)
91 | {:keys [status]}
92 | #_{:clj-kondo/ignore [:unresolved-var]}
93 | @(httpkit/post
94 | "http://0.0.0.0:9090/api/v1/generate"
95 | {:headers post-header
96 | :basic-auth ["username" "password"]
97 | :multipart (multipart-post-content endpoint "foo" "bar")})]
98 | (is (= 200 status))))
99 | (tc/stop! cont)
100 | (http/stop server))))
101 |
--------------------------------------------------------------------------------