├── app ├── config │ ├── __init__.py │ ├── settings │ │ ├── __init__.py │ │ ├── nginx.py │ │ ├── aws.py │ │ ├── local.py │ │ └── base.py │ ├── views.py │ ├── urls.py │ └── wsgi.py ├── engine │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── update_model.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0014_auto_20180613_0239.py │ │ ├── 0015_activity_prerequisite_activities.py │ │ ├── 0002_activity_url.py │ │ ├── 0011_collection_collection_id.py │ │ ├── 0006_auto_20180513_1843.py │ │ ├── 0010_auto_20180603_2303.py │ │ ├── 0013_auto_20180613_0211.py │ │ ├── 0009_auto_20180520_1840.py │ │ ├── 0012_auto_20180613_0211.py │ │ ├── 0005_auto_20180209_0242.py │ │ ├── 0004_auto_20180209_0241.py │ │ ├── 0008_auto_20180519_0406.py │ │ ├── 0007_auto_20180518_2356.py │ │ ├── 0003_auto_20180209_0240.py │ │ └── 0001_initial.py │ ├── fixtures │ │ ├── collections.json │ │ ├── learners.json │ │ ├── knowledge_components.json │ │ ├── engine_settings.json │ │ └── activities.json │ ├── apps.py │ ├── admin.py │ ├── urls.py │ ├── models.py │ └── serializers.py ├── tests │ ├── __init__.py │ ├── test_sequence.py │ ├── fixtures.py │ ├── test_hpl.py │ └── test_api.py ├── .dockerignore ├── pytest.ini ├── Dockerfile ├── run ├── Dockerfile_prod ├── requirements.txt ├── docker-compose.alosi-mount.yml ├── .gitignore ├── docker-compose.yml ├── docker-compose.prod.yml ├── manage.py └── README.md ├── writeup ├── writeup.pdf ├── lti-tool.png ├── bkt_bibliography.bib ├── iopart12.clo ├── acmcopyright.sty └── nips_2016.cls ├── data_description_for_python_prototype.pdf ├── .gitignore ├── .travis.yml ├── prototypes ├── r_prototype │ ├── evaluation │ │ ├── update_model.R │ │ ├── clean_slate.R │ │ ├── run_through.R │ │ ├── master.R │ │ ├── evaluate.R │ │ └── data_load.R │ ├── derivedData.R │ ├── master.R │ ├── testOnSuperEarths.R │ ├── propagator.R │ ├── masterSuperEarths.R │ ├── recommendation.R │ ├── load_data.R │ ├── fakeInitials.R │ ├── initialsSuperEarths.R │ ├── evaluate.R │ └── optimizer.R └── python_prototype │ ├── derivedData.py │ ├── sim.py │ ├── additiveFormulation.py │ ├── fakeInitials.py │ ├── multiplicativeFormulation.py │ └── empiricalEstimation.py ├── README.md └── LICENSE /app/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/engine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/config/settings/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/engine/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/engine/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/engine/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /writeup/writeup.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harvard-vpal/adaptive-engine/HEAD/writeup/writeup.pdf -------------------------------------------------------------------------------- /writeup/lti-tool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harvard-vpal/adaptive-engine/HEAD/writeup/lti-tool.png -------------------------------------------------------------------------------- /app/.dockerignore: -------------------------------------------------------------------------------- 1 | **/__pycache__ 2 | *.py[cod] 3 | .idea/ 4 | .git/ 5 | **/.DS_Store 6 | .env 7 | .pytest_cache/ 8 | static/ 9 | -------------------------------------------------------------------------------- /app/config/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | 3 | def health(request): 4 | return HttpResponse() 5 | -------------------------------------------------------------------------------- /app/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE = config.settings.local 3 | python_files = tests.py test_*.py *_tests.py 4 | -------------------------------------------------------------------------------- /data_description_for_python_prototype.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/harvard-vpal/adaptive-engine/HEAD/data_description_for_python_prototype.pdf -------------------------------------------------------------------------------- /app/config/settings/nginx.py: -------------------------------------------------------------------------------- 1 | """ 2 | Referenced in docker build for nginx image 3 | """ 4 | 5 | from config.settings.local import * 6 | 7 | STATIC_ROOT = '/www/static/' 8 | -------------------------------------------------------------------------------- /app/engine/fixtures/collections.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "engine.collection", 4 | "pk": 1, 5 | "fields": { 6 | "name": "My test collection" 7 | } 8 | } 9 | ] 10 | -------------------------------------------------------------------------------- /app/engine/apps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.apps import AppConfig 5 | 6 | 7 | class EngineConfig(AppConfig): 8 | name = 'engine' 9 | -------------------------------------------------------------------------------- /app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | RUN mkdir /app 5 | WORKDIR /app 6 | 7 | COPY ./requirements.txt /app/ 8 | RUN pip install -r requirements.txt 9 | COPY . /app/ 10 | 11 | EXPOSE 8000 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # OSX files 6 | .DS_Store 7 | 8 | # Python package files 9 | *.egg-info/ 10 | 11 | # ide 12 | .idea/ 13 | 14 | # pytest 15 | .pytest_cache/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: generic 4 | 5 | services: 6 | - docker 7 | 8 | install: 9 | - cd app 10 | - docker-compose build 11 | - docker-compose up -d postgres 12 | 13 | script: 14 | - docker-compose run web pytest 15 | -------------------------------------------------------------------------------- /app/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # shortcut for running "docker-compose run web ..." using docker-compose.alosi-mount.yml as override 4 | # usage: ./run pytest 5 | 6 | docker-compose -f docker-compose.yml -f docker-compose.alosi-mount.yml run web "$@" 7 | -------------------------------------------------------------------------------- /app/Dockerfile_prod: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | RUN mkdir /app 5 | WORKDIR /app 6 | 7 | COPY ./requirements.txt /app/ 8 | RUN pip install -r requirements.txt 9 | COPY ./entrypoint.sh /app/ 10 | COPY . /app/ 11 | 12 | ENTRYPOINT ["./entrypoint.sh"] 13 | EXPOSE 8000 14 | -------------------------------------------------------------------------------- /app/engine/fixtures/learners.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "engine.learner", 4 | "pk": 1, 5 | "fields": { 6 | "identifier": 1 7 | } 8 | }, 9 | { 10 | "model": "engine.learner", 11 | "pk": 2, 12 | "fields": { 13 | "identifier": 2 14 | } 15 | } 16 | ] -------------------------------------------------------------------------------- /app/config/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path 2 | from django.contrib import admin 3 | from config import views 4 | 5 | urlpatterns = [ 6 | path('admin/', admin.site.urls), 7 | path('health/', views.health), 8 | path('', include('engine.urls', namespace="engine")), 9 | ] 10 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | Django==2.0.8 2 | djangorestframework==3.8.2 3 | numpy==1.14.0 4 | requests==2.20.0 5 | gunicorn==19.7.1 6 | psycopg2==2.7.4 7 | alosi==2.1.1 8 | django_extensions==1.9.9 9 | pandas==0.22.0 10 | ipykernel==4.8.2 11 | pytest 12 | pytest-django 13 | boto3==1.9.29 14 | django-filter==2.0.0 15 | -------------------------------------------------------------------------------- /writeup/bkt_bibliography.bib: -------------------------------------------------------------------------------- 1 | @inproceedings{hawkins2014learning, 2 | title={Learning bayesian knowledge tracing parameters with a knowledge heuristic and empirical probabilities}, 3 | author={Hawkins, William J and Heffernan, Neil T and Baker, Ryan SJD}, 4 | booktitle={International Conference on Intelligent Tutoring Systems}, 5 | pages={150--155}, 6 | year={2014}, 7 | organization={Springer} 8 | } -------------------------------------------------------------------------------- /app/docker-compose.alosi-mount.yml: -------------------------------------------------------------------------------- 1 | # docker-compose override for local development with alosi lib 2 | # 3 | # specify path to local alosi library in .env file: 4 | # e.g. "ALOSI_LOCAL_LIBRARY=/Users/me/github/alosi/alosi" 5 | # 6 | # usage: docker-compose -f docker-compose.yml -f docker-compose.alosi-mount.yml 7 | 8 | 9 | version: '2' 10 | services: 11 | web: 12 | volumes: 13 | - ${ALOSI_LOCAL_LIBRARY}:/usr/local/lib/python3.6/site-packages/alosi 14 | -------------------------------------------------------------------------------- /app/engine/fixtures/knowledge_components.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "engine.knowledgecomponent", 4 | "pk": 1, 5 | "fields": { 6 | "name": "kc1" 7 | } 8 | }, 9 | { 10 | "model": "engine.knowledgecomponent", 11 | "pk": 2, 12 | "fields": { 13 | "name": "kc2" 14 | } 15 | }, 16 | { 17 | "model": "engine.knowledgecomponent", 18 | "pk": 3, 19 | "fields": { 20 | "name": "kc3" 21 | } 22 | } 23 | ] -------------------------------------------------------------------------------- /app/config/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for app project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /app/engine/migrations/0014_auto_20180613_0239.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-06-13 02:39 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('engine', '0013_auto_20180613_0211'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='activity', 15 | name='type', 16 | field=models.CharField(blank=True, default='', max_length=200), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # local database 6 | *.sqlite3 7 | 8 | # collected static files 9 | http_static/ 10 | static/ 11 | 12 | # Secure settings 13 | config/settings/secure.py 14 | config/settings/dev.py 15 | config/settings/test.py 16 | config/settings/prod.py 17 | 18 | # OSX files 19 | .DS_Store 20 | 21 | # Elastic Beanstalk Files 22 | .elasticbeanstalk/* 23 | !.elasticbeanstalk/*.cfg.yml 24 | !.elasticbeanstalk/*.global.yml 25 | 26 | # local config 27 | .env 28 | -------------------------------------------------------------------------------- /app/config/settings/aws.py: -------------------------------------------------------------------------------- 1 | from config.settings.base import * 2 | 3 | # for health check 4 | # see https://gist.github.com/dryan/8271687 5 | import requests 6 | EC2_PRIVATE_IP = None 7 | try: 8 | EC2_PRIVATE_IP = requests.get('http://169.254.169.254/latest/meta-data/local-ipv4', timeout=0.01).text 9 | except requests.exceptions.RequestException: 10 | pass 11 | 12 | if EC2_PRIVATE_IP: 13 | ALLOWED_HOSTS.append(EC2_PRIVATE_IP) 14 | 15 | SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') 16 | SESSION_COOKIE_SECURE = True 17 | -------------------------------------------------------------------------------- /app/engine/migrations/0015_activity_prerequisite_activities.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-07-31 18:17 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('engine', '0014_auto_20180613_0239'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='activity', 15 | name='prerequisite_activities', 16 | field=models.ManyToManyField(blank=True, to='engine.Activity'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /app/engine/migrations/0002_activity_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2018-02-08 03:48 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('engine', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='activity', 17 | name='url', 18 | field=models.CharField(default='', max_length=500), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /app/engine/migrations/0011_collection_collection_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-06-13 02:10 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('engine', '0010_auto_20180603_2303'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='collection', 15 | name='collection_id', 16 | field=models.CharField(max_length=200, null=True), 17 | preserve_default=False, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /app/engine/fixtures/engine_settings.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "engine.enginesettings", 4 | "pk": 1, 5 | "fields": { 6 | "name": "My engine settings", 7 | "epsilon": 1e-10, 8 | "eta": 0.0, 9 | "M": 0.0, 10 | "L_star": 2.2, 11 | "r_star": 0.0, 12 | "W_p": 5.0, 13 | "W_r": 3.0, 14 | "W_d": 1.0, 15 | "W_c": 1.0, 16 | "slip_probability": 0.15, 17 | "guess_probability":0.1, 18 | "trans_probability":0.1, 19 | "prior_knowledge_probability":0.2, 20 | "stop_on_mastery": true 21 | } 22 | } 23 | ] -------------------------------------------------------------------------------- /app/engine/migrations/0006_auto_20180513_1843.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11 on 2018-05-13 18:43 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('engine', '0005_auto_20180209_0242'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='activity', 17 | name='tags', 18 | field=models.TextField(blank=True, default=''), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /app/engine/migrations/0010_auto_20180603_2303.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-06-03 23:03 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('engine', '0009_auto_20180520_1840'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='knowledgecomponent', 15 | name='kc_id', 16 | field=models.CharField(default='default', max_length=200, unique=True), 17 | preserve_default=False, 18 | ) 19 | ] 20 | -------------------------------------------------------------------------------- /app/engine/migrations/0013_auto_20180613_0211.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-06-13 02:11 2 | 3 | from django.db import migrations, models 4 | import uuid 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('engine', '0012_auto_20180613_0211'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='collection', 16 | name='collection_id', 17 | field=models.CharField(max_length=200, unique=True), 18 | preserve_default=False, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /app/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # docker-compose file for local development 2 | 3 | version: '2' 4 | services: 5 | postgres: 6 | image: postgres 7 | environment: 8 | POSTGRES_PASSWORD: postgres 9 | volumes: 10 | - pgdata:/var/lib/postgresql/data/ 11 | ports: 12 | - "5432" 13 | web: 14 | image: engine_app 15 | build: 16 | context: . 17 | dockerfile: Dockerfile 18 | command: python manage.py runserver 0.0.0.0:8000 19 | volumes: 20 | - .:/app 21 | ports: 22 | - "8000:8000" 23 | links: 24 | - postgres 25 | 26 | volumes: 27 | pgdata: 28 | -------------------------------------------------------------------------------- /app/engine/migrations/0009_auto_20180520_1840.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-05-20 18:40 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('engine', '0008_auto_20180519_0406'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='learner', 16 | name='experimental_group', 17 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='engine.ExperimentalGroup'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /app/engine/fixtures/activities.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "engine.activity", 4 | "pk": 1, 5 | "fields": { 6 | "name": "test activity 1", 7 | "collection": 1, 8 | "knowledge_components": [1] 9 | } 10 | }, 11 | { 12 | "model": "engine.activity", 13 | "pk": 2, 14 | "fields": { 15 | "name": "test activity 2", 16 | "collection": 1, 17 | "knowledge_components": [1] 18 | } 19 | }, 20 | { 21 | "model": "engine.activity", 22 | "pk": 3, 23 | "fields": { 24 | "name": "test activity 3", 25 | "collection": 1, 26 | "knowledge_components": [2] 27 | } 28 | } 29 | ] -------------------------------------------------------------------------------- /app/docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | # docker-compose file for minimal deployment 2 | 3 | version: '2' 4 | services: 5 | postgres: 6 | image: postgres 7 | environment: 8 | POSTGRES_PASSWORD: postgres 9 | volumes: 10 | - pgdata:/var/lib/postgresql/data/ 11 | ports: 12 | - "5432:5432" 13 | web: 14 | build: 15 | context: . 16 | dockerfile: Dockerfile 17 | command: /usr/local/bin/gunicorn config.wsgi:application -w 2 -b :8000 --log-level=info --log-file - --access-logfile - 18 | volumes: 19 | - .:/app 20 | ports: 21 | - "80:8000" 22 | links: 23 | - postgres 24 | 25 | volumes: 26 | pgdata: 27 | -------------------------------------------------------------------------------- /app/engine/migrations/0012_auto_20180613_0211.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-06-13 02:11 2 | 3 | from django.db import migrations 4 | import uuid 5 | 6 | 7 | def gen_uuid(apps, schema_editor): 8 | Collection = apps.get_model('engine', 'Collection') 9 | for row in Collection.objects.all(): 10 | row.collection_id = uuid.uuid4() 11 | row.save(update_fields=['collection_id']) 12 | 13 | 14 | class Migration(migrations.Migration): 15 | 16 | dependencies = [ 17 | ('engine', '0011_collection_collection_id'), 18 | ] 19 | 20 | operations = [ 21 | migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop) 22 | ] 23 | -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/update_model.R: -------------------------------------------------------------------------------- 1 | #Optimize the BKT parameters 2 | time.start=proc.time()[3] 3 | 4 | ##Estimate on the training set 5 | est=estimate(relevance.threshold=eta, information.threshold=M,remove.degeneracy=T, training.set) 6 | cat("Elapsed seconds in estimating: ",round(proc.time()[3]-time.start,3),"\n") 7 | m.L.i=est$L.i ##Update the prior-knowledge matrix 8 | ind.pristine=which(m.exposure==0); 9 | ##Update the pristine elements of the current mastery probability matrix 10 | m.L=replace(m.L,ind.pristine,m.L.i[ind.pristine]) 11 | #Update the transit, guess, slip odds 12 | m.trans=est$trans 13 | m.guess=est$guess 14 | m.slip=est$slip 15 | 16 | source(file.path(dir_scripts,"derivedData.R")) -------------------------------------------------------------------------------- /app/engine/migrations/0005_auto_20180209_0242.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2018-02-09 02:42 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('engine', '0004_auto_20180209_0241'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name='activity', 17 | name='collection', 18 | ), 19 | migrations.AlterField( 20 | model_name='activity', 21 | name='collections', 22 | field=models.ManyToManyField(blank=True, to='engine.Collection'), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /app/engine/migrations/0004_auto_20180209_0241.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2018-02-09 02:41 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | def make_many_collections(apps, schema_editor): 8 | """ 9 | Adds the Collection object in Activity.collection to the 10 | many-to-many relationship in Activity.collections 11 | """ 12 | Activity = apps.get_model('engine','Activity') 13 | for activity in Activity.objects.all(): 14 | activity.collections.add(activity.collection) 15 | 16 | class Migration(migrations.Migration): 17 | 18 | dependencies = [ 19 | ('engine', '0003_auto_20180209_0240'), 20 | ] 21 | 22 | operations = [ 23 | migrations.RunPython(make_many_collections), 24 | ] 25 | -------------------------------------------------------------------------------- /app/engine/admin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.contrib import admin 5 | from .models import * 6 | 7 | 8 | class ActivityInline(admin.TabularInline): 9 | model = Collection.activity_set.through 10 | readonly_fields = ['activity'] 11 | 12 | 13 | @admin.register(Collection) 14 | class CollectionAdmin(admin.ModelAdmin): 15 | inlines = [ 16 | ActivityInline 17 | ] 18 | 19 | 20 | admin.site.register(Activity) 21 | admin.site.register(Learner) 22 | admin.site.register(Score) 23 | admin.site.register(KnowledgeComponent) 24 | admin.site.register(EngineSettings) 25 | admin.site.register(ExperimentalGroup) 26 | admin.site.register(PrerequisiteRelation) 27 | admin.site.register(Guess) 28 | admin.site.register(Slip) 29 | admin.site.register(Transit) 30 | admin.site.register(Mastery) 31 | -------------------------------------------------------------------------------- /app/engine/migrations/0008_auto_20180519_0406.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-05-19 04:06 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('engine', '0007_auto_20180518_2356'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='learner', 15 | name='tool_consumer_instance_guid', 16 | field=models.CharField(default='', max_length=200), 17 | ), 18 | migrations.AddField( 19 | model_name='learner', 20 | name='user_id', 21 | field=models.CharField(default='', max_length=200), 22 | ), 23 | migrations.AlterUniqueTogether( 24 | name='learner', 25 | unique_together={('user_id', 'tool_consumer_instance_guid')}, 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /app/engine/migrations/0007_auto_20180518_2356.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-05-18 23:56 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('engine', '0006_auto_20180513_1843'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='experimentalgroup', 16 | name='engine_settings', 17 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='engine.EngineSettings'), 18 | ), 19 | migrations.AlterField( 20 | model_name='learner', 21 | name='experimental_group', 22 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='engine.ExperimentalGroup'), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /app/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /app/engine/migrations/0003_auto_20180209_0240.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2018-02-09 02:40 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('engine', '0002_activity_url'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='activity', 18 | name='collections', 19 | field=models.ManyToManyField(blank=True, related_name='activities', to='engine.Collection'), 20 | ), 21 | migrations.AlterField( 22 | model_name='activity', 23 | name='collection', 24 | field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, related_name='activity', to='engine.Collection'), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /app/engine/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path 2 | from rest_framework import routers 3 | from . import api_v2 4 | 5 | app_name = 'engine' 6 | 7 | router_v2 = routers.DefaultRouter(trailing_slash=False) 8 | router_v2.register('activity', api_v2.ActivityViewSet) 9 | router_v2.register('collection', api_v2.CollectionViewSet) 10 | router_v2.register('score', api_v2.ScoreViewSet) 11 | router_v2.register('mastery', api_v2.MasteryViewSet) 12 | router_v2.register('knowledge_component', api_v2.KnowledgeComponentViewSet) 13 | router_v2.register('prerequisite_activity', api_v2.PrerequisiteActivityViewSet) 14 | router_v2.register('prerequisite_knowledge_component', api_v2.PrerequisiteKnowledgeComponentViewSet) 15 | router_v2.register('collection_activity', api_v2.CollectionActivityMemberViewSet) 16 | 17 | 18 | urlpatterns = [ 19 | path('api/v2/', include(router_v2.urls)), 20 | path('engine/api/', include(router_v2.urls)), # included for backcompatibility through sep 2018 21 | ] 22 | -------------------------------------------------------------------------------- /app/engine/management/commands/update_model.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from engine.engines import update_model 3 | 4 | class Command(BaseCommand): 5 | """ 6 | Runs engine model optimization 7 | 8 | Usage: 9 | python manage.py update_model [--eta] [--M] 10 | 11 | Example: 12 | python manage.py update_model --eta 0.0 --M 20.0 13 | """ 14 | 15 | help = 'Updates model' 16 | 17 | def add_arguments(self, parser): 18 | parser.add_argument('--eta', type=float, default=0.0) 19 | parser.add_argument('--M', type=float, default=0.0) 20 | 21 | def handle(self, *args, **options): 22 | self.stdout.write( 23 | 'Starting model update with parameters eta={} and M={}'.format( 24 | options['eta'],options['M'] 25 | ) 26 | ) 27 | 28 | update_model(eta=options['eta'], M=options['M']) 29 | 30 | self.stdout.write(self.style.SUCCESS('Successfully ran model update')) 31 | -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/clean_slate.R: -------------------------------------------------------------------------------- 1 | ################# 2 | ##Define the matrix which keeps track whether a LO for a user has ever been updated 3 | m.exposure<<-matrix(0,ncol=n.los, nrow=n.users) 4 | rownames(m.exposure)=users$id 5 | colnames(m.exposure)=los$id 6 | row.exposure<<- m.exposure[1,] 7 | 8 | ##Define the matrix of confidence: essentially how much information we had for the mastery estimate 9 | m.confidence<<-matrix(0,ncol=n.los, nrow=n.users) 10 | rownames(m.confidence)=users$id 11 | colnames(m.confidence)=los$id 12 | row.confidence<<- m.confidence[1,] 13 | 14 | ##Define the matrix of "user has seen a problem or not": rownames are problems. #### 15 | m.unseen<<-matrix(TRUE,nrow=n.users, ncol=n.probs); 16 | rownames(m.unseen)=users$id 17 | colnames(m.unseen)=probs$id 18 | row.unseen<<-m.unseen[1,] 19 | ## 20 | 21 | ##Define vector that will store the latest item seen by a user 22 | 23 | last.seen<<- rep("",n.users); 24 | names(last.seen)=users$id 25 | 26 | #Initialize the mastery matrix with the initial values 27 | m.L<<- m.L.i -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/run_through.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | source("clean_slate.R") 4 | 5 | time.start=proc.time()[3]; 6 | 7 | 8 | LogData$predicted=NA; 9 | LogData$med_mastery_odds=NA 10 | 11 | for(i in 1:nrow(transactions)){ 12 | problem=transactions$problem_id[i] 13 | score=transactions$score[i] 14 | time=transactions$time[i] 15 | u=transactions$user_id[i] 16 | LogData$predicted[i]=predictCorrectness(u=u,problem=problem) ##Predict probability of success 17 | 18 | bayesUpdate(u=u,problem=problem,score=score,time=time,write=FALSE) ##Update the user's mastery matrix and history 19 | LogData$med_mastery_odds[i]=median(m.L[u,]) 20 | 21 | } 22 | 23 | LogData$med_mastery_prob=LogData$med_mastery_odds/(1+LogData$med_mastery_odds) 24 | LogData=LogData[order(LogData$user_id,LogData$time),] 25 | LogData$nproblem=ave(LogData$score,LogData$user_id,FUN=seq_along) 26 | 27 | 28 | 29 | cat("Elapsed seconds in knowledge tracing: ",round(proc.time()[3]-time.start,3),"\n") 30 | 31 | -------------------------------------------------------------------------------- /prototypes/r_prototype/derivedData.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | 4 | inv.epsilon<<-1/epsilon 5 | 6 | log.epsilon<<--log(epsilon) 7 | 8 | ## Calculate the useful matrices of guess and slip probabilities and of negative logs of the odds. 9 | m.guess.neg.log<<- -log(m.guess) 10 | m.p.guess<<- m.guess/(m.guess+1) 11 | 12 | m.slip.neg.log<<- -log(m.slip) 13 | m.p.slip<<- m.slip/(m.slip+1) 14 | 15 | # m.trans.log <<- log(m.trans) 16 | # m.g.trans <<- m.trans.log-m.trans 17 | 18 | 19 | ##Define the matrix of mixed odds and of relevance m.k: 20 | m.x0<<- (m.slip*(1+m.guess)/(1+m.slip)) 21 | m.x1<<- ((1+m.guess)/(m.guess*(1+m.slip))) 22 | #m.x10<<-m.x1-m.x0 23 | m.x10<<-m.x1/m.x0 24 | 25 | m.k<<- -log(m.guess)-log(m.slip) 26 | 27 | 28 | 29 | 30 | ##Define a matrix of problem difficulties clones for all LOs (used in recommendation) 31 | m.difficulty<<-matrix(rep(difficulty,n.los),ncol=n.los, byrow = FALSE) 32 | rownames(m.difficulty)=probs$id 33 | colnames(m.difficulty)=los$id 34 | m.difficulty=t(m.difficulty) 35 | 36 | -------------------------------------------------------------------------------- /app/config/settings/local.py: -------------------------------------------------------------------------------- 1 | from config.settings.base import * 2 | 3 | SECRET_KEY = 'sp(j(ts6ri()muwz-$^i+k+jgjfv$jbgs@9oq@lzy6x5@lynqd' 4 | 5 | INSTALLED_APPS += [ 6 | 'django_extensions', 7 | ] 8 | 9 | ALLOWED_HOSTS += ['localhost', 'engine'] 10 | 11 | # Database 12 | # https://docs.djangoproject.com/en/1.11/ref/settings/#databases 13 | 14 | DATABASES = { 15 | 'default': { 16 | 'ENGINE': 'django.db.backends.postgresql', 17 | 'NAME': 'engine', 18 | 'USER': 'postgres', 19 | 'HOST': 'postgres', 20 | 'PASSWORD': 'postgres', 21 | 'PORT': 5432, 22 | } 23 | } 24 | 25 | # Logging settings 26 | LOGGING = { 27 | 'version': 1, 28 | 'disable_existing_loggers': False, 29 | 'handlers': { 30 | 'console': { 31 | 'class': 'logging.StreamHandler', 32 | }, 33 | }, 34 | 'loggers': { 35 | 'django': { 36 | 'handlers': ['console'], 37 | 'level': 'INFO', 38 | }, 39 | 'engine': { 40 | 'handlers': ['console'], 41 | 'level': 'DEBUG', 42 | }, 43 | 'alosi.engine': { 44 | 'handlers': ['console'], 45 | 'level': 'DEBUG', 46 | } 47 | }, 48 | } 49 | -------------------------------------------------------------------------------- /prototypes/r_prototype/master.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | 4 | library(plotly) 5 | source("propagator.R") 6 | source("optimizer.R") 7 | source("recommendation.R") 8 | 9 | ####Initialize with fake data#### 10 | source("fakeInitials.R") 11 | ##### 12 | 13 | m.L<<- m.L.i 14 | 15 | source("derivedData.R") 16 | 17 | TotalTime=50 18 | curve=as.data.frame(t(m.L["u1",])) 19 | user_ids=sample(users$id,TotalTime,replace=TRUE) 20 | ##Simulate user interactions: at each moment of time, a randomly picked user submits a problem 21 | learningCurve=matrix(NA,ncol=ncol(m.L),nrow=TotalTime) 22 | colnames(learningCurve)=colnames(m.L) 23 | 24 | for (t in 1:TotalTime){ 25 | 26 | u=user_ids[t] 27 | problem=recommend(u=u) ## Get the recommendation for the next question to serve to the user u. If there is no problem (list exhausted), will be NULL. 28 | if(!is.null(problem)){ 29 | score=predictCorrectness(u,problem) 30 | 31 | bayesUpdate(u=u,problem=problem,score=score, time=t) ##Update the user's mastery matrix and the rest 32 | 33 | } 34 | 35 | learningCurve[t,]=m.L[u,] 36 | cat(problem,'\n') 37 | } 38 | 39 | learningCurve=learningCurve/(1+learningCurve) 40 | 41 | #Optimize the BKT parameters 42 | est=estimate(relevance.threshold=eta, information.threshold=M,remove.degeneracy=TRUE) 43 | m.L.i=est$L.i ##Update the prior-knowledge matrix 44 | 45 | ind.pristine=which(m.exposure==0); ##Update the pristine elements of the current mastery probability matrix 46 | 47 | m.L=replace(m.L,ind.pristine,m.L.i[ind.pristine]) 48 | #Update the transit, guess, slip odds 49 | m.transit=est$transit 50 | m.guess=est$guess 51 | m.slip=est$slip 52 | source("derivedData.R") 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # adaptive-engine 2 | 3 | [![Travis CI build status](https://travis-ci.org/harvard-vpal/adaptive-engine.svg?branch=master)](https://travis-ci.org/harvard-vpal/adaptive-engine) 4 | 5 | ## About 6 | The ALOSI adaptive engine is a web application that powers the recommendation of learning resources to learners based on real-time activity. This application is designed to be used with the [Bridge for Adaptivity](https://github.com/harvard-vpal/bridge-adaptivity), which handles the serving of activities recommended by the engine. 7 | 8 | ## Contents 9 | This repository contains the Django web application code, and related documentation/writeups for the adaptive engine. 10 | 11 | Folder contents: 12 | * `app/` - Adaptive engine web application (python/django) code 13 | * `data/` - data for engine initialization and data processing/transform scripts 14 | * `monitoring/` - terraform files for setting up cloudwatch alarms on an elastic beanstalk deployment 15 | * `python_prototype/` - python prototype for adaptive engine 16 | * `r_prototype/` - R prototype for adaptive engine 17 | * `tests/` - Testing scripts, including load testing with Locust 18 | * `writeup/` - Writeup and LaTeX files to generate the document 19 | 20 | ## Getting started 21 | * [Web application folder and documentation](https://github.com/harvard-vpal/adaptive-engine/tree/master/app) 22 | * [Theoretical overview of the recommendation engine algorithm](https://github.com/harvard-vpal/adaptive-engine/blob/master/writeup/writeup.pdf) 23 | 24 | ## Related projects: 25 | * _alosi_ library: Python package for recommendation engine algorithm utilities, and APIs for ALOSI Bridge and Engine (https://github.com/harvard-vpal/alosi) 26 | * Bridge for Adaptivity: Application that handles serving of content recommended by this and other engines (https://github.com/harvard-vpal/bridge-adaptivity) 27 | -------------------------------------------------------------------------------- /prototypes/r_prototype/testOnSuperEarths.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | # K=1; #K-fold validation (if K=1, it will use all data for both training and validation) 3 | # kfoldRepeatNumber=1 #How many times to do the k-fold validation, to average out weirdness. 4 | # 5 | # ##Use "all" attempts, "first" attempts or "last" attempts 6 | # attempts="all" 7 | 8 | ##Use tagging by "SME" or "auto" 9 | taggingBy="auto" 10 | 11 | 12 | source("load_data.R") 13 | 14 | x.c.all=NULL 15 | x.p.all=NULL 16 | x.p.chance.all=NULL 17 | chance.all=NULL 18 | x.exposure.all=NULL 19 | ##Repeat k-fold validation 20 | 21 | for(kfoldrepeat in 1:kfoldRepeatNumber){ 22 | #Split users into K groups, roughly equal in number 23 | 24 | val_group=rep(1,n.users) 25 | if(K>1){ 26 | gg=1:n.users 27 | ind=NULL 28 | for (i in 2:K){ 29 | if(!is.null(ind)){ 30 | ind.i=sample(gg[-ind],round(n.users/K)); 31 | }else{ 32 | ind.i=sample(gg,round(n.users/K)); 33 | } 34 | val_group[ind.i]=i 35 | ind=c(ind,ind.i) 36 | 37 | } 38 | } 39 | ##NOW do K-fold validation 40 | 41 | for (fold in 1:K){ 42 | if(K>1){ 43 | validation.set=users$id[val_group==fold] 44 | training.set=users$id[val_group!=fold] 45 | }else{ 46 | validation.set=users$id 47 | training.set=users$id 48 | } 49 | before.optimizing=TRUE 50 | source("masterSuperEarths.R") 51 | before.optimizing=FALSE 52 | source("masterSuperEarths.R") 53 | source("evaluate.R") 54 | cat(fold,'out of',K,'folds done, iteration',kfoldrepeat,'\n') 55 | } 56 | 57 | 58 | } 59 | 60 | 61 | x.c=x.c.all 62 | x.p=x.p.all 63 | x.p.chance=x.p.chance.all 64 | chance=chance.all 65 | x.exposure=x.exposure.all 66 | eval.results=list(list(M=M,eta=eta,x.c=x.c,x.p=x.p,chance=chance, x.p.chance=x.p.chance,x.exposure=x.exposure)) 67 | # save(eval.results,file=paste0("eval_results_SCRAMBLED_",taggingBy,"tag_M_",M,"_",K,"_fold_",kfoldRepeatNumber,"_times_",attempts,"_attempts.RData")) 68 | -------------------------------------------------------------------------------- /prototypes/python_prototype/derivedData.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | import numpy as np 3 | 4 | def calculate_derived_data(self): 5 | 6 | ##Define infinity cutoff and the log cutoff: 7 | self.inv_epsilon=1.0/self.epsilon 8 | 9 | self.log_epsilon=-np.log(self.epsilon) 10 | 11 | ## Calculate the useful matrices of guess and slip probabilities and of negative logs of the odds. 12 | self.m_guess_neg_log= -np.log(self.m_guess) 13 | self.m_p_guess= self.m_guess/(self.m_guess+1.0) 14 | 15 | self.m_slip_neg_log= -np.log(self.m_slip) 16 | self.m_p_slip= self.m_slip/(self.m_slip+1.0) 17 | 18 | #m_trans_log = np.log(m_trans) 19 | #m_g_trans = m_trans_log-m_trans 20 | 21 | ##Define the matrix of mixed odds: 22 | 23 | #m_x0_add= np.log(m_slip*(1.0+m_guess)/(1.0+m_slip)) ##Additive formulation 24 | 25 | #Multiplicative formulation 26 | self.m_x0_mult= self.m_slip*(1.0+self.m_guess)/(1.0+self.m_slip) 27 | #m_x1_mult=(1.0+m_guess)/(m_guess*(1.0+m_slip)) 28 | self.m_x1_0_mult=((1.0+self.m_guess)/(self.m_guess*(1.0+self.m_slip)))/self.m_x0_mult 29 | 30 | 31 | #m_x1_0_mult= (1.0+m_guess)/(m_guess*(1.0+m_slip))-m_x0_mult 32 | 33 | 34 | ##Define the matrix of relevance m_k 35 | self.m_k= -np.log(self.m_guess)-np.log(self.m_slip) 36 | 37 | 38 | ##Normalize and prepare difficulty vector: 39 | 40 | #if(difficulty.max()!=difficulty.min()): 41 | # difficulty=(difficulty-difficulty.min())/(difficulty.max()-difficulty.min()) 42 | 43 | self.difficulty=np.minimum(np.maximum(self.difficulty,self.epsilon),1-self.epsilon) 44 | 45 | self.difficulty_mult=self.difficulty/(1.0-self.difficulty) 46 | self.difficulty_add=np.log(self.difficulty_mult) 47 | 48 | ##Define a matrix of problem difficulties clones for all LOs (used in recommendation) 49 | #m_difficulty_mult=np.tile(difficulty_mult,(n_los,1)) 50 | n_los = len(self.los) 51 | self.m_difficulty=np.tile(self.difficulty_add,(n_los,1)) 52 | -------------------------------------------------------------------------------- /prototypes/r_prototype/propagator.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | bayesUpdate=function(u, problem, score=1, time=1, attempts="all", write=TRUE){ 4 | 5 | options(stringsAsFactors = FALSE) 6 | 7 | if((attempts!="first")|((attempts=="first")&(m.timessubmitted[u,problem]==0))){ 8 | if(write){ 9 | transactions<<-rbind(transactions,data.frame(user_id=u,problem_id=problem,time=time,score=score)) 10 | } 11 | x=m.x0[problem,]*((m.x10[problem,])^score) 12 | L=m.L[u,]*x 13 | 14 | ##Add the transferred knowledge 15 | 16 | L=L+m.trans[problem,]*(L+1) 17 | } 18 | 19 | 20 | 21 | ##In case of maxing out to infinity or zero, apply cutoff. 22 | L[which(is.infinite(L))]=inv.epsilon 23 | L[which(L==0)]=epsilon 24 | m.L[u,]<<-L 25 | 26 | ##Bookkeeping: 27 | 28 | ##Record the problem's ID as the last seen by this user. 29 | last.seen[u]<<-problem 30 | 31 | ##Propagate the memory of all items for this user: 32 | m.item.memory[u,]<<-m.item.memory[u,]*m.forgetting[u,] 33 | ##Update the memory of the submitted item: 34 | m.item.memory[u,problem]<<-m.item.memory[u,problem]+1 35 | 36 | 37 | ##Record that item was submitted 38 | m.times.submitted[u,problem]<<-m.times.submitted[u,problem]+1 39 | ##Update exposure and confidence for this user/KC combinations 40 | m.exposure[u,]<<-m.exposure[u,]+m.tagging[problem,] 41 | m.confidence[u,]<<-m.confidence[u,]+m.k[problem,] 42 | } 43 | 44 | predictCorrectness=function(u, problem){ 45 | 46 | 47 | #This function calculates the probability of correctness on a problem, as a prediction based on student's current mastery. 48 | 49 | L=m.L[u,] 50 | p.slip=m.p.slip[problem,]; 51 | p.guess=m.p.guess[problem,]; 52 | 53 | x=(L*(1-p.slip)+p.guess)/(L*p.slip+1-p.guess); ##Odds by LO 54 | # x=(L*(1-p.slip)+p.guess)/(L+1); ##Odds by LO 55 | x=prod(x) ##Total odds 56 | 57 | p=x/(1+x) ##Convert odds to probability 58 | if(is.na(p)|is.infinite(p)){ 59 | p=1 60 | } 61 | return(p) 62 | 63 | } -------------------------------------------------------------------------------- /app/README.md: -------------------------------------------------------------------------------- 1 | # ALOSI adaptive engine web application 2 | This repo subfolder contains a Django web application the runs the ALOSI adaptive engine. 3 | 4 | ## Running the engine application locally 5 | 6 | ### Install prerequisites: 7 | * [Docker](https://docs.docker.com/install/) 8 | 9 | ### Setup application for local development: 10 | 11 | ``` 12 | # clone the repo locally 13 | git clone https://github.com/harvard-vpal/adaptive-engine 14 | 15 | # change into app directory 16 | cd app 17 | 18 | # build Docker images and start up application in background 19 | docker-compose up -d 20 | 21 | # apply database migrations 22 | docker-compose run engine python manage.py migrate 23 | ``` 24 | 25 | The engine should now be available at localhost:8000. Try opening localhost:8000/engine/api in a web browser. 26 | 27 | ## Application Configuration 28 | ### Creating a superuser 29 | To access the Django admin panel, a superuser needs to be created. 30 | ``` 31 | # Open an interactive shell in engine docker container 32 | docker-compose run engine bash 33 | 34 | # Create super user account 35 | python manage.py createsuperuser 36 | 37 | # ... answer the prompts ... 38 | 39 | # Ctrl-D to exit shell when finished 40 | 41 | ``` 42 | 43 | ### API Token generation 44 | 1. Open admin panel (localhost:8000/admin) and log in with user credentials 45 | 46 | 2. Create a new Token model associated with user (Token -> Add Token). An API token will be auto-generated in the 47 | `key` field of the new model. 48 | 49 | 50 | ## Running tests 51 | ``` 52 | docker-compose run web pytest 53 | ``` 54 | 55 | ## Running model update 56 | There is a custom django-admin command to update the engine model. One approach for automating the model update is to 57 | set up a cron job. Here's an example that runs the custom command via Docker (assumes the image is named "app", and 58 | uses custom settings located in `config/settings/eb_prod.py`) that updates the model every 2 hours: 59 | ``` 60 | 0 */2 * * * docker run app python manage.py update_model --eta=0.0 --M=20.0 --settings=config.settings.eb_prod 61 | ``` 62 | -------------------------------------------------------------------------------- /prototypes/python_prototype/sim.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | #Should L be mastery odds(for multiplicative) or logarithmic odds (for additive formulation)? 4 | 5 | from multiplicativeFormulation import MultiplicativeFormulation 6 | import numpy as np 7 | 8 | n_users=10 9 | n_los=8 10 | n_items=40 11 | n_modules=2 12 | 13 | # initialize users, los, items 14 | ##Store mappings of ids and names for users, LOs, items. These will serve as look-up tables for the rows and columns of data matrices 15 | users='u'+np.char.array(range(n_users)) 16 | los='l'+np.char.array(range(n_los)) 17 | items='p'+np.char.array(range(n_items)) 18 | 19 | engine = MultiplicativeFormulation( 20 | users = users, 21 | los=los, 22 | items=items, 23 | n_modules=n_modules, 24 | 25 | epsilon=1e-10, # a regularization cutoff, the smallest value of a mastery probability 26 | eta=0.0, ##Relevance threshold used in the BKT optimization procedure 27 | M=0.0, ##Information threshold user in the BKT optimization procedure 28 | L_star=2.2, #Threshold logarithmic odds. If mastery logarithmic odds are >= than L_star, the LO is considered mastered 29 | 30 | r_star=0.0, #Threshold for forgiving lower odds of mastering pre-requisite LOs. 31 | W_p=5.0, ##Importance of readiness in recommending the next item 32 | W_r=3.0, ##Importance of demand in recommending the next item 33 | W_d=1.0, ##Importance of appropriate difficulty in recommending the next item 34 | W_c=1.0, ##Importance of continuity in recommending the next item 35 | 36 | ##Values prior to estimating model: 37 | slip_probability=0.15, 38 | guess_probability=0.1, 39 | trans_probability=0.1, 40 | prior_knowledge_probability=0.2, 41 | 42 | los_per_item=2, ##Number of los per problem 43 | ) 44 | 45 | # initialize random user response data 46 | T=2000 47 | user_ids=np.random.choice(users,T) 48 | score=np.random.choice([0,1],T) 49 | 50 | for t in range(T): 51 | 52 | u=engine.mapUser(user_ids[t]) 53 | rec_item=engine.recommend(u) 54 | 55 | if rec_item!=None: 56 | engine.bayesUpdate(u,rec_item,score[t],t) 57 | 58 | print "updating model" 59 | engine.updateModel() 60 | 61 | -------------------------------------------------------------------------------- /prototypes/r_prototype/masterSuperEarths.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | 4 | library(plotly) 5 | source("/Users/ilr548/Dropbox/BKT/multiplicative formulation/propagator.R") 6 | source("/Users/ilr548/Dropbox/BKT/multiplicative formulation/optimizer.R") 7 | 8 | 9 | source("initialsSuperEarths.R") 10 | 11 | m.L<<- m.L.i 12 | 13 | source("/Users/ilr548/Dropbox/BKT/multiplicative formulation/derivedData.R") 14 | time.start=proc.time()[3]; 15 | if(!before.optimizing){ 16 | curve=matrix(NA,nrow=1,ncol=n.los) 17 | Pcheck$predicted=NA; 18 | for(i in 1:nrow(Pcheck)){ 19 | problem=Pcheck$problem_id[i] 20 | score=Pcheck$correctness[i] 21 | time=Pcheck$time[i] 22 | u=Pcheck$username[i] 23 | Pcheck$predicted[i]=predictCorrectness(u=u,problem=problem) ##Predict probability of success 24 | bayesUpdate(u=u,problem=problem,score=score,time=time) ##Update the user's mastery matrix and history 25 | 26 | ##This matrix tracks whether this is the situation when the user had no prior interaction with the learning objectives 27 | temp=(m.exposure[u,,drop=F]) %*% t(m.tagging[problem,,drop=F]) 28 | m.exposure.before.problem[u,problem]=temp 29 | 30 | # if(u=="DevonBMason"){ 31 | # curve=rbind(curve,m.L[u,]) 32 | # } 33 | 34 | 35 | } 36 | 37 | 38 | cat("Elapsed seconds in knowledge tracing: ",round(proc.time()[3]-time.start,3),"\n") 39 | } 40 | 41 | # 42 | # p=plot_ly() 43 | # for ( i in 1:ncol(curve)){ 44 | # p=p%>%add_trace(x=1:nrow(curve),y=curve[,i]/(1+curve[,i]),type="scatter",mode="markers+lines", name=los$name[i]) 45 | # } 46 | # p=p%>%layout(title="Learning curves of user 1", xaxis=list(title="Time"),yaxis=list(title="probability of mastery")) 47 | # print(p) 48 | 49 | 50 | if(before.optimizing){ 51 | #Optimize the BKT parameters 52 | time.start=proc.time()[3] 53 | 54 | ##Estimate on the training set 55 | est=estimate(relevance.threshold=eta, information.threshold=M,remove.degeneracy=T, training.set) 56 | cat("Elapsed seconds in estimating: ",round(proc.time()[3]-time.start,3),"\n") 57 | m.L.i=est$L.i ##Update the prior-knowledge matrix 58 | ind.pristine=which(m.exposure==0); 59 | ##Update the pristine elements of the current mastery probability matrix 60 | m.L=replace(m.L,ind.pristine,m.L.i[ind.pristine]) 61 | #Update the transit, guess, slip odds 62 | m.trans=est$trans 63 | m.guess=est$guess 64 | m.slip=est$slip 65 | } 66 | -------------------------------------------------------------------------------- /prototypes/r_prototype/recommendation.R: -------------------------------------------------------------------------------- 1 | recommend=function(u, module=1, stopOnMastery=FALSE,maxitems=1){ 2 | 3 | ##This function returns the id(s) of the next recommended problem(s). If none is recommended it returns NULL. 4 | 5 | ##Exclude from the pool items that have been submitted too many times and which do not belong to the scope 6 | ind.pool=which(((m.times.submitted[u,]=Nmin]; 38 | df_tag=subset(df_tag,df_tag$LO %in% los.select) 39 | 40 | 41 | 42 | ##SCRAMBLE: 43 | # df_tag$LO=sample(df_tag$LO,replace=FALSE) 44 | # df_tag$LO=1 45 | 46 | 47 | # los.select=c("Density1","Distance2","Exo-Transit2","Exo-Wobble2","Gravity1","Life-Needs1","Life-Planet1","Planets1","Planets2","Science2","Spectro1","Spectro3","Spectro4") 48 | # df_tag=subset(df_tag,df_tag$LO %in% los.select) 49 | # df_tag.probs.lo=subset(df_tag.probs.lo,df_tag.probs.lo$LO %in% los.select) 50 | 51 | 52 | ###### 53 | 54 | Pcheck=read.csv(file.path(datadir_charlesRiverX,"problem_check.csv.gz"),header=T, stringsAsFactors = F) 55 | options(digits.secs=8) 56 | Pcheck$time=as.POSIXct(Pcheck$time,tz="UTC") 57 | Pcheck$problem_id=gsub(moduleIDPrefix,"",Pcheck$module_id) 58 | 59 | Pcheck=subset(Pcheck,problem_id %in% df_tag$edx.id) 60 | Pcheck$correctness=0 61 | Pcheck$correctness[which(Pcheck$success=="correct")]=1 62 | 63 | #source("staffUserNames.R") 64 | load("staffUserNames.RData") 65 | Pcheck=subset(Pcheck,!(username %in% staff$username)) 66 | 67 | # source("buttefly.R") 68 | 69 | 70 | ############# 71 | 72 | if(attempts=="first"){ 73 | 74 | # #Leave the first attempts only: 75 | Pcheck$userproblem_id=paste(Pcheck$username,Pcheck$problem_id) 76 | temp=aggregate(Pcheck$time, by=list("userproblem_id"=Pcheck$userproblem_id), FUN=min) 77 | temp=merge(Pcheck,temp, by="userproblem_id") 78 | Pcheck=subset(temp,time==x); 79 | Pcheck$userproblem_id=NULL 80 | Pcheck$x=NULL 81 | } 82 | 83 | if(attempts=="last"){ 84 | 85 | #Leave the last attempts only: 86 | Pcheck$userproblem_id=paste(Pcheck$username,Pcheck$problem_id) 87 | temp=aggregate(Pcheck$time, by=list("userproblem_id"=Pcheck$userproblem_id), FUN=max) 88 | temp=merge(Pcheck,temp, by="userproblem_id") 89 | Pcheck=subset(temp,time==x); 90 | Pcheck$userproblem_id=NULL 91 | Pcheck$x=NULL 92 | } 93 | 94 | ############## 95 | 96 | 97 | Pcheck$time=as.numeric(Pcheck$time) 98 | 99 | 100 | Pcheck=Pcheck[order(Pcheck$time),] 101 | 102 | 103 | ##Define the lists of LOs and problems 104 | los=data.frame("id"=unique(df_tag$LO)); 105 | los$id=as.character(los$id) 106 | los$name=los$id 107 | los=los[order(los$id),] 108 | n.los=nrow(los) 109 | 110 | probs=data.frame("id"=unique(df_tag$edx.id)); 111 | probs$id=as.character(probs$id) 112 | # temp=unique(df_tag[,c("edx.id","display_name.x")]) 113 | # temp=unique(df_tag[]) 114 | # probs=merge(probs,temp,by.x="id",by.y="edx.id") 115 | # probs=plyr::rename(probs,c("display_name.x"="name")) 116 | 117 | probs$name=probs$id 118 | 119 | #probs=probs[order(probs$id),] 120 | n.probs=nrow(probs) 121 | 122 | 123 | 124 | #Define the list of users 125 | users=data.frame("id"=unique(Pcheck$username)) 126 | users$id=as.character(users$id) 127 | users$name=users$id 128 | n.users=nrow(users) 129 | 130 | cat("Data Loaded.", n.probs, "problems,", n.users, "users,", n.los, "KCs,", nrow(Pcheck),'submits\n') 131 | 132 | -------------------------------------------------------------------------------- /writeup/iopart12.clo: -------------------------------------------------------------------------------- 1 | %% 2 | %% This is file `iopart12.clo' 3 | %% 4 | %% This file is distributed in the hope that it will be useful, 5 | %% but WITHOUT ANY WARRANTY; without even the implied warranty of 6 | %% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | %% 8 | %% \CharacterTable 9 | %% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z 10 | %% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z 11 | %% Digits \0\1\2\3\4\5\6\7\8\9 12 | %% Exclamation \! Double quote \" Hash (number) \# 13 | %% Dollar \$ Percent \% Ampersand \& 14 | %% Acute accent \' Left paren \( Right paren \) 15 | %% Asterisk \* Plus \+ Comma \, 16 | %% Minus \- Point \. Solidus \/ 17 | %% Colon \: Semicolon \; Less than \< 18 | %% Equals \= Greater than \> Question mark \? 19 | %% Commercial at \@ Left bracket \[ Backslash \\ 20 | %% Right bracket \] Circumflex \^ Underscore \_ 21 | %% Grave accent \` Left brace \{ Vertical bar \| 22 | %% Right brace \} Tilde \~} 23 | \ProvidesFile{iopart12.clo}[1997/01/15 v1.0 LaTeX2e file (size option)] 24 | \renewcommand\normalsize{% 25 | \@setfontsize\normalsize\@xiipt{16}% 26 | \abovedisplayskip 12\p@ \@plus3\p@ \@minus7\p@ 27 | \abovedisplayshortskip \z@ \@plus3\p@ 28 | \belowdisplayshortskip 6.5\p@ \@plus3.5\p@ \@minus3\p@ 29 | \belowdisplayskip \abovedisplayskip 30 | \let\@listi\@listI} 31 | \normalsize 32 | \newcommand\small{% 33 | \@setfontsize\small\@xipt{14}% 34 | \abovedisplayskip 11\p@ \@plus3\p@ \@minus6\p@ 35 | \abovedisplayshortskip \z@ \@plus3\p@ 36 | \belowdisplayshortskip 6.5\p@ \@plus3.5\p@ \@minus3\p@ 37 | \def\@listi{\leftmargin\leftmargini 38 | \topsep 9\p@ \@plus3\p@ \@minus5\p@ 39 | \parsep 4.5\p@ \@plus2\p@ \@minus\p@ 40 | \itemsep \parsep}% 41 | \belowdisplayskip \abovedisplayskip 42 | } 43 | \newcommand\footnotesize{% 44 | % \@setfontsize\footnotesize\@xpt\@xiipt 45 | \@setfontsize\footnotesize\@xpt{13}% 46 | \abovedisplayskip 10\p@ \@plus2\p@ \@minus5\p@ 47 | \abovedisplayshortskip \z@ \@plus3\p@ 48 | \belowdisplayshortskip 6\p@ \@plus3\p@ \@minus3\p@ 49 | \def\@listi{\leftmargin\leftmargini 50 | \topsep 6\p@ \@plus2\p@ \@minus2\p@ 51 | \parsep 3\p@ \@plus2\p@ \@minus\p@ 52 | \itemsep \parsep}% 53 | \belowdisplayskip \abovedisplayskip 54 | } 55 | \newcommand\scriptsize{\@setfontsize\scriptsize\@viiipt{9.5}} 56 | \newcommand\tiny{\@setfontsize\tiny\@vipt\@viipt} 57 | \newcommand\large{\@setfontsize\large\@xivpt{18}} 58 | \newcommand\Large{\@setfontsize\Large\@xviipt{22}} 59 | \newcommand\LARGE{\@setfontsize\LARGE\@xxpt{25}} 60 | \newcommand\huge{\@setfontsize\huge\@xxvpt{30}} 61 | \let\Huge=\huge 62 | \if@twocolumn 63 | \setlength\parindent{14\p@} 64 | \else 65 | \setlength\parindent{18\p@} 66 | \fi 67 | \setlength\headheight{14\p@} 68 | \setlength\headsep{14\p@} 69 | \setlength\topskip{12\p@} 70 | \setlength\footskip{24\p@} 71 | \setlength\maxdepth{.5\topskip} 72 | \setlength\@maxdepth\maxdepth 73 | \setlength\textwidth{37.2pc} 74 | \setlength\textheight{56pc} 75 | \setlength\oddsidemargin {\z@} 76 | \setlength\evensidemargin {\z@} 77 | \setlength\marginparwidth {72\p@} 78 | \setlength\marginparsep{10\p@} 79 | \setlength\marginparpush{5\p@} 80 | \setlength\topmargin{-12pt} 81 | \setlength\footnotesep{8.4\p@} 82 | \setlength{\skip\footins} {10.8\p@ \@plus 4\p@ \@minus 2\p@} 83 | \setlength\floatsep {14\p@ \@plus 2\p@ \@minus 4\p@} 84 | \setlength\textfloatsep {24\p@ \@plus 2\p@ \@minus 4\p@} 85 | \setlength\intextsep {16\p@ \@plus 4\p@ \@minus 4\p@} 86 | \setlength\dblfloatsep {16\p@ \@plus 2\p@ \@minus 4\p@} 87 | \setlength\dbltextfloatsep{24\p@ \@plus 2\p@ \@minus 4\p@} 88 | \setlength\@fptop{0\p@} 89 | \setlength\@fpsep{10\p@ \@plus 1fil} 90 | \setlength\@fpbot{0\p@} 91 | \setlength\@dblfptop{0\p@} 92 | \setlength\@dblfpsep{10\p@ \@plus 1fil} 93 | \setlength\@dblfpbot{0\p@} 94 | \setlength\partopsep{3\p@ \@plus 2\p@ \@minus 2\p@} 95 | \def\@listI{\leftmargin\leftmargini 96 | \parsep=\z@ 97 | \topsep=6\p@ \@plus3\p@ \@minus3\p@ 98 | \itemsep=3\p@ \@plus2\p@ \@minus1\p@} 99 | \let\@listi\@listI 100 | \@listi 101 | \def\@listii {\leftmargin\leftmarginii 102 | \labelwidth\leftmarginii 103 | \advance\labelwidth-\labelsep 104 | \topsep=3\p@ \@plus2\p@ \@minus\p@ 105 | \parsep=\z@ 106 | \itemsep=\parsep} 107 | \def\@listiii{\leftmargin\leftmarginiii 108 | \labelwidth\leftmarginiii 109 | \advance\labelwidth-\labelsep 110 | \topsep=\z@ 111 | \parsep=\z@ 112 | \partopsep=\z@ 113 | \itemsep=\z@} 114 | \def\@listiv {\leftmargin\leftmarginiv 115 | \labelwidth\leftmarginiv 116 | \advance\labelwidth-\labelsep} 117 | \def\@listv{\leftmargin\leftmarginv 118 | \labelwidth\leftmarginv 119 | \advance\labelwidth-\labelsep} 120 | \def\@listvi {\leftmargin\leftmarginvi 121 | \labelwidth\leftmarginvi 122 | \advance\labelwidth-\labelsep} 123 | \endinput 124 | %% 125 | %% End of file `iopart12.clo'. -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/master.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | K=1; #K-fold validation (if K=1, it will use all data for both training and validation) 3 | kfoldRepeatNumber=1 #How many times to do the k-fold validation, to average out weirdness. 4 | 5 | saveResults=FALSE 6 | 7 | #Will remove timestamps prior to this 8 | start_date=as.POSIXct('2017-10-17',tz='UTC') 9 | 10 | ##Use tagging by "SME" or "auto" 11 | taggingBy="SME" 12 | options(stringsAsFactors = FALSE) 13 | 14 | 15 | ##Point to where the scripts propagator, optimizer and derivedData are: 16 | dir_scripts='..' 17 | dir_scripts='../multiplicative formulation' 18 | 19 | ###Point to the place where SMEs tables are: 20 | datadir='../SME_data' 21 | ##Where to write matrices (will create directory if missing; if NULL, will not write anywhere) 22 | writedir=NULL 23 | 24 | source(file.path(dir_scripts,"propagator.R")) 25 | source(file.path(dir_scripts,"optimizer.R")) 26 | 27 | 28 | 29 | 30 | 31 | ######Load the transaction log data (This needs to be changed according to your data situation)####### 32 | ###Required variables in the log: "username","problem_id","time","score". The "score" should be on 0-1 scale. 33 | 34 | tological=function(vec 35 | ,TRUEis=c("1","1.0","true","True","TRUE") ## Which entries count as TRUE, if the vector is character. 36 | ,NAis=c("","NA","na") ##Which entries count as NA, if the vector is character. 37 | ,NAmeans=FALSE ##What does NA mean? I.e. what it should be replaced with. 38 | ){ 39 | if((is.numeric(vec))|is.logical(vec)){ 40 | temp=as.logical(vec); 41 | temp[is.na(temp)]=NAmeans; 42 | }else{ 43 | temp=rep(FALSE,length(vec)); 44 | temp[vec %in% TRUEis]=TRUE; 45 | temp[vec %in% NAis]=NAmeans; 46 | } 47 | return(temp); 48 | } 49 | 50 | 51 | library(plyr) 52 | options(stringsAsFactors = FALSE) 53 | ddir='/Users/ilr548/Documents/AdaptiveEngineData' 54 | LogData=read.csv(file.path(ddir,'engine_score'),header=TRUE) 55 | engineLearner=read.csv(file.path(ddir,'engine_learner'),header=TRUE) 56 | engineActivity=read.csv(file.path(ddir,'engine_activity'),header=TRUE) 57 | engineCollection=plyr::rename(read.csv(file.path(ddir,'engine_collection'),header=TRUE),c('name'='module_name','max_problems'='problems_in_module')) 58 | engineActivity=merge(engineActivity,engineCollection,by.x='collection_id',by.y='id') 59 | 60 | 61 | 62 | engineActivity$include_adaptive=tological(engineActivity$include_adaptive) 63 | LogData=merge(LogData,engineLearner,by.x='learner_id',by.y = 'id') 64 | LogData=merge(LogData,engineActivity, by.x='activity_id',by.y='id') 65 | 66 | LogData=plyr::rename(LogData,c('timestamp'='time','activity_id'='problem_id','learner_id'='user_id')) 67 | 68 | options(digits.secs=8) 69 | LogData$time=as.POSIXct(LogData$time,tz="UTC") 70 | ##Subset to the events after the launch: 71 | LogData=subset(LogData,LogData$time>=start_date) 72 | LogData$time=as.numeric(LogData$time) 73 | 74 | LogData$user_id=as.character(LogData$user_id) 75 | 76 | LogData$module_name=factor(LogData$module_name,levels=engineCollection$module_name) 77 | 78 | ##IMPORTANT: order chronologically! 79 | LogData=LogData[order(LogData$time),] 80 | 81 | ######################################################### 82 | 83 | 84 | 85 | 86 | source("data_load.R") 87 | source(file.path(dir_scripts,"derivedData.R")) 88 | x.c.all=NULL 89 | x.p.all=NULL 90 | x.p.chance.all=NULL 91 | chance.all=NULL 92 | x.exposure.all=NULL 93 | 94 | ##Repeat k-fold validation 95 | for(kfoldrepeat in 1:kfoldRepeatNumber){ 96 | #Split users into K groups, roughly equal in number 97 | 98 | val_group=rep(1,n.users) 99 | if(K>1){ 100 | gg=1:n.users 101 | ind=NULL 102 | for (i in 2:K){ 103 | if(!is.null(ind)){ 104 | ind.i=sample(gg[-ind],round(n.users/K)); 105 | }else{ 106 | ind.i=sample(gg,round(n.users/K)); 107 | } 108 | val_group[ind.i]=i 109 | ind=c(ind,ind.i) 110 | 111 | } 112 | } 113 | 114 | ############################ 115 | ##Now do K-fold validation## 116 | ############################ 117 | for (fold in 1:K){ 118 | if(K>1){ 119 | validation.set=users$id[val_group==fold] 120 | training.set=users$id[val_group!=fold] 121 | }else{ 122 | validation.set=users$id 123 | training.set=users$id 124 | } 125 | before.optimizing=TRUE 126 | source('clean_slate.R') 127 | source('update_model.R') 128 | before.optimizing=FALSE 129 | source('run_through.R') 130 | source('evaluate.R') 131 | cat(fold,'out of',K,'folds done, iteration',kfoldrepeat,'\n') 132 | } 133 | 134 | 135 | } 136 | 137 | 138 | x.c=x.c.all 139 | x.p=x.p.all 140 | x.p.chance=x.p.chance.all 141 | chance=chance.all 142 | x.exposure=x.exposure.all 143 | eval.results=list(list(M=M,eta=eta,x.c=x.c,x.p=x.p,chance=chance, x.p.chance=x.p.chance,x.exposure=x.exposure)) 144 | if(saveResults){ 145 | save(eval.results,file=paste0("new_eval_results_",taggingBy,"tag_M_",M,"_",K,"_fold_",kfoldRepeatNumber,"_times.RData")) 146 | } 147 | -------------------------------------------------------------------------------- /prototypes/python_prototype/additiveFormulation.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | ##This function maps the user_id to the user index used by other functions, and also adds new users 4 | ##SYNCHRONIZATION IS IMPORTANT 5 | def mapUser(user_id): 6 | 7 | global users 8 | 9 | try: 10 | u=np.where(users==user_id)[0][0] 11 | except: 12 | global n_users, last_seen, m_L, m_exposure, m_unseen, m_correctness, m_timestamp 13 | u=n_users 14 | n_users+=1 15 | users=np.append(users,user_id) 16 | last_seen=np.append(last_seen,-1) 17 | m_L=np.vstack((m_L,L_i)) 18 | m_exposure=np.vstack((m_exposure,row_exposure)) 19 | m_unseen=np.vstack((m_unseen,row_unseen)) 20 | m_correctness=np.vstack((m_correctness,row_correctness)) 21 | m_timestamp=np.vstack((m_timestamp,row_timestamp)) 22 | 23 | 24 | return u 25 | 26 | 27 | def mapItem(item_id): 28 | 29 | global items 30 | 31 | item=np.where(items==item_id)[0][0] 32 | 33 | return(item) 34 | 35 | def bayesUpdate(u, item, score=1, time=0): 36 | 37 | 38 | #This function updates the user mastery and record of interactions that will be needed for recommendation and estimation of the BKT 39 | 40 | # global m_x0_add, m_k, m_L, m_trans, last_seen, m_unseen, m_correctness, m_timestamp, m_exposure, m_tagging, log_epsilon 41 | 42 | 43 | last_seen[u]=item 44 | self.m_correctness[u,item]=score 45 | self.m_timestamp[u,item]=time 46 | if m_unseen[u,item]: 47 | self.m_unseen[u,item]=False 48 | self.m_exposure[u,]+=self.m_tagging[item,] 49 | 50 | ##The increment of log-odds due to evidence of the problem, but before the transfer 51 | x=self.m_x0_add[item,]+score*self.m_k[item,] 52 | L=self.m_L[u,]+x 53 | 54 | ##Add the transferred knowledge 55 | trans=self.m_trans[item,] 56 | #L=np.log(trans+(trans+1)*np.exp(L)) 57 | L=np.log(trans+(trans+1)*np.exp(L)) 58 | 59 | L[np.isposinf(L)]=self.log_epsilon 60 | L[np.isneginf(L)]=-self.log_epsilon 61 | 62 | self.m_L[u,]=L 63 | #return{'L':L, 'x':x} 64 | 65 | 66 | 67 | 68 | #This function calculates the probability of correctness on a problem, as a prediction based on student's current mastery. 69 | def predictCorrectness(u, item): 70 | 71 | global m_L, m_p_slip, m_p_guess 72 | 73 | L=m_L[u,] 74 | p_slip=m_p_slip[item,]; 75 | p_guess=m_p_guess[item,]; 76 | 77 | odds=np.exp(L); 78 | 79 | x=(odds*(1.0-p_slip)+p_guess)/(odds*p_slip+1.0-p_guess); ##Odds by LO 80 | x=np.prod(x) ##Total odds 81 | 82 | p=x/(1+x) ##Convert odds to probability 83 | if np.isnan(p) or np.isinf(p): 84 | p=1.0 85 | 86 | return(p) 87 | 88 | 89 | 90 | ##This function returns the id of the next recommended problem. If none is recommended (list of problems exhausted or the user has reached mastery) it returns None. 91 | def recommend(u, module=1, stopOnMastery=True): 92 | 93 | global m_L, L_star, m_w, m_unseen, m_k, r_star, last_seen, m_difficulty_add, V_r, V_d, V_a, V_c, scope 94 | 95 | #Subset to the unseen problems and calculate problem readiness and demand 96 | ind_unseen=np.where(m_unseen[u,] & (scope==module)|(scope==0))[0] 97 | 98 | N=len(ind_unseen) 99 | 100 | if(N==0): ##This means we ran out of problems, so we stop 101 | next_item = None 102 | 103 | else: 104 | L=m_L[u,] 105 | 106 | #Calculate the user readiness for LOs 107 | 108 | m_r=np.dot(np.minimum(L-L_star,0), m_w); 109 | m_k_unseen=m_k[ind_unseen,] 110 | R=np.dot(m_k_unseen, np.minimum((m_r+r_star),0)) 111 | D=np.dot(m_k_unseen, np.maximum((L_star-L),0)) 112 | 113 | if last_seen[u]<0: 114 | C=np.repeat(0.0,N) 115 | else: 116 | C=np.dot(m_k_unseen, m_k[last_seen[u],]) 117 | 118 | #A=0.0 119 | d_temp=m_difficulty_add[:,ind_unseen] 120 | L_temp=np.tile(L,(N,1)).transpose() 121 | A=-np.diag(np.dot(m_k_unseen,np.abs(L_temp-d_temp))) 122 | 123 | if stopOnMastery and sum(D)==0: ##This means the user has reached threshold mastery in all LOs relevant to the problems in the homework, so we stop 124 | next_item=None 125 | else: 126 | 127 | temp=(A.max()-A.min()); 128 | if(temp!=0.0): 129 | A=A/temp 130 | 131 | temp=(D.max()-D.min()); 132 | if(temp!=0.0): 133 | D=D/temp 134 | 135 | temp=(R.max()-R.min()); 136 | if(temp!=0.0): 137 | R=R/temp 138 | 139 | temp=(C.max()-C.min()); 140 | if(temp!=0.0): 141 | C=C/temp 142 | 143 | next_item=ind_unseen[np.argmax(V_r*R+V_d*D+V_a*A+V_c*C)] 144 | 145 | 146 | return(next_item) 147 | 148 | 149 | 150 | #This function updates the BKT model using the estimates from student data. 151 | def updateModel(): 152 | 153 | global eta, M, L_i, m_exposure, m_L, m_L_i, m_trans, m_guess, m_slip 154 | est=estimate(eta, M) 155 | 156 | L_i=np.log(est['L_i']) 157 | m_L_i=np.tile(L_i,(m_L.shape[0],1)) 158 | 159 | ind_pristine=np.where(m_exposure==0.0) 160 | m_L[ind_pristine]=m_L_i[ind_pristine] 161 | m_trans=1.0*est[‘trans’] 162 | m_guess=1.0*est[‘guess’] 163 | m_slip=1.0*est[‘slip’] 164 | -------------------------------------------------------------------------------- /prototypes/r_prototype/fakeInitials.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | n.users<<-1 4 | n.los<<-8 5 | n.probs<<-40 6 | n.modules=2 7 | 8 | slip.probability=0.15 9 | guess.probability=0.1 10 | trans.probability=0.1 11 | prior.knowledge=0.2 12 | forgetting=exp(-1) 13 | 14 | ####Global variables#### 15 | epsilon<<-1e-10 # a regularization cutoff, the smallest value of a mastery probability 16 | eta=0 ##Relevance threshold used in the BKT optimization procedure 17 | M=20 ##Information threshold user in the BKT optimization procedure 18 | L.star<<- 3 #Threshold odds. If mastery odds are >= than L.star, the LO is considered mastered 19 | r.star<<- 0 #Threshold for forgiving lower odds of mastering pre-requisite LOs. 20 | 21 | ##Substrategy weights 22 | 23 | W<<-c( 24 | 'remediation'=3 25 | ,'continuity'=0.5 26 | ,'readiness'=1 27 | ,'difficulty'=1 28 | ,'memory'=1 29 | ,'suggested'=1 30 | ) 31 | 32 | ##List of substrategies: 33 | strategy<<-rep(list(NULL),length(W)) 34 | names(strategy)=names(W) 35 | 36 | 37 | options(stringsAsFactors = FALSE) 38 | 39 | 40 | users=data.frame("id"=paste0("u",1:n.users),"name"=paste0("user ",1:n.users), "group"=1) 41 | users$group[1:round(n.users*2/3)]=0 42 | users$id=as.character(users$id) 43 | los=data.frame("id"=paste0("l",1:n.los),"name"=paste0("LO ",1:n.los)) 44 | los$id=as.character(los$id) 45 | probs=data.frame("id"=paste0("p",1:n.probs),"name"=paste0("problem ",1:n.probs)) 46 | probs$id=as.character(probs$id) 47 | 48 | probs$required.next.id='' 49 | probs$suggested.next.id='' 50 | # probs$required.next.id[2]=c(probs$id[1]) 51 | probs$suggested.next.id=c(probs$id[-1],'') 52 | 53 | probs$maxsubmits.for.serving=c(NA,rep(1,nrow(probs)-1)) 54 | 55 | 56 | #Let problems be divided into several modules of adaptivity. In each module, only the items from that scope are used. 57 | 58 | scope<<-matrix(FALSE,nrow=n.probs, ncol=n.modules) 59 | scope[,1]=TRUE 60 | rownames(scope)=probs$id 61 | 62 | 63 | 64 | ##List which items should be used for training the BKT 65 | useForTraining=probs$id 66 | 67 | 68 | #Initialize the matrix of mastery odds 69 | 70 | L.i<<-exp(rep(0,n.los)) 71 | 72 | # Define the matrix of initial mastery by replicating the same row for each user 73 | m.L.i<<-matrix(rep(L.i,n.users),ncol=n.los, byrow = FALSE) 74 | rownames(m.L.i)=users$id 75 | colnames(m.L.i)=los$id 76 | 77 | ##Define pre-requisite matrix. rownames are pre-reqs. Assumed that the entries are in [0,1] interval #### 78 | m.w<<-matrix(runif(n.los^2),nrow=n.los); 79 | rownames(m.w)=los$id 80 | colnames(m.w)=los$id 81 | for(i in 1:nrow(m.w)){ 82 | 83 | for(j in 1:ncol(m.w)){ 84 | des=sample(c(TRUE,FALSE),1) 85 | if(des){ 86 | m.w[i,j]=0 87 | }else{ 88 | m.w[j,i]=0 89 | } 90 | } 91 | 92 | } 93 | ## 94 | 95 | 96 | ##Define the vector of difficulties #### 97 | difficulty<<-rep(0.5,n.probs); 98 | names(difficulty)=probs$id 99 | 100 | difficulty=pmin(pmax(difficulty,epsilon),1-epsilon) 101 | difficulty=log(difficulty/(1-difficulty)) 102 | 103 | ## 104 | 105 | 106 | ##Define the preliminary relevance matrix: problems tagged with LOs. rownames are problems. Assumed that the entries are in [0,1] interval #### 107 | 108 | los.per.problem=1 109 | 110 | temp=c(rep(1,los.per.problem),rep(0,n.los-los.per.problem)) 111 | m.tagging<<-matrix(0,nrow=n.probs, ncol=n.los); 112 | 113 | for(i in 1:n.probs){ 114 | m.tagging[i,]=sample(temp,replace=FALSE) 115 | } 116 | rownames(m.tagging)=probs$id 117 | colnames(m.tagging)=los$id 118 | 119 | 120 | ##Define the matrix of transit odds #### 121 | 122 | m.trans<<-(trans.probability/(1-trans.probability))*m.tagging 123 | ## 124 | 125 | ##Define the matrix of guess odds (and probabilities) #### 126 | m.guess<<-(guess.probability/(1-guess.probability))*matrix(1,nrow=n.probs, ncol = n.los); 127 | m.guess[which(m.tagging==0)]=1 128 | rownames(m.guess)=probs$id 129 | colnames(m.guess)=los$id 130 | ## 131 | 132 | ##Define the matrix of slip odds #### 133 | m.slip<<-(slip.probability/(1-slip.probability))*matrix(1,nrow=n.probs, ncol = n.los); 134 | m.slip[which(m.tagging==0)]=1 135 | rownames(m.slip)=probs$id 136 | colnames(m.slip)=los$id 137 | ## 138 | 139 | ##Define the matrix which keeps track whether a LO for a user has ever been updated 140 | m.exposure<<-matrix(0,ncol=n.los, nrow=n.users) 141 | rownames(m.exposure)=users$id 142 | colnames(m.exposure)=los$id 143 | row.exposure<<- m.exposure[1,] 144 | 145 | ##Define the matrix of confidence: essentially how much information we had for the mastery estimate 146 | m.confidence<<-matrix(0,ncol=n.los, nrow=n.users) 147 | rownames(m.confidence)=users$id 148 | colnames(m.confidence)=los$id 149 | row.confidence<<- m.confidence[1,] 150 | 151 | ##Define the matrix of "user has seen a problem or not": rownames are problems. #### 152 | m.times.submitted<<-matrix(0,nrow=n.users, ncol=n.probs); 153 | rownames(m.times.submitted)=users$id 154 | colnames(m.times.submitted)=probs$id 155 | row.times.submitted<<-m.times.submitted[1,] 156 | ## 157 | 158 | ##Define the matrix of item forgetting parameters 159 | m.forgetting<<-matrix(forgetting,nrow=nrow(users),ncol=nrow(probs)) 160 | rownames(m.forgetting)=users$id 161 | colnames(m.forgetting)=probs$id 162 | 163 | m.item.memory<<-matrix(0,nrow=nrow(users),ncol=nrow(probs)) 164 | rownames(m.item.memory)=users$id 165 | colnames(m.item.memory)=probs$id 166 | 167 | ##Define the data frame of interaction records 168 | transactions<<-data.frame() 169 | 170 | ##Define vector that will store the latest item seen by a user 171 | 172 | last.seen<<- rep("",n.users); 173 | names(last.seen)=users$id -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/evaluate.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | #This code is for the purpose of evaluating the predicting power of the algorithm. 4 | 5 | if(!exists("training.set")){training.set=users$id} 6 | if(!exists("validation.set")){validation.set=users$id} 7 | 8 | LogData.training=subset(LogData,LogData$user_id %in% training.set) 9 | LogData.validation=subset(LogData,LogData$user_id %in% validation.set) 10 | LogData.validation=LogData.validation[order(LogData.validation$time),] 11 | ##Overall chance 12 | chance=mean(LogData.training$score,na.rm=TRUE) 13 | 14 | ##Specific chance for each problem 15 | spec.chance=aggregate(LogData.training$score, by=list(problem_id=LogData.training$problem_id), FUN=mean, na.rm=TRUE) 16 | 17 | temp=merge(LogData.validation,spec.chance[,c("problem_id","x")]) 18 | temp=temp[order(temp$time),] 19 | x.p.chance=temp$x 20 | x.c=LogData.validation$score 21 | x.p=LogData.validation$predicted 22 | ind=which(!is.na(x.c)) 23 | x.c=x.c[ind] 24 | x.p=x.p[ind] 25 | x.p.chance=x.p.chance[ind] 26 | x.p=pmin(pmax(x.p,epsilon),1-epsilon) 27 | x.p.chance=pmin(pmax(x.p.chance,epsilon),1-epsilon) 28 | chance=rep(pmin(pmax(chance,epsilon),1-epsilon),length(x.c)) 29 | 30 | 31 | x.p.r=round(x.p) 32 | 33 | 34 | # if((!exists("eval.results"))|(!before.optimizing)){ 35 | if((!before.optimizing)){ 36 | 37 | x.c.all=c(x.c.all,x.c) 38 | x.p.all=c(x.p.all,x.p) 39 | x.p.chance.all=c(x.p.chance.all,x.p.chance) 40 | chance.all=c(chance.all,chance) 41 | } 42 | 43 | log.like=function(x.c,x.p){ 44 | 45 | if(length(x.p)==1){ 46 | x.p=rep(x.p,length(x.c)) 47 | } 48 | 49 | all= -(mean(x.c*log(x.p))+mean((1-x.c)*log(1-x.p)))/(2*log(2)) 50 | 51 | i=which(x.c==1) 52 | correct=-(mean(log(x.p[i])))/(2*log(2)) 53 | 54 | i=which(x.c==0) 55 | incorrect=-(mean(log(1-x.p[i])))/(2*log(2)) 56 | return(list(all=all,incorrect=incorrect,correct=correct)) 57 | # return( -(mean(x.c*log(x.p))+mean((1-x.c)*log(1-x.p)))/(2*log(2))) 58 | 59 | } 60 | 61 | 62 | 63 | 64 | show.eval=function(eval.results,i=1, rounding=3){ 65 | 66 | x.c=eval.results[[i]]$x.c 67 | x.p=eval.results[[i]]$x.p 68 | x.p.chance=eval.results[[i]]$x.p.chance 69 | chance=eval.results[[i]]$chance 70 | x.exposure=eval.results[[i]]$x.exposure 71 | 72 | # ind=which(x.exposure>=min.exposure); 73 | # 74 | # x.c=x.c[ind] 75 | # x.p=x.p[ind] 76 | # x.p.chance=x.p.chance[ind] 77 | # chance=chance[ind] 78 | x.p.r=round(x.p) 79 | 80 | cat("Number of observations used:",length(x.p),"\n") 81 | cat("M =",eval.results[[i]]$M,"eta =",eval.results[[i]]$eta,"\n") 82 | cat("-LL =",round(log.like(x.c,x.p)$all,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$all,rounding),"and",round(log.like(x.c,x.p.chance)$all,rounding),"respectively)\n") 83 | cat("-LL correct =",round(log.like(x.c,x.p)$correct,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$correct,rounding),"and",round(log.like(x.c,x.p.chance)$correct,rounding),"respectively)\n") 84 | cat("-LL incorrect =",round(log.like(x.c,x.p)$incorrect,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$incorrect,rounding),"and",round(log.like(x.c,x.p.chance)$incorrect,rounding),"respectively)\n") 85 | 86 | cat("MAE =",round(mean(abs(x.c-x.p)),rounding),"(overall and problem-specific learned chance would give",round(mean(abs(x.c-chance)),rounding),"and",round(mean(abs(x.c-x.p.chance)),rounding),"respectively)\n") 87 | 88 | cat("RMSE =",round(sqrt(mean((x.c-x.p)^2)),rounding),"(overall and problem-specific learned chance would give",round(sqrt(mean((x.c-chance)^2)),rounding),"and",round(sqrt(mean((x.c-x.p.chance)^2)),rounding),"respectively)\n") 89 | 90 | m=matrix(0,nrow=2,ncol=2); 91 | colnames(m)=c("Incorrect","Correct") 92 | rownames(m)=c("Predict Incorrect","Predict Correct") 93 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 94 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 95 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 96 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 97 | 98 | m=100*m/length(x.c) 99 | 100 | print("Confusion matrix:") 101 | print(round(m,1)) 102 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 103 | 104 | x.p.r=round(chance) 105 | m=matrix(0,nrow=2,ncol=2); 106 | colnames(m)=c("Incorrect","Correct") 107 | rownames(m)=c("Predict Incorrect","Predict Correct") 108 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 109 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 110 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 111 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 112 | 113 | m=100*m/length(x.c) 114 | print("Confusion matrix of overall chance:") 115 | print(round(m,1)) 116 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 117 | 118 | x.p.r=round(x.p.chance) 119 | m=matrix(0,nrow=2,ncol=2); 120 | colnames(m)=c("Incorrect","Correct") 121 | rownames(m)=c("Predict Incorrect","Predict Correct") 122 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 123 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 124 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 125 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 126 | 127 | m=100*m/length(x.c) 128 | print("Confusion matrix of specific chance:") 129 | print(round(m,1)) 130 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 131 | } 132 | 133 | -------------------------------------------------------------------------------- /prototypes/r_prototype/initialsSuperEarths.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | library(plyr) 3 | 4 | slip.probability=0.1 5 | guess.probability=0.1 6 | trans.probability=0.1 7 | prior.knowledge=0.13 8 | 9 | ####Global variables#### 10 | epsilon<<-1e-10 # a regularization cutoff, the smallest value of a mastery probability 11 | eta=0 ##Relevance threshold used in the BKT optimization procedure 12 | M=20 ##Information threshold user in the BKT optimization procedure 13 | L.star<<- 3 #Threshold odds. If mastery odds are >= than L.star, the LO is considered mastered 14 | r.star<<- 0 #Threshold for forgiving lower odds of mastering pre-requisite LOs. 15 | V.r<<-5 ##Importance of readiness in recommending the next item 16 | V.d<<-3 ##Importance of demand in recommending the next item 17 | V.a<<-1 ##Importance of appropriate difficulty in recommending the next item 18 | V.c<<-1 ##Importance of continuity in recommending the next item 19 | 20 | ##### 21 | 22 | if(!exists("before.optimizing")){before.optimizing=T} 23 | 24 | 25 | ########################### 26 | ### DEFINE DATA MATRICES### 27 | ########################### 28 | 29 | 30 | ##List which items should be used for training the BKT 31 | useForTraining=probs$id 32 | 33 | 34 | ##Relations among the LOs: 35 | ##Define pre-requisite matrix. rownames are pre-reqs. Assumed that the entries are in [0,1] interval #### 36 | m.w<<-matrix(0,nrow=n.los, ncol=n.los); 37 | rownames(m.w)=los$id 38 | colnames(m.w)=los$id 39 | 40 | ## 41 | 42 | m.tagging<<- matrix(0,nrow=n.probs,ncol=n.los) 43 | rownames(m.tagging)=probs$id 44 | colnames(m.tagging)=los$id 45 | 46 | for (i in 1:nrow(df_tag)){ 47 | 48 | temp=df_tag[i,] 49 | m.tagging[temp$edx.id,temp$LO]=1 50 | 51 | } 52 | 53 | ##Check that all problems are tagged and that all LOs are used: 54 | 55 | # ind=which(rowSums(m.tagging)==0) 56 | # if(length(ind)>0){ 57 | # cat("Problem without an LO: ",paste0(rownames(m.tagging)[ind],collapse=", "),"\n") 58 | # }else{ 59 | # cat("Problems without an LO: none\n") 60 | # } 61 | # ind=which(colSums(m.tagging)==0) 62 | # if(length(ind)>0){ 63 | # cat("LOs without a problem: ",paste0(colnames(m.tagging)[ind],collapse=", "),"\n") 64 | # }else{ 65 | # cat("LOs without a problem: none\n") 66 | # } 67 | 68 | 69 | ##Define the vector of difficulties #### 70 | difficulty<<-rep(1,n.probs); 71 | names(difficulty)=probs$id 72 | 73 | ## 74 | 75 | if(before.optimizing){ 76 | ##Define the matrix of transit odds #### 77 | 78 | m.trans<<- (trans.probability/(1-trans.probability))*m.tagging 79 | ## 80 | 81 | ##Define the matrix of guess odds #### 82 | m.guess<<-matrix(guess.probability/(1-guess.probability),nrow=n.probs, ncol = n.los); 83 | m.guess[which(m.tagging==0)]=1 84 | rownames(m.guess)=probs$id 85 | colnames(m.guess)=los$id 86 | ## 87 | 88 | ##Define the matrix of slip odds #### 89 | m.slip<<-matrix(slip.probability/(1-slip.probability),nrow=n.probs, ncol = n.los); 90 | m.slip[which(m.tagging==0)]=1 91 | rownames(m.slip)=probs$id 92 | colnames(m.slip)=los$id 93 | ## 94 | } 95 | 96 | 97 | 98 | if(before.optimizing){ 99 | #Initialize the matrix of mastery odds 100 | m.L.i<<-matrix((prior.knowledge/(1-prior.knowledge)),ncol=n.los, nrow=n.users) 101 | rownames(m.L.i)=users$id 102 | colnames(m.L.i)=los$id 103 | } 104 | 105 | ##Define the matrix which keeps track whether a LO for a user has ever been updated 106 | m.exposure<<-matrix(0,ncol=n.los, nrow=n.users) 107 | rownames(m.exposure)=users$id 108 | colnames(m.exposure)=los$id 109 | 110 | ##Define the matrix which keeps track whether a LO for a user has ever been updated 111 | m.exposure.before.problem<<-matrix(0,ncol=n.probs, nrow=n.users) 112 | rownames(m.exposure.before.problem)=users$id 113 | colnames(m.exposure.before.problem)=probs$id 114 | 115 | ##Define the matrix of confidence: essentially how much information we had for the mastery estimate 116 | m.confidence<<-matrix(0,ncol=n.los, nrow=n.users) 117 | rownames(m.confidence)=users$id 118 | colnames(m.confidence)=los$id 119 | row.confidence<<- m.confidence[1,] 120 | 121 | ##Define the matrix of "user has seen a problem or not": rownames are problems. #### 122 | m.unseen<<-matrix(T,nrow=n.users, ncol=n.probs); 123 | rownames(m.unseen)=users$id 124 | colnames(m.unseen)=probs$id 125 | ## 126 | 127 | 128 | ##Define the data frame of interaction records 129 | #transactions<<-data.frame() 130 | transactions<<-plyr::rename(Pcheck[,c("username","problem_id","time","correctness")],c("username"="user_id","correctness"="score")) 131 | 132 | # ##Define the matrix of results of user interactions with problems.#### 133 | # m.correctness<<-matrix(NA,nrow=n.users, ncol=n.probs); 134 | # # m.correctness<<-matrix(sample(c(T,F),n.users*n.probs,replace=T),nrow=n.users, ncol=n.probs); 135 | # rownames(m.correctness)=users$id 136 | # colnames(m.correctness)=probs$id 137 | # 138 | # m.predicted<<-m.correctness 139 | # 140 | # 141 | # ##Define the matrix of time stamps of results of user interactions with problems.#### 142 | # m.timestamp<<-matrix(NA,nrow=n.users, ncol=n.probs); 143 | # rownames(m.timestamp)=users$id 144 | # colnames(m.timestamp)=probs$id 145 | 146 | 147 | ##Define vector that will store the latest item seen by a user 148 | 149 | last.seen<<- rep("",n.users); 150 | names(last.seen)=users$id 151 | 152 | #Let problems be divided into several modules of adaptivity. In each module, only the items from that scope are used. 153 | ##Proposed code: 154 | # -1 - is not among the adaptively served ones 155 | # 0 - problem can be served in any module 156 | # n=1,2,3,... - problem can be served in the module n 157 | scope<<-rep(1, n.probs) 158 | # cat("Initialization complete\n") 159 | 160 | -------------------------------------------------------------------------------- /app/tests/test_hpl.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pytest 3 | from engine.models import Collection, KnowledgeComponent, Mastery, Learner, Activity, PrerequisiteRelation 4 | from .fixtures import engine_api 5 | from time import sleep 6 | import random 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | @pytest.fixture 12 | def hpl_test_resources(db): 13 | """ 14 | Create resources to simulate hpl use case: 15 | - KCs 16 | - collection containing two activities 17 | :param db: 18 | :return: 19 | """ 20 | #create collection 21 | collection = Collection.objects.create(collection_id='hpl') 22 | 23 | # create activities and associate with collection 24 | problems = [] 25 | activities = [ 26 | Activity( 27 | url='http://example.com/3_Lawrence', 28 | name='3_Lawrence', 29 | type='html' 30 | ), 31 | Activity( 32 | url='http://example.com/4_SNHU', 33 | name='4_SNHU', 34 | type='html' 35 | ) 36 | ] 37 | for activity in activities: 38 | activity.save() 39 | activity.collections.add(collection) 40 | 41 | # create knowledge components and relations 42 | kc_ids_1 = ['sect_academic','sect_consult','sect_policy','sect_prek12','role_admin','role_consult', 43 | 'role_counsel','role_curricdev','role_coach','role_orgspec','role_policy','role_profdev', 44 | 'role_research','role_teaching','prox_instruct','prox_leader','prox_system' 45 | ] 46 | kc_ids_2 = ['sect_academic','sect_consult','sect_corp','sect_highered','sect_mediatech','sect_nonprof', 47 | 'role_admin','role_advoc','role_consult','role_counsel','role_curricdev','role_coach', 48 | 'role_mediatech','role_orgspec','role_policy','role_profdev','role_teaching', 49 | 'prox_instruct','prox_leader','prox_system'] 50 | kc_ids = sorted(list(set(kc_ids_1 + kc_ids_2))) 51 | kcs = [KnowledgeComponent(kc_id=kc_id, name=kc_id, mastery_prior=0.5) for kc_id in kc_ids] 52 | for kc in kcs: 53 | kc.save() 54 | 55 | # tag activites with knowledge components 56 | activities[0].knowledge_components.set(KnowledgeComponent.objects.filter(kc_id__in=kc_ids_1)) 57 | activities[1].knowledge_components.set(KnowledgeComponent.objects.filter(kc_id__in=kc_ids_2)) 58 | 59 | return dict(collection=collection, kcs=kcs) 60 | 61 | @pytest.fixture 62 | def hpl_test_learner_lawrence(db): 63 | """ 64 | Create learner with masteries 65 | :param db: 66 | :return: 67 | """ 68 | # case: 69 | learner = Learner(user_id='hpl_test_learner_lawrence',tool_consumer_instance_guid='default') 70 | learner.save() 71 | Mastery.objects.bulk_create([ 72 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_highered'), value=0.9), 73 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_mediatech'), value=0.9), 74 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_prek12'), value=0.1), 75 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_policy'), value=0.1), 76 | ]) 77 | return learner 78 | 79 | @pytest.fixture 80 | def hpl_test_learner_snhu(db): 81 | """ 82 | Create learner with masteries 83 | :param db: 84 | :return: 85 | """ 86 | # case: 87 | learner = Learner(user_id='hpl_test_learner_snhu', tool_consumer_instance_guid='default') 88 | learner.save() 89 | Mastery.objects.bulk_create([ 90 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_highered'), value=0.2), 91 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_mediatech'), value=0.2), 92 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_prek12'), value=0.8), 93 | Mastery(learner=learner, knowledge_component=KnowledgeComponent.objects.get(kc_id='sect_policy'), value=0.8), 94 | ]) 95 | return learner 96 | 97 | def test_hpl_recommend_lawrence(engine_api, hpl_test_resources, hpl_test_learner_lawrence): 98 | """ 99 | Test recommendation behavior, given collection and prepopulated masteries for learner that should be 100 | expected to receive lawrence as recommendation 101 | """ 102 | collection = hpl_test_resources['collection'] 103 | learner = hpl_test_learner_lawrence 104 | r = engine_api.recommend( 105 | learner=dict( 106 | user_id=learner.user_id, 107 | tool_consumer_instance_guid=learner.tool_consumer_instance_guid 108 | ), 109 | collection=collection.collection_id, 110 | sequence=[] 111 | ) 112 | assert r.json()['source_launch_url'] == 'http://example.com/3_Lawrence' 113 | 114 | 115 | def test_hpl_recommend_snhu(engine_api, hpl_test_resources, hpl_test_learner_snhu): 116 | """ 117 | Test recommendation behavior, given collection and prepopulated masteries for learner that should be 118 | expected to receive snhu as recommendation 119 | """ 120 | collection = hpl_test_resources['collection'] 121 | learner = hpl_test_learner_snhu 122 | r = engine_api.recommend( 123 | learner=dict( 124 | user_id=learner.user_id, 125 | tool_consumer_instance_guid=learner.tool_consumer_instance_guid 126 | ), 127 | collection=collection.collection_id, 128 | sequence=[] 129 | ) 130 | log.warning([kc.kc_id for kc in hpl_test_resources['kcs']]) 131 | assert r.json()['source_launch_url'] == 'http://example.com/4_SNHU' 132 | -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluate.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | #This code is for the purpose of evaluating the predicting power of the algorithm. 4 | 5 | if(!exists("training.set")){training.set=users$id} 6 | if(!exists("validation.set")){validation.set=users$id} 7 | 8 | chance=mean(as.vector(m.correctness[training.set,]),na.rm=T) 9 | 10 | m.c.mean=m.correctness 11 | m.c.mean[validation.set,]=NA 12 | 13 | for(i in 1:ncol(m.c.mean)){ 14 | 15 | m.c.mean[,i]=mean(m.c.mean[,i],na.rm=T) 16 | 17 | } 18 | 19 | x.p.chance=as.vector(m.c.mean[validation.set,]) 20 | x.c=as.vector(m.correctness[validation.set,]); 21 | # x.incl=as.vector(m.include[validation.set,]); 22 | x.p=as.vector(m.predicted[validation.set,]); 23 | x.exposure=as.vector(m.exposure.before.problem[validation.set,]) 24 | # ind=which((!is.na(x.c))&x.incl) 25 | ind=which(!is.na(x.c)) 26 | x.c=x.c[ind] 27 | x.p=x.p[ind] 28 | x.exposure=x.exposure[ind] 29 | x.p.chance=x.p.chance[ind] 30 | x.p=pmin(pmax(x.p,epsilon),1-epsilon) 31 | x.p.chance=pmin(pmax(x.p.chance,epsilon),1-epsilon) 32 | chance=rep(pmin(pmax(chance,epsilon),1-epsilon),length(x.c)) 33 | 34 | 35 | x.p.r=round(x.p) 36 | 37 | 38 | # if((!exists("eval.results"))|(!before.optimizing)){ 39 | if((!before.optimizing)){ 40 | 41 | x.c.all=c(x.c.all,x.c) 42 | x.p.all=c(x.p.all,x.p) 43 | x.p.chance.all=c(x.p.chance.all,x.p.chance) 44 | chance.all=c(chance.all,chance) 45 | x.exposure.all=c(x.exposure.all,x.exposure) 46 | # if(!exists("eval.results")){ 47 | # 48 | # eval.results=list(list(M=NA,eta=NA,x.c=x.c,x.p=x.p,chance=chance, x.p.chance=x.p.chance,x.exposure=x.exposure)) 49 | # }else{ 50 | # eval.results[[length(eval.results)+1]]=list(M=M,eta=eta,x.c=x.c,x.p=x.p, chance=chance, x.p.chance=x.p.chance,x.exposure=x.exposure) 51 | # } 52 | } 53 | 54 | log.like=function(x.c,x.p){ 55 | 56 | if(length(x.p)==1){ 57 | x.p=rep(x.p,length(x.c)) 58 | } 59 | 60 | all= -(mean(x.c*log(x.p))+mean((1-x.c)*log(1-x.p)))/(2*log(2)) 61 | 62 | i=which(x.c==1) 63 | correct=-(mean(log(x.p[i])))/(2*log(2)) 64 | 65 | i=which(x.c==0) 66 | incorrect=-(mean(log(1-x.p[i])))/(2*log(2)) 67 | return(list(all=all,incorrect=incorrect,correct=correct)) 68 | # return( -(mean(x.c*log(x.p))+mean((1-x.c)*log(1-x.p)))/(2*log(2))) 69 | 70 | } 71 | 72 | 73 | 74 | 75 | show.eval=function(eval.results,i=1,min.exposure=1, rounding=3){ 76 | 77 | x.c=eval.results[[i]]$x.c 78 | x.p=eval.results[[i]]$x.p 79 | x.p.chance=eval.results[[i]]$x.p.chance 80 | chance=eval.results[[i]]$chance 81 | x.exposure=eval.results[[i]]$x.exposure 82 | 83 | ind=which(x.exposure>=min.exposure); 84 | 85 | x.c=x.c[ind] 86 | x.p=x.p[ind] 87 | x.p.chance=x.p.chance[ind] 88 | chance=chance[ind] 89 | x.p.r=round(x.p) 90 | 91 | cat("Number of observations used:",length(x.p),"\n") 92 | cat("M =",eval.results[[i]]$M,"eta =",eval.results[[i]]$eta,"min and max exposures:",min.exposure,max(x.exposure),"\n") 93 | cat("-LL =",round(log.like(x.c,x.p)$all,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$all,rounding),"and",round(log.like(x.c,x.p.chance)$all,rounding),"respectively)\n") 94 | cat("-LL correct =",round(log.like(x.c,x.p)$correct,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$correct,rounding),"and",round(log.like(x.c,x.p.chance)$correct,rounding),"respectively)\n") 95 | cat("-LL incorrect =",round(log.like(x.c,x.p)$incorrect,rounding),"(overall and problem-specific learned chance would give",round(log.like(x.c,chance)$incorrect,rounding),"and",round(log.like(x.c,x.p.chance)$incorrect,rounding),"respectively)\n") 96 | 97 | cat("MAE =",round(mean(abs(x.c-x.p)),rounding),"(overall and problem-specific learned chance would give",round(mean(abs(x.c-chance)),rounding),"and",round(mean(abs(x.c-x.p.chance)),rounding),"respectively)\n") 98 | 99 | cat("RMSE =",round(sqrt(mean((x.c-x.p)^2)),rounding),"(overall and problem-specific learned chance would give",round(sqrt(mean((x.c-chance)^2)),rounding),"and",round(sqrt(mean((x.c-x.p.chance)^2)),rounding),"respectively)\n") 100 | 101 | m=matrix(0,nrow=2,ncol=2); 102 | colnames(m)=c("Incorrect","Correct") 103 | rownames(m)=c("Predict Incorrect","Predict Correct") 104 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 105 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 106 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 107 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 108 | 109 | m=100*m/length(x.c) 110 | 111 | print("Confusion matrix:") 112 | print(round(m,1)) 113 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 114 | 115 | x.p.r=round(chance) 116 | m=matrix(0,nrow=2,ncol=2); 117 | colnames(m)=c("Incorrect","Correct") 118 | rownames(m)=c("Predict Incorrect","Predict Correct") 119 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 120 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 121 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 122 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 123 | 124 | m=100*m/length(x.c) 125 | print("Confusion matrix of overall chance:") 126 | print(round(m,1)) 127 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 128 | 129 | x.p.r=round(x.p.chance) 130 | m=matrix(0,nrow=2,ncol=2); 131 | colnames(m)=c("Incorrect","Correct") 132 | rownames(m)=c("Predict Incorrect","Predict Correct") 133 | m["Predict Incorrect","Incorrect"]=length(which((x.p.r==0)&(x.c==0))) 134 | m["Predict Incorrect","Correct"]=length(which((x.p.r==0)&(x.c==1))) 135 | m["Predict Correct","Incorrect"]=length(which((x.p.r==1)&(x.c==0))) 136 | m["Predict Correct","Correct"]=length(which((x.p.r==1)&(x.c==1))) 137 | 138 | m=100*m/length(x.c) 139 | print("Confusion matrix of specific chance:") 140 | print(round(m,1)) 141 | print(paste("True: ",round(m[1,1]+m[2,2],1))) 142 | } 143 | 144 | -------------------------------------------------------------------------------- /app/tests/test_api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pytest 3 | from engine.models import Collection, KnowledgeComponent, Mastery, Learner, Activity 4 | from .fixtures import engine_api, sequence_test_collection 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | @pytest.fixture 10 | def test_collection(db): 11 | """ 12 | Create empty test collection 13 | :param db: https://pytest-django.readthedocs.io/en/latest/helpers.html#db 14 | :return: Collection model instance 15 | """ 16 | collection = Collection.objects.create(collection_id='test_collection',name='foo') 17 | return collection 18 | 19 | 20 | @pytest.fixture 21 | def knowledge_component(db): 22 | """ 23 | Create knowledge component 24 | :param db: 25 | :return: knowledge component model instance 26 | """ 27 | return KnowledgeComponent.objects.create(kc_id='kc_id',name='kc name',mastery_prior=0.5) 28 | 29 | 30 | @pytest.fixture 31 | def knowledge_components(db): 32 | """ 33 | Create knowledge components 34 | :param db: 35 | :return: queryset 36 | """ 37 | objects = [ 38 | KnowledgeComponent( 39 | kc_id='kc1', 40 | name='kc 1', 41 | mastery_prior=0.5 42 | ), 43 | KnowledgeComponent( 44 | kc_id='kc2', 45 | name='kc 2', 46 | mastery_prior=0.6 47 | ), 48 | ] 49 | KnowledgeComponent.objects.bulk_create(objects) 50 | return KnowledgeComponent.objects.filter(kc_id__in=['kc1','kc2']) 51 | 52 | 53 | @pytest.fixture 54 | def activities(db): 55 | """ 56 | Create two activities 57 | :param db: 58 | :return: queryset of knowledge components 59 | """ 60 | objects = [ 61 | Activity( 62 | url='http://example.com/1', 63 | name='activity 1', 64 | ), 65 | Activity( 66 | url='http://example.com/2', 67 | name='activity 2', 68 | ), 69 | ] 70 | Activity.objects.bulk_create(objects) 71 | return Activity.objects.filter(url__in=['http://example.com/1','http://example.com/2']) 72 | 73 | 74 | def test_recommend(engine_api, test_collection): 75 | """ 76 | Recommend activity via api 77 | :param engine_api: alosi.EngineApi instance 78 | :param test_collection: Collection model instance 79 | """ 80 | r = engine_api.recommend( 81 | learner=dict( 82 | user_id='my_user_id', 83 | tool_consumer_instance_guid='default' 84 | ), 85 | collection=test_collection.collection_id, 86 | sequence=[] 87 | ) 88 | log.warning("response text: {}".format(r.text)) 89 | assert r.ok 90 | 91 | 92 | def test_create_knowledge_component(engine_api, test_collection): 93 | """ 94 | Creates KC via api 95 | :param engine_api: 96 | :param test_collection: 97 | :return: 98 | """ 99 | KC_ID = 'kc_id' 100 | KC_NAME = 'kc name' 101 | KC_MASTERY_PRIOR = 0.5 102 | r = engine_api.create_knowledge_component( 103 | kc_id = KC_ID, 104 | name = KC_NAME, 105 | mastery_prior = KC_MASTERY_PRIOR 106 | ) 107 | assert r.ok 108 | 109 | 110 | def test_knowledge_component_id_field(engine_api, knowledge_component): 111 | """ 112 | Tests that 'id' field is available in knowledge component list endpoint data 113 | :param engine_api: 114 | :param knowledge_component: 115 | :return: 116 | """ 117 | r = engine_api.request('GET', 'knowledge_component') # kc list endpoint 118 | assert 'id' in r.json()['results'][0] 119 | 120 | 121 | @pytest.mark.django_db 122 | def test_bulk_update_mastery(engine_api, knowledge_component): 123 | """ 124 | Update mastery for a new learner via api 125 | Tests that learner is created, mastery object is created, and mastery value is updated (checks db) 126 | :param engine_api: 127 | :param knowledge_component: 128 | :return: 129 | """ 130 | NEW_VALUE = 0.6 131 | LEARNER = { 132 | 'user_id': 'user_id', 133 | 'tool_consumer_instance_guid': 'tool_consumer_instance_guid' 134 | } 135 | data = [ 136 | { 137 | 'learner': LEARNER, 138 | 'knowledge_component': { 139 | 'kc_id': knowledge_component.kc_id 140 | }, 141 | 'value': NEW_VALUE 142 | } 143 | ] 144 | r = engine_api.bulk_update_mastery(data) 145 | assert r.ok 146 | 147 | learner = Learner.objects.get( 148 | user_id=LEARNER['user_id'], 149 | tool_consumer_instance_guid=LEARNER['tool_consumer_instance_guid'] 150 | ) 151 | 152 | mastery = Mastery.objects.get( 153 | learner=learner, 154 | knowledge_component=knowledge_component 155 | ) 156 | assert mastery.value == NEW_VALUE 157 | 158 | 159 | @pytest.mark.django_db 160 | def test_api_create_prerequisite_activity(engine_api, activities): 161 | """ 162 | Create prerequisite activity relation via api 163 | :param engine_api: 164 | :param knowledge_components: 165 | :return: 166 | """ 167 | data = dict( 168 | from_activity=activities[1].pk, 169 | to_activity=activities[0].pk 170 | ) 171 | r = engine_api.request('POST', 'prerequisite_activity', json=data) 172 | assert r.ok 173 | 174 | 175 | def test_api_create_prerequisite_activity_via_field(engine_api, activities): 176 | """ 177 | Modify prerequisite activity relation via activity field 178 | :param engine_api: engine api client 179 | :param activities: activity queryset 180 | :return: 181 | """ 182 | data = dict(prerequisite_activities=[activities[0].pk]) 183 | r = engine_api.request('PATCH', 'activity/{}'.format(activities[1].pk), json=data) 184 | assert r.ok 185 | 186 | 187 | def test_api_create_prerequisite_ka(engine_api, knowledge_components): 188 | """ 189 | Create prereq relation between two kcs via api 190 | :param engine_api: engine api client 191 | :param knowledge_components: kc queryset 192 | :return: 193 | """ 194 | data = dict( 195 | prerequisite=knowledge_components[0].pk, 196 | knowledge_component=knowledge_components[1].pk, 197 | value=1.0 198 | ) 199 | r = engine_api.request('POST', 'prerequisite_knowledge_component', json=data) 200 | assert r.ok 201 | 202 | 203 | def test_api_grade(engine_api, sequence_test_collection): 204 | """ 205 | Test that mastery-based grade generator is working 206 | 207 | :param engine_api: engine api fixture 208 | :param knowledge_components 209 | """ 210 | data = { 211 | 'learner': { 212 | 'user_id': 'user_id', 213 | 'tool_consumer_instance_guid': 'tool_consumer_instance_guid' 214 | } 215 | } 216 | r = engine_api.request('POST', f'collection/{sequence_test_collection.collection_id}/grade', json=data) 217 | assert r.ok 218 | -------------------------------------------------------------------------------- /prototypes/python_prototype/fakeInitials.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | import numpy as np 3 | import pandas as pd 4 | 5 | def initialize_variables(self, 6 | users=None, 7 | los=None, 8 | items=None, 9 | n_modules=2, 10 | 11 | epsilon=1e-10, # a regularization cutoff, the smallest value of a mastery probability 12 | eta=0.0, ##Relevance threshold used in the BKT optimization procedure 13 | M=0.0, ##Information threshold user in the BKT optimization procedure 14 | L_star=2.2, #Threshold logarithmic odds. If mastery logarithmic odds are >= than L_star, the LO is considered mastered 15 | 16 | r_star=0.0, #Threshold for forgiving lower odds of mastering pre-requisite LOs. 17 | W_p=5.0, ##Importance of readiness in recommending the next item 18 | W_r=3.0, ##Importance of demand in recommending the next item 19 | W_d=1.0, ##Importance of appropriate difficulty in recommending the next item 20 | W_c=1.0, ##Importance of continuity in recommending the next item 21 | 22 | ##Values prior to estimating model: 23 | slip_probability=0.15, 24 | guess_probability=0.1, 25 | trans_probability=0.1, 26 | prior_knowledge_probability=0.2, 27 | 28 | los_per_item=2, ##Number of los per problem 29 | ): 30 | 31 | self.epsilon=epsilon # a regularization cutoff, the smallest value of a mastery probability 32 | self.eta=eta ##Relevance threshold used in the BKT optimization procedure 33 | self.M=M ##Information threshold user in the BKT optimization procedure 34 | self.L_star=L_star #Threshold logarithmic odds. If mastery logarithmic odds are >= than L_star, the LO is considered mastered 35 | 36 | self.r_star=r_star #Threshold for forgiving lower odds of mastering pre-requisite LOs. 37 | self.W_p=W_p ##Importance of readiness in recommending the next item 38 | self.W_r=W_r ##Importance of demand in recommending the next item 39 | self.W_d=W_d ##Importance of appropriate difficulty in recommending the next item 40 | self.W_c=W_c ##Importance of continuity in recommending the next item 41 | 42 | 43 | # ##Store mappings of ids and names for users, LOs, items. These will serve as look-up tables for the rows and columns of data matrices 44 | self.users=users 45 | self.los=los 46 | self.items=items 47 | 48 | n_users = len(users) 49 | n_los = len(los) 50 | n_items = len(items) 51 | 52 | #Let problems be divided into several modules of adaptivity. In each module, only the items from that scope are used. 53 | self.scope=np.ones([n_items,n_modules],dtype=bool) 54 | self.scope[:,1]=False 55 | 56 | ##List which items should be used for training the BKT 57 | self.useForTraining=np.repeat(True, n_items) 58 | self.useForTraining=np.where(self.useForTraining)[0] 59 | 60 | 61 | #Initial mastery of all LOs (a row of the initial mastery matrix) 62 | #Logarithmic if additive formulation. 63 | 64 | self.L_i=np.repeat(prior_knowledge_probability/(1.0-prior_knowledge_probability),n_los) 65 | 66 | 67 | # Define the matrix of initial mastery by replicating the same row for each user 68 | self.m_L_i=np.tile(self.L_i,(n_users,1)) 69 | 70 | # Define a copy to update 71 | self.m_L=self.m_L_i.copy() 72 | 73 | ##Define fake pre-requisite matrix. rownames are pre-reqs. Assumed that the entries are in [0,1] interval #### 74 | self.m_w=np.random.rand(n_los,n_los) 75 | 76 | for i in range(self.m_w.shape[0]): 77 | for j in range(self.m_w.shape[1]): 78 | des=(np.random.rand()>0.5) 79 | if des: 80 | self.m_w[i,j]=0. 81 | else: 82 | self.m_w[j,i]=0. 83 | 84 | 85 | 86 | ##Define the vector of difficulties that will be visible to users, between 0 and 1 (but we'll check and normalize)#### 87 | self.difficulty=np.repeat(1.,n_items) 88 | 89 | ## 90 | 91 | 92 | ##Define the preliminary relevance matrix: problems tagged with LOs. rownames are problems. Assumed that the entries are 0 or 1 #### 93 | 94 | temp=np.append(np.repeat(1.0,los_per_item),np.repeat(0.0,n_los-los_per_item)) 95 | 96 | self.m_tagging=np.zeros([n_items,n_los]) 97 | 98 | for i in range(n_items) : 99 | self.m_tagging[i,]=np.random.choice(temp,size=len(temp),replace=False) 100 | 101 | 102 | ##CHeck that there are no zero rows or columns in tagging 103 | 104 | ind=np.where(~self.m_tagging.any(axis=0))[0] 105 | 106 | if(len(ind)>0): 107 | # print("LOs without a problem: ",los.id[ind]) 108 | print("LOs without a problem: ",los[ind]) 109 | else: 110 | print("LOs without a problem: none\n") 111 | 112 | 113 | 114 | ind=np.where(~self.m_tagging.any(axis=1))[0] 115 | 116 | if(len(ind)>0): 117 | # print("Problem without an LO: ",items.id[ind]) 118 | print("Problem without an LO: ",items[ind]) 119 | else: 120 | print("Problems without an LO: none\n") 121 | 122 | 123 | ##Define the matrix of transit odds #### 124 | self.m_trans=(trans_probability/(1.0-trans_probability))*self.m_tagging 125 | 126 | ##Define the matrix of guess odds #### 127 | self.m_guess=guess_probability/(1.0-guess_probability)*np.ones([n_items,n_los]); 128 | self.m_guess[np.where(self.m_tagging==0.0)]=1.0 129 | 130 | ##Define the matrix of slip odds #### 131 | self.m_slip=slip_probability/(1.0-slip_probability)*np.ones([n_items,n_los]); 132 | self.m_slip[np.where(self.m_tagging==0.0)]=1.0 133 | 134 | 135 | ##Define the matrix which keeps track whether a LO for a user has ever been updated 136 | #For convenience of adding users later, also define a row of each matrix 137 | self.m_exposure=np.zeros([n_users,n_los]) 138 | self.row_exposure=self.m_exposure[0,] 139 | 140 | #Define the matrix of confidence: essentially how much information we had for the mastery estimate 141 | self.m_confidence=np.zeros([n_users,n_los]) 142 | self.row_confidence=self.m_confidence[0,] 143 | 144 | 145 | ##Define the matrix of "user has seen a problem or not": rownames are problems. #### 146 | self.m_unseen=np.ones([n_users,n_items], dtype=bool) 147 | self.row_unseen=self.m_unseen[0,] 148 | ## 149 | ###Define the matrix of results of user interactions with problems.#### 150 | #m_correctness=np.empty([n_users,n_items]) 151 | #m_correctness[:]=np.nan 152 | #row_correctness=m_correctness[0,] 153 | # 154 | ###Define the matrix of time stamps of results of user interactions with problems.#### 155 | #m_timestamp=np.empty([n_users,n_items]) 156 | #m_timestamp[:]=np.nan 157 | #row_timestamp=m_timestamp[0,] 158 | 159 | #Initialize the data frame which will store the results of users submit-transactions (much like problem_check in Glenn's data) 160 | self.transactions=pd.DataFrame() 161 | 162 | 163 | ##Define vector that will store the latest item seen by a user 164 | self.last_seen=np.repeat(-1,n_users) 165 | 166 | -------------------------------------------------------------------------------- /prototypes/python_prototype/multiplicativeFormulation.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from fakeInitials import initialize_variables 7 | from derivedData import calculate_derived_data 8 | from empiricalEstimation import estimate 9 | 10 | class MultiplicativeFormulation(object): 11 | def __init__(self, **kwargs): 12 | initialize_variables(self, **kwargs) 13 | calculate_derived_data(self) 14 | 15 | 16 | def mapUser(self, user_id): 17 | """ 18 | This function maps the user_id to the user index used by other functions, and also adds new users 19 | SYNCHRONIZATION IS IMPORTANT 20 | """ 21 | # global users 22 | 23 | try: 24 | u=np.where(self.users==user_id)[0][0] 25 | except: 26 | """ 27 | Add a new user 28 | """ 29 | # global n_users, last_seen, m_L, m_exposure, m_unseen, m_correctness, m_timestamp 30 | n_users = len(self.users) 31 | u=n_users 32 | # n_users+=1 33 | self.users=np.append(self.users,user_id) 34 | self.last_seen=np.append(self.last_seen,-1) 35 | self.m_L=np.vstack((self.m_L,L_i)) 36 | self.m_exposure=np.vstack((self.m_exposure,row_exposure)) 37 | self.m_unseen=np.vstack((self.m_unseen,row_unseen)) 38 | # m_correctness=np.vstack((m_correctness,row_correctness)) 39 | # m_timestamp=np.vstack((m_timestamp,row_timestamp)) 40 | 41 | 42 | return u 43 | 44 | 45 | def mapItem(self,item_id): 46 | 47 | item=np.where(self.items==item_id)[0][0] 48 | 49 | return item 50 | 51 | def bayesUpdate(self, u, item, score=1.0,time=0, attempts='all'): 52 | 53 | 54 | #This function updates the user mastery and record of interactions that will be needed for recommendation and estimation of the BKT 55 | 56 | # global m_x0_mult, m_x1_0_mult, m_L, m_trans, last_seen, m_unseen, transactions, m_exposure, m_tagging, epsilon, inv_epsilon 57 | 58 | 59 | self.last_seen[u]=item 60 | # m_correctness[u,item]=score 61 | # m_timestamp[u,item]=time 62 | if self.m_unseen[u,item]: 63 | self.m_unseen[u,item]=False 64 | self.m_exposure[u,]+=self.m_tagging[item,] 65 | self.m_confidence[u,]+=self.m_k[item,] 66 | 67 | if attempts=='first': 68 | ##Record the transaction by appending a new row to the data frame "transactions": 69 | self.transactions=self.transactions.append(pd.DataFrame([[u,item,time,score]], columns=['user_id','problem_id','time','score']),ignore_index=True) 70 | ##The increment of odds due to evidence of the problem, but before the transfer 71 | x=self.m_x0_mult[item,]*np.power(self.m_x1_0_mult[item,],score) 72 | L=self.m_L[u,]*x 73 | ##Add the transferred knowledge 74 | L+=self.m_trans[item,]*(L+1) 75 | 76 | 77 | if attempts!='first': 78 | ##Record the transaction by appending a new row to the data frame "transactions": 79 | self.transactions=self.transactions.append(pd.DataFrame([[u,item,time,score]], columns=['user_id','problem_id','time','score']),ignore_index=True) 80 | ##The increment of odds due to evidence of the problem, but before the transfer 81 | x=self.m_x0_mult[item,]*np.power(self.m_x1_0_mult[item,],score) 82 | L=self.m_L[u,]*x 83 | ##Add the transferred knowledge 84 | L+=self.m_trans[item,]*(L+1) 85 | 86 | L[np.where(np.isposinf(L))]=self.inv_epsilon 87 | L[np.where(L==0.0)]=self.epsilon 88 | 89 | self.m_L[u,]=L 90 | 91 | #m_L[u,]+=trans[item,]*(L+1) 92 | 93 | 94 | 95 | 96 | #return{'L':L, 'x':x} 97 | 98 | 99 | 100 | #This function calculates the probability of correctness on a problem, as a prediction based on student's current mastery. 101 | def predictCorrectness(self, u, item): 102 | 103 | # global m_L, m_p_slip, m_p_guess 104 | 105 | L=self.m_L[u,] 106 | p_slip=self.m_p_slip[item,]; 107 | p_guess=self.m_p_guess[item,]; 108 | 109 | x=(L*(1.0-p_slip)+p_guess)/(L*p_slip+1.0-p_guess); ##Odds by LO 110 | x=np.prod(x) ##Total odds 111 | 112 | p=x/(1+x) ##Convert odds to probability 113 | if np.isnan(p) or np.isinf(p): 114 | p=1.0 115 | 116 | return(p) 117 | 118 | 119 | 120 | 121 | ##This function returns the id of the next recommended problem in an adaptive module. If none is recommended (list of problems exhausted or the user has reached mastery) it returns None. 122 | def recommend(self, u, module=0, stopOnMastery=False, normalize=False): 123 | 124 | # global m_L, L_star, m_w, m_unseen, m_k, r_star, last_seen, m_difficulty, W_r, W_d, W_p, W_c, scope 125 | 126 | #Subset to the unseen problems from the relevant scope 127 | #ind_unseen=np.where(m_unseen[u,] & ((scope==module)|(scope==0)))[0] 128 | ind_unseen=np.where(self.m_unseen[u,] & (self.scope[:,module]))[0] 129 | L=np.log(self.m_L[u,]) 130 | if stopOnMastery: 131 | m_k_unseen=self.m_k[ind_unseen,] 132 | R=np.dot(m_k_unseen, np.maximum((self.L_star-L),0)) 133 | ind_unseen=ind_unseen[R!=0.0] 134 | 135 | 136 | N=len(ind_unseen) 137 | 138 | if(N==0): ##This means we ran out of problems, so we stop 139 | next_item = None 140 | 141 | else: 142 | #L=np.log(m_L[u,]) 143 | 144 | #Calculate the user readiness for LOs 145 | 146 | m_r=np.dot(np.minimum(L-self.L_star,0), self.m_w); 147 | m_k_unseen=self.m_k[ind_unseen,] 148 | P=np.dot(m_k_unseen, np.minimum((m_r+self.r_star),0)) 149 | R=np.dot(m_k_unseen, np.maximum((self.L_star-L),0)) 150 | 151 | if self.last_seen[u]<0: 152 | C=np.repeat(0.0,N) 153 | else: 154 | C=np.sqrt(np.dot(m_k_unseen, self.m_k[self.last_seen[u],])) 155 | 156 | #A=0.0 157 | d_temp=self.m_difficulty[:,ind_unseen] 158 | L_temp=np.tile(L,(N,1)).transpose() 159 | D=-np.diag(np.dot(m_k_unseen,np.abs(L_temp-d_temp))) 160 | 161 | #if stopOnMastery and sum(D)==0: ##This means the user has reached threshold mastery in all LOs relevant to the problems in the homework, so we stop 162 | next_item=None 163 | #else: 164 | 165 | if normalize: 166 | temp=(D.max()-D.min()); 167 | if(temp!=0.0): 168 | D=D/temp 169 | temp=(R.max()-R.min()); 170 | if(temp!=0.0): 171 | R=R/temp 172 | temp=(P.max()-P.min()); 173 | if(temp!=0.0): 174 | P=P/temp 175 | temp=(C.max()-C.min()); 176 | if(temp!=0.0): 177 | C=C/temp 178 | 179 | next_item=ind_unseen[np.argmax(self.W_p*P+self.W_r*R+self.W_d*D+self.W_c*C)] 180 | 181 | 182 | return(next_item) 183 | 184 | 185 | def updateModel(self): 186 | 187 | # global eta, M, L_i, m_exposure, m_L, m_L_i, m_trans, m_guess, m_slip 188 | est=estimate(self, self.eta, self.M) 189 | 190 | self.L_i=1.0*est['L_i'] 191 | self.m_L_i=np.tile(self.L_i,(self.m_L.shape[0],1)) 192 | 193 | ind_pristine=np.where(self.m_exposure==0.0) 194 | self.m_L[ind_pristine]=self.m_L_i[ind_pristine] 195 | m_trans=1.0*est['trans'] 196 | m_guess=1.0*est['guess'] 197 | m_slip=1.0*est['slip'] 198 | # execfile('derivedData.py') 199 | calculate_derived_data(self) 200 | -------------------------------------------------------------------------------- /app/engine/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models 5 | 6 | 7 | def first_and_last_n_chars(s, n1=30, n2=30): 8 | """ 9 | Utility function to display first n1 characters and last n2 characters of a long string 10 | (Adjusts display if string is less than n1+n2 char long) 11 | :param s: string 12 | :return: string for display 13 | """ 14 | first_len = min(len(s), n1) 15 | first = s[:first_len] 16 | last_len = min(len(s) - len(first), n2) 17 | last = s[-last_len:] if last_len > 0 else '' 18 | 19 | if first_len == len(s): 20 | return first 21 | elif first_len + last_len == len(s): 22 | return "{}{}".format(first, last) 23 | else: 24 | return "{}...{}".format(first, last) 25 | 26 | 27 | class Collection(models.Model): 28 | """ 29 | Collection consists of multiple activities 30 | """ 31 | collection_id = models.CharField(max_length=200, unique=True) 32 | name = models.CharField(max_length=200) 33 | max_problems = models.PositiveIntegerField(null=True, blank=True) 34 | 35 | def __str__(self): 36 | return "Collection: {} ({})".format(self.collection_id, self.name) 37 | 38 | 39 | class KnowledgeComponent(models.Model): 40 | kc_id = models.CharField(max_length=200, unique=True) 41 | name = models.CharField(max_length=200) 42 | mastery_prior = models.FloatField(null=True, blank=True) 43 | 44 | def __str__(self): 45 | return "KC: {} ({})".format(self.kc_id, self.name) 46 | 47 | 48 | class PrerequisiteRelation(models.Model): 49 | prerequisite = models.ForeignKey( 50 | KnowledgeComponent, 51 | on_delete=models.CASCADE, 52 | related_name="dependent_relation" 53 | ) 54 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 55 | value = models.FloatField() 56 | 57 | def __str__(self): 58 | return "PrerequisiteRelation: {} (prereq) -> {} = {}".format( 59 | self.prerequisite.kc_id, 60 | self.knowledge_component.kc_id, 61 | self.value 62 | ) 63 | 64 | 65 | class Activity(models.Model): 66 | """ 67 | Activity model 68 | """ 69 | url = models.CharField(max_length=500, default='') 70 | name = models.CharField(max_length=200, default='') 71 | collections = models.ManyToManyField(Collection, blank=True) 72 | knowledge_components = models.ManyToManyField(KnowledgeComponent, blank=True) 73 | difficulty = models.FloatField(null=True,blank=True) 74 | tags = models.TextField(default='', blank=True) 75 | type = models.CharField(max_length=200, default='', blank=True) 76 | # whether to include as valid problem to recommend from adaptive engine 77 | include_adaptive = models.BooleanField(default=True) 78 | # order for non-adaptive problems 79 | nonadaptive_order = models.PositiveIntegerField(null=True, blank=True) 80 | # order for pre-adaptive problems 81 | preadaptive_order = models.PositiveIntegerField(null=True, blank=True) 82 | # prerequisite activities - used to designate activities that should be served before 83 | prerequisite_activities = models.ManyToManyField('self', blank=True, symmetrical=False) 84 | 85 | def __str__(self): 86 | return "Activity: {} ({})".format(first_and_last_n_chars(self.url, 40, 10), self.name) 87 | 88 | 89 | class EngineSettings(models.Model): 90 | name = models.CharField(max_length=200, default='') 91 | r_star = models.FloatField() # Threshold for forgiving lower odds of mastering pre-requisite LOs. 92 | L_star = models.FloatField() # Threshold logarithmic odds. If mastery logarithmic odds are >= than L_star, the LO is considered mastered 93 | W_p = models.FloatField() # Importance of readiness in recommending the next item 94 | W_r = models.FloatField() # Importance of demand in recommending the next item 95 | W_c = models.FloatField() # Importance of continuity in recommending the next item 96 | W_d = models.FloatField() # Importance of appropriate difficulty in recommending the next item 97 | 98 | def __str__(self): 99 | return "EngineSettings: {}".format(self.name if self.name else self.pk) 100 | 101 | 102 | class ExperimentalGroup(models.Model): 103 | name = models.CharField(max_length=200,default='') 104 | weight = models.FloatField(default=0) 105 | engine_settings = models.ForeignKey( 106 | EngineSettings, 107 | on_delete=models.SET_NULL, 108 | blank=True, 109 | null=True 110 | ) 111 | 112 | def __str__(self): 113 | return "Experimental Group {}".format(self.name if self.name else self.pk) 114 | 115 | 116 | class Learner(models.Model): 117 | """ 118 | User model for students 119 | """ 120 | user_id = models.CharField(max_length=200, default='') 121 | tool_consumer_instance_guid = models.CharField(max_length=200, default='') 122 | experimental_group = models.ForeignKey( 123 | ExperimentalGroup, 124 | on_delete=models.SET_NULL, 125 | null=True, 126 | blank=True, 127 | ) 128 | 129 | class Meta: 130 | unique_together = (('user_id', 'tool_consumer_instance_guid'),) 131 | 132 | def __str__(self): 133 | return "Learner: {}/{}".format( 134 | self.user_id or "", 135 | self.tool_consumer_instance_guid or "" 136 | ) 137 | 138 | 139 | class Score(models.Model): 140 | """ 141 | Score resulting from a learner's attempt on an activity 142 | """ 143 | learner = models.ForeignKey(Learner, on_delete=models.CASCADE) 144 | activity = models.ForeignKey(Activity, on_delete=models.CASCADE) 145 | # score value 146 | score = models.FloatField() 147 | # creation time 148 | timestamp = models.DateTimeField(null=True, auto_now_add=True) 149 | 150 | def __str__(self): 151 | return "Score: {} [{} - {}]".format( 152 | self.score, self.learner, self.activity) 153 | 154 | 155 | class Transit(models.Model): 156 | activity = models.ForeignKey(Activity, on_delete=models.CASCADE) 157 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 158 | value = models.FloatField() 159 | 160 | def __str__(self): 161 | return "Transit: {} [{} - {}]".format( 162 | self.value, self.activity, self.knowledge_component) 163 | 164 | 165 | class Guess(models.Model): 166 | activity = models.ForeignKey(Activity, on_delete=models.CASCADE) 167 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 168 | value = models.FloatField() 169 | 170 | def __str__(self): 171 | return "Guess: {} [{} - {}]".format( 172 | self.value, self.activity, self.knowledge_component) 173 | 174 | 175 | class Slip(models.Model): 176 | activity = models.ForeignKey(Activity, on_delete=models.CASCADE) 177 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 178 | value = models.FloatField() 179 | 180 | def __str__(self): 181 | return "Slip: {} [{} - {}]".format( 182 | self.value, self.activity, self.knowledge_component) 183 | 184 | 185 | class Mastery(models.Model): 186 | learner = models.ForeignKey(Learner, on_delete=models.CASCADE) 187 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 188 | value = models.FloatField() 189 | 190 | def __str__(self): 191 | return "Mastery: {} [{} - {}]".format( 192 | self.value, self.learner, self.knowledge_component) 193 | 194 | 195 | class Exposure(models.Model): 196 | learner = models.ForeignKey(Learner, on_delete=models.CASCADE) 197 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 198 | value = models.IntegerField() 199 | 200 | def __str__(self): 201 | return "Exposure: {} [{} - {}]".format( 202 | self.value, self.learner, self.knowledge_component) 203 | 204 | 205 | class Confidence(models.Model): 206 | learner = models.ForeignKey(Learner, on_delete=models.CASCADE) 207 | knowledge_component = models.ForeignKey(KnowledgeComponent, on_delete=models.CASCADE) 208 | value = models.FloatField() 209 | 210 | def __str__(self): 211 | return "Confidence: {} [{} - {}]".format( 212 | self.value, self.learner, self.knowledge_component) 213 | 214 | -------------------------------------------------------------------------------- /prototypes/r_prototype/optimizer.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | 4 | knowledge=function(prob_id,correctness, method="average"){ 5 | ##This function finds the empirical knowledge of a single user given a chronologically ordered sequence of items submitted. 6 | 7 | method=tolower(method) 8 | if(!(method %in% c("average","min","max"))){ 9 | method="average" 10 | } 11 | 12 | 13 | m.k.u=m.k[prob_id,,drop=FALSE] 14 | m.slip.u=m.slip.neg.log[prob_id,,drop=FALSE] 15 | m.guess.u=m.guess.neg.log[prob_id,,drop=FALSE] 16 | N=length(prob_id) 17 | 18 | z=matrix(0,nrow=N+1,ncol=ncol(m.k)); 19 | x=rep(0,N) 20 | z[1,]=(1-correctness) %*% m.slip.u; 21 | z[N+1,]=correctness %*% m.guess.u; 22 | if(N>1){ 23 | for(n in 1:(N-1)){ 24 | x[1:n]=correctness[1:n] 25 | x[(n+1):N]=1-correctness[(n+1):N] 26 | temp=rbind(m.guess.u[1:n,,drop=FALSE],m.slip.u[(n+1):N,,drop=FALSE]) 27 | z[n+1,]=x %*% temp 28 | } 29 | } 30 | knowledge=matrix(0,ncol=ncol(m.k),nrow=N); 31 | rownames(knowledge)=prob_id; 32 | colnames(knowledge)=colnames(m.k) 33 | 34 | for (j in 1:ncol(z)){ 35 | ind=which(z[,j]==min(z[,j])) 36 | 37 | if(method=="average"){ 38 | for (i in ind){ 39 | 40 | temp=rep(0,N); 41 | if (i==1){ 42 | temp=rep(1,N) 43 | }else if (i<=N){ 44 | temp[i:N]=1 45 | } 46 | 47 | knowledge[,j]=knowledge[,j]+temp 48 | 49 | } 50 | 51 | knowledge[,j]=knowledge[,j]/length(ind) ##We average the knowledge when there are multiple candidates (length(ind)>1) 52 | 53 | } 54 | 55 | if(method=="max"){ 56 | i=max(ind) 57 | temp=rep(0,N); 58 | if (i==1){ 59 | temp=rep(1,N) 60 | }else if (i<=N){ 61 | temp[i:N]=1 62 | } 63 | 64 | knowledge[,j]=temp 65 | } 66 | 67 | if(method=="min"){ 68 | i=min(ind) 69 | temp=rep(0,N); 70 | if (i==1){ 71 | temp=rep(1,N) 72 | }else if (i<=N){ 73 | temp[i:N]=1 74 | } 75 | 76 | knowledge[,j]=temp 77 | } 78 | 79 | } 80 | 81 | return(knowledge) 82 | 83 | 84 | } 85 | 86 | 87 | 88 | estimate=function(relevance.threshold=0, information.threshold=20,remove.degeneracy=TRUE, training.set=NULL){ 89 | 90 | ##This function estimates the matrices of the BKT parameters from the user interaction data. 91 | ##To account for the fact that NaN and Inf elements of the estimated matrices should not be used as updates, this function replaces such elements with the corresponding elements of the current BKT parameter matrices. 92 | ##Thus, the outputs of this function do not contain any non-numeric values and should be used to simply replace the current BKT parameter matrices. 93 | 94 | trans=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 95 | trans.denom=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 96 | guess=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 97 | guess.denom=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 98 | slip=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 99 | slip.denom=matrix(0,nrow=n.probs,ncol=n.los,dimnames=list(probs$id,los$id)) 100 | p.i=rep(0,n.los); 101 | p.i.denom=rep(0,n.los) 102 | 103 | if(is.null(training.set)){ 104 | training.set=users$id 105 | } 106 | 107 | for (u in training.set){ 108 | 109 | ##List problems that the user tried, in chronological order 110 | # temp=subset(transactions,(transactions$user_id==u)&(transactions$problem_id %in% useForTraining)) 111 | temp=subset(transactions,(transactions$user_id==u)) 112 | temp=temp[order(temp$time),] 113 | 114 | J=length(temp$problem_id) 115 | if(J>0){ 116 | 117 | m.k.u=m.k[temp$problem_id,,drop=FALSE] 118 | 119 | ##Calculate the sum of relevances of user's experience for a each learning objective 120 | if(J==1){ 121 | u.R=m.k.u 122 | }else{ 123 | u.R=colSums(as.matrix(m.k.u)) 124 | } 125 | 126 | ##Implement the relevance threshold: 127 | 128 | u.R=(u.R>relevance.threshold) 129 | 130 | m.k.u=(m.k.u>relevance.threshold) 131 | 132 | #u.knowledge=knowledge(prob_id, m.correctness[u,prob_id], method="average"); 133 | #u.correctness=m.correctness[u,prob_id] 134 | 135 | 136 | # u.correctness=temp$score 137 | # prob_id=temp$problem_id 138 | u.knowledge=knowledge(temp$problem_id, temp$score, method="average"); 139 | 140 | ##Contribute to the averaged initial knowledge. 141 | p.i=p.i+u.knowledge[1,]*u.R 142 | p.i.denom=p.i.denom+u.R 143 | 144 | # m.R.u=matrix(rep(u.R,J),nrow=J,byrow=T) 145 | 146 | 147 | 148 | 149 | ##Contribute to the trans, guess and slip probabilities (numerators and denominators separately). 150 | # if(J>1){ 151 | # u.trans.denom=(1-u.knowledge[-J,]) 152 | # trans[prob_id[-J],]=trans[prob_id[-J],]+(m.k.u[-J,]*u.trans.denom)*u.knowledge[-1,] ##Order of multiplication is important, otherwise the row names get shifted (R takes them from 1st factor) 153 | # trans.denom[prob_id[-J],]=trans.denom[prob_id[-J],]+m.k.u[-J,]*u.trans.denom 154 | # } 155 | # 156 | # guess[prob_id,]=guess[prob_id,]+(m.k.u*(1-u.knowledge))*u.correctness #This relies on the fact that R regards matrices as filled by column. This is not a matrix multiplication! 157 | # guess.denom[prob_id,]=guess.denom[prob_id,]+m.k.u*(1-u.knowledge) 158 | # 159 | # slip[prob_id,]=slip[prob_id,]+(m.k.u*u.knowledge)*(1-u.correctness) #This relies on the fact that R regards matrices as filled by column. This is not a matrix multiplication! 160 | # slip.denom[prob_id,]=slip.denom[prob_id,]+(m.k.u*u.knowledge) 161 | 162 | for (pr in 1:J){ 163 | 164 | prob_id=temp$problem_id[pr] 165 | 166 | if(pr=0.5)|(guess+slip>=1)) 224 | ind_s=which((slip>=0.5)|(guess+slip>=1)) 225 | 226 | guess[ind_g]=NA 227 | slip[ind_s]=NA 228 | # guess[which(guess>=0.5)]=NA 229 | # slip[which(slip>=0.5)]=NA 230 | 231 | } 232 | #Replicate the initial knowledge to all users: 233 | p.i=matrix(rep(p.i,n.users),nrow=n.users, byrow=TRUE) 234 | dimnames(p.i)=dimnames(m.L.i) 235 | 236 | 237 | #Convert to odds 238 | p.i=pmin(pmax(p.i,epsilon),1-epsilon) 239 | trans=pmin(pmax(trans,epsilon),1-epsilon) 240 | guess=pmin(pmax(guess,epsilon),1-epsilon) 241 | slip=pmin(pmax(slip,epsilon),1-epsilon) 242 | 243 | L.i=(p.i/(1-p.i)) 244 | trans=trans/(1-trans) 245 | guess=guess/(1-guess) 246 | slip=slip/(1-slip) 247 | 248 | ## Matrices contain NaNs or Infs in those elements that we do not want to update (it means we don't have sufficient data). Therefore, replace them by the previously stored values. 249 | 250 | ##Keep the versions with NAs in them: 251 | L.i.na=L.i 252 | trans.na=trans 253 | guess.na=guess 254 | slip.na=slip 255 | 256 | ind=which((is.na(L.i))|(is.infinite(L.i))) 257 | L.i=replace(L.i,ind,m.L.i[ind]) 258 | ind=which((is.na(trans))|(is.infinite(trans))) 259 | trans=replace(trans,ind,m.trans[ind]) 260 | ind=which((is.na(guess))|(is.infinite(guess))) 261 | guess=replace(guess,ind,m.guess[ind]) 262 | ind=which((is.na(slip))|(is.infinite(slip))) 263 | slip=replace(slip,ind,m.slip[ind]) 264 | 265 | 266 | 267 | return(list(L.i=L.i,trans=trans,guess=guess,slip=slip, L.i.na=L.i.na,trans.na=trans.na,guess.na=guess.na,slip.na=slip.na)) 268 | 269 | } 270 | -------------------------------------------------------------------------------- /prototypes/python_prototype/empiricalEstimation.py: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | 3 | ###################################### 4 | ####Estimation functions are below#### 5 | ###################################### 6 | 7 | import numpy as np 8 | 9 | def knowledge(self, problems,correctness): 10 | """ 11 | ##This function finds the empirical knowledge of a single user given a 12 | chronologically ordered sequence of items submitted. 13 | """ 14 | # global m_slip_neg_log, m_guess_neg_log, n_los 15 | n_los = len(self.los) 16 | 17 | m_slip_u=self.m_slip_neg_log[problems,] 18 | m_guess_u=self.m_guess_neg_log[problems,] 19 | N=len(problems) 20 | 21 | 22 | #### In case there is only one problem, need to make sure the result of subsetting is a 1-row matrix, not a 1D array. Later, for dot product to work out. 23 | m_slip_u=m_slip_u.reshape(N,n_los) 24 | m_guess_u=m_guess_u.reshape(N,n_los) 25 | #### 26 | 27 | z=np.zeros((N+1,n_los)) 28 | x=np.repeat(0.0,N) 29 | z[0,]=np.dot((1.0-correctness),m_slip_u) 30 | z[N,]=np.dot(correctness,m_guess_u) 31 | 32 | if N>1: 33 | 34 | for n in range(1,N): 35 | x[range(n)]=correctness[range(n)] 36 | x[range(n,N)]=1.0-correctness[range(n,N)] 37 | temp=np.vstack((m_guess_u[range(n),],m_slip_u[n:,])) 38 | z[n,]=np.dot(x, temp) 39 | 40 | knowl=np.zeros((N,n_los)) 41 | 42 | for j in range(n_los): 43 | 44 | ind=np.where(z[:,j]==min(z[:,j]))[0] 45 | 46 | for i in ind: 47 | 48 | temp=np.repeat(0.0,N) 49 | if (i==0): 50 | temp=np.repeat(1.0,N) 51 | elif (i1) 57 | 58 | return(knowl) 59 | 60 | #This function estimates the BKT model using empirical probabilities 61 | ##To account for the fact that NaN and Inf elements of the estimated matrices should not be used as updates, this function replaces such elements with the corresponding elements of the current BKT parameter matrices. 62 | ##Thus, the outputs of this function do not contain any non-numeric values and should be used to simply replace the current BKT parameter matrices. 63 | def estimate(self, relevance_threshold=0.01,information_threshold=20, remove_degeneracy=True): 64 | 65 | 66 | # global n_items,n_los, m_k, transactions, L_i, m_trans, m_guess, m_slip, self.epsilon, useForTraining, n_users 67 | n_items = len(self.items) 68 | n_los = len(self.los) 69 | n_users = len(self.users) 70 | 71 | trans=np.zeros((n_items,n_los)) 72 | trans_denom=trans.copy() 73 | guess=trans.copy() 74 | guess_denom=trans.copy() 75 | slip=trans.copy() 76 | slip_denom=trans.copy() 77 | p_i=np.repeat(0.,n_los) 78 | p_i_denom=p_i.copy() 79 | 80 | #if ~('training_set' in globals()): 81 | training_set=range(n_users) 82 | 83 | 84 | for u in training_set: 85 | 86 | ##List problems that the user tried, in chronological order 87 | 88 | # n_of_na=np.count_nonzero(np.isnan(m_timestamp[u,])) 89 | # problems=m_timestamp[u,].argsort() ##It is important here that argsort() puts NaNs at the end, so we remove them from there 90 | # if n_of_na>0: 91 | # problems=problems[:-n_of_na] ##These are indices of items submitted by user u, in chronological order. 92 | # 93 | # problems=np.intersect1d(problems, useForTraining) 94 | 95 | 96 | temp=self.transactions.loc[(self.transactions.user_id==u)&(self.transactions.problem_id.isin(self.useForTraining))] 97 | temp=temp.sort_values('time') 98 | temp.index=range(np.shape(temp)[0]) 99 | ## Now temp is the data frame of submits of a particular user u, arranged in chronological order. In particular, temp.problem_id is the list of problems in chronological order. 100 | 101 | J=np.shape(temp)[0] 102 | if(J>0): 103 | m_k_u=self.m_k[temp.problem_id,] 104 | ##Reshape for the case J=1, we still want m_k_u to be a 2D array, not 1D. 105 | m_k_u=m_k_u.reshape(J,n_los) 106 | 107 | #Calculate the sum of relevances of user's experience for a each learning objective 108 | if(J==1): 109 | u_R=m_k_u[0] 110 | else: 111 | u_R=np.sum(m_k_u,axis=0) 112 | 113 | ##Implement the relevance threshold: zero-out what is not above it, set the rest to 1 114 | #u_R=u_R*(u_R>relevance_threshold) 115 | #u_R[u_R>0]=1 116 | u_R=(u_R>relevance_threshold) 117 | #m_k_u=m_k_u*(m_k_u>relevance_threshold) 118 | #m_k_u[m_k_u>0]=1 119 | m_k_u=(m_k_u>relevance_threshold) 120 | 121 | #u_correctness=m_correctness[u,problems] 122 | #u_knowledge=knowledge(problems, u_correctness) 123 | u_knowledge=knowledge(self, temp.problem_id, temp.score) 124 | #Now prepare the matrix by replicating the correctness column for each LO. 125 | #u_correctness=np.tile(u_correctness,(n_los,1)).transpose() 126 | 127 | 128 | ##Contribute to the averaged initial knowledge. 129 | p_i+=u_knowledge[0,]*u_R 130 | p_i_denom+=u_R 131 | 132 | 133 | ##Contribute to the trans, guess and slip probabilities (numerators and denominators separately). 134 | for pr in range(J): 135 | prob_id=temp.problem_id[pr] 136 | 137 | shorthand=m_k_u[pr,]*(1.0-u_knowledge[pr,]) 138 | 139 | guess[prob_id,]+=shorthand*temp.score[pr] 140 | guess_denom[prob_id,]+=shorthand 141 | 142 | shorthand=m_k_u[pr,]-shorthand ##equals m_k_u*u_knowledge 143 | slip[prob_id,]+=shorthand*(1.0-temp.score[pr]) 144 | slip_denom[prob_id,]+=shorthand 145 | 146 | if pr<(J-1): 147 | shorthand=m_k_u[pr,]*(1.0-u_knowledge[pr,]) 148 | trans[prob_id,]+=shorthand*u_knowledge[pr+1,] 149 | trans_denom[prob_id,]+=shorthand 150 | 151 | 152 | ##Normalize the results over users. 153 | ind=np.where(p_i_denom!=0) 154 | p_i[ind]/=p_i_denom[ind] 155 | ind=np.where(trans_denom!=0) 156 | trans[ind]/=trans_denom[ind] 157 | ind=np.where(guess_denom!=0) 158 | guess[ind]/=guess_denom[ind] 159 | ind=np.where(slip_denom!=0) 160 | slip[ind]/=slip_denom[ind] 161 | 162 | 163 | ##Replace with nans where denominators are below information cutoff 164 | p_i[(p_i_denom=0.5) | (guess+slip>=1)) 173 | ind_s=np.where((slip>=0.5) | (guess+slip>=1)) 174 | 175 | guess[ind_g]=np.nan 176 | slip[ind_s]=np.nan 177 | 178 | 179 | #Convert to odds (logarithmic in case of p.i): 180 | p_i=np.minimum(np.maximum(p_i,self.epsilon),1.0-self.epsilon) 181 | trans=np.minimum(np.maximum(trans,self.epsilon),1.0-self.epsilon) 182 | guess=np.minimum(np.maximum(guess,self.epsilon),1.0-self.epsilon) 183 | slip=np.minimum(np.maximum(slip,self.epsilon),1.0-self.epsilon) 184 | 185 | #L=np.log(p_i/(1-p_i)) 186 | L=p_i/(1.0-p_i) 187 | trans=trans/(1.0-trans) 188 | guess=guess/(1.0-guess) 189 | slip=slip/(1.0-slip) 190 | 191 | ##Keep the versions with NAs in them: 192 | L_i_nan=L.copy() 193 | trans_nan=trans.copy() 194 | guess_nan=guess.copy() 195 | slip_nan=slip.copy() 196 | 197 | 198 | ind=np.where(np.isnan(L) | np.isinf(L)) 199 | L[ind]=self.L_i[ind] 200 | ind=np.where(np.isnan(trans) | np.isinf(trans)) 201 | trans[ind]=self.m_trans[ind] 202 | ind=np.where(np.isnan(guess) | np.isinf(guess)) 203 | guess[ind]=self.m_guess[ind] 204 | ind=np.where(np.isnan(slip) | np.isinf(slip)) 205 | slip[ind]=self.m_slip[ind] 206 | 207 | 208 | return { 209 | 'L_i':L, 210 | 'trans':trans, 211 | 'guess':guess, 212 | 'slip':slip, 213 | 'L_i_nan':L_i_nan, 214 | 'trans_nan':trans_nan, 215 | 'guess_nan':guess_nan, 216 | 'slip_nan':slip_nan 217 | } 218 | -------------------------------------------------------------------------------- /app/engine/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2017-10-01 20:59 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Activity', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 21 | ('name', models.CharField(default='', max_length=200)), 22 | ('difficulty', models.FloatField(blank=True, null=True)), 23 | ('tags', models.TextField(default='')), 24 | ('type', models.CharField(default='', max_length=200)), 25 | ('include_adaptive', models.BooleanField(default=True)), 26 | ('nonadaptive_order', models.PositiveIntegerField(blank=True, null=True)), 27 | ('preadaptive_order', models.PositiveIntegerField(blank=True, null=True)), 28 | ], 29 | ), 30 | migrations.CreateModel( 31 | name='Collection', 32 | fields=[ 33 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 34 | ('name', models.CharField(max_length=200)), 35 | ('max_problems', models.PositiveIntegerField(blank=True, null=True)), 36 | ], 37 | ), 38 | migrations.CreateModel( 39 | name='Confidence', 40 | fields=[ 41 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 42 | ('value', models.FloatField()), 43 | ], 44 | ), 45 | migrations.CreateModel( 46 | name='EngineSettings', 47 | fields=[ 48 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 49 | ('name', models.CharField(default='', max_length=200)), 50 | ('r_star', models.FloatField()), 51 | ('L_star', models.FloatField()), 52 | ('W_p', models.FloatField()), 53 | ('W_r', models.FloatField()), 54 | ('W_c', models.FloatField()), 55 | ('W_d', models.FloatField()), 56 | ], 57 | ), 58 | migrations.CreateModel( 59 | name='ExperimentalGroup', 60 | fields=[ 61 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 62 | ('name', models.CharField(default='', max_length=200)), 63 | ('weight', models.FloatField(default=0)), 64 | ('engine_settings', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='engine.EngineSettings')), 65 | ], 66 | ), 67 | migrations.CreateModel( 68 | name='Exposure', 69 | fields=[ 70 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 71 | ('value', models.IntegerField()), 72 | ], 73 | ), 74 | migrations.CreateModel( 75 | name='Guess', 76 | fields=[ 77 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 78 | ('value', models.FloatField()), 79 | ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Activity')), 80 | ], 81 | ), 82 | migrations.CreateModel( 83 | name='KnowledgeComponent', 84 | fields=[ 85 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 86 | ('name', models.CharField(max_length=200)), 87 | ('mastery_prior', models.FloatField(blank=True, null=True)), 88 | ], 89 | ), 90 | migrations.CreateModel( 91 | name='Learner', 92 | fields=[ 93 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 94 | ('experimental_group', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='engine.ExperimentalGroup')), 95 | ], 96 | ), 97 | migrations.CreateModel( 98 | name='Mastery', 99 | fields=[ 100 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 101 | ('value', models.FloatField()), 102 | ('knowledge_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent')), 103 | ('learner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Learner')), 104 | ], 105 | ), 106 | migrations.CreateModel( 107 | name='PrerequisiteRelation', 108 | fields=[ 109 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 110 | ('value', models.FloatField()), 111 | ('knowledge_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent')), 112 | ('prerequisite', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dependent_relation', to='engine.KnowledgeComponent')), 113 | ], 114 | ), 115 | migrations.CreateModel( 116 | name='Score', 117 | fields=[ 118 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 119 | ('score', models.FloatField()), 120 | ('timestamp', models.DateTimeField(auto_now_add=True, null=True)), 121 | ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Activity')), 122 | ('learner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Learner')), 123 | ], 124 | ), 125 | migrations.CreateModel( 126 | name='Slip', 127 | fields=[ 128 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 129 | ('value', models.FloatField()), 130 | ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Activity')), 131 | ('knowledge_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent')), 132 | ], 133 | ), 134 | migrations.CreateModel( 135 | name='Transit', 136 | fields=[ 137 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 138 | ('value', models.FloatField()), 139 | ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Activity')), 140 | ('knowledge_component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent')), 141 | ], 142 | ), 143 | migrations.AddField( 144 | model_name='guess', 145 | name='knowledge_component', 146 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent'), 147 | ), 148 | migrations.AddField( 149 | model_name='exposure', 150 | name='knowledge_component', 151 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent'), 152 | ), 153 | migrations.AddField( 154 | model_name='exposure', 155 | name='learner', 156 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Learner'), 157 | ), 158 | migrations.AddField( 159 | model_name='confidence', 160 | name='knowledge_component', 161 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.KnowledgeComponent'), 162 | ), 163 | migrations.AddField( 164 | model_name='confidence', 165 | name='learner', 166 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='engine.Learner'), 167 | ), 168 | migrations.AddField( 169 | model_name='activity', 170 | name='collection', 171 | field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='engine.Collection'), 172 | ), 173 | migrations.AddField( 174 | model_name='activity', 175 | name='knowledge_components', 176 | field=models.ManyToManyField(blank=True, to='engine.KnowledgeComponent'), 177 | ), 178 | ] 179 | -------------------------------------------------------------------------------- /writeup/acmcopyright.sty: -------------------------------------------------------------------------------- 1 | %% 2 | %% This is file `acmcopyright.sty', 3 | %% generated with the docstrip utility. 4 | %% 5 | %% The original source files were: 6 | %% 7 | %% acmcopyright.dtx (with options: `style') 8 | %% 9 | %% IMPORTANT NOTICE: 10 | %% 11 | %% For the copyright see the source file. 12 | %% 13 | %% Any modified versions of this file must be renamed 14 | %% with new filenames distinct from acmcopyright.sty. 15 | %% 16 | %% For distribution of the original source see the terms 17 | %% for copying and modification in the file acmcopyright.dtx. 18 | %% 19 | %% This generated file may be distributed as long as the 20 | %% original source files, as listed above, are part of the 21 | %% same distribution. (The sources need not necessarily be 22 | %% in the same archive or directory.) 23 | %% \CharacterTable 24 | %% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z 25 | %% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z 26 | %% Digits \0\1\2\3\4\5\6\7\8\9 27 | %% Exclamation \! Double quote \" Hash (number) \# 28 | %% Dollar \$ Percent \% Ampersand \& 29 | %% Acute accent \' Left paren \( Right paren \) 30 | %% Asterisk \* Plus \+ Comma \, 31 | %% Minus \- Point \. Solidus \/ 32 | %% Colon \: Semicolon \; Less than \< 33 | %% Equals \= Greater than \> Question mark \? 34 | %% Commercial at \@ Left bracket \[ Backslash \\ 35 | %% Right bracket \] Circumflex \^ Underscore \_ 36 | %% Grave accent \` Left brace \{ Vertical bar \| 37 | %% Right brace \} Tilde \~} 38 | \NeedsTeXFormat{LaTeX2e} 39 | \ProvidesPackage{acmcopyright} 40 | [2014/06/29 v1.2 Copyright statemens for ACM classes] 41 | \newif\if@printcopyright 42 | \@printcopyrighttrue 43 | \newif\if@printpermission 44 | \@printpermissiontrue 45 | \newif\if@acmowned 46 | \@acmownedtrue 47 | \RequirePackage{xkeyval} 48 | \define@choicekey*{ACM@}{acmcopyrightmode}[% 49 | \acm@copyrightinput\acm@copyrightmode]{none,acmcopyright,acmlicensed,% 50 | rightsretained,usgov,usgovmixed,cagov,cagovmixed,% 51 | licensedusgovmixed,licensedcagovmixed,othergov,licensedothergov}{% 52 | \@printpermissiontrue 53 | \@printcopyrighttrue 54 | \@acmownedtrue 55 | \ifnum\acm@copyrightmode=0\relax % none 56 | \@printpermissionfalse 57 | \@printcopyrightfalse 58 | \@acmownedfalse 59 | \fi 60 | \ifnum\acm@copyrightmode=2\relax % acmlicensed 61 | \@acmownedfalse 62 | \fi 63 | \ifnum\acm@copyrightmode=3\relax % rightsretained 64 | \@acmownedfalse 65 | \fi 66 | \ifnum\acm@copyrightmode=4\relax % usgov 67 | \@printpermissiontrue 68 | \@printcopyrightfalse 69 | \@acmownedfalse 70 | \fi 71 | \ifnum\acm@copyrightmode=6\relax % cagov 72 | \@acmownedfalse 73 | \fi 74 | \ifnum\acm@copyrightmode=8\relax % licensedusgovmixed 75 | \@acmownedfalse 76 | \fi 77 | \ifnum\acm@copyrightmode=9\relax % licensedcagovmixed 78 | \@acmownedfalse 79 | \fi 80 | \ifnum\acm@copyrightmode=10\relax % othergov 81 | \@acmownedtrue 82 | \fi 83 | \ifnum\acm@copyrightmode=11\relax % licensedothergov 84 | \@acmownedfalse 85 | \@printcopyrightfalse 86 | \fi} 87 | \def\setcopyright#1{\setkeys{ACM@}{acmcopyrightmode=#1}} 88 | \setcopyright{acmcopyright} 89 | \def\@copyrightowner{% 90 | \ifcase\acm@copyrightmode\relax % none 91 | \or % acmcopyright 92 | ACM. 93 | \or % acmlicensed 94 | Copyright held by the owner/author(s). Publication rights licensed to 95 | ACM. 96 | \or % rightsretained 97 | Copyright held by the owner/author(s). 98 | \or % usgov 99 | \or % usgovmixed 100 | ACM. 101 | \or % cagov 102 | Crown in Right of Canada. 103 | \or %cagovmixed 104 | ACM. 105 | \or %licensedusgovmixed 106 | Copyright held by the owner/author(s). Publication rights licensed to 107 | ACM. 108 | \or %licensedcagovmixed 109 | Copyright held by the owner/author(s). Publication rights licensed to 110 | ACM. 111 | \or % othergov 112 | ACM. 113 | \or % licensedothergov 114 | \fi} 115 | \def\@copyrightpermission{% 116 | \ifcase\acm@copyrightmode\relax % none 117 | \or % acmcopyright 118 | Permission to make digital or hard copies of all or part of this 119 | work for personal or classroom use is granted without fee provided 120 | that copies are not made or distributed for profit or commercial 121 | advantage and that copies bear this notice and the full citation on 122 | the first page. Copyrights for components of this work owned by 123 | others than ACM must be honored. Abstracting with credit is 124 | permitted. To copy otherwise, or republish, to post on servers or to 125 | redistribute to lists, requires prior specific permission 126 | and\hspace*{.5pt}/or a fee. Request permissions from 127 | permissions@acm.org. 128 | \or % acmlicensed 129 | Permission to make digital or hard copies of all or part of this 130 | work for personal or classroom use is granted without fee provided 131 | that copies are not made or distributed for profit or commercial 132 | advantage and that copies bear this notice and the full citation on 133 | the first page. Copyrights for components of this work owned by 134 | others than the author(s) must be honored. Abstracting with credit 135 | is permitted. To copy otherwise, or republish, to post on servers 136 | or to redistribute to lists, requires prior specific permission 137 | and\hspace*{.5pt}/or a fee. Request permissions from 138 | permissions@acm.org. 139 | \or % rightsretained 140 | Permission to make digital or hard copies of part or all of this work 141 | for personal or classroom use is granted without fee provided that 142 | copies are not made or distributed for profit or commercial advantage 143 | and that copies bear this notice and the full citation on the first 144 | page. Copyrights for third-party components of this work must be 145 | honored. For all other uses, contact the 146 | owner\hspace*{.5pt}/author(s). 147 | \or % usgov 148 | This paper is authored by an employee(s) of the United States 149 | Government and is in the public domain. Non-exclusive copying or 150 | redistribution is allowed, provided that the article citation is 151 | given and the authors and agency are clearly identified as its 152 | source. 153 | \or % usgovmixed 154 | ACM acknowledges that this contribution was authored or co-authored 155 | by an employee, or contractor of the national government. As such, 156 | the Government retains a nonexclusive, royalty-free right to 157 | publish or reproduce this article, or to allow others to do so, for 158 | Government purposes only. Permission to make digital or hard copies 159 | for personal or classroom use is granted. Copies must bear this 160 | notice and the full citation on the first page. Copyrights for 161 | components of this work owned by others than ACM must be 162 | honored. To copy otherwise, distribute, republish, or post, 163 | requires prior specific permission and\hspace*{.5pt}/or a 164 | fee. Request permissions from permissions@acm.org. 165 | \or % cagov 166 | This article was authored by employees of the Government of Canada. 167 | As such, the Canadian government retains all interest in the 168 | copyright to this work and grants to ACM a nonexclusive, 169 | royalty-free right to publish or reproduce this article, or to allow 170 | others to do so, provided that clear attribution is given both to 171 | the authors and the Canadian government agency employing them. 172 | Permission to make digital or hard copies for personal or classroom 173 | use is granted. Copies must bear this notice and the full citation 174 | on the first page. Copyrights for components of this work owned by 175 | others than the Canadain Government must be honored. To copy 176 | otherwise, distribute, republish, or post, requires prior specific 177 | permission and\hspace*{.5pt}/or a fee. Request permissions from 178 | permissions@acm.org. 179 | \or % cagovmixed 180 | ACM acknowledges that this contribution was co-authored by an 181 | affiliate of the national government of Canada. As such, the Crown 182 | in Right of Canada retains an equal interest in the copyright. 183 | Reprints must include clear attribution to ACM and the author's 184 | government agency affiliation. Permission to make digital or hard 185 | copies for personal or classroom use is granted. Copies must bear 186 | this notice and the full citation on the first page. Copyrights for 187 | components of this work owned by others than ACM must be honored. 188 | To copy otherwise, distribute, republish, or post, requires prior 189 | specific permission and\hspace*{.5pt}/or a fee. Request permissions 190 | from permissions@acm.org. 191 | \or % licensedusgovmixed 192 | Publication rights licensed to ACM. ACM acknowledges that this 193 | contribution was authored or co-authored by an employee, contractor 194 | or affiliate of the United States government. As such, the 195 | Government retains a nonexclusive, royalty-free right to publish or 196 | reproduce this article, or to allow others to do so, for Government 197 | purposes only. 198 | \or % licensedcagovmixed 199 | Publication rights licensed to ACM. ACM acknowledges that this 200 | contribution was authored or co-authored by an employee, contractor 201 | or affiliate of the national government of Canada. As such, the 202 | Government retains a nonexclusive, royalty-free right to publish or 203 | reproduce this article, or to allow others to do so, for Government 204 | purposes only. 205 | \or % othergov 206 | ACM acknowledges that this contribution was authored or co-authored 207 | by an employee, contractor or affiliate of a national government. As 208 | such, the Government retains a nonexclusive, royalty-free right to 209 | publish or reproduce this article, or to allow others to do so, for 210 | Government purposes only. 211 | \or % licensedothergov 212 | Publication rights licensed to ACM. ACM acknowledges that this 213 | contribution was authored or co-authored by an employee, contractor 214 | or affiliate of a national government. As such, the Government 215 | retains a nonexclusive, royalty-free right to publish or reproduce 216 | this article, or to allow others to do so, for Government purposes 217 | only. 218 | \fi} 219 | \endinput 220 | %% 221 | %% End of file `acmcopyright.sty'. 222 | -------------------------------------------------------------------------------- /prototypes/r_prototype/evaluation/data_load.R: -------------------------------------------------------------------------------- 1 | ##Author: Ilia Rushkin, VPAL Research, Harvard University, Cambridge, MA, USA 2 | # datadir='/Users/ilr548/Dropbox/AdaptiveEngine/SME_data' 3 | # ##Where to write matrices (will create directory if missing; if NULL, will not write anywhere) 4 | # writedir='/Users/ilr548/Dropbox/AdaptiveEngine/SME_data/tagging_data' 5 | 6 | 7 | #Codes for converting categorical to numerical data 8 | prereq_weight_code=c('Weak'=0.33,'Moderate'=0.66,'Strong'=1.0,'default value'=1.0) 9 | guess_code=c('Low'=0.08, 'Low '=0.08, 'Moderate'=0.12,'High'=0.15, 'default value'=0.1) 10 | slip_code=c('Low'=0.1, 'Low '=0.1, 'Moderate'=0.15,'High'=0.20, 'default value'=0.15) 11 | trans_code=c('Low'=0.08, 'Low '=0.08, 'Moderate'=0.12,'High'=0.15, 'default value'=0.1) 12 | difficulty_code=c('Easy'=0.3, 'Reg'=0.5, 'Difficult'=0.8, 'default value'=0.5) 13 | 14 | prior.knowledge.probability=0.2 15 | 16 | 17 | ####Global variables#### 18 | epsilon<<-1e-10 # a regularization cutoff, the smallest value of a mastery probability 19 | eta=0 ##Relevance threshold used in the BKT optimization procedure 20 | M=20 ##Information threshold user in the BKT optimization procedure 21 | L.star<<- 2.2 #Threshold odds. If mastery odds are >= than L.star, the LO is considered mastered 22 | r.star<<- 0 #Threshold for forgiving lower odds of mastering pre-requisite LOs. 23 | 24 | V.r<<-1 ##Importance of readiness in recommending the next item 25 | V.a<<-0.5 ##Importance of appropriate difficulty in recommending the next item 26 | 27 | V.d<<-2 ##Importance of demand in recommending the next item 28 | V.c<<-1 ##Importance of continuity in recommending the next item 29 | 30 | 31 | 32 | 33 | library("RBGL"); 34 | library("Rgraphviz"); 35 | items_KC=read.csv(file.path(datadir,'Adaptive Engine Data - Essential Stats - Items-KC.csv'), header=TRUE,stringsAsFactors = FALSE) 36 | 37 | KcColumn='LO.short.name' 38 | itemColumn='Item.ID' 39 | guessColumn='Guess.probability..chance.of.answering.correctly.despite.not.knowing.the.LO.' 40 | transColumn='Learning.value..chance.of.learning.the.LO.from.this.item.' 41 | locationColumn="Pre.Post.If.location.provided..this.question.is.fixed..this.is.where.it.currently.appears.in.the.course.and.should.be.part.of.the.adaptive.testing.in.that.location." 42 | slipColumn=NA 43 | items_KC=subset(items_KC,!((items_KC[,itemColumn]=='')|(is.na(items_KC[,itemColumn])))) 44 | 45 | ##Order item IDs numerically 46 | items_KC=items_KC[order(items_KC[,itemColumn]),] 47 | ## 48 | 49 | items_KC[,itemColumn]=as.character(items_KC[,itemColumn]) 50 | 51 | ##Provide items that will be served as pre-test for control group with no adaptivity 52 | ind_nonadpt=which(grepl('Final',items_KC[,locationColumn])|grepl('Module',items_KC[,locationColumn])) 53 | ind=which(items_KC[,locationColumn]=='Final') 54 | 55 | ind1=rep(NA,length(ind)) 56 | for(i in 1:length(ind)){ 57 | 58 | if(!grepl('Module',items_KC[ind[i]-1,locationColumn])){ 59 | ind1[i]=ind[i]-1 60 | }else{ 61 | if(!grepl('Module',items_KC[ind[i]+1,locationColumn])){ 62 | ind1[i]=ind[i]+1 63 | } 64 | } 65 | 66 | } 67 | 68 | ind1=ind1[!is.na(ind1)] 69 | items_KC_export=items_KC 70 | items_KC_export[ind1,locationColumn]="Pretest" 71 | items_KC_export=items_KC_export[sort(c(ind1,ind_nonadpt)),] 72 | 73 | names(items_KC_export)[which(names(items_KC_export)==locationColumn)]='Module' 74 | 75 | # write.csv(items_KC_export,file='items_KC_Group_C_marked.csv',row.names = FALSE) 76 | 77 | 78 | 79 | 80 | items=read.csv(file.path(datadir,'Adaptive Engine Data - Essential Stats - Items.csv'), header=TRUE,stringsAsFactors = FALSE) 81 | moduleColumn='Module_Liberty' 82 | difficultyColumn='Difficulty.Level' 83 | items[,itemColumn]=as.character(items[,itemColumn]) 84 | 85 | items_KC$index=1:nrow(items_KC) 86 | items_KC=merge(items_KC,items[,c(itemColumn,moduleColumn, difficultyColumn)], by=itemColumn) 87 | items_KC=items_KC[order(items_KC$index),] 88 | items_KC$index=NULL 89 | 90 | 91 | 92 | kgraph=read.csv(file.path(datadir,'Adaptive Engine Data - Essential Stats - KC-KC.csv'), header=TRUE,stringsAsFactors = FALSE) 93 | preColumn='Pre.req.LO.short.name' 94 | postColumn='Post.req.LO.short.name' 95 | strengthColumn='Connection.strength' 96 | 97 | ##Convert categorical to numerical 98 | kgraph$weight=NA 99 | for(i in 1:length(prereq_weight_code)){ 100 | kgraph$weight[kgraph[,strengthColumn]==names(prereq_weight_code)[i]]=prereq_weight_code[i] 101 | } 102 | kgraph$weight[is.na(kgraph$weight)]=prereq_weight_code['default value'] 103 | 104 | items_KC$guess=NA 105 | if(guessColumn %in% names(items_KC)){ 106 | for(i in 1:length(guess_code)){ 107 | items_KC$guess[items_KC[,guessColumn]==names(guess_code)[i]]=guess_code[i] 108 | } 109 | } 110 | items_KC$guess[is.na(items_KC$guess)]=guess_code['default value'] 111 | 112 | items_KC$slip=NA 113 | if(slipColumn %in% names(items_KC)){ 114 | for(i in 1:length(slip_code)){ 115 | items_KC$slip[items_KC[,slipColumn]==names(slip_code)[i]]=slip_code[i] 116 | } 117 | } 118 | items_KC$slip[is.na(items_KC$slip)]=slip_code['default value'] 119 | 120 | items_KC$trans=NA 121 | if(transColumn %in% names(items_KC)){ 122 | for(i in 1:length(guess_code)){ 123 | items_KC$trans[items_KC[,transColumn]==names(trans_code)[i]]=trans_code[i] 124 | } 125 | } 126 | items_KC$trans[is.na(items_KC$trans)]=trans_code['default value'] 127 | 128 | items_KC$diff=NA 129 | if(difficultyColumn %in% names(items_KC)){ 130 | for(i in 1:length(difficulty_code)){ 131 | items_KC$diff[items_KC[,difficultyColumn]==names(difficulty_code)[i]]=difficulty_code[i] 132 | } 133 | } 134 | items_KC$diff[is.na(items_KC$diff)]=difficulty_code['default value'] 135 | 136 | 137 | 138 | ##Store problems and KCs, modules lists: 139 | 140 | #los=data.frame("id"=unique(c(items_KC[,KcColumn],kgraph[,preColumn],kgraph[,postColumn]))) 141 | los=data.frame("id"=unique(items_KC[,KcColumn])) 142 | los$id=as.character(los$id) 143 | n.los=nrow(los) 144 | probs=data.frame("id"=unique(items_KC[,itemColumn])) 145 | probs$id=as.character(probs$id) 146 | n.probs=nrow(probs) 147 | modules=data.frame('id'=sort(unique(items_KC[,moduleColumn]))) 148 | modules$id=as.character(modules$id) 149 | n.modules=nrow(modules) 150 | 151 | #########matrix of pre-requisite relations:####### 152 | 153 | 154 | ##If by mistake a link is given more than once, keep the version with the highest weight. 155 | kgraph$prepost=paste(kgraph[,preColumn],kgraph[,postColumn]) 156 | temp=aggregate(kgraph$weight, by=list(prepost=kgraph$prepost), max, na.rm=TRUE) 157 | names(temp)[2]='weight' 158 | kgraph$weight=NULL 159 | kgraph=merge(temp,kgraph, by='prepost') 160 | kgraph$prepost=NULL 161 | 162 | #Remove pre-requisites to self 163 | kgraph=subset(kgraph,kgraph[,preColumn]!=kgraph[,postColumn]) 164 | 165 | g=graphNEL(unique(c(kgraph[,preColumn],kgraph[,postColumn])),edgemode="directed"); 166 | g=addEdge(kgraph[,preColumn],kgraph[,postColumn],g,kgraph$weight); 167 | 168 | cycles=strongComp(g); 169 | cycles=cycles[lengths(cycles)>1]; 170 | if(length(cycles)>0){ 171 | cat("Loops found in the knowledge graph:\n"); 172 | print(cycles); 173 | }else{ 174 | cat("No loops found in the knowledge graph.\n"); 175 | } 176 | 177 | #Define pre-requisite matrix. rownames are pre-reqs. 178 | m.w<<-matrix(0,nrow=n.los, ncol=n.los); 179 | rownames(m.w)=los$id 180 | colnames(m.w)=los$id 181 | for(i in 1:nrow(kgraph)){ 182 | m.w[kgraph[i,preColumn],kgraph[i,postColumn]]=kgraph$weight[i] 183 | } 184 | ########END of matrix of pre-requisite relations######### 185 | 186 | 187 | #####Tagging matrices##### 188 | 189 | scope<<-matrix(FALSE, nrow=n.probs, ncol=n.modules) 190 | rownames(scope)=probs$id 191 | colnames(scope)=modules$id 192 | 193 | m.tagging<<-matrix(0,nrow=n.probs, ncol=n.los) 194 | rownames(m.tagging)=probs$id 195 | colnames(m.tagging)=los$id 196 | 197 | 198 | m.guess<<-matrix(1,nrow=n.probs, ncol=n.los) 199 | rownames(m.guess)=probs$id 200 | colnames(m.guess)=los$id 201 | 202 | m.slip<<-matrix(1,nrow=n.probs, ncol=n.los) 203 | rownames(m.slip)=probs$id 204 | colnames(m.slip)=los$id 205 | 206 | m.trans<<-matrix(0,nrow=n.probs, ncol=n.los) 207 | rownames(m.trans)=probs$id 208 | colnames(m.trans)=los$id 209 | 210 | difficulty<<-rep(1,n.probs) 211 | names(difficulty)=probs$id 212 | 213 | for(i in 1:nrow(items_KC)){ 214 | m.tagging[items_KC[i,itemColumn],items_KC[i,KcColumn]]=1 215 | scope[items_KC[i,itemColumn],items_KC[i,moduleColumn]]=TRUE 216 | m.guess[items_KC[i,itemColumn],items_KC[i,KcColumn]]=items_KC$guess[i]/(1-items_KC$guess[i]) 217 | m.slip[items_KC[i,itemColumn],items_KC[i,KcColumn]]=items_KC$slip[i]/(1-items_KC$slip[i]) 218 | m.trans[items_KC[i,itemColumn],items_KC[i,KcColumn]]=items_KC$trans[i]/(1-items_KC$trans[i]) 219 | difficulty[items_KC[i,itemColumn]]=log(items_KC$diff[i]/(1-items_KC$diff[i])) 220 | } 221 | 222 | L.i<<-rep(prior.knowledge.probability/(1-prior.knowledge.probability),n.los) 223 | 224 | 225 | if(!is.null(writedir)){ 226 | if(!file.exists(writedir)){ 227 | dir.create(writedir) 228 | } 229 | 230 | write.table(los$id,file=file.path(writedir,'KCs.csv'), row.names = FALSE, col.names = FALSE, sep=',') 231 | write.table(modules$id,file=file.path(writedir,'modules.csv'), row.names = FALSE, col.names = FALSE, sep=',') 232 | write.table(probs$id,file=file.path(writedir,'items.csv'), row.names = FALSE, col.names = FALSE, sep=',') 233 | write.table(m.w,file=file.path(writedir,'m_w.csv'), row.names = FALSE, col.names = FALSE, sep=',') 234 | write.table(difficulty,file=file.path(writedir,'difficulty.csv'), row.names = FALSE, col.names = FALSE, sep=',') 235 | write.table(m.tagging,file=file.path(writedir,'m_tagging.csv'), row.names = FALSE, col.names = FALSE, sep=',') 236 | write.table(scope,file=file.path(writedir,'scope.csv'), row.names = FALSE, col.names = FALSE, sep=',') 237 | write.table(m.guess,file=file.path(writedir,'m_guess.csv'), row.names = FALSE, col.names = FALSE, sep=',') 238 | write.table(m.slip,file=file.path(writedir,'m_slip.csv'), row.names = FALSE, col.names = FALSE, sep=',') 239 | write.table(m.trans,file=file.path(writedir,'m_trans.csv'), row.names = FALSE, col.names = FALSE, sep=',') 240 | write.table(t(L.i),file=file.path(writedir,'L_i.csv'), row.names = FALSE, col.names = FALSE, sep=',') 241 | } 242 | ##Loading the transactions 243 | transactions<<-LogData[,c("user_id","problem_id","time","score")] 244 | ##Initial knowledge: 245 | # Define the matrix of initial mastery by replicating the same row for each user 246 | 247 | options(stringsAsFactors = FALSE) 248 | users<<-data.frame("id"=unique(transactions$user_id),"name"=unique(transactions$user_id), "group"=1) 249 | users$id=as.character(users$id) 250 | n.users<<-nrow(users) 251 | 252 | m.L.i<<-matrix(rep(L.i,n.users),ncol=n.los, byrow = FALSE) 253 | rownames(m.L.i)=users$id 254 | colnames(m.L.i)=los$id 255 | 256 | 257 | -------------------------------------------------------------------------------- /writeup/nips_2016.cls: -------------------------------------------------------------------------------- 1 | % partial rewrite of the LaTeX2e package for submissions to the 2 | % Conference on Neural Information Processing Systems (NIPS): 3 | % 4 | % - uses more LaTeX conventions 5 | % - line numbers at submission time replaced with aligned numbers from 6 | % lineno package 7 | % - \nipsfinalcopy replaced with [final] package option 8 | % - automatically loads times package for authors 9 | % - loads natbib automatically; this can be suppressed with the 10 | % [nonatbib] package option 11 | % - adds foot line to first page identifying the conference 12 | % 13 | % Roman Garnett (garnett@wustl.edu) and the many authors of 14 | % nips15submit_e.sty, including MK and drstrip@sandia 15 | % 16 | % last revision: August 2016 17 | 18 | \NeedsTeXFormat{LaTeX2e} 19 | \ProvidesPackage{nips_2016}[2016/08/08 NIPS 2016 submission/camera-ready style file] 20 | 21 | % declare final option, which creates camera-ready copy 22 | \newif\if@nipsfinal\@nipsfinalfalse 23 | \DeclareOption{final}{ 24 | \@nipsfinaltrue 25 | } 26 | 27 | % declare nonatbib option, which does not load natbib in case of 28 | % package clash (users can pass options to natbib via 29 | % \PassOptionsToPackage) 30 | \newif\if@natbib\@natbibtrue 31 | \DeclareOption{nonatbib}{ 32 | \@natbibfalse 33 | } 34 | 35 | \ProcessOptions\relax 36 | 37 | % fonts 38 | \renewcommand{\rmdefault}{ptm} 39 | \renewcommand{\sfdefault}{phv} 40 | 41 | % change this every year for notice string at bottom 42 | \newcommand{\@nipsordinal}{30th} 43 | \newcommand{\@nipsyear}{2016} 44 | \newcommand{\@nipslocation}{Barcelona, Spain} 45 | 46 | % handle tweaks for camera-ready copy vs. submission copy 47 | \if@nipsfinal 48 | \newcommand{\@noticestring}{% 49 | \@nipsordinal\/ Conference on Neural Information Processing Systems 50 | (NIPS \@nipsyear), \@nipslocation.% 51 | } 52 | \else 53 | \newcommand{\@noticestring}{% 54 | Submitted to \@nipsordinal\/ Conference on Neural Information 55 | Processing Systems (NIPS \@nipsyear). Do not distribute.% 56 | } 57 | 58 | % line numbers for submission 59 | \RequirePackage{lineno} 60 | \linenumbers 61 | 62 | % fix incompatibilities between lineno and amsmath, if required, by 63 | % transparently wrapping linenomath environments around amsmath 64 | % environments 65 | \AtBeginDocument{% 66 | \@ifpackageloaded{amsmath}{% 67 | \newcommand*\patchAmsMathEnvironmentForLineno[1]{% 68 | \expandafter\let\csname old#1\expandafter\endcsname\csname #1\endcsname 69 | \expandafter\let\csname oldend#1\expandafter\endcsname\csname end#1\endcsname 70 | \renewenvironment{#1}% 71 | {\linenomath\csname old#1\endcsname}% 72 | {\csname oldend#1\endcsname\endlinenomath}% 73 | }% 74 | \newcommand*\patchBothAmsMathEnvironmentsForLineno[1]{% 75 | \patchAmsMathEnvironmentForLineno{#1}% 76 | \patchAmsMathEnvironmentForLineno{#1*}% 77 | }% 78 | \patchBothAmsMathEnvironmentsForLineno{equation}% 79 | \patchBothAmsMathEnvironmentsForLineno{align}% 80 | \patchBothAmsMathEnvironmentsForLineno{flalign}% 81 | \patchBothAmsMathEnvironmentsForLineno{alignat}% 82 | \patchBothAmsMathEnvironmentsForLineno{gather}% 83 | \patchBothAmsMathEnvironmentsForLineno{multline}% 84 | }{} 85 | } 86 | \fi 87 | 88 | % load natbib unless told otherwise 89 | \if@natbib 90 | \RequirePackage{natbib} 91 | \fi 92 | 93 | % set page geometry 94 | \usepackage[verbose=true,letterpaper]{geometry} 95 | \AtBeginDocument{ 96 | \newgeometry{ 97 | textheight=9in, 98 | textwidth=5.5in, 99 | top=1in, 100 | headheight=12pt, 101 | headsep=25pt, 102 | footskip=30pt 103 | } 104 | \@ifpackageloaded{fullpage} 105 | {\PackageWarning{nips_2016}{fullpage package not allowed! Overwriting formatting.}} 106 | {} 107 | } 108 | 109 | \widowpenalty=10000 110 | \clubpenalty=10000 111 | \flushbottom 112 | \sloppy 113 | 114 | % font sizes with reduced leading 115 | \renewcommand{\normalsize}{% 116 | \@setfontsize\normalsize\@xpt\@xipt 117 | \abovedisplayskip 7\p@ \@plus 2\p@ \@minus 5\p@ 118 | \abovedisplayshortskip \z@ \@plus 3\p@ 119 | \belowdisplayskip \abovedisplayskip 120 | \belowdisplayshortskip 4\p@ \@plus 3\p@ \@minus 3\p@ 121 | } 122 | \normalsize 123 | \renewcommand{\small}{% 124 | \@setfontsize\small\@ixpt\@xpt 125 | \abovedisplayskip 6\p@ \@plus 1.5\p@ \@minus 4\p@ 126 | \abovedisplayshortskip \z@ \@plus 2\p@ 127 | \belowdisplayskip \abovedisplayskip 128 | \belowdisplayshortskip 3\p@ \@plus 2\p@ \@minus 2\p@ 129 | } 130 | \renewcommand{\footnotesize}{\@setfontsize\footnotesize\@ixpt\@xpt} 131 | \renewcommand{\scriptsize}{\@setfontsize\scriptsize\@viipt\@viiipt} 132 | \renewcommand{\tiny}{\@setfontsize\tiny\@vipt\@viipt} 133 | \renewcommand{\large}{\@setfontsize\large\@xiipt{14}} 134 | \renewcommand{\Large}{\@setfontsize\Large\@xivpt{16}} 135 | \renewcommand{\LARGE}{\@setfontsize\LARGE\@xviipt{20}} 136 | \renewcommand{\huge}{\@setfontsize\huge\@xxpt{23}} 137 | \renewcommand{\Huge}{\@setfontsize\Huge\@xxvpt{28}} 138 | 139 | % sections with less space 140 | \providecommand{\section}{} 141 | \renewcommand{\section}{% 142 | \@startsection{section}{1}{\z@}% 143 | {-2.0ex \@plus -0.5ex \@minus -0.2ex}% 144 | { 1.5ex \@plus 0.3ex \@minus 0.2ex}% 145 | {\large\bf\raggedright}% 146 | } 147 | \providecommand{\subsection}{} 148 | \renewcommand{\subsection}{% 149 | \@startsection{subsection}{2}{\z@}% 150 | {-1.8ex \@plus -0.5ex \@minus -0.2ex}% 151 | { 0.8ex \@plus 0.2ex}% 152 | {\normalsize\bf\raggedright}% 153 | } 154 | \providecommand{\subsubsection}{} 155 | \renewcommand{\subsubsection}{% 156 | \@startsection{subsubsection}{3}{\z@}% 157 | {-1.5ex \@plus -0.5ex \@minus -0.2ex}% 158 | { 0.5ex \@plus 0.2ex}% 159 | {\normalsize\bf\raggedright}% 160 | } 161 | \providecommand{\paragraph}{} 162 | \renewcommand{\paragraph}{% 163 | \@startsection{paragraph}{4}{\z@}% 164 | {1.5ex \@plus 0.5ex \@minus 0.2ex}% 165 | {-1em}% 166 | {\normalsize\bf}% 167 | } 168 | \providecommand{\subparagraph}{} 169 | \renewcommand{\subparagraph}{% 170 | \@startsection{subparagraph}{5}{\z@}% 171 | {1.5ex \@plus 0.5ex \@minus 0.2ex}% 172 | {-1em}% 173 | {\normalsize\bf}% 174 | } 175 | \providecommand{\subsubsubsection}{} 176 | \renewcommand{\subsubsubsection}{% 177 | \vskip5pt{\noindent\normalsize\rm\raggedright}% 178 | } 179 | 180 | % float placement 181 | \renewcommand{\topfraction }{0.85} 182 | \renewcommand{\bottomfraction }{0.4} 183 | \renewcommand{\textfraction }{0.1} 184 | \renewcommand{\floatpagefraction}{0.7} 185 | 186 | \newlength{\@nipsabovecaptionskip}\setlength{\@nipsabovecaptionskip}{7\p@} 187 | \newlength{\@nipsbelowcaptionskip}\setlength{\@nipsbelowcaptionskip}{\z@} 188 | 189 | \setlength{\abovecaptionskip}{\@nipsabovecaptionskip} 190 | \setlength{\belowcaptionskip}{\@nipsbelowcaptionskip} 191 | 192 | % swap above/belowcaptionskip lengths for tables 193 | \renewenvironment{table} 194 | {\setlength{\abovecaptionskip}{\@nipsbelowcaptionskip}% 195 | \setlength{\belowcaptionskip}{\@nipsabovecaptionskip}% 196 | \@float{table}} 197 | {\end@float} 198 | 199 | % footnote formatting 200 | \setlength{\footnotesep }{6.65\p@} 201 | \setlength{\skip\footins}{9\p@ \@plus 4\p@ \@minus 2\p@} 202 | \renewcommand{\footnoterule}{\kern-3\p@ \hrule width 12pc \kern 2.6\p@} 203 | \setcounter{footnote}{0} 204 | 205 | % paragraph formatting 206 | \setlength{\parindent}{\z@} 207 | \setlength{\parskip }{5.5\p@} 208 | 209 | % list formatting 210 | \setlength{\topsep }{4\p@ \@plus 1\p@ \@minus 2\p@} 211 | \setlength{\partopsep }{1\p@ \@plus 0.5\p@ \@minus 0.5\p@} 212 | \setlength{\itemsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@} 213 | \setlength{\parsep }{2\p@ \@plus 1\p@ \@minus 0.5\p@} 214 | \setlength{\leftmargin }{3pc} 215 | \setlength{\leftmargini }{\leftmargin} 216 | \setlength{\leftmarginii }{2em} 217 | \setlength{\leftmarginiii}{1.5em} 218 | \setlength{\leftmarginiv }{1.0em} 219 | \setlength{\leftmarginv }{0.5em} 220 | \def\@listi {\leftmargin\leftmargini} 221 | \def\@listii {\leftmargin\leftmarginii 222 | \labelwidth\leftmarginii 223 | \advance\labelwidth-\labelsep 224 | \topsep 2\p@ \@plus 1\p@ \@minus 0.5\p@ 225 | \parsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@ 226 | \itemsep \parsep} 227 | \def\@listiii{\leftmargin\leftmarginiii 228 | \labelwidth\leftmarginiii 229 | \advance\labelwidth-\labelsep 230 | \topsep 1\p@ \@plus 0.5\p@ \@minus 0.5\p@ 231 | \parsep \z@ 232 | \partopsep 0.5\p@ \@plus 0\p@ \@minus 0.5\p@ 233 | \itemsep \topsep} 234 | \def\@listiv {\leftmargin\leftmarginiv 235 | \labelwidth\leftmarginiv 236 | \advance\labelwidth-\labelsep} 237 | \def\@listv {\leftmargin\leftmarginv 238 | \labelwidth\leftmarginv 239 | \advance\labelwidth-\labelsep} 240 | \def\@listvi {\leftmargin\leftmarginvi 241 | \labelwidth\leftmarginvi 242 | \advance\labelwidth-\labelsep} 243 | 244 | % create title 245 | \providecommand{\maketitle}{} 246 | \renewcommand{\maketitle}{% 247 | \par 248 | \begingroup 249 | \renewcommand{\thefootnote}{\fnsymbol{footnote}} 250 | % for perfect author name centering 251 | \renewcommand{\@makefnmark}{\hbox to \z@{$^{\@thefnmark}$\hss}} 252 | % The footnote-mark was overlapping the footnote-text, 253 | % added the following to fix this problem (MK) 254 | \long\def\@makefntext##1{% 255 | \parindent 1em\noindent 256 | \hbox to 1.8em{\hss $\m@th ^{\@thefnmark}$}##1 257 | } 258 | \thispagestyle{empty} 259 | \@maketitle 260 | \@thanks 261 | \@notice 262 | \endgroup 263 | \let\maketitle\relax 264 | \let\thanks\relax 265 | } 266 | 267 | % rules for title box at top of first page 268 | \newcommand{\@toptitlebar}{ 269 | \hrule height 4\p@ 270 | \vskip 0.25in 271 | \vskip -\parskip% 272 | } 273 | \newcommand{\@bottomtitlebar}{ 274 | \vskip 0.29in 275 | \vskip -\parskip 276 | \hrule height 1\p@ 277 | \vskip 0.09in% 278 | } 279 | 280 | % create title (includes both anonymized and non-anonymized versions) 281 | \providecommand{\@maketitle}{} 282 | \renewcommand{\@maketitle}{% 283 | \vbox{% 284 | \hsize\textwidth 285 | \linewidth\hsize 286 | \vskip 0.1in 287 | \@toptitlebar 288 | \centering 289 | {\LARGE\bf \@title\par} 290 | \@bottomtitlebar 291 | \if@nipsfinal 292 | \def\And{% 293 | \end{tabular}\hfil\linebreak[0]\hfil% 294 | \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces% 295 | } 296 | \def\AND{% 297 | \end{tabular}\hfil\linebreak[4]\hfil% 298 | \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces% 299 | } 300 | \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\@author\end{tabular}% 301 | \else 302 | \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@} 303 | Anonymous Author(s) \\ 304 | Affiliation \\ 305 | Address \\ 306 | \texttt{email} \\ 307 | \end{tabular}% 308 | \fi 309 | \vskip 0.3in \@minus 0.1in 310 | } 311 | } 312 | 313 | % add conference notice to bottom of first page 314 | \newcommand{\ftype@noticebox}{8} 315 | \newcommand{\@notice}{% 316 | % give a bit of extra room back to authors on first page 317 | \enlargethispage{2\baselineskip}% 318 | \@float{noticebox}[b]% 319 | \footnotesize\@noticestring% 320 | \end@float% 321 | } 322 | 323 | % abstract styling 324 | \renewenvironment{abstract}% 325 | {% 326 | \vskip 0.075in% 327 | \centerline% 328 | {\large\bf Abstract}% 329 | \vspace{0.5ex}% 330 | \begin{quote}% 331 | } 332 | { 333 | \par% 334 | \end{quote}% 335 | \vskip 1ex% 336 | } 337 | 338 | \endinput 339 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 President and Fellows of Harvard College 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /app/engine/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers, validators 2 | from .models import * 3 | 4 | 5 | class LearnerSerializer(serializers.ModelSerializer): 6 | """ 7 | Used as a nested serializer for learner foreign key field 8 | Acts as a "compound lookup field" 9 | Supports lookups from creation-like request methods to related objects 10 | """ 11 | class Meta: 12 | model = Learner 13 | fields = ('user_id', 'tool_consumer_instance_guid') 14 | 15 | def run_validators(self, value): 16 | """ 17 | Modify base run_validators() to not enforce unique_together validator on user_id/tool_consumer_instance_guid 18 | field combination. This is so that creation-like methods (e.g. put/post) on related objects (e.g. mastery) 19 | can reference an existing learner (using the user_id/consumer_id pair). 20 | """ 21 | for validator in self.validators: 22 | if isinstance(validator, validators.UniqueTogetherValidator): 23 | self.validators.remove(validator) 24 | super().run_validators(value) 25 | 26 | 27 | class KnowledgeComponentFieldSerializer(serializers.ModelSerializer): 28 | """ 29 | Used as a nested serializer for knowledge_component foreign key field 30 | """ 31 | # override default serializer field used so that custom field validator can be defined, 32 | # while ignoring default unique validator 33 | kc_id = serializers.CharField() 34 | 35 | class Meta: 36 | model = KnowledgeComponent 37 | fields = ('kc_id',) 38 | 39 | def validate_kc_id(self, value): 40 | """ 41 | Custom validation on kc_id field, that verifies that a KnowledgeComponent with the given kc_id exists. 42 | :param value: 43 | :return: 44 | """ 45 | if KnowledgeComponent.objects.filter(kc_id=value).exists(): 46 | return value 47 | else: 48 | raise serializers.ValidationError("Knowledge component with specified kc_id does not exist") 49 | 50 | 51 | class CollectionFieldSerializer(serializers.ModelSerializer): 52 | """ 53 | Used as a nested serializer for collection foreign key field 54 | """ 55 | # override default serializer field used so that custom field validator can be defined, 56 | # while ignoring default unique validator 57 | collection_id = serializers.CharField() 58 | 59 | class Meta: 60 | model = Collection 61 | fields = ('collection_id',) 62 | 63 | def validate_collection_id(self, value): 64 | """ 65 | Specified collection must exist (create not supported with this serializer) 66 | :param value: 67 | :return: 68 | """ 69 | if Collection.objects.filter(collection_id=value).exists(): 70 | return value 71 | else: 72 | raise serializers.ValidationError("object with specified id does not exist") 73 | 74 | 75 | class CollectionSerializer(serializers.ModelSerializer): 76 | """ 77 | Collection model serializer 78 | """ 79 | class Meta: 80 | model = Collection 81 | fields = '__all__' 82 | lookup_field = 'collection_id' # lookup based on collection_id slug field 83 | 84 | 85 | class ActivitySerializer(serializers.ModelSerializer): 86 | """ 87 | Activity model serializer 88 | """ 89 | source_launch_url = serializers.CharField(source='url') 90 | tags = serializers.CharField(allow_null=True, allow_blank=True, default='') 91 | 92 | def validate_tags(self, value): 93 | """ 94 | Convert null value into empty string 95 | """ 96 | if value is None: 97 | return '' 98 | else: 99 | return value 100 | 101 | class Meta: 102 | model = Activity 103 | fields = ('id', 'collections', 'source_launch_url', 'name', 'difficulty', 'tags', 'knowledge_components', 104 | 'prerequisite_activities') 105 | 106 | 107 | class MasterySerializer(serializers.ModelSerializer): 108 | """ 109 | Mastery model serializer 110 | """ 111 | learner = LearnerSerializer() 112 | knowledge_component = KnowledgeComponentFieldSerializer() 113 | 114 | class Meta: 115 | model = Mastery 116 | fields = ('learner', 'knowledge_component', 'value') 117 | 118 | def create(self, validated_data): 119 | """ 120 | Defines write behavior for nested serializers 121 | Supports auto creation of related learner if they do not exist yet 122 | Does not support auto creation of new knowledge components if they do not exist yet 123 | TODO does it make sense to move related object auto creation to view perform_create() instead? 124 | :param validated_data: validated incoming data (serializer.validated_data) 125 | :return: 126 | """ 127 | # create referenced learner if it doesn't exist already 128 | learner_data = validated_data.pop('learner') 129 | learner, created = Learner.objects.get_or_create(**learner_data) 130 | # get referenced knowledge component 131 | knowledge_component_data = validated_data.pop('knowledge_component') 132 | knowledge_component = KnowledgeComponent.objects.get(**knowledge_component_data) 133 | # create mastery, but act as an update if mastery object for learner/kc already exists 134 | mastery, created = Mastery.objects.get_or_create( 135 | learner=learner, 136 | knowledge_component=knowledge_component, 137 | defaults=validated_data 138 | ) 139 | # update the value field if mastery object for learner/kc already exists 140 | if not created: 141 | mastery.value = validated_data['value'] 142 | mastery.save(update_fields=['value']) 143 | return mastery 144 | 145 | 146 | class ScoreSerializer(serializers.ModelSerializer): 147 | """ 148 | Score model serializer 149 | """ 150 | learner = LearnerSerializer() 151 | activity = serializers.SlugRelatedField( 152 | slug_field='url', 153 | queryset=Activity.objects.all() 154 | ) 155 | 156 | class Meta: 157 | model = Score 158 | fields = ('learner', 'activity', 'score') 159 | 160 | def create(self, validated_data): 161 | """ 162 | Defines write behavior for nested serializer 163 | Supports auto creation of related learner if doesn't exist yet 164 | 165 | :param validated_data: validated incoming data (serializer.validated_data) 166 | :return: Score model instance 167 | """ 168 | # create related learner if it doesn't exist already 169 | learner_data = validated_data.pop('learner') 170 | learner, created = Learner.objects.get_or_create(**learner_data) 171 | # get related activity 172 | activity = validated_data.pop('activity') 173 | # create the score object 174 | score = Score.objects.create( 175 | learner=learner, 176 | activity=activity, 177 | score=validated_data.pop('score') 178 | ) 179 | return score 180 | 181 | 182 | class KnowledgeComponentSerializer(serializers.ModelSerializer): 183 | """ 184 | KnowledgeComponent model serializer 185 | """ 186 | class Meta: 187 | model = KnowledgeComponent 188 | fields = ('id', 'kc_id', 'name', 'mastery_prior') 189 | lookup_field = 'kc_id' # lookup based on kc_id slug field 190 | 191 | 192 | class ActivityRecommendationSerializer(serializers.ModelSerializer): 193 | """ 194 | Serializer for recommendation response data 195 | """ 196 | source_launch_url = serializers.CharField(source='url') 197 | 198 | class Meta: 199 | model = Activity 200 | fields = ('source_launch_url',) 201 | 202 | 203 | class SequenceActivitySerializer(serializers.Serializer): 204 | """ 205 | Serializer for activity in a sequence list 206 | (used for parsing sequence list in recommendation request) 207 | """ 208 | activity = serializers.CharField(source='url') 209 | score = serializers.FloatField(allow_null=True) 210 | is_problem = serializers.BooleanField(required=False) 211 | 212 | class Meta: 213 | model = Activity 214 | fields = ('activity', 'score', 'is_problem') 215 | 216 | 217 | class ActivityRecommendationRequestSerializer(serializers.Serializer): 218 | """ 219 | Serializer for incoming activity recommendation request data 220 | """ 221 | learner = LearnerSerializer() 222 | collection = serializers.SlugRelatedField( 223 | slug_field='collection_id', 224 | queryset=Collection.objects.all() 225 | ) 226 | sequence = SequenceActivitySerializer(many=True) 227 | 228 | 229 | class CollectionActivityListSerializer(serializers.ListSerializer): 230 | 231 | def update(self, instance, validated_data): 232 | """ 233 | Assumes collection instance or id is passed into serializer context at initialization, 234 | and is available at self.instance.context 235 | Adds activities to the collection if they are not already in collection, 236 | and create new activities or updates fields of existing activities if needed. 237 | "instance" argument is the queryset of activities currently in the collection 238 | """ 239 | # Maps for id->instance and id->data item. 240 | activity_mapping = {activity.url: activity for activity in instance} 241 | data_mapping = {item['url']: item for item in validated_data} 242 | 243 | # Perform creations, updates and additions to collection 244 | results = [] 245 | for activity_url, data in data_mapping.items(): 246 | # check if activity with url id exists anywhere 247 | activity, created = Activity.objects.update_or_create(data, url=activity_url) 248 | # make sure it is added to collection if within collection context 249 | activity.collections.add(self.context['collection']) 250 | results.append(activity) 251 | 252 | # Perform removals from collection. 253 | for activity_url, activity in activity_mapping.items(): 254 | if activity_url not in data_mapping: 255 | activity.collections.remove(self.context['collection']) 256 | 257 | return results 258 | 259 | 260 | class CollectionActivitySerializer(serializers.ModelSerializer): 261 | """ 262 | Represents activity in the context of a collection 263 | Separate serializers so that addition/deletion to collection doesn't affect 264 | membership of activity in other collections 265 | TODO probably override init to get collection id in 266 | """ 267 | source_launch_url = serializers.CharField(source='url') 268 | tags = serializers.CharField(allow_null=True, allow_blank=True, default='') 269 | 270 | def validate_tags(self, value): 271 | """ 272 | Convert null value into empty string 273 | """ 274 | if value is None: 275 | return '' 276 | else: 277 | return value 278 | 279 | class Meta: 280 | model = Activity 281 | fields = ('source_launch_url', 'name', 'difficulty', 'tags') 282 | list_serializer_class = CollectionActivityListSerializer 283 | 284 | 285 | class PrerequisiteActivitySerializer(serializers.ModelSerializer): 286 | """ 287 | Model serializer for Activity.prerequisite_activities.through 288 | """ 289 | class Meta: 290 | model = Activity.prerequisite_activities.through 291 | # from_activity: dependent activity 292 | # to_activity: prerequisite activity 293 | fields = ('id','from_activity','to_activity') 294 | 295 | 296 | class PrerequisiteRelationSerializer(serializers.ModelSerializer): 297 | """ 298 | Model serializer for PrerequisiteRelation 299 | """ 300 | class Meta: 301 | model = PrerequisiteRelation 302 | fields = ('prerequisite','knowledge_component','value') 303 | 304 | 305 | class CollectionActivityMemberSerializer(serializers.ModelSerializer): 306 | """ 307 | Serializer for Collection-Activity membership relation 308 | """ 309 | class Meta: 310 | model = Activity.collections.through 311 | fields = ['id', 'activity', 'collection'] 312 | --------------------------------------------------------------------------------