├── MANIFEST.in ├── pymushra ├── __init__.py ├── templates │ ├── error │ │ ├── error.html │ │ └── 404.html │ ├── admin │ │ ├── table.html │ │ ├── info.html │ │ └── list.html │ ├── stats │ │ └── mushra.html │ └── base.html ├── wsgi.py ├── utils.py ├── cli.py ├── stats.py ├── casting.py └── service.py ├── tests ├── send_data.sh └── test_casting.py ├── pyproject.toml ├── LICENSE ├── README.md ├── .gitignore └── DEPLOYMENT.md /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft pymushra/templates 2 | global-exclude *.pyc 3 | -------------------------------------------------------------------------------- /pymushra/__init__.py: -------------------------------------------------------------------------------- 1 | from .casting import collection_to_df 2 | 3 | __all__ = ["collection_to_df"] 4 | -------------------------------------------------------------------------------- /pymushra/templates/error/error.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |

{{ type }} Error

5 |

6 | {{ message }} 7 |

8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /pymushra/templates/admin/table.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |
5 | {{ table|safe }} 6 |
7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /pymushra/templates/error/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |

404 Error

5 |

6 | The page you were looking for could not be found. 7 |

8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /pymushra/wsgi.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | from tinydb import TinyDB 3 | from pymushra.service import app as application 4 | 5 | application.config['admin_allowlist'] = ["127.0.0.1"] 6 | application.config['webmushra_dir'] = os.path.join(os.getcwd(), "webmushra") 7 | 8 | application.config['db'] = TinyDB( 9 | os.path.join(os.getcwd(), "db/webmushra.json") 10 | ) 11 | -------------------------------------------------------------------------------- /tests/send_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | curl -X POST -d 'sessionJSON={"testId":"test","participant":{"name":["email","age","gender"],"response":["asd","30","female"]},"trials":[{"id":"lss1","type":"likert_single_stimulus","responses":[{"stimulus":"C1","stimulusRating":"not at all","time":1771},{"stimulus":"C2","stimulusRating":"not a lot","time":803},{"stimulus":"C3","stimulusRating":"not a lot","time":712}]}],"config":"configs/default.yaml"}' http://localhost:5000/service/write.php 4 | -------------------------------------------------------------------------------- /pymushra/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fnmatch 3 | 4 | 5 | def get_configs(path): 6 | """ Retrieves all config files from webMUSHRA config directory 7 | 8 | """ 9 | return [ 10 | file for file in os.listdir(path) if fnmatch.fnmatch(file, "*.yaml") 11 | ] 12 | 13 | 14 | def flatten_columns(columns): 15 | """ Transforms hierarchical column names to concatenated single-level 16 | columns 17 | 18 | """ 19 | return ['_'.join(col).strip() for col in columns] 20 | -------------------------------------------------------------------------------- /tests/test_casting.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, absolute_import, print_function 2 | 3 | from pymushra import casting 4 | 5 | 6 | def test_casting(): 7 | val = {"test": "Test1", "participants": [{"age": "30", "name": "Nils"}, 8 | {"age": "30", "name": "Nils", "as": None, "yo": ""}], "yoo": "True", "foo": 100} 9 | expected = {"test": "Test1", "participants": [{"age": 30, "name": "Nils"}, 10 | {"age": 30, "name": "Nils", "as": None, "yo": ""}], "yoo": True, "foo": 100} 11 | 12 | assert expected == casting.cast_recursively(val) 13 | -------------------------------------------------------------------------------- /pymushra/templates/admin/info.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |

TestID: {{ testId }}

5 | 6 | 7 | {% for config in configs %} 8 | 9 | 12 | 15 | 16 | {% endfor %} 17 |
10 | {{ config }} 11 | 13 | Run Experiment 14 |
18 | 19 | {% endblock %} 20 | -------------------------------------------------------------------------------- /pymushra/templates/stats/mushra.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |

{{ url }}

5 | 6 |

7 | Numer of participants: {{ n_post }} ({{ n_pre - n_post }} outliers removed) 8 |

9 | 10 |

Preconditions

11 |

12 |

Test of normality (KS):

13 | {% for stimulus, p in ks.items() %} 14 | Condition {{ stimulus }}: p = {{ p }} {{ significance_stars(p) | safe }}
15 | {% endfor %} 16 |

Test of equal variances:

17 | Levene test: p = {{ lev_p }} {{ significance_stars(lev_p) | safe }} 18 |

ANOVA

19 |

20 | 

{{ aov }}
21 |

22 |

Pairwise Comparison Test

23 |

24 | 

{{ pair }}
25 |

26 | {% endblock %} 27 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "pymushra" 3 | version = "0.4" 4 | description = "webMUSHRA server in Python" 5 | authors = [ 6 | {name = "Nils Werner", email = "nils.werner@audiolabs-erlangen.de"}, 7 | {name = "Fabian-Robert Stöter", email = "fabian-robert.stoeter@audiolabs-erlangen.de"}, 8 | ] 9 | license = "MIT" 10 | license-files = [ "LICENSE" ] 11 | readme = "README.md" 12 | requires-python = ">=3.7" 13 | dependencies = [ 14 | "click", 15 | "flask>=2.2.5", 16 | "ipython", 17 | "matplotlib>=2.0.0", 18 | "numpy", 19 | "pandas", 20 | "patsy", 21 | "scipy", 22 | "seaborn", 23 | "statsmodels", 24 | "tinydb>=3.0.0", 25 | "tinyrecord", 26 | ] 27 | 28 | [project.scripts] 29 | pymushra = "pymushra.cli:cli" 30 | 31 | [dependency-groups] 32 | dev = [ 33 | "pytest", 34 | ] 35 | 36 | [build-system] 37 | requires = ["hatchling"] 38 | build-backend = "hatchling.build" 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2021 Nils Werner 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pyMUSHRA 2 | ======== 3 | 4 | Description 5 | ----------- 6 | 7 | pyMUSHRA is a python web application which hosts [webMUSHRA](https://github.com/audiolabs/webMUSHRA) experiments 8 | and collects the data with python. 9 | 10 | Quick Start 11 | ----------- 12 | 13 | You can quickly spin up a pyMUSHRA server [using `uv`](https://docs.astral.sh/uv/guides/tools/) or [`pipx`](https://github.com/pypa/pipx): 14 | 15 | uvx pymushra server -w 16 | pipx run pymushra server -w 17 | 18 | or install in a project using 19 | 20 | uv add pymushra 21 | pip install pymushra 22 | 23 | Then open 24 | 25 | Debugging 26 | --------- 27 | 28 | You may use the additional tools 29 | 30 | uvx pymushra db 31 | 32 | to load and inspect the TinyDB connection and 33 | 34 | uvx pymushra df [collection] 35 | 36 | to inspect the Pandas DataFrame export the TinyDB collection. 37 | 38 | Server Installation 39 | ------------------- 40 | 41 | For a long-running pyMUSHRA installation, please do not use the builtin server but instead use a proper 42 | HTTP server, like Apache or Nginx. See [DEPLOYMENT.md](DEPLOYMENT.md) for installation instructions. 43 | -------------------------------------------------------------------------------- /pymushra/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | pyMUSHRA 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
16 |
17 |

pyMUSHRA

18 |
19 | 20 |
21 | {% block content %} 22 | {% endblock content %} 23 |
24 | {% block script %} 25 | {% endblock script %} 26 |
27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | #### TINYDB 107 | 108 | *.json 109 | 110 | webmushra/ 111 | db/ 112 | -------------------------------------------------------------------------------- /pymushra/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import click 3 | import IPython 4 | from tinydb import TinyDB 5 | from . import service, casting 6 | 7 | 8 | @click.group() 9 | @click.option('--webmushra-path', '-w', default="webmushra") 10 | @click.option('--db-path', '-d', default="db/webmushra.json") 11 | @click.pass_context 12 | def cli(ctx, webmushra_path, db_path): 13 | ctx.obj = { 14 | 'webmushra_path': webmushra_path, 15 | 'db_path': db_path 16 | } 17 | 18 | 19 | @cli.command() 20 | @click.option('--port', '-p', default=5000) 21 | @click.option('--admin-allow', '-a', default=["127.0.0.1"], multiple=True) 22 | @click.pass_context 23 | def server(ctx, port, admin_allow): 24 | service.app.config['webmushra_dir'] = os.path.join( 25 | os.getcwd(), ctx.obj['webmushra_path'] 26 | ) 27 | 28 | service.app.config['admin_allowlist'] = admin_allow 29 | 30 | with TinyDB(ctx.obj['db_path'], create_dirs=True) as service.app.config['db']: 31 | service.app.run(debug=True, host='0.0.0.0', port=port) 32 | 33 | 34 | @cli.command() 35 | @click.pass_context 36 | def db(ctx): 37 | with TinyDB(ctx.obj['db_path'], create_dirs=True) as db: 38 | collections = db.tables() # noqa: F841 39 | 40 | click.echo(""" 41 | Available variables: 42 | 43 | db : TinyDB database 44 | The database you selected using the --db-path switch 45 | collections : list 46 | The list of all available collections 47 | """) 48 | 49 | IPython.embed() 50 | 51 | 52 | @cli.command() 53 | @click.argument('collection_name') 54 | @click.pass_context 55 | def df(ctx, collection_name): 56 | with TinyDB(ctx.obj['db_path'], create_dirs=True) as db: 57 | collection = db.table(collection_name) 58 | df = casting.collection_to_df(collection) # noqa: F841 59 | 60 | click.echo(""" 61 | Available variables: 62 | 63 | db : TinyDB database 64 | The database you selected using the --db_name switch 65 | collection : TinyDB collection 66 | The collection you selected using the command line argument 67 | df : DataFrame 68 | The dataframe generated from the collection 69 | """) 70 | 71 | IPython.embed() 72 | -------------------------------------------------------------------------------- /DEPLOYMENT.md: -------------------------------------------------------------------------------- 1 | Deploying 2 | --------- 3 | 4 | ### User Setup 5 | 6 | Create a `webmushra` service user 7 | 8 | useradd --system --shell /usr/bin/nologin webmushra 9 | 10 | ### Apache 11 | 12 | Importing SciPy into a WSGI script has proven to be tricky, so you need to change the `WSGIScriptAlias` line a bit: 13 | 14 | WSGIDaemonProcess webmushra user=webmushra group=webmushra python-path=/path/to/venv/pymushra/pymushra:/path/to/venv/env/lib/python3.6/site-packages home=/path/to/venv 15 | 16 | Alias /webmushra/admin /path/to/venv/pymushra/pymushra/wsgi.py/admin 17 | Alias /webmushra/collect /path/to/venv/pymushra/pymushra/wsgi.py/collect 18 | Alias /webmushra/download /path/to/venv/pymushra/pymushra/wsgi.py/download 19 | Alias /webmushra /path/to/venv/webmushra 20 | 21 | 22 | WSGIProcessGroup webmushra 23 | WSGIApplicationGroup %{GLOBAL} 24 | Require all granted 25 | WSGIScriptReloading On 26 | 27 | Options ExecCGI 28 | AddHandler wsgi-script .py 29 | 30 | 31 | 32 | Require all granted 33 | 34 | 35 | #### Debugging 36 | 37 | See 38 | 39 | tail -f /var/log/apache2/errors.log 40 | 41 | ### Nginx 42 | 43 | Install uWSGI 44 | 45 | pip install uwsgi 46 | 47 | Create a hosts file `/etc/nginx/sites-enabled/webmushra` 48 | 49 | server { 50 | listen 80; 51 | server_name ext1.myserver.de; 52 | 53 | location = /webmushra { 54 | rewrite ^(.*[^/])$ $1/ permanent; 55 | } 56 | 57 | location /webmushra { 58 | include uwsgi_params; 59 | uwsgi_pass unix:/tmp/webmushra.sock; 60 | } 61 | 62 | location /webmushra/admin { 63 | allow 131.188.0.0/16; 64 | deny all; 65 | } 66 | 67 | } 68 | 69 | link it to `/etc/nginx/sites-available` and restart Nginx `systemctl restart nginx`. 70 | 71 | Create a Systemd service file `/etc/systemd/system/webmushra.service` 72 | 73 | [Unit] 74 | Description=uWSGI instance to serve pyMUSHRA 75 | 76 | [Service] 77 | ExecStart=/path/to/venv/bin/uwsgi --ini /path/to/venv/uwsgi.ini' 78 | User=webmushra 79 | Group=webmushra 80 | Restart=on-failure 81 | KillSignal=SIGQUIT 82 | Type=notify 83 | StandardError=syslog 84 | NotifyAccess=all 85 | 86 | [Install] 87 | WantedBy=multi-user.target 88 | 89 | Then create a `uwsgi.ini` file 90 | 91 | [uwsgi] 92 | mount = /webmushra=wsgi:application 93 | logto = log/%n.log 94 | virtualenv = /path/to/venv 95 | 96 | manage-script-name = true 97 | 98 | socket = /tmp/webmushra.sock 99 | chmod-socket = 666 100 | 101 | touch-reload = wsgi.py 102 | 103 | and start the service using `systemctl start webmushra.service`. 104 | 105 | #### Debugging 106 | 107 | See 108 | 109 | tail -f /path/to/venv/log/webmushra.log 110 | -------------------------------------------------------------------------------- /pymushra/templates/admin/list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 |

Results

5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {% for collection in collections %} 14 | 15 | 18 | 21 | 24 | 37 | 38 | {% endfor %} 39 | {% if not collections %} 40 | 41 | 44 | 45 | {% endif %} 46 |
Test IDNLast submissionOptions
16 | {{ collection.id }} 17 | 19 | {{ collection.participants }} 20 | 22 | {{ collection.last_submission|datetime }} 23 | 25 | Info 26 | Statistics 27 |
28 | 29 | 34 |
35 | Preview 36 |
42 | No results found 43 |
47 | 48 | 49 | 50 |
51 |

Experiments

52 | 53 | 54 | {% for config in configs %} 55 | 56 | 59 | 62 | 63 | {% endfor %} 64 | {% if not configs %} 65 | 66 | 69 | 70 | {% endif %} 71 |
57 | {{ config }} 58 | 60 | Run 61 |
67 | No experiments found 68 |
72 |
73 | 74 | {% endblock %} 75 | -------------------------------------------------------------------------------- /pymushra/stats.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import seaborn as sns 3 | from io import BytesIO 4 | import statsmodels.stats.diagnostic 5 | import statsmodels.stats.anova 6 | import statsmodels.stats.multicomp 7 | import matplotlib as mpl 8 | import base64 9 | from flask import render_template 10 | from statsmodels.formula.api import ols 11 | from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas 12 | from matplotlib.figure import Figure 13 | 14 | try: 15 | from urllib import quote 16 | except ImportError: 17 | from urllib.parse import quote 18 | 19 | 20 | def significance_class(p, alpha=0.05): 21 | if p < 0.0001: 22 | return 4 23 | elif (p < 0.001): 24 | return 3 25 | elif (p < 0.01): 26 | return 2 27 | elif (p < 0.05): 28 | return 1 29 | else: 30 | return 0 31 | 32 | 33 | def significance_stars(p, alpha=0.05): 34 | return ''.join(['*'] * significance_class(p, alpha)) 35 | 36 | 37 | def detect_outliers(df, method=None): 38 | if method is None: 39 | # add simple median based outlier detection here 40 | pass 41 | elif method == "mushra": 42 | 43 | # get 'bad' responses according to mushra rec 44 | bad_responses = df[ 45 | (df.responses_stimulus == 'reference') & 46 | (df.responses_score < 90) 47 | ] 48 | 49 | # select allowed number of bad trials 50 | allwd_bad_trials = len(df.wm_id.unique()) * 0.15 51 | 52 | # build condition 53 | condi = bad_responses.groupby(['questionaire_uuid']).count().wm_id > \ 54 | allwd_bad_trials 55 | 56 | # return cleaned data 57 | return df[~df[('questionaire_uuid')].isin(condi.index)] 58 | else: 59 | return df 60 | 61 | 62 | def render_mushra(testid, df): 63 | if len(df) == 0: 64 | raise ValueError("Dataset was empty") 65 | 66 | df = df[df['wm_type'] == 'mushra'] 67 | 68 | # counting participants 69 | n_pre = df.drop_duplicates(('questionaire_uuid')).count()[ 70 | ('questionaire_uuid') 71 | ] 72 | 73 | df = detect_outliers(df, method="mushra") 74 | 75 | # counting participants 76 | n_post = df.drop_duplicates(('questionaire_uuid')).count()[ 77 | ('questionaire_uuid') 78 | ] 79 | 80 | # Filter Dataframe to only get the systems under test 81 | disallowed_conditions = ['reference', 'anchor35', 'anchor70'] 82 | # all conditions NOT in allowed conditions 83 | df_dut = df[~df[('responses_stimulus')].isin(disallowed_conditions)] 84 | 85 | # Kolmogorov Smirnov Test for normality 86 | ks_ps = {} 87 | for stimulus in df_dut[('responses_stimulus')].unique(): 88 | ks, ks_p = statsmodels.stats.diagnostic.kstest_normal( 89 | df_dut[df_dut[('responses_stimulus')] == stimulus][ 90 | ('responses_score') 91 | ] 92 | ) 93 | ks_ps[stimulus] = ks_p 94 | 95 | # Levene test for equal variances. 96 | groups = [] 97 | for stimulus in df_dut[('responses_stimulus')].unique(): 98 | groups.append( 99 | df_dut[df_dut[('responses_stimulus')] == stimulus][ 100 | ('responses_score') 101 | ].to_numpy() 102 | ) 103 | W, lev_p = scipy.stats.levene( 104 | *tuple(groups) 105 | ) 106 | 107 | # ANOVA 108 | lm = ols('responses_score ~ responses_stimulus', df_dut).fit() 109 | aov_table = statsmodels.stats.anova.anova_lm(lm) 110 | 111 | pc_table = statsmodels.stats.multicomp.pairwise_tukeyhsd( 112 | df_dut[('responses_score')], df_dut[('responses_stimulus')]) 113 | 114 | # render boxplot to base64 115 | boxplot_uri = render_boxplot(testid=testid, df=df) 116 | return render_template( 117 | 'stats/mushra.html', 118 | url=testid, 119 | n_pre=n_pre, 120 | n_post=n_post, 121 | alpha=0.05, 122 | ks=ks_ps, 123 | lev_p=lev_p, 124 | aov=aov_table, 125 | pair=pc_table, 126 | boxplot_uri=boxplot_uri 127 | ) 128 | 129 | 130 | def render_boxplot(testid, df): 131 | fig = Figure(facecolor=(0, 0, 0, 0)) 132 | ax = fig.add_subplot(111) 133 | 134 | sns.set_style("whitegrid") 135 | sns.set_style("darkgrid", { 136 | 'axes.facecolor': (1, 1, 1, 0), 137 | 'figure.facecolor': (1, 1, 1, 0), 138 | 'axes.edgecolor': '.8', 139 | 'grid.color': '.8', 140 | 'ytick.minor.size': 0, 141 | 'ytick.color': '0', 142 | 143 | }) 144 | 145 | sns.boxplot( 146 | df[('responses_stimulus')], 147 | df[('responses_score')], 148 | ax=ax, 149 | ) 150 | 151 | ax.get_yaxis().set_minor_locator(mpl.ticker.AutoMinorLocator()) 152 | ax.grid(b=True, which='minor', color='0.8', linewidth=0.5) 153 | ax.set_ylim([0, 101]) 154 | 155 | sns.despine(left=True, ax=ax, trim=False) 156 | 157 | canvas = FigureCanvas(fig) 158 | 159 | png_output = BytesIO() 160 | canvas.print_png(png_output) 161 | 162 | png_output.seek(0) # rewind the data 163 | 164 | uri = 'data:image/png;base64,' + quote( 165 | base64.b64encode(png_output.getbuffer()) 166 | ) 167 | return uri 168 | -------------------------------------------------------------------------------- /pymushra/casting.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import uuid 3 | import itertools 4 | import datetime 5 | from . import utils 6 | 7 | 8 | def escape_objects(df, columns=None): 9 | df = df.copy() 10 | 11 | if columns is None: 12 | columns = [ 13 | ('questionaire', 'uuid'), 14 | ('wm', 'id',), 15 | ] 16 | 17 | # Add flattened columns too, so we catch JSON and CSV column names 18 | columns = columns + utils.flatten_columns(columns) 19 | 20 | for col in columns: 21 | try: 22 | df[col] = df[col].astype(str) 23 | except KeyError: 24 | pass 25 | 26 | return df 27 | 28 | 29 | def collection_to_df(collection): 30 | """ Transform TinyDB collection to DataFrame 31 | 32 | Parameters 33 | ---------- 34 | collection : TinyDB collection 35 | The collection to transform. The entire collection is taken. 36 | 37 | Returns 38 | ------- 39 | d : DataFrame 40 | The DataFrame 41 | 42 | Notes 43 | ----- 44 | 45 | Turns dataset inside out: 46 | 47 | .. code-block:: yaml 48 | 49 | Trial: Something 50 | Questionaire: 51 | Name: Nils 52 | Responses: # Notice the list here 53 | - Stimulus: C3 54 | Score: 100 55 | - Stimulus: C1 56 | Score: 80 57 | 58 | must become 59 | 60 | .. code-block:: yaml 61 | 62 | - Trial: Something # Notice the list here 63 | Questionaire: 64 | Name: Nils 65 | Responses: 66 | Stimulus: C3 67 | Score: 100 68 | - Trial: Something 69 | Questionaire: 70 | Name: Nils 71 | Responses: 72 | Stimulus: C1 73 | Score: 80 74 | 75 | For each row in responses we need an aditional row in our DataFrame 76 | 77 | """ 78 | rawdata = list(collection.all()) 79 | 80 | if not rawdata: 81 | return pd.DataFrame() 82 | 83 | dataset = [] 84 | 85 | for trial in rawdata: 86 | for response in trial['responses']: 87 | outitem = {} 88 | 89 | for key, item in response.items(): 90 | outitem[('responses', key)] = item 91 | 92 | for key, item in trial['questionaire'].items(): 93 | outitem[('questionaire', key)] = item 94 | 95 | for key, item in trial.items(): 96 | if key not in ('responses', 'questionaire'): 97 | outitem[('wm', key)] = item 98 | 99 | outitem[('wm', 'id')] = str(outitem[('wm', 'id')]) 100 | dataset.append(outitem) 101 | 102 | columns = list(set(itertools.chain(*map(lambda x: x.keys(), dataset)))) 103 | 104 | df = pd.DataFrame( 105 | dataset, 106 | columns=pd.MultiIndex.from_tuples(columns) 107 | ) 108 | 109 | df[('wm', 'date')] = pd.to_datetime(df[('wm', 'date')]) 110 | 111 | return df 112 | 113 | 114 | def json_to_dict(payload): 115 | """ Transform webMUSHRA JSON dict to sane structure 116 | 117 | Parameters 118 | ---------- 119 | payload : dict_like 120 | The container to be transformed 121 | 122 | Returns 123 | ------- 124 | d : dict_like 125 | The transformed container 126 | 127 | Notes 128 | ----- 129 | 130 | Actions taken: 131 | 132 | 1. One dataset per trial is generated 133 | 2. Config from global payload is inserted into all datasets 134 | 3. TestId from global payload is inserted into all datasets 135 | 4. date is added to all datasets 136 | 5. Questionaire structure 137 | 138 | .. code-block:: python 139 | 140 | {'name': ['firstname', 'age'], 'response': ['Nils', 29]} 141 | 142 | becomes 143 | 144 | .. code-block:: python 145 | 146 | {'firstname': 'Nils', 'age': 29} 147 | 148 | 6. UUID4 field is added to questionaire 149 | 150 | """ 151 | questionaire = payload['participant'] 152 | questionaire = dict( 153 | zip(questionaire['name'], questionaire['response']) 154 | ) 155 | questionaire['uuid'] = str(uuid.uuid4()) 156 | insert = [] 157 | 158 | for trial in payload['trials']: 159 | data = trial 160 | 161 | data['config'] = payload['config'] 162 | data['testId'] = payload['testId'] 163 | data['date'] = str(datetime.datetime.now()) 164 | data['questionaire'] = questionaire 165 | 166 | insert.append(data) 167 | 168 | return insert 169 | 170 | 171 | def bool_or_fail(v): 172 | """ A special variant of :code:`bool` that raises :code:`ValueError`s 173 | if the provided value was not :code:`True` or :code:`False`. 174 | 175 | This prevents overeager casting like :code:`bool("bla") -> True` 176 | 177 | Parameters 178 | ---------- 179 | v : mixed 180 | Value to be cast 181 | 182 | Returns 183 | ------- 184 | b : boolean 185 | The result of the cast 186 | 187 | """ 188 | try: 189 | if v.lower() == 'true': 190 | return True 191 | elif v.lower() == 'false': 192 | return True 193 | except Exception: 194 | pass 195 | raise ValueError() 196 | 197 | 198 | def cast_recursively(d, castto=None): 199 | """ Traverse list or dict recursively, trying to cast their items. 200 | 201 | Parameters 202 | ---------- 203 | d : iterable or dict_like 204 | The container to be casted 205 | castto : list of callables 206 | The types to cast to. Defaults to :code:`bool_or_fail, int, float` 207 | 208 | Returns 209 | ------- 210 | d : iterable or dict_like 211 | The transformed container 212 | 213 | """ 214 | if castto is None: 215 | castto = (bool_or_fail, int, float) 216 | 217 | if isinstance(d, dict): 218 | return { 219 | k: cast_recursively(v, castto=castto) 220 | for k, v in d.items() 221 | } 222 | elif isinstance(d, list): 223 | return [cast_recursively(v, castto=castto) for v in d] 224 | else: 225 | for tp in castto: 226 | try: 227 | return tp(d) 228 | except (ValueError, TypeError): 229 | pass 230 | return d 231 | -------------------------------------------------------------------------------- /pymushra/service.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, absolute_import, print_function 2 | 3 | import os 4 | import json 5 | import pickle 6 | from flask import Flask, request, send_from_directory, send_file, \ 7 | render_template, redirect, url_for, abort 8 | from tinyrecord import transaction 9 | from functools import wraps 10 | 11 | from . import stats, casting, utils 12 | 13 | from io import BytesIO 14 | try: 15 | from io import StringIO 16 | except ImportError: 17 | from StringIO import StringIO 18 | 19 | app = Flask(__name__) 20 | 21 | def only_admin_allowlist(f): 22 | @wraps(f) 23 | def wrapped(*args, **kwargs): 24 | if request.remote_addr in app.config['admin_allowlist']: 25 | return f(*args, **kwargs) 26 | else: 27 | return abort(403) 28 | return wrapped 29 | 30 | 31 | @app.route('/') 32 | @app.route('/') 33 | def home(url='index.html'): 34 | return send_from_directory(app.config['webmushra_dir'], url) 35 | 36 | 37 | @app.route('/service/write.php', methods=['POST']) 38 | @app.route('//collect', methods=['POST']) 39 | @app.route('/collect', methods=['POST']) 40 | def collect(testid=''): 41 | if request.headers['Content-Type'].startswith( 42 | 'application/x-www-form-urlencoded' 43 | ): 44 | try: 45 | db = app.config['db'] 46 | payload = json.loads(request.form['sessionJSON']) 47 | payload = casting.cast_recursively(payload) 48 | insert = casting.json_to_dict(payload) 49 | 50 | collection = db.table(payload['trials'][0]['testId']) 51 | with transaction(collection): 52 | inserted_ids = collection.insert_multiple(insert) 53 | print(inserted_ids) 54 | 55 | return { 56 | 'error': False, 57 | 'message': "Saved as ids %s" % ','.join(map(str, inserted_ids)) 58 | } 59 | except Exception as e: 60 | return { 61 | 'error': True, 62 | 'message': "An error occurred: %s" % str(e) 63 | } 64 | else: 65 | return "415 Unsupported Media Type", 415 66 | 67 | 68 | @app.route('/admin/') 69 | @app.route('/admin/list') 70 | @only_admin_allowlist 71 | def admin_list(): 72 | db = app.config['db'] 73 | collection_names = db.tables() 74 | 75 | collection_dfs = [ 76 | casting.collection_to_df(db.table(name)) for name in collection_names 77 | ] 78 | 79 | print(collection_dfs) 80 | 81 | collections = [ 82 | { 83 | 'id': name, 84 | 'participants': len(df['questionaire', 'uuid'].unique()), 85 | 'last_submission': df['wm', 'date'].max(), 86 | } for name, df in zip(collection_names, collection_dfs) 87 | if len(df) > 0 88 | ] 89 | 90 | configs = utils.get_configs( 91 | os.path.join(app.config['webmushra_dir'], "configs") 92 | ) 93 | 94 | return render_template( 95 | "admin/list.html", 96 | collections=collections, 97 | configs=configs 98 | ) 99 | 100 | 101 | @app.route('/admin/delete//') 102 | @only_admin_allowlist 103 | def admin_delete(testid): 104 | app.config['db'].drop_table(testid) 105 | return redirect(url_for('admin_list')) 106 | 107 | 108 | @app.route('/admin/info//') 109 | @only_admin_allowlist 110 | def admin_info(testid): 111 | collection = app.config['db'].table(testid) 112 | df = casting.collection_to_df(collection) 113 | try: 114 | configs = df['wm']['config'].unique().tolist() 115 | except KeyError: 116 | configs = [] 117 | 118 | configs = map(os.path.basename, configs) 119 | 120 | return render_template( 121 | "admin/info.html", 122 | testId=testid, 123 | configs=configs 124 | ) 125 | 126 | 127 | @app.route('/admin/latest//') 128 | @only_admin_allowlist 129 | def admin_latest(testid): 130 | collection = app.config['db'].table(testid) 131 | latest = sorted(collection.all(), key=lambda x: x['date'], reverse=True)[0] 132 | return latest 133 | 134 | 135 | @app.route('/admin/stats//') 136 | @only_admin_allowlist 137 | def admin_stats(testid, stats_type='mushra'): 138 | collection = app.config['db'].table(testid) 139 | df = casting.collection_to_df(collection) 140 | df.columns = utils.flatten_columns(df.columns) 141 | # analyse mushra experiment 142 | try: 143 | if stats_type == "mushra": 144 | return stats.render_mushra(testid, df) 145 | except ValueError as e: 146 | return render_template( 147 | 'error/error.html', type="Value", message=str(e) 148 | ) 149 | return render_template('error/404.html'), 404 150 | 151 | 152 | @app.route( 153 | '/admin/download/.', 154 | defaults={'show_as': 'download'}) 155 | @app.route( 156 | '/admin/download//.', 157 | defaults={'show_as': 'download'}) 158 | @app.route( 159 | '/download//.', 160 | defaults={'show_as': 'download'}) 161 | @app.route( 162 | '/download/.', 163 | defaults={'show_as': 'download'}) 164 | @app.route( 165 | '/admin/show/.', 166 | defaults={'show_as': 'text'}) 167 | @app.route( 168 | '/admin/show//.', 169 | defaults={'show_as': 'text'}) 170 | @only_admin_allowlist 171 | def download(testid, show_as, statstype=None, filetype='csv'): 172 | allowed_types = ('csv', 'pickle', 'json', 'html') 173 | 174 | if show_as == 'download': 175 | as_attachment = True 176 | else: 177 | as_attachment = False 178 | 179 | if filetype not in allowed_types: 180 | return render_template( 181 | 'error/error.html', 182 | type="Value", 183 | message="File type must be in %s" % ','.join(allowed_types) 184 | ) 185 | 186 | if filetype == "pickle" and not as_attachment: 187 | return render_template( 188 | 'error/error.html', 189 | type="Value", 190 | message="Pickle data cannot be viewed" 191 | ) 192 | 193 | collection = app.config['db'].table(testid) 194 | df = casting.collection_to_df(collection) 195 | 196 | if statstype is not None: 197 | # subset by statstype 198 | df = df[df[('wm', 'type')] == statstype] 199 | 200 | # Merge hierarchical columns 201 | if filetype not in ("pickle", "html"): 202 | df.columns = utils.flatten_columns(df.columns.values) 203 | 204 | if len(df) == 0: 205 | return render_template( 206 | 'error/error.html', 207 | type="Value", 208 | message="Data Frame was empty" 209 | ) 210 | 211 | if filetype == "csv": 212 | # We need to escape certain objects in the DF to prevent Segfaults 213 | mem = StringIO() 214 | casting.escape_objects(df).to_csv( 215 | mem, 216 | sep=";", 217 | index=False, 218 | encoding='utf-8' 219 | ) 220 | 221 | elif filetype == "html": 222 | mem = StringIO() 223 | df.sort_index(axis=1).to_html(mem, classes="table table-striped") 224 | 225 | elif filetype == "pickle": 226 | mem = BytesIO() 227 | pickle.dump(df, mem) 228 | 229 | elif filetype == "json": 230 | mem = StringIO() 231 | # We need to escape certain objects in the DF to prevent Segfaults 232 | casting.escape_objects(df).to_json(mem, orient='records') 233 | 234 | mem.seek(0) 235 | 236 | if (as_attachment or filetype != "html") and not isinstance(mem, BytesIO): 237 | mem2 = BytesIO() 238 | mem2.write(mem.getvalue().encode('utf-8')) 239 | mem2.seek(0) 240 | mem = mem2 241 | 242 | if as_attachment: 243 | return send_file( 244 | mem, 245 | download_name="%s.%s" % (testid, filetype), 246 | as_attachment=True, 247 | max_age=-1 248 | ) 249 | else: 250 | if filetype == "html": 251 | return render_template('admin/table.html', table=mem.getvalue()) 252 | else: 253 | return send_file( 254 | mem, 255 | mimetype="text/plain", 256 | cache_timeout=-1 257 | ) 258 | 259 | 260 | @app.context_processor 261 | def utility_processor(): 262 | def significance_stars(p, alpha=0.05): 263 | return ''.join( 264 | [ 265 | '' 267 | ] * stats.significance_class(p, alpha) 268 | ) 269 | 270 | return dict(significance_stars=significance_stars) 271 | 272 | 273 | @app.template_filter('datetime') 274 | def datetime_filter(value, format='%x %X'): 275 | return value.strftime(format) 276 | --------------------------------------------------------------------------------