├── .gitattributes ├── CALEngine ├── Dockerfile ├── Makefile ├── README.md ├── api.py ├── runAll.sh ├── src │ ├── bmi.cc │ ├── bmi.h │ ├── bmi_cli.cc │ ├── bmi_doc_scal.cc │ ├── bmi_doc_scal.h │ ├── bmi_fcgi.cc │ ├── bmi_forget.h │ ├── bmi_online_learning.cc │ ├── bmi_online_learning.h │ ├── bmi_para.cc │ ├── bmi_para.h │ ├── bmi_para_scal.cc │ ├── bmi_para_scal.h │ ├── bmi_precision_delay.cc │ ├── bmi_precision_delay.h │ ├── bmi_recency_weighting.h │ ├── bmi_reduced_ranking.cc │ ├── bmi_reduced_ranking.h │ ├── bmi_reduced_ranking_dynamic.h │ ├── classifier.cc │ ├── classifier.h │ ├── corpus_parser.cc │ ├── dataset.cc │ ├── dataset.h │ ├── features.cc │ ├── features.h │ ├── sofiaml │ │ ├── sf-sparse-vector.cc │ │ └── sf-sparse-vector.h │ └── utils │ │ ├── feature_parser.cc │ │ ├── feature_parser.h │ │ ├── feature_writer.cc │ │ ├── feature_writer.h │ │ ├── features.h │ │ ├── porter.c │ │ ├── simple-cmd-line-helper.h │ │ ├── text_utils.cc │ │ ├── text_utils.h │ │ ├── utils.cc │ │ └── utils.h └── tests │ ├── test_feature_parsers.cc │ └── test_scorer.cc ├── HiCAL.yml ├── HiCALWeb ├── .coveragerc ├── .editorconfig ├── .gitattributes ├── .gitignore ├── .pylintrc ├── .travis.yml ├── LICENSE ├── README.rst ├── __init__.py ├── compose │ ├── local │ │ └── django │ │ │ ├── Dockerfile │ │ │ ├── celery │ │ │ ├── beat │ │ │ │ └── start.sh │ │ │ └── worker │ │ │ │ └── start.sh │ │ │ └── start.sh │ └── production │ │ ├── caddy │ │ ├── Caddyfile │ │ └── Dockerfile │ │ ├── django │ │ ├── Dockerfile │ │ ├── celery │ │ │ ├── beat │ │ │ │ └── start.sh │ │ │ └── worker │ │ │ │ └── start.sh │ │ ├── entrypoint.sh │ │ └── gunicorn.sh │ │ └── postgres │ │ ├── Dockerfile │ │ ├── backup.sh │ │ ├── list-backups.sh │ │ └── restore.sh ├── config │ ├── __init__.py │ ├── settings │ │ ├── README.md │ │ ├── __init__.py │ │ ├── base.py │ │ ├── local.py │ │ ├── production.py │ │ └── test.py │ ├── urls.py │ └── wsgi.py ├── env.example ├── hicalweb │ ├── CAL │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── exceptions.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ └── CAL │ │ │ │ └── CAL.html │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── __init__.py │ ├── allauth │ │ ├── __init__.py │ │ └── forms.py │ ├── contrib │ │ ├── __init__.py │ │ └── sites │ │ │ ├── __init__.py │ │ │ └── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ ├── interfaces │ │ ├── CAL │ │ │ ├── __init__.py │ │ │ └── functions.py │ │ ├── DocumentSnippetEngine │ │ │ ├── __init__.py │ │ │ └── functions.py │ │ ├── Iterative │ │ │ ├── __init__.py │ │ │ └── functions.py │ │ ├── SearchEngine │ │ │ ├── __init__.py │ │ │ └── functions.py │ │ └── __init__.py │ ├── iterative │ │ ├── __init__.py │ │ ├── apps.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── templates │ │ │ └── iterative │ │ │ │ └── iterative.html │ │ ├── urls.py │ │ └── views.py │ ├── judgment │ │ ├── README.md │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── forms.py │ │ ├── helpers.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ ├── judgments_to_csv.py │ │ │ │ ├── judgments_trec_to_csv.py │ │ │ │ └── restore_CAL_sessions.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_judgement_task.py │ │ │ ├── 0003_auto_20180726_2004.py │ │ │ ├── 0004_judgement_source.py │ │ │ ├── 0005_auto_20200531_1714.py │ │ │ ├── 0006_auto_20200531_2249.py │ │ │ ├── 0007_auto_20200531_2359.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ └── judgment │ │ │ │ └── judgments.html │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── middleware │ │ ├── __init__.py │ │ └── pagefile.py │ ├── progress │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── fixtures │ │ │ ├── __init__.py │ │ │ └── init_tasksetting.json │ │ ├── forms.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_task_username.py │ │ │ ├── 0003_auto_20180723_1041.py │ │ │ ├── 0004_task_max_number_of_judgments.py │ │ │ ├── 0005_task_strategy.py │ │ │ ├── 0006_auto_20180804_1908.py │ │ │ ├── 0007_auto_20200528_1346.py │ │ │ ├── 0008_auto_20200603_2255.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ └── progress │ │ │ │ ├── home.html │ │ │ │ └── sessions.html │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── search │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── helpers.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ └── search │ │ │ │ ├── search.html │ │ │ │ └── search_list.html │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── static │ │ ├── css │ │ │ ├── bootstrap-flex.css │ │ │ ├── font-awesome-all.css │ │ │ ├── font-awesome.min.css │ │ │ ├── hical-bootstrap.css │ │ │ ├── hical-bootstrap.css.map │ │ │ ├── hical-bootstrap.min.css │ │ │ ├── hical-bootstrap.min.css.map │ │ │ ├── nprogress.css │ │ │ ├── pace.css │ │ │ └── project.css │ │ ├── fonts │ │ │ └── .gitkeep │ │ ├── images │ │ │ ├── cal.png │ │ │ ├── cal_para.png │ │ │ ├── favicon.ico │ │ │ ├── hical.png │ │ │ ├── hicalsmall.png │ │ │ └── search.png │ │ ├── js │ │ │ ├── CAL.js │ │ │ ├── Queue.src.js │ │ │ ├── bootstrap.min.js │ │ │ ├── easytimer.js │ │ │ ├── idle.js │ │ │ ├── jquery.bootpag.min.js │ │ │ ├── jquery.mark.min.js │ │ │ ├── moment.min.js │ │ │ ├── mousetrap.min.js │ │ │ ├── pace.min.js │ │ │ ├── popper.min.js │ │ │ ├── project.js │ │ │ ├── search.js │ │ │ ├── timer.js │ │ │ └── uuid-lib.js │ │ ├── sass │ │ │ ├── hical-bootstrap.min.scss │ │ │ └── project.scss │ │ └── webfonts │ │ │ ├── fa-brands-400.eot │ │ │ ├── fa-brands-400.svg │ │ │ ├── fa-brands-400.ttf │ │ │ ├── fa-brands-400.woff │ │ │ ├── fa-brands-400.woff2 │ │ │ ├── fa-regular-400.eot │ │ │ ├── fa-regular-400.svg │ │ │ ├── fa-regular-400.ttf │ │ │ ├── fa-regular-400.woff │ │ │ ├── fa-regular-400.woff2 │ │ │ ├── fa-solid-900.eot │ │ │ ├── fa-solid-900.svg │ │ │ ├── fa-solid-900.ttf │ │ │ ├── fa-solid-900.woff │ │ │ └── fa-solid-900.woff2 │ ├── templates │ │ ├── 403_csrf.html │ │ ├── 404.html │ │ ├── 500.html │ │ ├── account │ │ │ ├── account_inactive.html │ │ │ ├── base.html │ │ │ ├── email.html │ │ │ ├── email_confirm.html │ │ │ ├── login.html │ │ │ ├── logout.html │ │ │ ├── password_change.html │ │ │ ├── password_reset.html │ │ │ ├── password_reset_done.html │ │ │ ├── password_reset_from_key.html │ │ │ ├── password_reset_from_key_done.html │ │ │ ├── password_set.html │ │ │ ├── signup.html │ │ │ ├── signup_closed.html │ │ │ ├── verification_sent.html │ │ │ └── verified_email_required.html │ │ ├── base.html │ │ ├── bootstrap4 │ │ │ ├── field.html │ │ │ └── layout │ │ │ │ └── field_errors_block.html │ │ ├── old_base.html │ │ ├── pages │ │ │ ├── about.html │ │ │ ├── home.html │ │ │ ├── search.html │ │ │ └── sessions.html │ │ ├── raven-config.html │ │ └── users │ │ │ ├── user_detail.html │ │ │ ├── user_form.html │ │ │ └── user_list.html │ ├── time │ │ └── Idle.js │ │ │ ├── core.html │ │ │ ├── idle.js │ │ │ └── timer.js │ ├── topic │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── fixtures │ │ │ ├── __init__.py │ │ │ ├── init_topics.json │ │ │ └── topics.xml │ │ ├── forms.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ └── import_topics.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templates │ │ │ └── topic │ │ │ │ ├── create.html │ │ │ │ ├── details.html │ │ │ │ └── list.html │ │ ├── tests.py │ │ ├── urls.py │ │ └── views.py │ ├── users │ │ ├── __init__.py │ │ ├── adapters.py │ │ ├── admin.py │ │ ├── apps.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── templatetags │ │ │ ├── __init__.py │ │ │ └── has_group.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── factories.py │ │ │ ├── test_admin.py │ │ │ ├── test_models.py │ │ │ ├── test_urls.py │ │ │ └── test_views.py │ │ ├── urls.py │ │ └── views.py │ └── util │ │ ├── __init__.py │ │ ├── helpers.py │ │ └── topics │ │ ├── init.json │ │ ├── shortQuery.tsv │ │ ├── topic_formattor.py │ │ └── topics ├── logs │ ├── README.md │ ├── django_treccoreweb_format.json │ └── user_logs.py ├── manage.py ├── pytest.ini ├── requirements │ ├── base.txt │ ├── local.txt │ ├── production.txt │ └── test.txt ├── setup.cfg └── utility │ ├── install_os_dependencies.sh │ ├── install_python_dependencies.sh │ ├── requirements-jessie.apt │ ├── requirements-trusty.apt │ └── requirements-xenial.apt ├── LICENSE ├── README.md ├── config └── nginx │ ├── fastcgi.conf │ ├── mime.types │ ├── nginx.conf │ └── uwsgi_params └── images └── hical.png /.gitattributes: -------------------------------------------------------------------------------- 1 | *.css linguist-detectable=false 2 | *.html linguist-detectable=false 3 | -------------------------------------------------------------------------------- /CALEngine/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | RUN apt-get update 4 | RUN apt-get install -y \ 5 | libfcgi-dev spawn-fcgi g++ libarchive-dev make git 6 | 7 | RUN mkdir -p /src/ 8 | -------------------------------------------------------------------------------- /CALEngine/Makefile: -------------------------------------------------------------------------------- 1 | CXX = g++ 2 | CXXFLAGS = -Wall -pthread --std=c++14 -lfcgi -lfcgi++ -larchive 3 | ifeq ($(DEBUG), 1) 4 | CXXFLAGS += -g3 -O0 -DDEBUG 5 | else 6 | CXXFLAGS += -O3 7 | endif 8 | 9 | OBJ_DIR ?= obj 10 | SRC_DIRS ?= src 11 | TEST_DIRS ?= tests 12 | 13 | # Modify BIN_SRCS to add targets 14 | BIN_SRCS := $(SRC_DIRS)/bmi_fcgi.cc $(SRC_DIRS)/bmi_cli.cc $(SRC_DIRS)/corpus_parser.cc 15 | BIN_OBJS := $(BIN_SRCS:%=$(OBJ_DIR)/%.o) 16 | BIN_TARGETS := $(notdir $(basename $(BIN_SRCS))) 17 | 18 | TEST_SRCS := $(shell find $(TEST_DIRS) -name *.cc) 19 | TEST_TARGETS := $(notdir $(basename $(TEST_SRCS))) 20 | 21 | SRCS := $(shell find $(SRC_DIRS) -name '*.cc') 22 | SRCS := $(filter-out $(BIN_SRCS),$(SRCS)) 23 | OBJS := $(SRCS:%=$(OBJ_DIR)/%.o) 24 | 25 | DEPS := $(OBJS:.o=.d) 26 | DEP_FLAGS = -MMD -MP 27 | 28 | 29 | all: $(BIN_TARGETS) 30 | 31 | $(BIN_TARGETS): % : $(OBJ_DIR)/$(SRC_DIRS)/%.cc.o $(OBJS) 32 | $(CXX) $(OBJS) $(OBJ_DIR)/$(SRC_DIRS)/$@.cc.o -o $@ $(CXXFLAGS) 33 | 34 | $(TEST_TARGETS): % : $(OBJ_DIR)/$(TEST_DIRS)/%.cc.o $(OBJS) 35 | $(CXX) $(CXXFLAGS) $(OBJS) $(OBJ_DIR)/$(TEST_DIRS)/$@.cc.o -o $(TEST_DIRS)/$@ 36 | cd $(TEST_DIRS) && (./$@; cd ..) 37 | 38 | $(OBJ_DIR)/%.cc.o: %.cc 39 | $(MKDIR_P) $(dir $@) 40 | $(CXX) $(DEP_FLAGS) $(CXXFLAGS) -c $< -o $@ 41 | 42 | test: $(TEST_TARGETS) 43 | 44 | clean: 45 | rm -f $(BIN_TARGETS) 46 | rm -r $(OBJ_DIR) 47 | 48 | print-% : ; @echo $* = $($*) 49 | 50 | -include $(DEPS) 51 | 52 | MKDIR_P ?= mkdir -p 53 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_doc_scal.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "bmi_doc_scal.h" 5 | using namespace std; 6 | BMI_doc_scal::BMI_doc_scal(Seed _seed, 7 | Dataset *_documents, 8 | int _num_threads, 9 | int _training_iterations, int _N, vector> &seed_judgments) 10 | :BMI(_seed, _documents, _num_threads, -1, false, _training_iterations, false) 11 | { 12 | N = _N; 13 | T = N; 14 | R = 0; 15 | judgments_per_iteration = B; 16 | for(auto &seed_judgment: seed_judgments){ 17 | add_to_training_cache(documents->get_index(seed_judgment.first), seed_judgment.second); 18 | } 19 | perform_iteration(); 20 | stratums.push_back(vector()); 21 | for(int doc_id: judgment_queue) 22 | stratums.back().push_back(doc_id); 23 | B = B + ceil(B/10.0); 24 | } 25 | 26 | void BMI_doc_scal::record_judgment_batch(vector> _judgments){ 27 | lock_guard lock(judgment_list_mutex); 28 | for(const auto &judgment: _judgments){ 29 | size_t id = documents->get_index(judgment.first); 30 | add_to_training_cache(id, judgment.second); 31 | for(int i = (int)judgment_queue.size() - 1; i >= 0; i--){ 32 | if(judgment_queue[i] == id){ 33 | judgment_queue.erase(judgment_queue.begin() + i); 34 | if(judgment.second > 0) R++; 35 | break; 36 | } 37 | } 38 | } 39 | 40 | if(judgment_queue.size() == 0){ 41 | cerr<<"Refreshing"< selector(batch.size()); 57 | for(int i = 0; i < selector.size(); i++) 58 | selector[i] = (i < n?1:0); 59 | shuffle(batch.begin(), batch.end(), rand_generator); 60 | for(int i = 0; i < batch.size(); i++){ 61 | if(selector[i]) judgment_queue.push_back(batch[i]); 62 | else judgments[batch[i]] = -2; 63 | } 64 | B = B + ceil(B/10.0); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_doc_scal.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_DOC_SCAL_H 2 | #define BMI_DOC_SCAL_H 3 | #include 4 | #include "bmi.h" 5 | #include 6 | using namespace std; 7 | class BMI_doc_scal:public BMI { 8 | int B = 1; 9 | int T, N; 10 | int R; 11 | vector> stratums; 12 | public: 13 | BMI_doc_scal(Seed seed, 14 | Dataset *documents, 15 | int num_threads, 16 | int training_iterations, int N, std::vector> &seed_judgments); 17 | 18 | virtual void record_judgment_batch(std::vector> judgments); 19 | 20 | string strata_to_json(const vector strata){ 21 | string ret = ""; 22 | for(int doc_id: strata){ 23 | if(ret.size() != 0) 24 | ret += ","; 25 | ret += "\"" + documents->get_id(doc_id) + "\""; 26 | } 27 | return "[" + ret + "]"; 28 | } 29 | 30 | virtual string get_log() { 31 | string ret = ""; 32 | for(auto &strata: stratums){ 33 | if(ret.size() != 0){ 34 | ret += ","; 35 | } 36 | ret += strata_to_json(strata); 37 | } 38 | ret = "[" + ret + "]"; 39 | ret = "{ \"stratums\": " + ret + "}"; 40 | return ret; 41 | } 42 | }; 43 | #endif // BMI_DOC_SCAL_H 44 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_online_learning.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "bmi_online_learning.h" 3 | using namespace std; 4 | 5 | BMI_online_learning::BMI_online_learning(Seed _seed, 6 | Dataset *_documents, 7 | int _num_threads, 8 | int _judgments_per_iteration, 9 | bool _async_mode, 10 | size_t _refresh_period, 11 | float _delta, 12 | int _training_iterations) 13 | :BMI(_seed, _documents, _num_threads, _judgments_per_iteration, _async_mode, _training_iterations, false), 14 | refresh_period(_refresh_period), delta(_delta) 15 | { 16 | perform_iteration(); 17 | } 18 | vector BMI_online_learning::perform_training_iteration(){ 19 | lock_guard lock_training(training_mutex); 20 | 21 | { 22 | lock_guard lock(training_cache_mutex); 23 | for(pair training: training_cache){ 24 | if(!is_it_refresh_time()){ 25 | float p = 1 / (1 + exp(-documents->inner_product(training.first, this->weight))); 26 | int is_rel = (training.second > 0); 27 | for(FeatureValuePair feature: documents->get_sf_sparse_vector(training.first).features_){ 28 | this->weight[feature.id_] += (is_rel - p) * delta; 29 | } 30 | } 31 | judgments[training.first] = training.second; 32 | } 33 | training_cache.clear(); 34 | } 35 | 36 | // Training 37 | 38 | if(is_it_refresh_time()){ 39 | auto start = std::chrono::steady_clock::now(); 40 | this->weight = train(); 41 | auto duration = std::chrono::duration_cast 42 | (std::chrono::steady_clock::now() - start); 43 | cerr<<"Training finished in "<rescore(this->weight, num_threads, 51 | judgments_per_iteration + (async_mode ? extra_judgment_docs : 0), judgments); 52 | auto duration = std::chrono::duration_cast 53 | (std::chrono::steady_clock::now() - start); 54 | cerr<<"Rescored "<size()<<" documents in "< 5 | #include "bmi.h" 6 | 7 | class BMI_online_learning:public BMI { 8 | protected: 9 | size_t refresh_period; 10 | vector weight; 11 | float delta; 12 | 13 | protected: 14 | virtual bool is_it_refresh_time() { 15 | return (state.cur_iteration % refresh_period == 0) || weight.size() == 0; 16 | } 17 | 18 | public: 19 | BMI_online_learning(Seed seed, 20 | Dataset *documents, 21 | int num_threads, 22 | int judgments_per_iteration, 23 | bool async_mode, 24 | size_t refresh_period, 25 | float delta, 26 | int training_iterations); 27 | std::vector perform_training_iteration(); 28 | }; 29 | 30 | #endif // BMI_ONLINE_LEARNING_H 31 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_para.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "bmi_para.h" 7 | #include "utils/utils.h" 8 | 9 | using namespace std; 10 | BMI_para::BMI_para(Seed _seed, 11 | Dataset *_documents, 12 | ParagraphDataset *_paragraphs, 13 | int _num_threads, 14 | int _judgments_per_iteration, 15 | bool _async_mode, 16 | int _training_iterations) 17 | :BMI(_seed, _documents, _num_threads, _judgments_per_iteration, _async_mode, _training_iterations, false), 18 | paragraphs(_paragraphs) 19 | { 20 | perform_iteration(); 21 | } 22 | 23 | void BMI_para::record_judgment(string doc_id, int judgment){ 24 | record_judgment_batch({{doc_id.substr(0, doc_id.find(".")), judgment}}); 25 | } 26 | 27 | vector BMI_para::perform_training_iteration(){ 28 | lock_guard lock_training(training_mutex); 29 | sync_training_cache(); 30 | 31 | // Training 32 | TIMER_BEGIN(training); 33 | auto weights = train(); 34 | TIMER_END(training); 35 | 36 | // Scoring 37 | TIMER_BEGIN(rescoring); 38 | auto results = paragraphs->rescore(weights, num_threads, 39 | judgments_per_iteration + (async_mode ? extra_judgment_docs : 0), 40 | judgments); 41 | TIMER_END(rescoring); 42 | 43 | return results; 44 | } 45 | 46 | std::vector> BMI_para::get_ranklist(){ 47 | vector> ret_results; 48 | unordered_set doc_id_seen; 49 | auto results = get_ranking_dataset()->rescore(train(), num_threads, 50 | get_ranking_dataset()->size(), map()); 51 | 52 | for(auto result: results){ 53 | string para_id = get_ranking_dataset()->get_sf_sparse_vector(result).doc_id; 54 | string doc_id = para_id.substr(0, para_id.find('.')); 55 | if(doc_id_seen.find(doc_id) == doc_id_seen.end()){ 56 | ret_results.push_back({doc_id, 0}); 57 | doc_id_seen.insert(doc_id); 58 | } 59 | } 60 | return ret_results; 61 | } 62 | 63 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_para.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_PARA_H 2 | #define BMI_PARA_H 3 | 4 | #include 5 | #include 6 | #include "bmi.h" 7 | 8 | class BMI_para:public BMI { 9 | protected: 10 | ParagraphDataset *paragraphs; 11 | 12 | public: 13 | BMI_para(Seed seed, 14 | Dataset *documents, 15 | ParagraphDataset *paragraphs, 16 | int num_threads, 17 | int judgments_per_iteration, 18 | bool async_mode, 19 | int training_iterations); 20 | 21 | virtual void record_judgment(std::string doc_id, int judgment); 22 | Dataset *get_ranking_dataset() {return paragraphs;}; 23 | vector> get_ranklist(); 24 | std::vector perform_training_iteration(); 25 | 26 | bool is_judged(int id) { 27 | std::string para = (paragraphs->get_sf_sparse_vector(id)).doc_id; 28 | id = documents->get_index(para.substr(0, para.find("."))); 29 | return judgments.find(id) != judgments.end() || training_cache.find(id) != training_cache.end(); 30 | } 31 | }; 32 | 33 | #endif // BMI_PARA_H 34 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_para_scal.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "bmi_para_scal.h" 4 | using namespace std; 5 | 6 | BMI_para_scal::BMI_para_scal(Seed _seed, 7 | Dataset *_documents, 8 | ParagraphDataset *_paragraphs, 9 | int _num_threads, 10 | int _training_iterations, int _N, vector> &seed_judgments) 11 | :BMI_para(_seed, _documents, _paragraphs, _num_threads, -1, false, _training_iterations) 12 | { 13 | N = _N; 14 | T = N; 15 | R = 0; 16 | judgments_per_iteration = B; 17 | for(auto &seed_judgment: seed_judgments){ 18 | add_to_training_cache(documents->get_index(seed_judgment.first), seed_judgment.second); 19 | } 20 | perform_iteration(); 21 | stratums.push_back(vector()); 22 | for(int doc_id: judgment_queue) 23 | stratums.back().push_back(doc_id); 24 | B = B + ceil(B/10.0); 25 | } 26 | 27 | void BMI_para_scal::record_judgment_batch(vector> _judgments){ 28 | lock_guard lock(judgment_list_mutex); 29 | for(const auto &judgment: _judgments){ 30 | size_t id = documents->get_index(judgment.first); 31 | add_to_training_cache(id, judgment.second); 32 | for(int i = (int)judgment_queue.size() - 1; i >= 0; i--){ 33 | if(paragraphs->translate_index(judgment_queue[i]) == id){ 34 | judgment_queue.erase(judgment_queue.begin() + i); 35 | if(judgment.second > 0) R++; 36 | break; 37 | } 38 | } 39 | } 40 | 41 | if(judgment_queue.size() == 0){ 42 | cerr<<"Refreshing"< selector(batch.size()); 58 | for(int i = 0; i < selector.size(); i++) 59 | selector[i] = (i < n?1:0); 60 | shuffle(batch.begin(), batch.end(), rand_generator); 61 | for(int i = 0; i < batch.size(); i++){ 62 | if(selector[i]) judgment_queue.push_back(batch[i]); 63 | else judgments[paragraphs->translate_index(batch[i])] = -2; 64 | } 65 | B = B + ceil(B/10.0); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_para_scal.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_PARA_SCAL_H 2 | #define BMI_PARA_SCAL_H 3 | 4 | #include 5 | #include "bmi_para.h" 6 | 7 | class BMI_para_scal:public BMI_para { 8 | int B = 1; 9 | int T, N; 10 | int R; 11 | vector> stratums; 12 | public: 13 | BMI_para_scal(Seed seed, 14 | Dataset *documents, 15 | ParagraphDataset *paragraphs, 16 | int num_threads, 17 | int training_iterations, int N, std::vector> &seed_judgments); 18 | 19 | virtual void record_judgment_batch(std::vector> judgments); 20 | 21 | string strata_to_json(const vector strata){ 22 | string ret = ""; 23 | for(int doc_id: strata){ 24 | if(ret.size() != 0) 25 | ret += ","; 26 | ret += "\"" + paragraphs->get_id(doc_id) + "\""; 27 | } 28 | return "[" + ret + "]"; 29 | } 30 | 31 | virtual string get_log() { 32 | string ret = ""; 33 | for(auto &strata: stratums){ 34 | if(ret.size() != 0){ 35 | ret += ","; 36 | } 37 | ret += strata_to_json(strata); 38 | } 39 | ret = "[" + ret + "]"; 40 | ret = "{ \"stratums\": " + ret + "}"; 41 | return ret; 42 | } 43 | }; 44 | 45 | #endif // BMI_PARA_SCAL_H 46 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_precision_delay.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "bmi_precision_delay.h" 5 | #include "utils/utils.h" 6 | using namespace std; 7 | BMI_precision_delay::BMI_precision_delay(Seed _seed, 8 | Dataset *_documents, 9 | int _num_threads, 10 | bool _async_mode, 11 | float _threshold, 12 | int _window, 13 | int _training_iterations) 14 | :BMI(_seed, _documents, _num_threads, 1, _async_mode, _training_iterations, false), 15 | threshold(_threshold), window(_window) 16 | { 17 | perform_iteration(); 18 | } 19 | 20 | vector BMI_precision_delay::perform_training_iteration(){ 21 | lock_guard lock_training(training_mutex); 22 | 23 | sync_training_cache(); 24 | 25 | // Training 26 | if(!skip_training){ 27 | TIMER_BEGIN(training); 28 | weights = train(); 29 | TIMER_END(training); 30 | } 31 | 32 | 33 | // Scoring 34 | TIMER_BEGIN(rescoring); 35 | auto results = documents->rescore(weights, num_threads, 36 | judgments_per_iteration + (async_mode ? extra_judgment_docs : 0), judgments); 37 | TIMER_END(rescoring); 38 | 39 | return results; 40 | } 41 | 42 | void BMI_precision_delay::record_judgment_batch(std::vector> _judgments){ 43 | state.next_iteration_target += 1; 44 | int last_rel; 45 | for(const auto &judgment: _judgments){ 46 | size_t id = documents->get_index(judgment.first); 47 | if(judgment_queue.size() > 0 && id == judgment_queue.back()){ 48 | judgment_queue.pop_back(); 49 | } 50 | add_to_training_cache(id, judgment.second); 51 | q.push(judgment.second); 52 | tot++; 53 | if(q.size() > window){ 54 | int x = q.front(); 55 | q.pop(); 56 | if(x > 0) 57 | rel -= 1; 58 | } 59 | if(judgment.second > 0) 60 | rel += 1; 61 | last_rel = judgment.second; 62 | } 63 | 64 | if(!async_mode){ 65 | if(last_rel <= 0 && rel / (float)q.size() < threshold){ 66 | if(judgments_per_iteration > 1) 67 | judgments_per_iteration /= 2; 68 | perform_iteration(); 69 | }else if(judgment_queue.size() == 0 || get_doc_to_judge(1).size() == 0){ 70 | judgments_per_iteration *= 2; 71 | skip_training = true; 72 | add_to_judgment_list(perform_training_iteration()); 73 | skip_training = false; 74 | } 75 | }else{ 76 | auto t = std::thread(&BMI_precision_delay::perform_iteration_async, this); 77 | t.detach(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_precision_delay.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_PRECISION_DELAY_H 2 | #define BMI_PRECISION_DELAY_H 3 | 4 | #include "bmi.h" 5 | 6 | class BMI_precision_delay:public BMI { 7 | protected: 8 | float threshold; 9 | int window; 10 | std::queue q; 11 | int rel = 0; 12 | int tot = 0; 13 | void record_judgment_batch(std::vector> _judgments); 14 | vector weights; 15 | bool skip_training = false; 16 | vector perform_training_iteration(); 17 | 18 | public: 19 | BMI_precision_delay(Seed seed, 20 | Dataset *documents, 21 | int num_threads, 22 | bool async_mode, 23 | float threshold, 24 | int window, 25 | int training_iterations); 26 | }; 27 | 28 | #endif // BMI_PRECISION_DELAY_H 29 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_recency_weighting.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_RECENCY_WEIGHTING_H 2 | #define BMI_RECENCY_WEIGHTING_H 3 | #include 4 | #include 5 | #include 6 | #include "bmi.h" 7 | #include "classifier.h" 8 | 9 | class BMI_recency_weighting:public BMI { 10 | std::unordered_map judgment_order; 11 | const float max_relative_weight; 12 | int cur_time = 0; 13 | protected: 14 | virtual vector train(); 15 | public: 16 | BMI_recency_weighting(Seed _seed, 17 | Dataset *_documents, 18 | int _num_threads, 19 | int _judgments_per_iteration, 20 | bool _async_mode, 21 | float _max_relative_weight, 22 | int _training_iterations) 23 | :BMI(_seed, _documents, _num_threads, _judgments_per_iteration, _async_mode, _training_iterations, false), 24 | max_relative_weight(_max_relative_weight) { 25 | perform_iteration(); 26 | } 27 | 28 | virtual void record_judgment(std::string doc_id, int judgment){ 29 | size_t id = documents->get_index(doc_id); 30 | judgment_order[&documents->get_sf_sparse_vector(id)] = ++cur_time; 31 | record_judgment_batch({{doc_id, judgment}}); 32 | } 33 | }; 34 | 35 | vector BMI_recency_weighting::train(){ 36 | std::uniform_int_distribution distribution(0, documents->size()-1); 37 | for(int i = 0;iget_sf_sparse_vector(idx); 40 | } 41 | 42 | std::cerr<<"Training on "< bool {return this->judgment_order[a] < this->judgment_order[b];}); 46 | std::sort(negatives.begin()+100, negatives.end(), [this](const SfSparseVector *a, const SfSparseVector *b) -> bool {return this->judgment_order[a] < this->judgment_order[b];}); 47 | 48 | std::cerr<<"Training on "<get_dimensionality()); 51 | } 52 | 53 | #endif // BMI_RECENCY_WEIGHTING_H 54 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_reduced_ranking.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_REDUCED_RANKING_H 2 | #define BMI_REDUCED_RANKING_H 3 | 4 | #include 5 | #include "bmi.h" 6 | 7 | class BMI_reduced_ranking:public BMI { 8 | protected: 9 | Dataset *subset; 10 | vector indices; 11 | size_t subset_size; 12 | size_t refresh_period; 13 | 14 | protected: 15 | virtual bool is_it_refresh_time() { 16 | return (state.cur_iteration % refresh_period == 0); 17 | } 18 | 19 | void score_docs_insertion_sort(const vector &weights, 20 | int st, int end, 21 | std::pair *top_docs); 22 | 23 | vector subset_rescore(const vector &weights); 24 | 25 | public: 26 | BMI_reduced_ranking(Seed seed, 27 | Dataset *documents, 28 | int num_threads, 29 | int judgments_per_iteration, 30 | bool async_mode, 31 | size_t subset_size, 32 | size_t refresh_period, 33 | int training_iterations); 34 | std::vector perform_training_iteration(); 35 | }; 36 | 37 | #endif // BMI_REDUCED_RANKING_H 38 | -------------------------------------------------------------------------------- /CALEngine/src/bmi_reduced_ranking_dynamic.h: -------------------------------------------------------------------------------- 1 | #ifndef BMI_REDUCED_RANKING_DYNAMIC_H 2 | #define BMI_REDUCED_RANKING_DYNAMIC_H 3 | 4 | #include 5 | #include "bmi_reduced_ranking.h" 6 | 7 | class BMI_reduced_ranking_dynamic:public BMI_reduced_ranking { 8 | int next_refresh = 1; 9 | int cur_subset_size = 1; 10 | protected: 11 | bool is_it_refresh_time() { 12 | return (state.cur_iteration % refresh_period == 0); 13 | } 14 | 15 | public: 16 | BMI_reduced_ranking_dynamic(Seed seed, 17 | Dataset *documents, 18 | int num_threads, 19 | int judgments_per_iteration, 20 | bool async_mode, 21 | size_t subset_size) 22 | :BMI_reduced_ranking(seed, documents, num_threads, judgments_per_iteration, async_mode, subset_size, -1) 23 | { 24 | 25 | }; 26 | }; 27 | 28 | #endif // BMI_REDUCED_RANKING_DYNAMIC_H 29 | -------------------------------------------------------------------------------- /CALEngine/src/classifier.h: -------------------------------------------------------------------------------- 1 | #ifndef CLASSIFIER_H 2 | #define CLASSIFIER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "sofiaml/sf-sparse-vector.h" 8 | 9 | static thread_local std::mt19937 rand_generator; 10 | 11 | class Classifier { 12 | public: 13 | virtual vector train(const std::vector &positives, 14 | const std::vector &negatives, int dimensionality) = 0; 15 | }; 16 | 17 | class LRPegasosClassifier:public Classifier { 18 | const float lambda = 0.0001; 19 | const int num_iters; 20 | 21 | protected: 22 | virtual int RandInt(int num_vals) { 23 | std::uniform_int_distribution distribution(0, num_vals-1); 24 | return distribution(rand_generator); 25 | } 26 | 27 | public: 28 | LRPegasosClassifier(int _num_iters):num_iters(_num_iters) {} 29 | 30 | virtual vector train(const std::vector &positives, 31 | const std::vector &negatives, int dimensionality); 32 | }; 33 | 34 | class LRPegasosWeightedRecencyClassifier:public LRPegasosClassifier { 35 | int c; 36 | protected: 37 | virtual int RandInt(int num_vals) { 38 | num_vals--; 39 | if(num_vals == 0)return 0; 40 | double m = (c-1)/double(num_vals); 41 | double new_max = (m*(num_vals+1)*(num_vals+1) - m*(num_vals+1) + 2*(num_vals+1))/2; 42 | std::uniform_real_distribution distribution(0, new_max); 43 | double r = distribution(rand_generator); 44 | if(m < 1e-10) 45 | return int(r); 46 | double r_2 = (m-2 + sqrt((2-m)*(2-m)+8*r*m))/(2*m); 47 | return int(r_2); 48 | } 49 | 50 | public: 51 | LRPegasosWeightedRecencyClassifier(int _c, int _num_iters):c(_c), LRPegasosClassifier(_num_iters){} 52 | }; 53 | 54 | #endif // CLASSIFIER_H 55 | -------------------------------------------------------------------------------- /CALEngine/src/features.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "features.h" 9 | #include "utils/text_utils.h" 10 | using namespace std; 11 | 12 | unordered_map features::get_tf(const vector &words){ 13 | unordered_map tf_map; 14 | for(string word: words){ 15 | if(tf_map.find(word) == tf_map.end()) 16 | tf_map[word] = 0; 17 | tf_map[word]++; 18 | } 19 | return tf_map; 20 | } 21 | 22 | SfSparseVector features::get_features(const string &text, const Dataset &dataset, double max_norm){ 23 | vector features; 24 | vector> tmp_features; 25 | 26 | double sum = 0; 27 | auto &dictionary = dataset.get_dictionary(); 28 | for(pair term: get_tf(BMITokenizer().tokenize(text))){ 29 | auto it = dictionary.find(term.first); 30 | if(it != dictionary.end()){ 31 | int id = it->second.id; 32 | int df = it->second.df; 33 | int tf = term.second; 34 | tmp_features.push_back({id, ((1+log(tf)) * log(dataset.size()/(float)df))}); 35 | sum += tmp_features.back().second * tmp_features.back().second; 36 | } 37 | } 38 | sum = sqrt(sum); 39 | 40 | for(auto &feature: tmp_features){ 41 | features.push_back({feature.first, (float)(feature.second/max(max_norm, sum))}); 42 | } 43 | sort(features.begin(), features.end(), [](auto &a, auto &b) -> bool{return a.id_ < b.id_;}); 44 | return SfSparseVector("Q", features); 45 | } 46 | -------------------------------------------------------------------------------- /CALEngine/src/features.h: -------------------------------------------------------------------------------- 1 | #ifndef FEATURES_H 2 | #define FEATURES_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "utils/features.h" 8 | #include "sofiaml/sf-sparse-vector.h" 9 | #include "dataset.h" 10 | 11 | namespace features{ 12 | inline bool is_valid(char ch){ 13 | return isalpha(ch) || isdigit(ch); 14 | } 15 | 16 | void init(const std::string &fname); 17 | 18 | std::unordered_map get_tf(const vector &words); 19 | 20 | // Extract features from given text 21 | SfSparseVector get_features(const std::string &text, const Dataset &dataset, double max_norm=1); 22 | 23 | } 24 | #endif // FEATURES_H 25 | -------------------------------------------------------------------------------- /CALEngine/src/utils/feature_parser.h: -------------------------------------------------------------------------------- 1 | #ifndef FEATURE_PARSER_H 2 | #define FEATURE_PARSER_H 3 | 4 | #include "features.h" 5 | #include "../sofiaml/sf-sparse-vector.h" 6 | #include 7 | #include 8 | #include 9 | 10 | class FeatureParser{ 11 | protected: 12 | static const char DELIM_CHAR = '\n'; 13 | FILE *fp; 14 | std::unordered_map dictionary; 15 | public: 16 | FeatureParser(const string &fname){ fp = fopen(fname.c_str(), "rb"); setvbuf(fp, NULL, _IOFBF, 1 << 25); } 17 | virtual std::unique_ptr next() = 0; 18 | std::unordered_map get_dictionary() { return dictionary; } 19 | 20 | ~FeatureParser(){fclose(fp);} 21 | }; 22 | 23 | class BinFeatureParser:public FeatureParser { 24 | uint32_t num_records; 25 | public: 26 | BinFeatureParser(const string &file_name); 27 | BinFeatureParser(const string &file_name, const string &df_file_name); 28 | std::unique_ptr next() override; 29 | }; 30 | 31 | class SVMlightFeatureParser:public FeatureParser { 32 | char *buffer; 33 | size_t buffer_size; 34 | public: 35 | SVMlightFeatureParser(const string &file_name, const string &df_file_name); 36 | bool read_line(); 37 | std::unique_ptr next() override; 38 | ~SVMlightFeatureParser(){delete [] buffer;} 39 | }; 40 | #endif // FEATURE_PARSER_H 41 | -------------------------------------------------------------------------------- /CALEngine/src/utils/feature_writer.cc: -------------------------------------------------------------------------------- 1 | #include "feature_writer.h" 2 | using namespace std; 3 | 4 | BinFeatureWriter::BinFeatureWriter(const string &file_name, const vector> &dictionary): FeatureWriter(file_name){ 5 | // Leave space for end-of-dictionary pointer 6 | fseeko(fp, sizeof(uint32_t), SEEK_CUR); 7 | 8 | // Write dictionary terms with df 9 | for(auto &term: dictionary) { 10 | fwrite(term.first.c_str(), 1, min(term.first.length(), (size_t)MAX_TERM_LEN - 1), fp); 11 | fputc(0, fp); 12 | fwrite(&term.second, sizeof(uint32_t), 1, fp); 13 | } 14 | 15 | // Go back to start and write the end-of-dictionary pointer 16 | dict_end_offset = ftello(fp); 17 | fseeko(fp, 0, SEEK_SET); 18 | fwrite(&dict_end_offset, sizeof(uint32_t), 1, fp); 19 | 20 | // Go to the end and leave space for number of documents 21 | fseeko(fp, dict_end_offset, SEEK_SET); 22 | fseeko(fp, sizeof(uint32_t), SEEK_CUR); 23 | } 24 | 25 | SVMlightFeatureWriter::SVMlightFeatureWriter(const string &file_name, const string &df_file_name, const vector> &dictionary): FeatureWriter(file_name){ 26 | if(df_file_name.length() > 0){ 27 | FILE *df_fp = fopen(df_file_name.c_str(), "wb"); 28 | for(auto &term: dictionary) { 29 | fprintf(df_fp, "%d %s\n", term.second, term.first.c_str()); 30 | } 31 | fclose(df_fp); 32 | } 33 | } 34 | 35 | void BinFeatureWriter::write(const SfSparseVector &spv){ 36 | fwrite(spv.doc_id.c_str(), 1, spv.doc_id.length(), fp); 37 | fputc(DELIM_CHAR, fp); 38 | 39 | uint32_t num_pairs = 0; 40 | off_t record_len_offset = ftello(fp); 41 | fseeko(fp, sizeof(uint32_t), SEEK_CUR); 42 | 43 | for(auto &fpv: spv.features_){ 44 | if(fpv.id_ != 0){ 45 | num_pairs++; 46 | fwrite(&fpv.id_, sizeof(uint32_t), 1, fp); 47 | fwrite(&fpv.value_, sizeof(float), 1, fp); 48 | } 49 | } 50 | 51 | off_t backup_offset = ftello(fp); 52 | fseeko(fp, record_len_offset, SEEK_SET); 53 | fwrite(&num_pairs, sizeof(uint32_t), 1, fp); 54 | fseeko(fp, backup_offset, SEEK_SET); 55 | num_records++; 56 | } 57 | 58 | void BinFeatureWriter::finish(){ 59 | fseeko(fp, dict_end_offset, SEEK_SET); 60 | fwrite(&num_records, sizeof(uint32_t), 1, fp); 61 | fflush(fp); 62 | } 63 | 64 | void SVMlightFeatureWriter::write(const SfSparseVector &spv){ 65 | fprintf(fp, "%s", spv.doc_id.c_str()); 66 | for(auto &fpv: spv.features_){ 67 | if(fpv.id_ != 0) 68 | fprintf(fp, " %d:%.8f", fpv.id_, fpv.value_); 69 | } 70 | fprintf(fp, "\n"); 71 | } 72 | 73 | void FeatureWriter::write_dataset(const Dataset &dataset) { 74 | for(size_t i = 0; i < dataset.size(); i++){ 75 | write(dataset.get_sf_sparse_vector(i)); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /CALEngine/src/utils/feature_writer.h: -------------------------------------------------------------------------------- 1 | #ifndef FEATURE_WRITER_H 2 | #define FEATURE_WRITER_H 3 | 4 | #include "features.h" 5 | #include "../sofiaml/sf-sparse-vector.h" 6 | #include "../dataset.h" 7 | #include 8 | #include 9 | 10 | class FeatureWriter{ 11 | protected: 12 | static const char DELIM_CHAR = '\n'; 13 | FILE *fp; 14 | public: 15 | FeatureWriter(const string &fname){ fp = fopen(fname.c_str(), "wb"); setvbuf(fp, nullptr, _IOFBF, 1 << 25); } 16 | virtual void write(const SfSparseVector &spv) = 0; 17 | void write_dataset(const Dataset &dataset); 18 | ~FeatureWriter(){fclose(fp);} 19 | virtual void finish() = 0; 20 | }; 21 | 22 | class BinFeatureWriter:public FeatureWriter { 23 | uint32_t num_records = 0; 24 | uint32_t dict_end_offset; 25 | public: 26 | BinFeatureWriter(const string &file_name, const std::vector> &dictionary); 27 | void write(const SfSparseVector &spv) override; 28 | // Write final headers 29 | void finish() override; 30 | }; 31 | 32 | class SVMlightFeatureWriter:public FeatureWriter { 33 | public: 34 | SVMlightFeatureWriter(const string &file_name, const string &df_file_name, const std::vector> &dictionary); 35 | void write(const SfSparseVector &spv) override; 36 | void finish() override {} 37 | }; 38 | #endif // FEATURE_WRITER_H 39 | -------------------------------------------------------------------------------- /CALEngine/src/utils/features.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_FEATURES_H 2 | #define UTILS_FEATURES_H 3 | 4 | #define MAX_TERM_LEN 64 5 | struct TermInfo{ 6 | int id; 7 | int df; 8 | }; 9 | 10 | #endif // UTILS_FEATURES_H 11 | -------------------------------------------------------------------------------- /CALEngine/src/utils/text_utils.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "text_utils.h" 4 | #include "porter.c" 5 | 6 | using namespace std; 7 | bool AlphaFilter::filter(const std::string &token) { 8 | if(token.length() < 2) 9 | return false; 10 | for(char c: token){ 11 | if(!isalpha(c)) 12 | return false; 13 | } 14 | return true; 15 | } 16 | 17 | bool MinLengthFilter::filter(const string &token) { 18 | return token.length() >= min_length; 19 | } 20 | 21 | std::string PorterTransform::transform(const std::string &token) { 22 | char temp_str[token.length()+1]; 23 | strcpy(temp_str, token.c_str()); 24 | 25 | int end = stem(temp_str, 0, (int)token.length()-1); 26 | return std::string(temp_str, end+1); 27 | } 28 | 29 | std::string LowerTransform::transform(const std::string &token) { 30 | string transformed_token = token; 31 | std::transform(transformed_token.begin(), transformed_token.end(), transformed_token.begin(), ::tolower); 32 | return transformed_token; 33 | } 34 | 35 | std::vector BMITokenizer::tokenize(const std::string &text) { 36 | vector tokens; 37 | int st = 0; 38 | while(st < (int)text.length()){ 39 | int end = 0; 40 | while(isalnum(text[st+end])){ 41 | end++; 42 | } 43 | if(end > 0){ 44 | auto token = text.substr(st, end); 45 | if(alpha_filter.filter(token)){ 46 | token = porter_transform.transform(lower_transform.transform(token)); 47 | if(min_length_filter.filter(token)) 48 | tokens.push_back(token); 49 | } 50 | } 51 | st += end + 1; 52 | } 53 | return tokens; 54 | } 55 | -------------------------------------------------------------------------------- /CALEngine/src/utils/text_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef TEXT_UTILS_H 2 | #define TEXT_UTILS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | /* 11 | * Intentionally not optimized for sake of maintenance and flexibility 12 | * Todo: use char* for internal filters and transforms 13 | */ 14 | 15 | 16 | // Filter declarations start here 17 | class Filter { 18 | public: 19 | virtual bool filter(const std::string &token) = 0; 20 | }; 21 | 22 | class AlphaFilter:public Filter { 23 | public: 24 | bool filter(const std::string &token) override; 25 | }; 26 | 27 | class MinLengthFilter:public Filter { 28 | size_t min_length; 29 | public: 30 | MinLengthFilter(size_t _min_length):min_length(_min_length){}; 31 | bool filter(const std::string &token) override; 32 | }; 33 | 34 | 35 | // Transformer declarations start here 36 | class Transform { 37 | public: 38 | virtual std::string transform(const std::string &token) = 0; 39 | }; 40 | 41 | class PorterTransform:public Transform { 42 | public: 43 | std::string transform(const std::string &token) override; 44 | }; 45 | 46 | class LowerTransform:public Transform { 47 | public: 48 | std::string transform(const std::string &token) override; 49 | }; 50 | 51 | 52 | // Tokenizer declarations start here 53 | class Tokenizer { 54 | public: 55 | virtual std::vector tokenize(const std::string &text) = 0; 56 | }; 57 | 58 | class BMITokenizer:public Tokenizer { 59 | AlphaFilter alpha_filter = AlphaFilter(); 60 | MinLengthFilter min_length_filter = MinLengthFilter(2); 61 | PorterTransform porter_transform = PorterTransform(); 62 | LowerTransform lower_transform = LowerTransform(); 63 | public: 64 | std::vector tokenize(const std::string &text) override; 65 | }; 66 | #endif // TEXT_UTILS_H 67 | -------------------------------------------------------------------------------- /CALEngine/src/utils/utils.cc: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | 3 | void msg(const char *message){ 4 | fprintf(stderr, "%s", message); 5 | fprintf(stderr, "\n"); 6 | } 7 | 8 | void fail(const char *message, int e){ 9 | msg(message); 10 | exit(e); 11 | } 12 | 13 | void msg(const std::string &str){ 14 | msg(str.c_str()); 15 | } 16 | 17 | void fail(const std::string &str, int e){ 18 | fail(str.c_str(), e); 19 | } 20 | -------------------------------------------------------------------------------- /CALEngine/src/utils/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H 2 | #define UTILS_H 3 | 4 | #include 5 | #include 6 | 7 | #define TIMER_BEGIN(key) \ 8 | auto start##key = std::chrono::steady_clock::now(); 9 | 10 | #define TIMER_END(key) \ 11 | std::cerr<<"("<< #key <<"): "<<(std::chrono::duration_cast (std::chrono::steady_clock::now() - start##key)).count()<<"ms\n"; 12 | 13 | void msg(const char *message); 14 | void fail(const char *message, int e); 15 | void msg(const std::string &str); 16 | void fail(const std::string &str, int e); 17 | #endif // UTILS_H 18 | -------------------------------------------------------------------------------- /CALEngine/tests/test_scorer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "../src/scorer.h" 5 | #include "../src/utils/feature_parser.h" 6 | using namespace std; 7 | 8 | int main(int argc, char *argv[]){ 9 | string bin_file = "data/oldreut.bin"; 10 | int threads = 1; 11 | auto start = std::chrono::steady_clock::now(); 12 | cerr<<"Loading document features on memory"< 15 | (std::chrono::steady_clock::now() - start); 16 | 17 | cerr<<"Read "<size()<<" docs in "< results; 22 | set judgments; 23 | for(int j = 0;jsize()); 25 | } 26 | vector weights(dataset->get_dimensionality()); 27 | for(float &wt: weights) 28 | wt = (float)rand()/(float)(RAND_MAX); 29 | 30 | start = std::chrono::steady_clock::now(); 31 | Scorer::rescore_documents(*dataset, weights, threads, 100, judgments); 32 | duration = std::chrono::duration_cast 33 | (std::chrono::steady_clock::now() - start); 34 | cerr<<"Rescored "<size()<<" documents in "<&2 echo "Postgres is unavailable - sleeping" 40 | sleep 1 41 | done 42 | 43 | >&2 echo "Postgres is up - continuing..." 44 | exec $cmd 45 | -------------------------------------------------------------------------------- /HiCALWeb/compose/production/django/gunicorn.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | 8 | python /app/manage.py collectstatic --noinput 9 | /usr/local/bin/gunicorn config.wsgi -w 4 -b 0.0.0.0:5000 --chdir=/app 10 | -------------------------------------------------------------------------------- /HiCALWeb/compose/production/postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:9.6 2 | 3 | COPY ./compose/production/postgres/backup.sh /usr/local/bin/backup 4 | RUN chmod +x /usr/local/bin/backup 5 | 6 | COPY ./compose/production/postgres/restore.sh /usr/local/bin/restore 7 | RUN chmod +x /usr/local/bin/restore 8 | 9 | COPY ./compose/production/postgres/list-backups.sh /usr/local/bin/list-backups 10 | RUN chmod +x /usr/local/bin/list-backups 11 | -------------------------------------------------------------------------------- /HiCALWeb/compose/production/postgres/backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | 8 | # we might run into trouble when using the default `postgres` user, e.g. when dropping the postgres 9 | # database in restore.sh. Check that something else is used here 10 | if [ "$POSTGRES_USER" == "postgres" ] 11 | then 12 | echo "creating a backup as the postgres user is not supported, make sure to set the POSTGRES_USER environment variable" 13 | exit 1 14 | fi 15 | 16 | # export the postgres password so that subsequent commands don't ask for it 17 | export PGPASSWORD=$POSTGRES_PASSWORD 18 | 19 | echo "creating backup" 20 | echo "---------------" 21 | 22 | FILENAME=backup_$(date +'%Y_%m_%dT%H_%M_%S').sql.gz 23 | pg_dump -h postgres -U $POSTGRES_USER | gzip > /backups/$FILENAME 24 | 25 | echo "successfully created backup $FILENAME" 26 | -------------------------------------------------------------------------------- /HiCALWeb/compose/production/postgres/list-backups.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | 8 | echo "listing available backups" 9 | echo "-------------------------" 10 | ls /backups/ 11 | -------------------------------------------------------------------------------- /HiCALWeb/compose/production/postgres/restore.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | 8 | # we might run into trouble when using the default `postgres` user, e.g. when dropping the postgres 9 | # database in restore.sh. Check that something else is used here 10 | if [ "$POSTGRES_USER" == "postgres" ] 11 | then 12 | echo "restoring as the postgres user is not supported, make sure to set the POSTGRES_USER environment variable" 13 | exit 1 14 | fi 15 | 16 | # export the postgres password so that subsequent commands don't ask for it 17 | export PGPASSWORD=$POSTGRES_PASSWORD 18 | 19 | # check that we have an argument for a filename candidate 20 | if [[ $# -eq 0 ]] ; then 21 | echo 'usage:' 22 | echo ' docker-compose -f production.yml run postgres restore ' 23 | echo '' 24 | echo 'to get a list of available backups, run:' 25 | echo ' docker-compose -f production.yml run postgres list-backups' 26 | exit 1 27 | fi 28 | 29 | # set the backupfile variable 30 | BACKUPFILE=/backups/$1 31 | 32 | # check that the file exists 33 | if ! [ -f $BACKUPFILE ]; then 34 | echo "backup file not found" 35 | echo 'to get a list of available backups, run:' 36 | echo ' docker-compose -f production.yml run postgres list-backups' 37 | exit 1 38 | fi 39 | 40 | echo "beginning restore from $1" 41 | echo "-------------------------" 42 | 43 | # delete the db 44 | # deleting the db can fail. Spit out a comment if this happens but continue since the db 45 | # is created in the next step 46 | echo "deleting old database $POSTGRES_USER" 47 | if dropdb -h postgres -U $POSTGRES_USER $POSTGRES_USER 48 | then echo "deleted $POSTGRES_USER database" 49 | else echo "database $POSTGRES_USER does not exist, continue" 50 | fi 51 | 52 | # create a new database 53 | echo "creating new database $POSTGRES_USER" 54 | createdb -h postgres -U $POSTGRES_USER $POSTGRES_USER -O $POSTGRES_USER 55 | 56 | # restore the database 57 | echo "restoring database $POSTGRES_USER" 58 | gunzip -c $BACKUPFILE | psql -h postgres -U $POSTGRES_USER 59 | -------------------------------------------------------------------------------- /HiCALWeb/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/config/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/config/settings/README.md: -------------------------------------------------------------------------------- 1 | HiCALWeb Setting 2 | =========== 3 | 4 | Project's setting can be modified here. 5 | Configurations such as models, user, and retrieval components can be changed here. 6 | 7 | # Components IPs 8 | 9 | 10 | # COMPONENTS IPS *REQUIRED* 11 | # ------------------------------------------------------------------------------ 12 | CAL_SERVER_IP = 'nginx' 13 | CAL_SERVER_PORT = '9001' 14 | SEARCH_SERVER_IP = '129.97.25.37' 15 | SEARCH_SERVER_PORT = '80' 16 | DOCUMENTS_URL = 'http://nginx:9000/doc' 17 | PARA_URL = 'http://nginx:9000/para' 18 | 19 | -------------------------------------------------------------------------------- /HiCALWeb/config/settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/config/settings/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/config/settings/local.py: -------------------------------------------------------------------------------- 1 | """ 2 | Local settings 3 | 4 | - Run in Debug mode 5 | 6 | - Use console backend for emails 7 | 8 | - Add Django Debug Toolbar 9 | - Add django-extensions as app 10 | """ 11 | 12 | from .base import * # noqa 13 | 14 | # DEBUG 15 | # ------------------------------------------------------------------------------ 16 | DEBUG = env.bool('DJANGO_DEBUG', default=True) 17 | TEMPLATES[0]['OPTIONS']['debug'] = DEBUG 18 | 19 | # SECRET CONFIGURATION 20 | # ------------------------------------------------------------------------------ 21 | # See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key 22 | # Note: This key only used for development and testing. 23 | SECRET_KEY = env('DJANGO_SECRET_KEY', default='W]/aCD]oj.nAtSj9o!~quZmVmlv_I+`)q`T\d+)/$', 17 | views.GetLatestAJAXView.as_view(), 18 | name='get_latest'), 19 | 20 | ] 21 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/middleware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/middleware/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/middleware/pagefile.py: -------------------------------------------------------------------------------- 1 | from django.utils.deprecation import MiddlewareMixin 2 | 3 | 4 | class PageHTMLMiddleware(MiddlewareMixin): 5 | """ 6 | Adds file name of the rendered template to context of the response for certain views. 7 | """ 8 | 9 | def process_request(self, request): 10 | # Add page_file field to request 11 | request.page_file = None 12 | 13 | def process_view(self, request, view, *args, **kwargs): 14 | try: 15 | request.page_file = view.view_class.template_name 16 | except AttributeError: 17 | pass 18 | 19 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'progress.apps.ProgressConfig' 2 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from hicalweb.progress.models import Session 4 | 5 | 6 | class NoAddNoDeleteAdmin(admin.ModelAdmin): 7 | def has_add_permission(self, request): 8 | return False 9 | 10 | def has_delete_permission(self, request, obj=None): 11 | return False 12 | 13 | 14 | admin.site.register(Session, NoAddNoDeleteAdmin) 15 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ProgressConfig(AppConfig): 5 | name = 'hicalweb.progress' 6 | verbose_name = "Progress and Users Tasks" 7 | 8 | 9 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/progress/fixtures/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/fixtures/init_tasksetting.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "progress.tasksetting", 4 | "pk": 1, 5 | "fields": { 6 | "show_search": true, 7 | "toggle_doc": true, 8 | "only_show_doc": false 9 | } 10 | }, 11 | { 12 | "model": "progress.tasksetting", 13 | "pk": 2, 14 | "fields": { 15 | "show_search": true, 16 | "toggle_doc": false, 17 | "only_show_doc": false 18 | } 19 | }, 20 | { 21 | "model": "progress.tasksetting", 22 | "pk": 3, 23 | "fields": { 24 | "show_search": false, 25 | "toggle_doc": false, 26 | "only_show_doc": false 27 | } 28 | }, 29 | { 30 | "model": "progress.tasksetting", 31 | "pk": 4, 32 | "fields": { 33 | "show_search": false, 34 | "toggle_doc": true, 35 | "only_show_doc": false 36 | } 37 | }, 38 | { 39 | "model": "progress.tasksetting", 40 | "pk": 5, 41 | "fields": { 42 | "show_search": false, 43 | "toggle_doc": false, 44 | "only_show_doc": true 45 | } 46 | } 47 | ] 48 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-27 00:04 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | import uuid 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | ('topic', '0001_initial'), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='Task', 21 | fields=[ 22 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 23 | ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)), 24 | ('timespent', models.FloatField(default=0)), 25 | ('last_activity', models.FloatField(blank=True, default=None, null=True)), 26 | ('topic', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='topic.Topic')), 27 | ], 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0002_task_username.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-27 00:04 3 | from __future__ import unicode_literals 4 | 5 | from django.conf import settings 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 16 | ('progress', '0001_initial'), 17 | ] 18 | 19 | operations = [ 20 | migrations.AddField( 21 | model_name='task', 22 | name='username', 23 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0003_auto_20180723_1041.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-23 14:41 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('progress', '0002_task_username'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='task', 18 | name='created_at', 19 | field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), 20 | preserve_default=False, 21 | ), 22 | migrations.AddField( 23 | model_name='task', 24 | name='updated_at', 25 | field=models.DateTimeField(auto_now=True), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0004_task_max_number_of_judgments.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-26 23:18 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('progress', '0003_auto_20180723_1041'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='task', 17 | name='max_number_of_judgments', 18 | field=models.IntegerField(default=15), 19 | preserve_default=False, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0005_task_strategy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-26 23:27 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('progress', '0004_task_max_number_of_judgments'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='task', 17 | name='strategy', 18 | field=models.CharField(choices=[('para', 'Paragraph (CAL)'), ('para_scal', 'Paragraph (SCAL)')], default='para', max_length=64), 19 | preserve_default=False, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0006_auto_20180804_1908.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-08-04 23:08 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('progress', '0005_task_strategy'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='task', 17 | name='show_full_document_content', 18 | field=models.BooleanField(default=True), 19 | preserve_default=False, 20 | ), 21 | migrations.AlterField( 22 | model_name='task', 23 | name='strategy', 24 | field=models.CharField(choices=[('doc', 'Full document (CAL)'), ('para', 'Paragraph (CAL)'), ('para_scal', 'Paragraph (S-CAL)')], max_length=64), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0007_auto_20200528_1346.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2020-05-28 17:46 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('progress', '0006_auto_20180804_1908'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='task', 17 | name='strategy', 18 | field=models.CharField(choices=[('doc', 'Document (CAL)'), ('para', 'Paragraph (CAL)'), ('doc_scal', 'Document (S-CAL)'), ('para_scal', 'Paragraph (S-CAL)')], max_length=64), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/0008_auto_20200603_2255.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2020-06-04 02:55 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('progress', '0007_auto_20200528_1346'), 12 | ] 13 | 14 | operations = [ 15 | migrations.RenameModel( 16 | old_name='Task', 17 | new_name='Session', 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/progress/migrations/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/models.py: -------------------------------------------------------------------------------- 1 | from config.settings.base import AUTH_USER_MODEL as User 2 | import uuid 3 | 4 | from django.db import models 5 | 6 | from hicalweb.CAL.exceptions import CALError 7 | from hicalweb.interfaces.CAL import functions as CALFunctions 8 | from hicalweb.topic.models import Topic 9 | 10 | 11 | class Session(models.Model): 12 | STRATEGY_CHOICES = ( 13 | ('doc', 'Document (CAL)'), 14 | ('para', 'Paragraph (CAL)'), 15 | ('doc_scal', 'Document (S-CAL)'), 16 | ('para_scal', 'Paragraph (S-CAL)'), 17 | ) 18 | 19 | username = models.ForeignKey(User) 20 | topic = models.ForeignKey(Topic) 21 | uuid = models.UUIDField(default=uuid.uuid4, 22 | editable=False) 23 | 24 | # max number of judgments you wish for this task. 0 or negative to have no max. 25 | max_number_of_judgments = models.IntegerField(null=False, blank=False) 26 | strategy = models.CharField(max_length=64, 27 | choices=STRATEGY_CHOICES, 28 | null=False, 29 | blank=False) 30 | # For paragraphs strategies 31 | show_full_document_content = models.BooleanField(null=False, 32 | blank=False) 33 | # current task active time (in seconds) 34 | timespent = models.FloatField(default=0) 35 | # last activity timestamp 36 | last_activity = models.FloatField(default=None, null=True, blank=True) 37 | 38 | created_at = models.DateTimeField(auto_now_add=True, 39 | editable=False) 40 | updated_at = models.DateTimeField(auto_now=True) 41 | 42 | def save(self, *args, **kwargs): 43 | if not self.pk: 44 | try: 45 | CALFunctions.add_session(str(self.uuid), 46 | self.topic.seed_query, 47 | self.strategy) 48 | except (CALError, ConnectionRefusedError, Exception) as e: 49 | # TODO: log error 50 | pass 51 | super(Session, self).save(*args, **kwargs) 52 | 53 | def __unicode__(self): 54 | return "".format(self.username, self.topic.number) 55 | 56 | def __str__(self): 57 | return self.__unicode__() 58 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/progress/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from hicalweb.progress import views 4 | 5 | urlpatterns = [ 6 | url(r'^$', views.Home.as_view(), 7 | name='home'), 8 | url(r'^sessions/$', views.SessionListView.as_view(), 9 | name='sessions'), 10 | url(r'^practice/$', 11 | views.PracticeView.as_view(), 12 | name='practice'), 13 | url(r'^practice_complete/$', 14 | views.PracticeCompleteView.as_view(), 15 | name='practice_complete'), 16 | 17 | # Ajax views 18 | url(r'^post_ctrlf/$', 19 | views.CtrlFAJAXView.as_view(), 20 | name='post_ctrlf'), 21 | url(r'^post_find_keystroke/$', 22 | views.FindKeystrokeAJAXView.as_view(), 23 | name='post_find_keystroke'), 24 | url(r'^post_visit/$', 25 | views.VisitAJAXView.as_view(), 26 | name='post_visit'), 27 | url(r'^post_log/$', 28 | views.MessageAJAXView.as_view(), 29 | name='post_log_msg'), 30 | 31 | url(r'^get_session_details/$', 32 | views.SessionDetailsAJAXView.as_view(), 33 | name='get_session_details'), 34 | ] 35 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/search/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/admin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/search/admin.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class SearchConfig(AppConfig): 5 | name = 'hicalweb.search' 6 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/helpers.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Q 2 | from hicalweb.judgment.models import Judgment 3 | 4 | 5 | def join_judgments(document_values, document_ids, user, task): 6 | """ 7 | Adds the relevance judgment of the document to the document_values dict. 8 | If document has not been judged yet, `isJudged` will be False. 9 | :param user: 10 | :param task: 11 | :param document_values: 12 | :param document_ids: 13 | :return: document_values with extra information about the document 14 | """ 15 | judged_docs = Judgment.objects.filter(user=user, 16 | task=task, 17 | doc_id__in=document_ids, 18 | relevance__isnull=False) 19 | 20 | judged_docs = {j.doc_id: j for j in judged_docs} 21 | for id in document_values: 22 | is_judged = True if id in judged_docs else False 23 | 24 | if is_judged: 25 | judgment_object = judged_docs.get(id) 26 | document_values[id]['rel'] = judgment_object.relevance 27 | document_values[id]['isJudged'] = is_judged 28 | document_values[id]['relevance_judgment'] = judgment_object.relevance 29 | 30 | return document_values 31 | 32 | 33 | def padder(doc_ids): 34 | """ 35 | Pads zeros to ids to be length of 7 36 | :param doc_ids: list of document ids 37 | :return: doc ids padded with 0 to be length of 7. 38 | """ 39 | 40 | for idx, doc_id in enumerate(doc_ids): 41 | doc_id = doc_id.zfill(7) 42 | doc_ids[idx] = doc_id 43 | 44 | return doc_ids 45 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/search/migrations/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/search/models.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/search/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from hicalweb.search import views 4 | 5 | urlpatterns = [ 6 | url(r'^$', views.SearchHomePageView.as_view(), 7 | name='main'), 8 | url(r'^get_docs/$', 9 | views.SearchListView.as_view(), name='get_docs'), 10 | url(r'^get_single_doc/$', 11 | views.SearchGetDocAJAXView.as_view(), 12 | name='get_doc'), 13 | 14 | # Ajax views 15 | url(r'^post_search_status/$', views.SearchInputStatusAJAXView.as_view(), 16 | name='post_search_status'), 17 | url(r'^post_keystroke/$', views.SearchKeystrokeAJAXView.as_view(), 18 | name='post_keystroke'), 19 | url(r'^post_search_request/$', views.SearchButtonView.as_view(), 20 | name='post_search_request'), 21 | url(r'^post_search_doc_open/$', views.SearchSearchDocOpenedView.as_view(), 22 | name='post_search_doc_open'), 23 | ] 24 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/css/nprogress.css: -------------------------------------------------------------------------------- 1 | /* Make clicks pass-through */ 2 | #nprogress { 3 | pointer-events: none; 4 | } 5 | 6 | #nprogress .bar { 7 | background: #0275d8; 8 | 9 | position: fixed; 10 | z-index: 1031; 11 | top: 0; 12 | left: 0; 13 | 14 | width: 100%; 15 | height: 2px; 16 | } 17 | 18 | /* Fancy blur effect */ 19 | #nprogress .peg { 20 | display: block; 21 | position: absolute; 22 | right: 0px; 23 | width: 100px; 24 | height: 100%; 25 | box-shadow: 0 0 10px #0275d8, 0 0 5px #0275d8; 26 | opacity: 1.0; 27 | 28 | -webkit-transform: rotate(3deg) translate(0px, -4px); 29 | -ms-transform: rotate(3deg) translate(0px, -4px); 30 | transform: rotate(3deg) translate(0px, -4px); 31 | } 32 | 33 | /* Remove these to get rid of the spinner */ 34 | #nprogress .spinner { 35 | display: block; 36 | position: fixed; 37 | z-index: 1031; 38 | top: 10px; 39 | right: 3px; 40 | } 41 | 42 | #nprogress .spinner-icon { 43 | width: 18px; 44 | height: 18px; 45 | box-sizing: border-box; 46 | 47 | border: solid 2px transparent; 48 | border-top-color: #0275d8; 49 | border-left-color: #0275d8; 50 | border-radius: 50%; 51 | 52 | -webkit-animation: nprogress-spinner 400ms linear infinite; 53 | animation: nprogress-spinner 400ms linear infinite; 54 | } 55 | 56 | .nprogress-custom-parent { 57 | overflow: hidden; 58 | position: relative; 59 | } 60 | 61 | .nprogress-custom-parent #nprogress .spinner, 62 | .nprogress-custom-parent #nprogress .bar { 63 | position: absolute; 64 | } 65 | 66 | @-webkit-keyframes nprogress-spinner { 67 | 0% { -webkit-transform: rotate(0deg); } 68 | 100% { -webkit-transform: rotate(360deg); } 69 | } 70 | @keyframes nprogress-spinner { 71 | 0% { transform: rotate(0deg); } 72 | 100% { transform: rotate(360deg); } 73 | } 74 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/css/pace.css: -------------------------------------------------------------------------------- 1 | .pace { 2 | -webkit-pointer-events: none; 3 | pointer-events: none; 4 | -webkit-user-select: none; 5 | -moz-user-select: none; 6 | user-select: none; 7 | } 8 | 9 | .pace-inactive { 10 | display: none; 11 | } 12 | 13 | .pace .pace-progress { 14 | background: #6c757d; 15 | position: fixed; 16 | z-index: 2000; 17 | top: 0; 18 | right: 100%; 19 | width: 100%; 20 | height: 2px; 21 | } 22 | 23 | .pace .pace-progress-inner { 24 | display: block; 25 | position: absolute; 26 | right: 0px; 27 | width: 100px; 28 | height: 100%; 29 | box-shadow: 0 0 10px #6c757d, 0 0 5px #6c757d; 30 | opacity: 1.0; 31 | -webkit-transform: rotate(3deg) translate(0px, -4px); 32 | -moz-transform: rotate(3deg) translate(0px, -4px); 33 | -ms-transform: rotate(3deg) translate(0px, -4px); 34 | -o-transform: rotate(3deg) translate(0px, -4px); 35 | transform: rotate(3deg) translate(0px, -4px); 36 | } 37 | 38 | .pace .pace-activity { 39 | display: block; 40 | position: fixed; 41 | z-index: 2000; 42 | top: 95%; 43 | right: 15px; 44 | width: 24px; 45 | height: 24px; 46 | border: solid 2px transparent; 47 | border-top-color: #6c757d; 48 | border-left-color: #6c757d; 49 | border-radius: 10px; 50 | -webkit-animation: pace-spinner 400ms linear infinite; 51 | -moz-animation: pace-spinner 400ms linear infinite; 52 | -ms-animation: pace-spinner 400ms linear infinite; 53 | -o-animation: pace-spinner 400ms linear infinite; 54 | animation: pace-spinner 400ms linear infinite; 55 | } 56 | 57 | @-webkit-keyframes pace-spinner { 58 | 0% { 59 | -webkit-transform: rotate(0deg); 60 | transform: rotate(0deg); 61 | } 62 | 100% { 63 | -webkit-transform: rotate(360deg); 64 | transform: rotate(360deg); 65 | } 66 | } 67 | 68 | @-moz-keyframes pace-spinner { 69 | 0% { 70 | -moz-transform: rotate(0deg); 71 | transform: rotate(0deg); 72 | } 73 | 100% { 74 | -moz-transform: rotate(360deg); 75 | transform: rotate(360deg); 76 | } 77 | } 78 | 79 | @-o-keyframes pace-spinner { 80 | 0% { 81 | -o-transform: rotate(0deg); 82 | transform: rotate(0deg); 83 | } 84 | 100% { 85 | -o-transform: rotate(360deg); 86 | transform: rotate(360deg); 87 | } 88 | } 89 | 90 | @-ms-keyframes pace-spinner { 91 | 0% { 92 | -ms-transform: rotate(0deg); 93 | transform: rotate(0deg); 94 | } 95 | 100% { 96 | -ms-transform: rotate(360deg); 97 | transform: rotate(360deg); 98 | } 99 | } 100 | 101 | @keyframes pace-spinner { 102 | 0% { 103 | transform: rotate(0deg); 104 | transform: rotate(0deg); 105 | } 106 | 100% { 107 | transform: rotate(360deg); 108 | transform: rotate(360deg); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/fonts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/fonts/.gitkeep -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/cal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/cal.png -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/cal_para.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/cal_para.png -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/favicon.ico -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/hical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/hical.png -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/hicalsmall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/hicalsmall.png -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/images/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/static/images/search.png -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/js/project.js: -------------------------------------------------------------------------------- 1 | /* Project specific Javascript goes here. */ 2 | 3 | /* 4 | Formatting hack to get around crispy-forms unfortunate hardcoding 5 | in helpers.FormHelper: 6 | 7 | if template_pack == 'bootstrap4': 8 | grid_colum_matcher = re.compile('\w*col-(xs|sm|md|lg|xl)-\d+\w*') 9 | using_grid_layout = (grid_colum_matcher.match(self.label_class) or 10 | grid_colum_matcher.match(self.field_class)) 11 | if using_grid_layout: 12 | items['using_grid_layout'] = True 13 | 14 | Issues with the above approach: 15 | 16 | 1. Fragile: Assumes Bootstrap 4's API doesn't change (it does) 17 | 2. Unforgiving: Doesn't allow for any variation in template design 18 | 3. Really Unforgiving: No way to override this behavior 19 | 4. Undocumented: No mention in the documentation, or it's too hard for me to find 20 | */ 21 | $('.form-group').removeClass('row'); 22 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/js/search.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by mustafa on 2017-05-08. 3 | */ 4 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/js/timer.js: -------------------------------------------------------------------------------- 1 | function AwayTimer(showAlert){ 2 | if(showAlert === undefined) 3 | showAlert = false; 4 | 5 | this.awayTime = 0; 6 | this.backTime = 0; 7 | this.hiddenTime = 0; 8 | this.totalAwayTime = 0; 9 | this.startTime = Date.now(); 10 | 11 | var parent = this; 12 | 13 | parent.resetTimer= function(){ 14 | parent.startTime = Date.now(); 15 | parent.awayTime = 0; 16 | parent.totalAwayTime = 0; 17 | }; 18 | 19 | var awayCallback = function() { 20 | parent.awayTime = Date.now(); 21 | if(showAlert){ 22 | showAlarm("Are you still here? " + 23 | "If your are still available, please click ok to continue."); 24 | } 25 | }; 26 | 27 | var awayBackCallback = function() { 28 | parent.backTime = Date.now(); 29 | totalTime =0; 30 | if(parent.awayTime != 0){ 31 | totalTime = parent.backTime-parent.awayTime; 32 | } 33 | parent.totalAwayTime = parent.totalAwayTime+ totalTime; 34 | parent.awayTime = 0; 35 | parent.backTime = 0; 36 | }; 37 | 38 | var hiddenCallback = function() { 39 | parent.hiddenTime = Date.now(); 40 | }; 41 | 42 | var visibleCallback = function(){ 43 | parent.backTime = Date.now(); 44 | totalTime = 0; 45 | if(parent.awayTime != 0 && hiddenTime != 0){ 46 | if(parent.awayTime > hiddenTime){ 47 | totalTime = parent.backTime-parent.hiddenTime; 48 | }else{ 49 | totalTime = parent.backTime-parent.awayTime; 50 | } 51 | }else if(parent.hiddenTime != 0){ 52 | totalTime = parent.backTime-parent.hiddenTime; 53 | }else if(parent.awayTime != 0){ 54 | totalTime = parent.backTime-parent.awayTime; 55 | } 56 | 57 | 58 | parent.totalAwayTime+=totalTime; 59 | parent.awayTime = 0; 60 | parent.backTime = 0; 61 | parent.hiddenTime = 0; 62 | }; 63 | 64 | parent.idle = new Idle({ 65 | onHidden : hiddenCallback, 66 | onVisible : visibleCallback, 67 | onAway : awayCallback, 68 | onAwayBack : awayBackCallback, 69 | awayTimeout : 60000 //away with default value of the textbox 70 | }).start(); 71 | return parent; 72 | } 73 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/static/js/uuid-lib.js: -------------------------------------------------------------------------------- 1 | var validator = new RegExp("^[a-z0-9]{32}$", "i"); 2 | 3 | function gen(count) { 4 | var out = ""; 5 | for (var i=0; iForbidden (403) 7 | 8 |

CSRF verification failed. Request aborted.

9 | {% endblock content %} 10 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %}Page not found{% endblock %} 4 | 5 | {% block content %} 6 |

Page not found

7 | 8 |

This is not the page you were looking for.

9 | {% endblock content %} 10 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %}Server Error{% endblock %} 4 | 5 | {% block content %} 6 |

Ooops!!! 500

7 | 8 |

Looks like something went wrong!

9 | 10 |

We track these errors automatically, but if the problem persists feel free to contact us. In the meantime, try refreshing.

11 | {% endblock content %} 12 | 13 | 14 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/account_inactive.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% trans "Account Inactive" %}{% endblock %} 6 | 7 | {% block main %} 8 | {% block inner %} 9 |

{% trans "Account Inactive" %}

10 | 11 |

{% trans "This account is inactive." %}

12 | {% endblock %} 13 | {% endblock %} 14 | 15 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/base.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block title %}{% block head_title %}{% endblock head_title %}{% endblock title %} 3 | 4 | {% block sidebar %}{% endblock %} 5 | {% block nav-header %}{% endblock %} 6 | {% block nav-right%}{% endblock %} 7 | {% block nav-left%}{% endblock %} 8 | 9 | 10 | {% block main %} 11 | {% block content %} 12 |
13 |
14 | {% block inner %}{% endblock %} 15 |
16 |
17 | {% endblock %} 18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/email_confirm.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n account %} 4 | 5 | {% block head_title %}{% trans "Confirm E-mail Address" %}{% endblock %} 6 | 7 | 8 | {% block inner %} 9 |

{% trans "Confirm E-mail Address" %}

10 | 11 | {% if confirmation %} 12 | 13 | {% user_display confirmation.email_address.user as user_display %} 14 | 15 |

{% blocktrans with confirmation.email_address.email as email %}Please confirm that {{ email }} is an e-mail address for user {{ user_display }}.{% endblocktrans %}

16 | 17 |
18 | {% csrf_token %} 19 | 20 |
21 | 22 | {% else %} 23 | 24 | {% url 'account_email' as email_url %} 25 | 26 |

{% blocktrans %}This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request.{% endblocktrans %}

27 | 28 | {% endif %} 29 | 30 | {% endblock %} 31 | 32 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/login.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | {% load i18n static crispy_forms_tags %} 3 | 4 | {% block head_title %}{% trans "Sign In" %}{% endblock %} 5 | 6 | {% block main %} 7 | {% block inner %} 8 | 9 |
10 | 11 |
12 |
13 |

A System for Efficient High-Recall Retrieval

14 |
15 |
16 | 17 |

{% trans "Sign In" %}

18 | 19 |

{% blocktrans %}If you have not created an account yet, then please 20 | sign up first. {% endblocktrans %}

21 |

If you would like to practice using the system, click on the 22 | practice button above.

23 | 24 | 32 | 33 |
34 | 35 |
36 | 37 | 38 | {% endblock %} 39 | {% endblock %} 40 | 41 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/logout.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% trans "Sign Out" %}{% endblock %} 6 | 7 | {% block help_modal_text %} 8 | {% endblock %} 9 | 10 | {% block main %} 11 | {% block inner %} 12 |

{% trans "Sign Out" %}

13 | 14 |

{% trans 'Are you sure you want to sign out?' %}

15 | 16 |
17 | {% csrf_token %} 18 | {% if redirect_field_value %} 19 | 20 | {% endif %} 21 | 22 |
23 | 24 | 25 | {% endblock %} 26 | {% endblock %} 27 | 28 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_change.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n crispy_forms_tags %} 4 | 5 | {% block head_title %}{% trans "Change Password" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Change Password" %}

9 | 10 |
11 | {% csrf_token %} 12 | {{ form|crispy }} 13 | 14 |
15 | {% endblock %} 16 | 17 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_reset.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n account crispy_forms_tags %} 4 | 5 | {% block head_title %}{% trans "Password Reset" %}{% endblock %} 6 | 7 | {% block inner %} 8 | 9 |

{% trans "Password Reset" %}

10 | {% if user.is_authenticated %} 11 | {% include "account/snippets/already_logged_in.html" %} 12 | {% endif %} 13 | 14 |

{% trans "Forgotten your password? Enter your e-mail address below, and we'll send you an e-mail allowing you to reset it." %}

15 | 16 |
17 | {% csrf_token %} 18 | {{ form|crispy }} 19 | 20 |
21 | 22 |

{% blocktrans %}Please contact us if you have any trouble resetting your password.{% endblocktrans %}

23 | {% endblock %} 24 | 25 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_reset_done.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n account %} 4 | 5 | {% block head_title %}{% trans "Password Reset" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Password Reset" %}

9 | 10 | {% if user.is_authenticated %} 11 | {% include "account/snippets/already_logged_in.html" %} 12 | {% endif %} 13 | 14 |

{% blocktrans %}We have sent you an e-mail. Please contact us if you do not receive it within a few minutes.{% endblocktrans %}

15 | {% endblock %} 16 | 17 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_reset_from_key.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n crispy_forms_tags %} 4 | 5 | {% block head_title %}{% trans "Change Password" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% if token_fail %}{% trans "Bad Token" %}{% else %}{% trans "Change Password" %}{% endif %}

9 | 10 | {% if token_fail %} 11 | {% url 'account_reset_password' as passwd_reset_url %} 12 |

{% blocktrans %}The password reset link was invalid, possibly because it has already been used. Please request a new password reset.{% endblocktrans %}

13 | {% else %} 14 | {% if form %} 15 |
16 | {% csrf_token %} 17 | {{ form|crispy }} 18 | 19 |
20 | {% else %} 21 |

{% trans 'Your password is now changed.' %}

22 | {% endif %} 23 | {% endif %} 24 | {% endblock %} 25 | 26 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_reset_from_key_done.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | {% block head_title %}{% trans "Change Password" %}{% endblock %} 5 | 6 | {% block inner %} 7 |

{% trans "Change Password" %}

8 |

{% trans 'Your password is now changed.' %}

9 | {% endblock %} 10 | 11 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/password_set.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n crispy_forms_tags %} 4 | 5 | {% block head_title %}{% trans "Set Password" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Set Password" %}

9 | 10 |
11 | {% csrf_token %} 12 | {{ form|crispy }} 13 | 14 |
15 | {% endblock %} 16 | 17 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/signup.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n crispy_forms_tags %} 4 | 5 | {% block head_title %}{% trans "Signup" %}{% endblock %} 6 | 7 | {% block help_modal_text %} 8 |

9 | Create a user name and password to start using the system. You can also click on the practice button to use the practice account. 10 |

11 | {% endblock %} 12 | 13 | 14 | {% block main %} 15 | {% block inner %} 16 |

{% trans "Sign Up" %}

17 | 18 |

{% blocktrans %}Already have an account? Then please sign in.{% endblocktrans %}

19 | 20 | 28 | 29 | {% endblock %} 30 | {% endblock %} 31 | 32 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/signup_closed.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% trans "Sign Up Closed" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Sign Up Closed" %}

9 | 10 |

{% trans "We are sorry, but the sign up is currently closed." %}

11 | {% endblock %} 12 | 13 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/verification_sent.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% trans "Verify Your E-mail Address" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Verify Your E-mail Address" %}

9 | 10 |

{% blocktrans %}We have sent an e-mail to you for verification. Follow the link provided to finalize the signup process. Please contact us if you do not receive it within a few minutes.{% endblocktrans %}

11 | 12 | {% endblock %} 13 | 14 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/account/verified_email_required.html: -------------------------------------------------------------------------------- 1 | {% extends "account/base.html" %} 2 | 3 | {% load i18n %} 4 | 5 | {% block head_title %}{% trans "Verify Your E-mail Address" %}{% endblock %} 6 | 7 | {% block inner %} 8 |

{% trans "Verify Your E-mail Address" %}

9 | 10 | {% url 'account_email' as email_url %} 11 | 12 |

{% blocktrans %}This part of the site requires us to verify that 13 | you are who you claim to be. For this purpose, we require that you 14 | verify ownership of your e-mail address. {% endblocktrans %}

15 | 16 |

{% blocktrans %}We have sent an e-mail to you for 17 | verification. Please click on the link inside this e-mail. Please 18 | contact us if you do not receive it within a few minutes.{% endblocktrans %}

19 | 20 |

{% blocktrans %}Note: you can still change your e-mail address.{% endblocktrans %}

21 | 22 | 23 | {% endblock %} 24 | 25 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/bootstrap4/field.html: -------------------------------------------------------------------------------- 1 | 2 | {% load crispy_forms_field %} 3 | 4 | {% if field.is_hidden %} 5 | {{ field }} 6 | {% else %} 7 | {% if field|is_checkbox %} 8 |
9 | {% if label_class %} 10 |
11 | {% endif %} 12 | {% endif %} 13 | <{% if tag %}{{ tag }}{% else %}div{% endif %} id="div_{{ field.auto_id }}" {% if not field|is_checkbox %}class="form-group{% if using_grid_layout %} row{% endif %}{% else %}class="checkbox{% endif %}{% if wrapper_class %} {{ wrapper_class }}{% endif %}{% if form_show_errors%}{% if field.errors %} has-danger{% endif %}{% endif %}{% if field.css_classes %} {{ field.css_classes }}{% endif %}"> 14 | {% if field.label and not field|is_checkbox and form_show_labels %} 15 | 18 | {% endif %} 19 | 20 | {% if field|is_checkboxselectmultiple %} 21 | {% include 'bootstrap4/layout/checkboxselectmultiple.html' %} 22 | {% endif %} 23 | 24 | {% if field|is_radioselect %} 25 | {% include 'bootstrap4/layout/radioselect.html' %} 26 | {% endif %} 27 | 28 | {% if not field|is_checkboxselectmultiple and not field|is_radioselect %} 29 | {% if field|is_checkbox and form_show_labels %} 30 | 35 | {% else %} 36 |
37 | {% crispy_field field %} 38 |
39 | {% include 'bootstrap4/layout/help_text_and_errors.html' %} 40 | {% endif %} 41 | {% endif %} 42 | 43 | {% if field|is_checkbox %} 44 | {% if label_class %} 45 |
46 | {% endif %} 47 |
48 | {% endif %} 49 | {% endif %} 50 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/bootstrap4/layout/field_errors_block.html: -------------------------------------------------------------------------------- 1 | 2 | {% if form_show_errors and field.errors %} 3 | {% for error in field.errors %} 4 |

{{ error }}

5 | {% endfor %} 6 | {% endif %} 7 | 8 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/pages/about.html: -------------------------------------------------------------------------------- 1 | {% extends "old_base.html" %} 2 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/pages/home.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load i18n %} 3 | 4 | {% block pagetitle %}Home{% endblock %} 5 | 6 | 7 | {% block sidebar %} 8 | {% endblock %} 9 | 10 | {% block nav-header %} 11 |
  • 12 | Welcome to HiCAL Web 13 |
    • 14 | {% endblock %} 15 | 16 | {% block nav-right%} 17 | {% if request.user.is_authenticated %} 18 |
  • 19 | Topic 20 |
    • 21 | {% endif %} 22 | {% endblock %} 23 | 24 | 25 | {% block main %} 26 | 27 |
    28 |
    29 |
    30 |
    31 |

    HiCAL Web

    32 |
    Please choose from the following:
    33 | 34 |
    35 |
    36 | Home 37 | Tutorial 38 |
    39 | {% if request.user.is_authenticated %} 40 | Search 41 | CAL 42 | {# Stat Under construction#} 43 |
    44 | Sign out 45 | {% else %} 46 | Sign Up 47 | Sign In 48 | {% endif %} 49 | 50 |
    51 | 52 |
    53 | 54 |
    55 |
    56 | 57 | 58 | {% endblock %} 59 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/pages/search.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block pagetitle %}Search{% endblock %} 4 | 5 | 6 | {% block sidebar %} 7 |
  • 8 | S(current) 9 |
    • 10 |
  • 11 | J 12 |
    • 13 | {% comment %} 14 |
  • 15 | A 16 |
    • 17 | {% endcomment %} 18 | {% endblock %} 19 | 20 | {% block nav-header-title %} 21 |
  • 22 | 23 | TrecSearch: 24 | 25 |
    • 26 | {% endblock %} 27 | 28 | {% block main %} 29 |
    30 |
    31 |

    TrecSearch

    32 |
    33 |
    34 |
    35 | 36 | 37 | 38 | 39 |
    40 |
    41 | 42 |
    43 | 44 | {% for _ in "xxxxxxxxxxxxxxxxxxxx" %} 45 |
    46 |
    47 |
    48 |
    {% lorem 3 w random %}
    49 |
    http://www.url.com
    50 |
    
 {% lorem 30 w random %} 

    51 |
    52 |
    53 |
    54 |
    55 | 58 | 61 | 64 |
    65 |
    66 | 67 |
    68 | 69 |
    70 |
    71 | {% endfor %} 72 | 73 |
    74 | 75 | {% endblock %} 76 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/raven-config.html: -------------------------------------------------------------------------------- 1 | {% load raven %} 2 | 3 | 62 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/users/user_detail.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load static %} 3 | 4 | {% block title %}User: {{ object.username }}{% endblock %} 5 | 6 | {% block content %} 7 |
    8 | 9 |
    10 |
    11 | 12 |

    {{ object.username }}

    13 | {% if object.name %} 14 |

    {{ object.name }}

    15 | {% endif %} 16 |
    17 |
    18 | 19 | {% if object == request.user %} 20 | 21 |
    22 | 23 |
    24 | My Info 25 | E-Mail 26 | 27 |
    28 | 29 |
    30 | 31 | {% endif %} 32 | 33 | 34 |
    35 | {% endblock content %} 36 | 37 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/users/user_form.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load crispy_forms_tags %} 3 | 4 | {% block title %}{{ user.username }}{% endblock %} 5 | 6 | {% block content %} 7 |

    {{ user.username }}

    8 |
    9 | {% csrf_token %} 10 | {{ form|crispy }} 11 |
    12 |
    13 | 14 |
    15 |
    16 |
    17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/templates/users/user_list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load static i18n %} 3 | {% block title %}Members{% endblock %} 4 | 5 | {% block content %} 6 |
    7 |

    Users

    8 | 9 |
    10 | {% for user in user_list %} 11 | 12 |

    {{ user.username }}

    13 |     14 | {% endfor %} 15 |
    16 |
    17 | {% endblock content %} 18 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/time/Idle.js/core.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Idle.Js Test 5 | 6 | 7 | 8 | 12 | 23 | 24 | 25 | 27 |
      28 | 29 | 30 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/time/Idle.js/timer.js: -------------------------------------------------------------------------------- 1 | function AwayTimer(){ 2 | this.awayTime = 0; 3 | this.backTime = 0; 4 | this.hiddenTime = 0; 5 | this.totalAwayTime = 0; 6 | this.startTime = Date.now(); 7 | 8 | var parent = this; 9 | function showAlarm(msg) { 10 | alert(msg); 11 | document.body.click(); 12 | } 13 | 14 | 15 | parent.resetTimer= function(){ 16 | parent.startTime = Date.now(); 17 | parent.awayTime = 0; 18 | parent.totalAwayTime = 0; 19 | }; 20 | 21 | var awayCallback = function() { 22 | 23 | parent.awayTime = Date.now(); 24 | 25 | showAlarm("You have been away for more than 1 minute. Please stay on the interface and continue judgements."); 26 | 27 | }; 28 | var awayBackCallback = function() { 29 | parent.backTime = Date.now(); 30 | totalTime =0; 31 | if(parent.awayTime != 0){ 32 | totalTime = parent.backTime-parent.awayTime; 33 | } 34 | parent.totalAwayTime = parent.totalAwayTime+ totalTime; 35 | parent.awayTime = 0; 36 | parent.backTime = 0; 37 | }; 38 | var hiddenCallback = function() { 39 | parent.hiddenTime = Date.now(); 40 | }; 41 | var visibleCallback = function(){ 42 | parent.backTime = Date.now(); 43 | totalTime = 0; 44 | if(parent.awayTime != 0 && hiddenTime != 0){ 45 | if(parent.awayTime > hiddenTime){ 46 | totalTime = parent.backTime-parent.hiddenTime; 47 | }else{ 48 | totalTime = parent.backTime-parent.awayTime; 49 | } 50 | }else if(parent.hiddenTime != 0){ 51 | totalTime = parent.backTime-parent.hiddenTime; 52 | }else if(parent.awayTime != 0){ 53 | totalTime = parent.backTime-parent.awayTime; 54 | } 55 | 56 | 57 | parent.totalAwayTime+=totalTime; 58 | parent.awayTime = 0; 59 | parent.backTime = 0; 60 | parent.hiddenTime = 0; 61 | }; 62 | 63 | parent.idle = new Idle({ 64 | onHidden : hiddenCallback, 65 | onVisible : visibleCallback, 66 | onAway : awayCallback, 67 | onAwayBack : awayBackCallback, 68 | awayTimeout : 5000 //away with default value of the textbox 69 | }).start(); 70 | return parent; 71 | } 72 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'topic.apps.TopicConfig' 2 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from hicalweb.topic.models import Topic 3 | 4 | 5 | class TopicAdmin(admin.ModelAdmin): 6 | readonly_fields = ('seed_query', ) 7 | 8 | class Meta: 9 | model = Topic 10 | 11 | def has_add_permission(self, request): 12 | return False 13 | 14 | def has_delete_permission(self, request, obj=None): 15 | return False 16 | 17 | def get_readonly_fields(self, request, obj=None): 18 | if obj: # obj is not None, so this is an edit 19 | return ['seed_query', ] # Return a list or tuple of readonly fields' names 20 | else: # This is an addition 21 | return [] 22 | 23 | admin.site.register(Topic, TopicAdmin) 24 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class TopicConfig(AppConfig): 5 | name = 'hicalweb.topic' 6 | 7 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/topic/fixtures/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/forms.py: -------------------------------------------------------------------------------- 1 | from crispy_forms.layout import Submit 2 | from crispy_forms.helper import FormHelper 3 | 4 | from django import forms 5 | from hicalweb.topic.models import Topic 6 | 7 | 8 | class TopicForm(forms.ModelForm): 9 | """ 10 | Form for creating a new Topic 11 | 12 | """ 13 | submit_name = 'submit-topic-form' 14 | 15 | class Meta: 16 | model = Topic 17 | fields = ['title', 'number', 'seed_query', 'description'] 18 | 19 | def __init__(self, *args, **kwargs): 20 | super(TopicForm, self).__init__(*args, **kwargs) 21 | instance = getattr(self, 'instance', None) 22 | if instance and instance.pk: 23 | self.fields['seed_query'].widget.attrs['readonly'] = True 24 | self.fields['number'].widget.attrs['readonly'] = True 25 | 26 | self.helper = FormHelper(self) 27 | self.helper.layout.append( 28 | Submit(self.submit_name, u'Submit', 29 | css_class='btn btn-primary') 30 | ) 31 | 32 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/topic/management/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/management/commands/import_topics.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import re 3 | 4 | from bs4 import BeautifulSoup 5 | from django.core.management.base import BaseCommand 6 | 7 | from hicalweb.topic.models import Topic 8 | 9 | topic_path = "../../fixtures/topics.txt" 10 | 11 | 12 | class Command(BaseCommand): 13 | """ 14 | Imports pre-defined topics into database. 15 | 16 | Topics needs to be formatted in TREC XML format. For example: 17 | 18 | 19 | 20 | 3 21 | acupuncture epilepsy 22 | 23 | Can acupuncture be effective for people with epilepsy? 24 | 25 | 26 | Acupuncture, a traditional Chinese treatment which is applied by inserting 27 | thin needles in certain locations of body, has been used as a treatment for 28 | epilepsy. There are reports that it reduces the regularity and severity of 29 | epileptic episodes (seizures). A relevant document should discuss whether 30 | acupuncture can be used to treat epilepsy or control seizures and epilepsy 31 | symptoms. 32 | 33 | 34 | 35 | 36 | """ 37 | help = 'Import topics in standard TREC XML format' 38 | 39 | def handle(self, *args, **option): 40 | self.stdout.write(self.style.SUCCESS("Importing topics...")) 41 | 42 | soup = BeautifulSoup(codecs.open(topic_path, "r"), "lxml") 43 | exists = 0 44 | imported = 0 45 | for topic in soup.findAll("topic"): 46 | topic_id = topic.findNext("number").getText().strip() 47 | topic_id = re.sub("[^0-9]", "", topic_id) 48 | topic_id = int(topic_id) if topic_id else None 49 | query = topic.findNext("query").getText().strip() 50 | description = topic.findNext("description").getText().strip() 51 | narrative = topic.findNext("narrative").getText().strip() 52 | 53 | if not Topic.objects.filter(number=topic_id, title=query).exists(): 54 | Topic.objects.create( 55 | number=topic_id, 56 | title=query.capitalize(), 57 | description=description, 58 | narrative=narrative, 59 | seed_query="{} {}".format(query, description) 60 | ) 61 | imported += 1 62 | else: 63 | exists += 1 64 | 65 | self.stdout.write( 66 | self.style.SUCCESS( 67 | "Imported {} topics. {} topics already exist.".format(imported, exists) 68 | ) 69 | ) 70 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.7 on 2018-07-27 00:04 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Topic', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('number', models.PositiveIntegerField(blank=True, null=True)), 21 | ('title', models.CharField(max_length=512)), 22 | ('seed_query', models.CharField(max_length=512)), 23 | ('description', models.TextField(blank=True, null=True)), 24 | ('display_description', models.TextField(blank=True, null=True)), 25 | ('narrative', models.TextField(blank=True, null=True)), 26 | ('created_at', models.DateTimeField(auto_now_add=True)), 27 | ('updated_at', models.DateTimeField(auto_now=True)), 28 | ], 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/topic/migrations/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Topic(models.Model): 5 | number = models.PositiveIntegerField(null=True, 6 | blank=True) 7 | title = models.CharField(null=False, 8 | blank=False, 9 | max_length=512) 10 | seed_query = models.CharField(null=False, 11 | blank=False, 12 | max_length=512) 13 | description = models.TextField(null=True, 14 | blank=True) 15 | display_description = models.TextField(null=True, 16 | blank=True) 17 | narrative = models.TextField(null=True, 18 | blank=True) 19 | 20 | created_at = models.DateTimeField(auto_now_add=True, 21 | editable=False) 22 | updated_at = models.DateTimeField(auto_now=True) 23 | 24 | def __unicode__(self): 25 | return "{} {}".format(self.number, self.title) 26 | 27 | def __str__(self): 28 | return self.__unicode__() 29 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/templates/topic/create.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load crispy_forms_tags %} 3 | 4 | {% block pagetitle %}Topic create{% endblock %} 5 | 6 | {% block main %} 7 |

      Add topic

      8 |
      9 |
      10 |
      11 |
      12 | {% csrf_token %} 13 | {% crispy form %} 14 |
      15 |
      16 |
      17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/templates/topic/details.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load crispy_forms_tags %} 3 | 4 | {% block pagetitle %}Topic detail{% endblock %} 5 | 6 | {% block main %} 7 |
      8 |
      9 |

      Your current topic

      10 |
      11 |
      12 | List created topics 13 | Add new topic 14 |
      15 |
      16 |
      17 |
      18 |
      19 |
      20 | {% csrf_token %} 21 | {% crispy form %} 22 |
      23 |
      24 |
      25 | {% endblock %} 26 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/templates/topic/list.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | {% block pagetitle %}Topics list{% endblock %} 4 | 5 | {% block main %} 6 |

      Current active topic

      7 | 8 | {% if user.current_task.topic %} 9 |
      10 |
      11 | 12 |
      13 |
      14 | {{ user.current_task.topic.title }} 15 |
      16 |
      17 | {{ user.current_task.topic.seed_query }} 18 |
      19 |
      20 | 21 |
      22 |
      23 |       24 |
      25 |
      26 | {% endif %} 27 | 28 |
      29 |
      30 |

      Your created topics

      31 |
      32 |
      33 | Add new topic 34 |
      35 |
      36 | 37 | {% for topic in object_list %} 38 | {% if topic == user.current_task.topic %}{% else %} 39 |
      40 |
      41 |
      42 |
      43 | {{ topic.title }} 44 |
      45 |
      46 | {{ topic.seed_query }} 47 |
      48 |
      49 |
      50 | {% csrf_token %} 51 | 52 |
      53 |
      54 | {% csrf_token %} 55 | 56 |
      57 |
      58 |
      59 |
      60 |
      61 | {% endif %} 62 | {% endfor %} 63 | 64 | {% endblock %} 65 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/topic/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from hicalweb.topic import views 4 | 5 | urlpatterns = [ 6 | url(r'^(?P[\w.@+-]+)$', views.TopicView.as_view(), name='detail'), 7 | url(r'^create/$', views.TopicCreateView.as_view(), name='create'), 8 | url(r'^list/$', views.TopicListView.as_view(), name='list'), 9 | url(r'^activate/$', views.TopicActivateView.as_view(), name='activate'), 10 | url(r'^delete/$', views.TopicDeleteView.as_view(), name='delete'), 11 | ] 12 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/users/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/adapters.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from allauth.account.adapter import DefaultAccountAdapter 3 | from allauth.socialaccount.adapter import DefaultSocialAccountAdapter 4 | 5 | 6 | class AccountAdapter(DefaultAccountAdapter): 7 | def is_open_for_signup(self, request): 8 | return getattr(settings, 'ACCOUNT_ALLOW_REGISTRATION', True) 9 | 10 | 11 | class SocialAccountAdapter(DefaultSocialAccountAdapter): 12 | def is_open_for_signup(self, request, sociallogin): 13 | return getattr(settings, 'ACCOUNT_ALLOW_REGISTRATION', True) 14 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/admin.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from django.contrib.auth.admin import UserAdmin as AuthUserAdmin 4 | from django.contrib.auth.forms import UserChangeForm 5 | from django.contrib.auth.forms import UserCreationForm 6 | 7 | from .models import User 8 | 9 | 10 | class MyUserChangeForm(UserChangeForm): 11 | class Meta(UserChangeForm.Meta): 12 | model = User 13 | 14 | 15 | class MyUserCreationForm(UserCreationForm): 16 | 17 | error_message = UserCreationForm.error_messages.update({ 18 | 'duplicate_username': 'This username has already been taken.' 19 | }) 20 | 21 | class Meta(UserCreationForm.Meta): 22 | model = User 23 | 24 | def clean_username(self): 25 | username = self.cleaned_data["username"] 26 | try: 27 | User.objects.get(username=username) 28 | except User.DoesNotExist: 29 | return username 30 | raise forms.ValidationError(self.error_messages['duplicate_username']) 31 | 32 | 33 | @admin.register(User) 34 | class MyUserAdmin(AuthUserAdmin): 35 | form = MyUserChangeForm 36 | add_form = MyUserCreationForm 37 | fieldsets = ( 38 | ('User Profile', {'fields': ('name', 'current_task')}), 39 | ) + AuthUserAdmin.fieldsets 40 | list_display = ('username', 'name', 'is_superuser') 41 | search_fields = ['name'] 42 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class UsersConfig(AppConfig): 5 | name = 'hicalweb.users' 6 | verbose_name = "Users" 7 | 8 | def ready(self): 9 | """Override this to put in: 10 | Users system checks 11 | Users signal registration 12 | """ 13 | pass 14 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/users/migrations/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import AbstractUser 2 | from django.core.urlresolvers import reverse 3 | from django.db import models 4 | from django.utils.encoding import python_2_unicode_compatible 5 | from django.utils.translation import ugettext_lazy as _ 6 | 7 | from hicalweb.progress.models import Session 8 | 9 | 10 | @python_2_unicode_compatible 11 | class User(AbstractUser): 12 | 13 | # First Name and Last Name do not cover name patterns 14 | # around the globe. 15 | name = models.CharField(_('Name of User'), blank=True, max_length=255) 16 | current_task = models.ForeignKey(Session, blank=True, null=True) 17 | 18 | def __str__(self): 19 | return self.username 20 | 21 | def get_absolute_url(self): 22 | return reverse('users:detail', kwargs={'username': self.username}) 23 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/users/templatetags/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/templatetags/has_group.py: -------------------------------------------------------------------------------- 1 | from django import template 2 | from django.contrib.auth.models import Group 3 | 4 | register = template.Library() 5 | 6 | 7 | @register.filter(name='has_group') 8 | def has_group(user, group_name): 9 | try: 10 | group = Group.objects.get(name=group_name) 11 | except: 12 | return False # group doesn't exist, so for sure the user isn't part of the group 13 | 14 | return user.groups.filter(name=group_name).exists() 15 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/users/tests/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/factories.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | 4 | class UserFactory(factory.django.DjangoModelFactory): 5 | username = factory.Sequence(lambda n: 'user-{0}'.format(n)) 6 | email = factory.Sequence(lambda n: 'user-{0}@example.com'.format(n)) 7 | password = factory.PostGenerationMethodCall('set_password', 'password') 8 | 9 | class Meta: 10 | model = 'users.User' 11 | django_get_or_create = ('username', ) 12 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/test_admin.py: -------------------------------------------------------------------------------- 1 | from test_plus.test import TestCase 2 | 3 | from ..admin import MyUserCreationForm 4 | 5 | 6 | class TestMyUserCreationForm(TestCase): 7 | 8 | def setUp(self): 9 | self.user = self.make_user('notalamode', 'notalamodespassword') 10 | 11 | def test_clean_username_success(self): 12 | # Instantiate the form with a new username 13 | form = MyUserCreationForm({ 14 | 'username': 'alamode', 15 | 'password1': '7jefB#f@Cc7YJB]2v', 16 | 'password2': '7jefB#f@Cc7YJB]2v', 17 | }) 18 | # Run is_valid() to trigger the validation 19 | valid = form.is_valid() 20 | self.assertTrue(valid) 21 | 22 | # Run the actual clean_username method 23 | username = form.clean_username() 24 | self.assertEqual('alamode', username) 25 | 26 | def test_clean_username_false(self): 27 | # Instantiate the form with the same username as self.user 28 | form = MyUserCreationForm({ 29 | 'username': self.user.username, 30 | 'password1': 'notalamodespassword', 31 | 'password2': 'notalamodespassword', 32 | }) 33 | # Run is_valid() to trigger the validation, which is going to fail 34 | # because the username is already taken 35 | valid = form.is_valid() 36 | self.assertFalse(valid) 37 | 38 | # The form.errors dict should contain a single error called 'username' 39 | self.assertTrue(len(form.errors) == 1) 40 | self.assertTrue('username' in form.errors) 41 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from test_plus.test import TestCase 2 | 3 | 4 | class TestUser(TestCase): 5 | 6 | def setUp(self): 7 | self.user = self.make_user() 8 | 9 | def test__str__(self): 10 | self.assertEqual( 11 | self.user.__str__(), 12 | 'testuser' # This is the default username for self.make_user() 13 | ) 14 | 15 | def test_get_absolute_url(self): 16 | self.assertEqual( 17 | self.user.get_absolute_url(), 18 | '/users/testuser/' 19 | ) 20 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/test_urls.py: -------------------------------------------------------------------------------- 1 | from django.core.urlresolvers import reverse, resolve 2 | 3 | from test_plus.test import TestCase 4 | 5 | 6 | class TestUserURLs(TestCase): 7 | """Test URL patterns for users app.""" 8 | 9 | def setUp(self): 10 | self.user = self.make_user() 11 | 12 | def test_list_reverse(self): 13 | """users:list should reverse to /users/.""" 14 | self.assertEqual(reverse('users:list'), '/users/') 15 | 16 | def test_list_resolve(self): 17 | """/users/ should resolve to users:list.""" 18 | self.assertEqual(resolve('/users/').view_name, 'users:list') 19 | 20 | def test_redirect_reverse(self): 21 | """users:redirect should reverse to /users/~redirect/.""" 22 | self.assertEqual(reverse('users:redirect'), '/users/~redirect/') 23 | 24 | def test_redirect_resolve(self): 25 | """/users/~redirect/ should resolve to users:redirect.""" 26 | self.assertEqual( 27 | resolve('/users/~redirect/').view_name, 28 | 'users:redirect' 29 | ) 30 | 31 | def test_detail_reverse(self): 32 | """users:detail should reverse to /users/testuser/.""" 33 | self.assertEqual( 34 | reverse('users:detail', kwargs={'username': 'testuser'}), 35 | '/users/testuser/' 36 | ) 37 | 38 | def test_detail_resolve(self): 39 | """/users/testuser/ should resolve to users:detail.""" 40 | self.assertEqual(resolve('/users/testuser/').view_name, 'users:detail') 41 | 42 | def test_update_reverse(self): 43 | """users:update should reverse to /users/~update/.""" 44 | self.assertEqual(reverse('users:update'), '/users/~update/') 45 | 46 | def test_update_resolve(self): 47 | """/users/~update/ should resolve to users:update.""" 48 | self.assertEqual( 49 | resolve('/users/~update/').view_name, 50 | 'users:update' 51 | ) 52 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/tests/test_views.py: -------------------------------------------------------------------------------- 1 | from django.test import RequestFactory 2 | 3 | from test_plus.test import TestCase 4 | 5 | from ..views import ( 6 | UserRedirectView, 7 | UserUpdateView 8 | ) 9 | 10 | 11 | class BaseUserTestCase(TestCase): 12 | 13 | def setUp(self): 14 | self.user = self.make_user() 15 | self.factory = RequestFactory() 16 | 17 | 18 | class TestUserRedirectView(BaseUserTestCase): 19 | 20 | def test_get_redirect_url(self): 21 | # Instantiate the view directly. Never do this outside a test! 22 | view = UserRedirectView() 23 | # Generate a fake request 24 | request = self.factory.get('/fake-url') 25 | # Attach the user to the request 26 | request.user = self.user 27 | # Attach the request to the view 28 | view.request = request 29 | # Expect: '/users/testuser/', as that is the default username for 30 | # self.make_user() 31 | self.assertEqual( 32 | view.get_redirect_url(), 33 | '/users/testuser/' 34 | ) 35 | 36 | 37 | class TestUserUpdateView(BaseUserTestCase): 38 | 39 | def setUp(self): 40 | # call BaseUserTestCase.setUp() 41 | super(TestUserUpdateView, self).setUp() 42 | # Instantiate the view directly. Never do this outside a test! 43 | self.view = UserUpdateView() 44 | # Generate a fake request 45 | request = self.factory.get('/fake-url') 46 | # Attach the user to the request 47 | request.user = self.user 48 | # Attach the request to the view 49 | self.view.request = request 50 | 51 | def test_get_success_url(self): 52 | # Expect: '/users/testuser/', as that is the default username for 53 | # self.make_user() 54 | self.assertEqual( 55 | self.view.get_success_url(), 56 | '/users/testuser/' 57 | ) 58 | 59 | def test_get_object(self): 60 | # Expect: self.user, as that is the request's user object 61 | self.assertEqual( 62 | self.view.get_object(), 63 | self.user 64 | ) 65 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | url( 7 | regex=r'^$', 8 | view=views.UserListView.as_view(), 9 | name='list' 10 | ), 11 | url( 12 | regex=r'^~redirect/$', 13 | view=views.UserRedirectView.as_view(), 14 | name='redirect' 15 | ), 16 | url( 17 | regex=r'^(?P[\w.@+-]+)/$', 18 | view=views.UserDetailView.as_view(), 19 | name='detail' 20 | ), 21 | url( 22 | regex=r'^~update/$', 23 | view=views.UserUpdateView.as_view(), 24 | name='update' 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/users/views.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.mixins import LoginRequiredMixin 2 | from django.core.urlresolvers import reverse 3 | from django.views.generic import DetailView 4 | from django.views.generic import ListView 5 | from django.views.generic import RedirectView 6 | from django.views.generic import UpdateView 7 | 8 | from .models import User 9 | 10 | 11 | class UserDetailView(LoginRequiredMixin, DetailView): 12 | model = User 13 | # These next two lines tell the view to index lookups by username 14 | slug_field = 'username' 15 | slug_url_kwarg = 'username' 16 | 17 | 18 | class UserRedirectView(LoginRequiredMixin, RedirectView): 19 | permanent = False 20 | 21 | def get_redirect_url(self): 22 | return reverse('users:detail', 23 | kwargs={'username': self.request.user.username}) 24 | 25 | 26 | class UserUpdateView(LoginRequiredMixin, UpdateView): 27 | 28 | fields = ['name', ] 29 | 30 | # we already imported User in the view code above, remember? 31 | model = User 32 | 33 | # send the user back to their own page after a successful update 34 | def get_success_url(self): 35 | return reverse('users:detail', 36 | kwargs={'username': self.request.user.username}) 37 | 38 | def get_object(self): 39 | # Only get the User record for the user making the request 40 | return User.objects.get(username=self.request.user.username) 41 | 42 | 43 | class UserListView(LoginRequiredMixin, ListView): 44 | model = User 45 | # These next two lines tell the view to index lookups by username 46 | slug_field = 'username' 47 | slug_url_kwarg = 'username' 48 | -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/util/__init__.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/util/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/HiCALWeb/hicalweb/util/helpers.py -------------------------------------------------------------------------------- /HiCALWeb/hicalweb/util/topics/topic_formattor.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | # Our own modified descrptions 4 | nistTopicDict = {} 5 | with open("shortQuery.tsv") as f: 6 | for line in f: 7 | topicid = int(line.split('\t')[0]) 8 | desc = line.split('\t')[-2] 9 | nistTopicDict[topicid] = desc 10 | 11 | with open("topics") as f: 12 | lines = f.readlines() 13 | 14 | topics = {} 15 | it = iter(lines) 16 | try: 17 | while True: 18 | l = next(it) 19 | if l == '\n': 20 | num = int(next(it).split(" Number: ")[1].replace("", "").rstrip()) 21 | next(it) 22 | title = next(it).rstrip() 23 | next(it) 24 | next(it) 25 | desc = next(it).rstrip() 26 | next(it) 27 | next(it) 28 | narr = next(it).rstrip() 29 | 30 | topic = { 31 | "num": num, 32 | "title": title, 33 | "desc": desc, 34 | "narr": narr 35 | } 36 | topics[num] = topic 37 | except StopIteration: 38 | pass 39 | 40 | 41 | # reformat to 42 | # [ 43 | # { 44 | # "model": "topic.topic", 45 | # "pk": 1, 46 | # "fields": { 47 | # "number": 336, 48 | # "title": "Black Bear Attacks", 49 | # "seed_query": "Black Bear Attacks Black Bear Attacks", 50 | # "description": "Black Bear Attacks", 51 | # "display_description": "Black Bear Attacks are relevant", 52 | # "narrative": "Black Bear Attacks", 53 | # "created_at": "2017-06-22T20:18:35.279Z", 54 | # "updated_at": "2017-06-22T20:18:35.279Z" 55 | # } 56 | # }, 57 | # ... 58 | # ] 59 | l = [] 60 | pk = 1 61 | for t in topics: 62 | display_description = None 63 | # if t in nistTopicDict: 64 | # display_description = nistTopicDict[t] 65 | display_description = topics[t]["desc"] 66 | d = { 67 | "model": "topic.topic", 68 | "pk": pk, 69 | "fields": { 70 | "number": t, 71 | "title": topics[t]["title"], 72 | "seed_query": "{} {}".format(topics[t]["title"], topics[t]["desc"]), 73 | "description": topics[t]["desc"], 74 | "display_description": display_description, 75 | "narrative": topics[t]["narr"], 76 | "created_at": "2017-06-22T20:18:35.279Z", 77 | "updated_at": "2017-06-22T20:18:35.279Z" 78 | } 79 | } 80 | pk += 1 81 | l.append(d) 82 | 83 | print(json.dumps(l)) 84 | -------------------------------------------------------------------------------- /HiCALWeb/logs/README.md: -------------------------------------------------------------------------------- 1 | # Log files 2 | 3 | server and web logs are stored here. 4 | -------------------------------------------------------------------------------- /HiCALWeb/logs/django_treccoreweb_format.json: -------------------------------------------------------------------------------- 1 | { 2 | "hicalweb_log": { 3 | "title" : "hicalweb's log format", 4 | "description" : "Log format for hicalweb", 5 | "regex" : { 6 | "basic" : { 7 | "pattern": "^\\[(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{6})\\] (?\\w+) \\[(?[^:]+):(?\\d+)\\] (?.*)$" 8 | } 9 | }, 10 | "level-field" : "level", 11 | "level" : { 12 | "info": "INFO", 13 | "error" : "ERROR", 14 | "warning" : "WARNING" 15 | }, 16 | "value" : { 17 | "code_line" : { 18 | "kind" : "integer", 19 | "foreign-key": true 20 | }, 21 | "jmsg" : { 22 | "kind" : "json", 23 | "hidden": "true" 24 | } 25 | }, 26 | "sample" : [ 27 | { 28 | "line": "[2017-06-22 17:32:22.000287] INFO [topic.views.post:81] [{\"user\": \"a\", \"client_time\": 1498152742245, \"result\": {\"message\": \"Topics create page visit.\"}}]" 29 | } 30 | ] 31 | 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /HiCALWeb/logs/user_logs.py: -------------------------------------------------------------------------------- 1 | import click 2 | import json 3 | import re 4 | 5 | import sys 6 | from django.core.management.base import OutputWrapper 7 | from django_extensions.management.color import color_style 8 | 9 | LOG_RE = "^\\[(?P\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{6})\\] (?P\\w+) \\[(?P[^:]+):(?P\\d+)\\] (?P.*)$" 10 | h = re.compile(LOG_RE) 11 | 12 | stdout = OutputWrapper(sys.stdout) 13 | stderr = OutputWrapper(sys.stderr) 14 | style = color_style() 15 | stderr.style_func = style.ERROR 16 | 17 | @click.command() 18 | @click.argument('input', type=click.Path(exists=True)) 19 | @click.argument('user', type=str) 20 | def user_logs(input, user): 21 | logs = [] 22 | stdout.write(style.SUCCESS("Parsing '{}' logs from '{}' ".format(user, input))) 23 | with open(input, 'r') as f: 24 | errors_count = 0 25 | for l in f: 26 | matches = h.match(l) 27 | level = matches.group('level') 28 | try: 29 | jmsg = json.loads(matches.group('jmsg'))[0] 30 | log_user = jmsg['user'] 31 | if log_user != user: 32 | continue 33 | client_time = jmsg['client_time'] 34 | result = jmsg['result'] 35 | logs.append((client_time, level, user, result)) 36 | except: 37 | errors_count += 1 38 | stdout.write(style.ERROR("{}: Could not parse log message: {}".format( 39 | errors_count, matches.group('jmsg')))) 40 | 41 | out = "{}.{}".format(user, input) 42 | with open(out, 'wt') as f: 43 | for l in logs: 44 | line = "{} {} {} : {}\n".format(l[0], l[1], l[2], l[3]) 45 | f.write(line) 46 | 47 | stdout.write(style.SUCCESS("Parsing {} from {} completed. " 48 | "Output written to {}".format(user, input, out))) 49 | 50 | if __name__ == '__main__': 51 | user_logs() 52 | -------------------------------------------------------------------------------- /HiCALWeb/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == '__main__': 6 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') 7 | 8 | try: 9 | from django.core.management import execute_from_command_line 10 | except ImportError: 11 | # The above import may fail for some other reason. Ensure that the 12 | # issue is really that Django is missing to avoid masking other 13 | # exceptions on Python 2. 14 | try: 15 | import django # noqa 16 | except ImportError: 17 | raise ImportError( 18 | "Couldn't import Django. Are you sure it's installed and " 19 | "available on your PYTHONPATH environment variable? Did you " 20 | "forget to activate a virtual environment?" 21 | ) 22 | raise 23 | 24 | # This allows easy placement of apps within the interior 25 | # hicalweb directory. 26 | current_path = os.path.dirname(os.path.abspath(__file__)) 27 | sys.path.append(os.path.join(current_path, 'hicalweb')) 28 | 29 | execute_from_command_line(sys.argv) 30 | -------------------------------------------------------------------------------- /HiCALWeb/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE=config.settings.test 3 | -------------------------------------------------------------------------------- /HiCALWeb/requirements/base.txt: -------------------------------------------------------------------------------- 1 | # Wheel 0.25+ needed to install certain packages on CPython 3.5+ 2 | # like Pillow and psycopg2 3 | # See http://bitly.com/wheel-building-fails-CPython-35 4 | # Verified bug on Python 3.5.1 5 | wheel==0.29.0 6 | 7 | 8 | # Bleeding edge Django 9 | django==1.10.7 # pyup: >=1.10,<1.11 10 | 11 | # Configuration 12 | django-environ==0.4.3 13 | 14 | 15 | 16 | # Forms 17 | django-braces==1.11.0 18 | django-crispy-forms==1.9.1 19 | 20 | # Models 21 | django-model-utils==3.0.0 22 | 23 | # Images 24 | Pillow==4.1.1 25 | 26 | # Password storage 27 | argon2-cffi==16.3.0 28 | 29 | # For user registration, either via email or social 30 | # Well-built with regular release cycles! 31 | django-allauth==0.32.0 32 | 33 | 34 | # Python-PostgreSQL Database Adapter 35 | psycopg2==2.8.0 36 | 37 | # Unicode slugification 38 | awesome-slugify==1.6.5 39 | 40 | # Time zones support 41 | pytz==2017.2 42 | 43 | # Redis support 44 | django-redis==4.8.0 45 | redis>=2.10.5 46 | 47 | 48 | # Your custom requirements go here 49 | httplib2 50 | xmltodict 51 | lxml 52 | raven 53 | 54 | uwsgi 55 | 56 | beautifulsoup4==4.4.0 57 | -------------------------------------------------------------------------------- /HiCALWeb/requirements/local.txt: -------------------------------------------------------------------------------- 1 | # Local development dependencies go here 2 | -r base.txt 3 | 4 | coverage==4.3.4 5 | django-coverage-plugin==1.5.0 6 | 7 | Sphinx==1.5.5 8 | django-extensions==1.7.8 9 | Werkzeug==0.12.1 10 | django-test-plus==1.0.17 11 | factory-boy==2.8.1 12 | 13 | django-debug-toolbar==1.7 14 | 15 | # improved REPL 16 | ipdb==0.10.3 17 | 18 | pytest-django==3.1.2 19 | pytest-sugar==0.8.0 20 | -------------------------------------------------------------------------------- /HiCALWeb/requirements/production.txt: -------------------------------------------------------------------------------- 1 | # Pro-tip: Try not to put anything here. Avoid dependencies in 2 | # production that aren't in development. 3 | -r base.txt 4 | 5 | 6 | 7 | # WSGI Handler 8 | # ------------------------------------------------ 9 | gevent==1.2.1 10 | gunicorn==19.7.1 11 | 12 | # Static and Media Storage 13 | # ------------------------------------------------ 14 | boto==2.46.1 15 | django-storages-redux==1.3.2 16 | Collectfast==0.5.2 17 | 18 | # Email backends for Mailgun, Postmark, SendGrid and more 19 | # ------------------------------------------------------- 20 | django-anymail==0.9 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /HiCALWeb/requirements/test.txt: -------------------------------------------------------------------------------- 1 | # Test dependencies go here. 2 | -r base.txt 3 | 4 | 5 | 6 | coverage==4.3.4 7 | flake8==3.3.0 # pyup: != 2.6.0 8 | django-test-plus==1.0.17 9 | factory-boy==2.8.1 10 | 11 | # pytest 12 | pytest-django==3.1.2 13 | pytest-sugar==0.8.0 14 | -------------------------------------------------------------------------------- /HiCALWeb/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 90 3 | exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules 4 | 5 | [pycodestyle] 6 | max-line-length = 120 7 | exclude=.tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules 8 | -------------------------------------------------------------------------------- /HiCALWeb/utility/install_python_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WORK_DIR="$(dirname "$0")" 4 | PROJECT_DIR="$(dirname "$WORK_DIR")" 5 | 6 | pip --version >/dev/null 2>&1 || { 7 | echo >&2 -e "\npip is required but it's not installed." 8 | echo >&2 -e "You can install it by running the following command:\n" 9 | echo >&2 "wget https://bootstrap.pypa.io/get-pip.py --output-document=get-pip.py; chmod +x get-pip.py; sudo -H python3 get-pip.py" 10 | echo >&2 -e "\n" 11 | echo >&2 -e "\nFor more information, see pip documentation: https://pip.pypa.io/en/latest/" 12 | exit 1; 13 | } 14 | 15 | virtualenv --version >/dev/null 2>&1 || { 16 | echo >&2 -e "\nvirtualenv is required but it's not installed." 17 | echo >&2 -e "You can install it by running the following command:\n" 18 | echo >&2 "sudo -H pip3 install virtualenv" 19 | echo >&2 -e "\n" 20 | echo >&2 -e "\nFor more information, see virtualenv documentation: https://virtualenv.pypa.io/en/latest/" 21 | exit 1; 22 | } 23 | 24 | if [ -z "$VIRTUAL_ENV" ]; then 25 | echo >&2 -e "\nYou need activate a virtualenv first" 26 | echo >&2 -e 'If you do not have a virtualenv created, run the following command to create and automatically activate a new virtualenv named "venv" on current folder:\n' 27 | echo >&2 -e "virtualenv venv --python=\`which python3\`" 28 | echo >&2 -e "\nTo leave/disable the currently active virtualenv, run the following command:\n" 29 | echo >&2 "deactivate" 30 | echo >&2 -e "\nTo activate the virtualenv again, run the following command:\n" 31 | echo >&2 "source venv/bin/activate" 32 | echo >&2 -e "\nFor more information, see virtualenv documentation: https://virtualenv.pypa.io/en/latest/" 33 | echo >&2 -e "\n" 34 | exit 1; 35 | else 36 | 37 | pip install -r $PROJECT_DIR/requirements/local.txt 38 | pip install -r $PROJECT_DIR/requirements/test.txt 39 | 40 | fi 41 | -------------------------------------------------------------------------------- /HiCALWeb/utility/requirements-jessie.apt: -------------------------------------------------------------------------------- 1 | ##basic build dependencies of various Django apps for Debian Jessie 8.x 2 | #build-essential metapackage install: make, gcc, g++, 3 | build-essential 4 | #required to translate 5 | gettext 6 | python3-dev 7 | 8 | ##shared dependencies of: 9 | ##Pillow, pylibmc 10 | zlib1g-dev 11 | 12 | ##Postgresql and psycopg2 dependencies 13 | libpq-dev 14 | 15 | ##Pillow dependencies 16 | libtiff5-dev 17 | libjpeg62-turbo-dev 18 | libfreetype6-dev 19 | liblcms2-dev 20 | libwebp-dev 21 | 22 | ##django-extensions 23 | graphviz-dev 24 | -------------------------------------------------------------------------------- /HiCALWeb/utility/requirements-trusty.apt: -------------------------------------------------------------------------------- 1 | ##basic build dependencies of various Django apps for Ubuntu Trusty 14.04 2 | #build-essential metapackage install: make, gcc, g++, 3 | build-essential 4 | #required to translate 5 | gettext 6 | python3-dev 7 | 8 | ##shared dependencies of: 9 | ##Pillow, pylibmc 10 | zlib1g-dev 11 | 12 | ##Postgresql and psycopg2 dependencies 13 | libpq-dev 14 | 15 | ##Pillow dependencies 16 | libtiff4-dev 17 | libjpeg8-dev 18 | libfreetype6-dev 19 | liblcms1-dev 20 | libwebp-dev 21 | 22 | ##django-extensions 23 | graphviz-dev 24 | -------------------------------------------------------------------------------- /HiCALWeb/utility/requirements-xenial.apt: -------------------------------------------------------------------------------- 1 | ##basic build dependencies of various Django apps for Ubuntu Xenial 16.04 2 | #build-essential metapackage install: make, gcc, g++, 3 | build-essential 4 | #required to translate 5 | gettext 6 | python3-dev 7 | 8 | ##shared dependencies of: 9 | ##Pillow, pylibmc 10 | zlib1g-dev 11 | 12 | ##Postgresql and psycopg2 dependencies 13 | libpq-dev 14 | 15 | ##Pillow dependencies 16 | libtiff5-dev 17 | libjpeg8-dev 18 | libfreetype6-dev 19 | liblcms2-dev 20 | libwebp-dev 21 | 22 | ##django-extensions 23 | graphviz-dev 24 | -------------------------------------------------------------------------------- /config/nginx/fastcgi.conf: -------------------------------------------------------------------------------- 1 | 2 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 3 | fastcgi_param QUERY_STRING $query_string; 4 | fastcgi_param REQUEST_METHOD $request_method; 5 | fastcgi_param CONTENT_TYPE $content_type; 6 | fastcgi_param CONTENT_LENGTH $content_length; 7 | 8 | fastcgi_param SCRIPT_NAME $fastcgi_script_name; 9 | fastcgi_param REQUEST_URI $request_uri; 10 | fastcgi_param URI $uri; 11 | fastcgi_param DOCUMENT_URI $document_uri; 12 | fastcgi_param DOCUMENT_ROOT $document_root; 13 | fastcgi_param SERVER_PROTOCOL $server_protocol; 14 | fastcgi_param REQUEST_SCHEME $scheme; 15 | fastcgi_param HTTPS $https if_not_empty; 16 | 17 | fastcgi_param GATEWAY_INTERFACE CGI/1.1; 18 | fastcgi_param SERVER_SOFTWARE nginx/$nginx_version; 19 | 20 | fastcgi_param REMOTE_ADDR $remote_addr; 21 | fastcgi_param REMOTE_PORT $remote_port; 22 | fastcgi_param SERVER_ADDR $server_addr; 23 | fastcgi_param SERVER_PORT $server_port; 24 | fastcgi_param SERVER_NAME $server_name; 25 | 26 | # PHP only, required if PHP was built with --enable-force-cgi-redirect 27 | fastcgi_param REDIRECT_STATUS 200; 28 | -------------------------------------------------------------------------------- /config/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user root; 2 | worker_processes auto; 3 | worker_cpu_affinity auto; 4 | pcre_jit on; 5 | 6 | events { 7 | worker_connections 2048; 8 | } 9 | 10 | 11 | http { 12 | log_format upstream_time '$remote_addr - [$time_local] ' 13 | '"$request" $status $body_bytes_sent ' 14 | 'rt=$request_time'; 15 | access_log /var/log/nginx/access.log upstream_time; 16 | server { 17 | listen 9001; 18 | location /CAL/ { 19 | fastcgi_split_path_info ^/CAL(/.*)(\?.*)?$; 20 | fastcgi_param RELATIVE_URI $fastcgi_script_name; 21 | fastcgi_pass cal:8002; 22 | include /etc/nginx/fastcgi.conf; 23 | } 24 | } 25 | 26 | upstream django { 27 | server django:8001; # for a file socket 28 | } 29 | 30 | server { 31 | listen 9000; 32 | 33 | location /static { 34 | autoindex on; 35 | include /etc/nginx/mime.types; 36 | alias /app/staticfiles; 37 | add_header 'Cache-Control' 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; 38 | expires off; 39 | } 40 | location / { 41 | include uwsgi_params; # the uwsgi_params file you installed 42 | uwsgi_pass django; 43 | } 44 | location /doc { 45 | autoindex off; 46 | alias /data/docs; 47 | } 48 | location /para { 49 | autoindex off; 50 | alias /data/para; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /config/nginx/uwsgi_params: -------------------------------------------------------------------------------- 1 | uwsgi_param QUERY_STRING $query_string; 2 | uwsgi_param REQUEST_METHOD $request_method; 3 | uwsgi_param CONTENT_TYPE $content_type; 4 | uwsgi_param CONTENT_LENGTH $content_length; 5 | uwsgi_param REQUEST_URI $request_uri; 6 | uwsgi_param PATH_INFO $document_uri; 7 | uwsgi_param DOCUMENT_ROOT $document_root; 8 | uwsgi_param SERVER_PROTOCOL $server_protocol; 9 | uwsgi_param REQUEST_SCHEME $scheme; 10 | uwsgi_param HTTPS $https if_not_empty; 11 | uwsgi_param REMOTE_ADDR $remote_addr; 12 | uwsgi_param REMOTE_PORT $remote_port; 13 | uwsgi_param SERVER_PORT $server_port; 14 | uwsgi_param SERVER_NAME $server_name; -------------------------------------------------------------------------------- /images/hical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hical/HiCAL/b2e6aa56c0908f2c8af8f10162a17657b131a50c/images/hical.png --------------------------------------------------------------------------------