├── README.md ├── Slide_MSRS_ACM CIKM 2019.pdf ├── Slide_MSRS_ACM RecSys 2019.pdf ├── UBMSRS.jar ├── UBMSRS ├── .classpath ├── .project ├── bin │ ├── .gitignore │ ├── log4j.properties │ ├── msrs │ │ ├── baseline │ │ │ ├── Rankp.class │ │ │ └── ubrec │ │ │ │ ├── RunUBRec.class │ │ │ │ ├── UBRec.class │ │ │ │ └── UBRecProblem.class │ │ ├── demo │ │ │ ├── Config.class │ │ │ └── UBMSRS.class │ │ ├── learnalpha │ │ │ ├── EduProblem.class │ │ │ ├── EduRec.class │ │ │ └── RunEduProblem_LearnAlpha.class │ │ ├── learnexpectations │ │ │ ├── EduProblem.class │ │ │ ├── EduRec.class │ │ │ └── RunEduProblem_LearnExpectations.class │ │ ├── onestage │ │ │ ├── EduProblem.class │ │ │ └── EduRec.class │ │ └── twostage │ │ │ ├── EduProblem.class │ │ │ └── EduRec.class │ └── setting.conf ├── data │ ├── Sample of Outputs_LearnAlpha.txt │ ├── Sample of Outputs_LearnExpectations.txt │ ├── Sample of Outputs_OneStage.txt │ ├── Sample of Outputs_TwoStage.txt │ ├── expectations_student_learned_by_UBRec_NDCG_0.214.csv │ ├── ratings_instructor.csv │ ├── ratings_student.csv │ ├── ratings_student_candidates.csv │ ├── ratings_student_test.csv │ └── ratings_student_train.csv ├── jar │ ├── UBMSRS.jar │ └── setting.conf ├── lib │ ├── JMetal-4.3.jar │ ├── MOEAFramework-2.12.jar │ ├── commons-cli-1.2.jar │ ├── commons-codec-1.8.jar │ ├── commons-lang3-3.1.jar │ ├── commons-math3-3.4.1.jar │ ├── guava-15.0.jar │ ├── happy.coding.utils-1.2.5.jar │ ├── jcommon-1.0.20.jar │ ├── jfreechart-1.0.15.jar │ ├── log4j-1.2.16.jar │ ├── rsyntaxtextarea.jar │ ├── slf4j-api-1.6.4.jar │ └── slf4j-log4j12-1.6.4.jar ├── logs │ └── demo.log └── src │ └── main │ ├── java │ └── msrs │ │ ├── baseline │ │ ├── Rankp.java │ │ └── ubrec │ │ │ ├── RunUBRec.java │ │ │ ├── UBRec.java │ │ │ └── UBRecProblem.java │ │ ├── demo │ │ ├── Config.java │ │ └── UBMSRS.java │ │ ├── learnalpha │ │ ├── EduProblem.java │ │ ├── EduRec.java │ │ └── RunEduProblem_LearnAlpha.java │ │ ├── learnexpectations │ │ ├── EduProblem.java │ │ ├── EduRec.java │ │ └── RunEduProblem_LearnExpectations.java │ │ ├── onestage │ │ ├── EduProblem.java │ │ ├── EduRec.java │ │ └── RunEduProblem_OneStage.java │ │ └── twostage │ │ ├── EduProblem.java │ │ ├── EduRec.java │ │ └── RunEduProblem_TwoStage.java │ └── resources │ ├── log4j.properties │ └── setting.conf ├── data ├── Sample of Outputs_OneStage.txt ├── Sample of Outputs_TwoStage.txt ├── expectations_student_learned_by_UBRec_NDCG_0.214.csv ├── ratings_instructor.csv ├── ratings_student.csv ├── ratings_student_candidates.csv ├── ratings_student_test.csv └── ratings_student_train.csv └── setting.conf /README.md: -------------------------------------------------------------------------------- 1 | # Tutorial: Multi-Stakeholder Recommender Systems (MSRS) 2 | ### Introduction 3 | ------------------- 4 | Recommender systems are able to produce a list of recommended items tailored to user preferences, while the end user is the only stakeholder in the system. However, there could be multiple stakeholders in several applications or domains, e.g., e-commerce, advertising, educations, dating, job seeking, and so forth. It is useful to produce the recommendations by balancing the needs of different stakeholders. This tutorial covers the introductions to multi-stakeholder recommender systems (MSRS), introduces multiple case studies, discusses the corresponding methods and challenges to develop MSRS. Particularly, a demo based on the MOEA framework will be given in the talk. 5 | 6 | ### Demo 7 | ------------------- 8 | The demo shows case of utility-based multi-stakeholder recommender systems (UBMSRS) by using an educational data as case study. 9 | 10 | * **Setting**: To simplify the problem, the demo performs a hold-out evaluation by using 70% as training and 30% as testing. Recommendation performance was evaluated based on F1 and NDCG over the top-N recommendations, while you can change the value of N in the configuration file "setting.conf". Items with rating > 3 are considered as "relevant" items in the top-N recommendations. 11 | * **MOEA**: In terms of the multi-objective optimizers (MOO), we use the library [MOEA](https://github.com/MOEAFramework/MOEAFramework). We use the default learning parameters in MOEA. To further tune up the models, change the source codes by referring to the [Beginner's guide to the MOEA Framework](http://moeaframework.org/documentation.html) 12 | * **Baselines**: **UBRec** refers to the utility-based multi-criteria recommendation model which considers student preferences only. **Rankp** refers to the simple ranking based on the utility of items from the perspective of instructors or professors only. These baseline approaches will produce different objectives as shown at the end in the setting.conf 13 | * **How to run**: Change the setting in setting.conf, and run "java -jar UBMSRS.jar -c setting.conf" by using JRE 8 or above 14 | 15 | ### Setting.conf 16 | ------------------- 17 | * **data.path**: tell the demo where you data sets are. Make sure all necessary data are in the same folder. 18 | * **expectation.learn**: turn on to learn student expectations by using MOO. Otherwise, load student expectations 19 | * **runbaseline**: demo will run baseline approaches first, if it is turned on 20 | * **expectation.filename**: assign the file of student expectations you want to load 21 | * **topN**: assign the value of N for top-N recommendations 22 | * **maxeval**: assign the maximal number of evaluations in the MOO 23 | * **maxf1, maxndcg, ...**: these are the best performance from the baseline approaches. You do not need to change these values unless you found better baseline results 24 | 25 | ### Data Sets 26 | ------------------- 27 | Data files and explanations 28 | * **ratings_student.csv** and **ratings_instructor.csv** are the raw rating data given by students and the instructor respectively 29 | * **ratings_student_train.csv** and **ratings_student_test.csv** are the splitted data by using 70% as training and 30% as testing 30 | * **ratings_student_candidates.csv** contains the pair of users and items, while the items are the candidate items to be recommended according to the train/test split. We have used the independent biased matrix factorization to predict the multi-criteria ratings in in this data. 31 | * **expectations_student_learned_by_UBRec_NDCG_0.214.csv** is the file of learned student expectations by using the baseline UBRec. The optimal NDCG is 0.214. You can also save a new one if you can learn better expectations. 32 | 33 | We collected this educational data for multi-stakeholder recommendations, and the information of student identifications have been removed. If you reuse this data for the purpose of research, please cite the following papers. 34 | 35 | * Yong Zheng, J.R. Toribio. The role of transparency in multi-stakeholder educational recommendations. User Model User-Adap Inter 31, 513–540 (2021). (**Note: there is a user study in this paper**) 36 | * Yong Zheng, Nastaran Ghane, and Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations." Adjunct Proceedings of the ACM conference on User Modelling, Adaptation and Personalization (UMAP). ACM. 2019. 37 | * Yong Zheng. "Multi-Stakeholder Personalized Learning with Preference Corrections." Proceedings of the 18th IEEE International Conference on Advanced Learning Technologies (ICALT). IEEE. 2019. 38 | 39 | ### Tutorials 40 | ------------------- 41 | We have tutorials at ACM RecSys 2019 and ACM CIKM 2019. 42 | 43 | * Yong Zheng. "Multi-Stakeholder Recommendations: Case Studies, Methods and Challenges", Proceedings of the 13th ACM Conference on Recommender Systems, Copenhagen, Denmark, September 19th, 2019 44 | * Muthusamy Chelliah, Yong Zheng, Sudeshna Sarkar, Vishal Kakkar. "Recommendation for Multi-Stakeholders and through Neural Review Mining", Proceedings of the 28th ACM International Conference on Information and Knowledge Management (CIKM), Beijing, China, Nov, 2019 45 | -------------------------------------------------------------------------------- /Slide_MSRS_ACM CIKM 2019.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/Slide_MSRS_ACM CIKM 2019.pdf -------------------------------------------------------------------------------- /Slide_MSRS_ACM RecSys 2019.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/Slide_MSRS_ACM RecSys 2019.pdf -------------------------------------------------------------------------------- /UBMSRS.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS.jar -------------------------------------------------------------------------------- /UBMSRS/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /UBMSRS/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | UBMSRS 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.jdt.core.javanature 16 | 17 | 18 | -------------------------------------------------------------------------------- /UBMSRS/bin/.gitignore: -------------------------------------------------------------------------------- 1 | /log4j.properties 2 | /msrs/ 3 | /setting.conf 4 | -------------------------------------------------------------------------------- /UBMSRS/bin/log4j.properties: -------------------------------------------------------------------------------- 1 | # Logging level 2 | demo.log=logs/ 3 | log4j.rootLogger=INFO, file, CONSOLE 4 | 5 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 6 | 7 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.CONSOLE.layout.ConversionPattern= [%-5p] %x \u2013 %m%n 9 | 10 | #- size rotation with log cleanup. 11 | log4j.appender.file=org.apache.log4j.RollingFileAppender 12 | log4j.appender.file.MaxFileSize=4MB 13 | log4j.appender.file.MaxBackupIndex=9 14 | 15 | #- File to log to and log format 16 | log4j.appender.file.File=${demo.log}/demo.log 17 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 18 | log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n 19 | 20 | log4j.logger.org.apache.zookeeper=WARN 21 | log4j.logger.org.apache.hadoop=WARN 22 | 23 | # set to INFO to enable infostream log messages 24 | log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/baseline/Rankp.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/baseline/Rankp.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/baseline/ubrec/RunUBRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/baseline/ubrec/RunUBRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/baseline/ubrec/UBRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/baseline/ubrec/UBRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/baseline/ubrec/UBRecProblem.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/baseline/ubrec/UBRecProblem.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/demo/Config.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/demo/Config.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/demo/UBMSRS.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/demo/UBMSRS.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnalpha/EduProblem.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnalpha/EduProblem.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnalpha/EduRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnalpha/EduRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnalpha/RunEduProblem_LearnAlpha.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnalpha/RunEduProblem_LearnAlpha.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnexpectations/EduProblem.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnexpectations/EduProblem.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnexpectations/EduRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnexpectations/EduRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/learnexpectations/RunEduProblem_LearnExpectations.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/learnexpectations/RunEduProblem_LearnExpectations.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/onestage/EduProblem.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/onestage/EduProblem.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/onestage/EduRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/onestage/EduRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/twostage/EduProblem.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/twostage/EduProblem.class -------------------------------------------------------------------------------- /UBMSRS/bin/msrs/twostage/EduRec.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/bin/msrs/twostage/EduRec.class -------------------------------------------------------------------------------- /UBMSRS/bin/setting.conf: -------------------------------------------------------------------------------- 1 | 2 | # setup the path or the folder 3 | # just put all the data in this folder 4 | data.path.wins=d:\\data\\ 5 | data.path.lins=/home/user/ 6 | 7 | # turn on learning student expectations 8 | # the demo will load expectations if you turn off learning 9 | expectation.learn=on 10 | 11 | # the demo will run baseline approaches first if you turn on it 12 | runbaseline=on 13 | 14 | # to run baseline Rankp, we need such an expectation file, default file is "expectations_student_learned_by_UBRec_NDCG_0.214.csv" 15 | # you can also load other expectation file you learned by UBRec or MOO 16 | expectation.filename=expectations_student_learned_by_UBRec_NDCG_0.214.csv 17 | 18 | # how many items the system will recommend 19 | topN=10 20 | 21 | # maximal number of evaluations 22 | maxeval=5000 23 | 24 | # max metrics from UBRec and Rankp 25 | # they are used to calculate the loss 26 | maxf1=0.107 27 | maxndcg=0.214 28 | maxutil_instructor=0.241 29 | maxutil_student=0.745 -------------------------------------------------------------------------------- /UBMSRS/data/Sample of Outputs_LearnAlpha.txt: -------------------------------------------------------------------------------- 1 | Local path of your data: f:\data\ 2 | 3 | UBRec: Baseline UBRec has been executed. 4 | UBRec: F1 = .087, NDCG = .184, Utility_List_Student U(s, L) = .783, Utility_List_Instructor U(p, L) = .137, Loss = .182 5 | UBRec: Student expectations have been learned and stored to f:\data\expectations_student_learned_by_UBRec.csv 6 | 7 | Rankp: Baseline Rankp has been executed. 8 | Rankp: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss = .203 9 | 10 | ------------------------------------------------------------------------------ 11 | 12 | Running MSRS by using eMOEA as the optimizer... 13 | eMOEA: F1 = .078, NDCG = .134, Utility_List_Student U(s, L) = .710, Utility_List_Instructor U(p, L) = .213, Loss = .163, Alpha = .737 14 | 15 | Running MSRS by using NSGAII as the optimizer... 16 | NSGAII: F1 = .084, NDCG = .147, Utility_List_Student U(s, L) = .715, Utility_List_Instructor U(p, L) = .196, Loss = .164, Alpha = .823 17 | 18 | Running MSRS by using NSGAIII as the optimizer... 19 | NSGAIII: F1 = .085, NDCG = .148, Utility_List_Student U(s, L) = .715, Utility_List_Instructor U(p, L) = .195, Loss = .163, Alpha = .827 20 | 21 | Running MSRS by using MSOPS as the optimizer... 22 | MSOPS: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss = .203, Alpha = .000 23 | 24 | Running MSRS by using SMPSO as the optimizer... 25 | SMPSO: F1 = .085, NDCG = .148, Utility_List_Student U(s, L) = .715, Utility_List_Instructor U(p, L) = .196, Loss = .162, Alpha = .823 26 | 27 | Running MSRS by using OMOPSO as the optimizer... 28 | OMOPSO: F1 = .078, NDCG = .134, Utility_List_Student U(s, L) = .710, Utility_List_Instructor U(p, L) = .213, Loss = .163, Alpha = .737 29 | 30 | ------------------------------------------------------------------------------ 31 | 32 | The best model was learned by SMPSO. Setting: expectation.learn=off 33 | SMPSO: F1 = .085, NDCG = .148, Utility_List_Student U(s, L) = .715, Utility_List_Instructor U(p, L) = .196, Loss = .162, Alpha = .823 34 | -------------------------------------------------------------------------------- /UBMSRS/data/Sample of Outputs_LearnExpectations.txt: -------------------------------------------------------------------------------- 1 | Local path of your data: f:\data\ 2 | 3 | UBRec: Baseline UBRec has been executed. 4 | UBRec: F1 = .093, NDCG = .184, Utility_List_Student U(s, L) = .724, Utility_List_Instructor U(p, L) = .133, Loss = .205 5 | UBRec: Student expectations have been learned and stored to f:\data\expectations_student_learned_by_UBRec.csv 6 | 7 | Rankp: Baseline Rankp has been executed. 8 | Rankp: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss = .203 9 | 10 | ------------------------------------------------------------------------------ 11 | 12 | Running MSRS by using eMOEA as the optimizer... 13 | eMOEA: F1 = .117, NDCG = .215, Utility_List_Student U(s, L) = .810, Utility_List_Instructor U(p, L) = .214, Loss = -.007, Alpha = .708 14 | 15 | Running MSRS by using NSGAII as the optimizer... 16 | NSGAII: F1 = .088, NDCG = .149, Utility_List_Student U(s, L) = .828, Utility_List_Instructor U(p, L) = .219, Loss = .072, Alpha = .710 17 | 18 | Running MSRS by using NSGAIII as the optimizer... 19 | NSGAIII: F1 = .125, NDCG = .232, Utility_List_Student U(s, L) = .854, Utility_List_Instructor U(p, L) = .168, Loss = .009, Alpha = .760 20 | 21 | Running MSRS by using MSOPS as the optimizer... 22 | MSOPS: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .687, Utility_List_Instructor U(p, L) = .241, Loss = .194, Alpha = .000 23 | 24 | Running MSRS by using SMPSO as the optimizer... 25 | SMPSO: F1 = .056, NDCG = .080, Utility_List_Student U(s, L) = .914, Utility_List_Instructor U(p, L) = .239, Loss = .111, Alpha = .438 26 | 27 | Running MSRS by using OMOPSO as the optimizer... 28 | OMOPSO: F1 = .075, NDCG = .118, Utility_List_Student U(s, L) = .831, Utility_List_Instructor U(p, L) = .230, Loss = .101, Alpha = .650 29 | 30 | ------------------------------------------------------------------------------ 31 | 32 | The best model was learned by eMOEA. Setting: expectation.learn=on 33 | eMOEA: F1 = .117, NDCG = .215, Utility_List_Student U(s, L) = .810, Utility_List_Instructor U(p, L) = .214, Loss = -.007, Alpha = .708 34 | eMOEA: Student expectations have been learned and stored to f:\data\expectations_student_learned_by_BestMOO.csv 35 | -------------------------------------------------------------------------------- /UBMSRS/data/Sample of Outputs_OneStage.txt: -------------------------------------------------------------------------------- 1 | [INFO ] – Local path of your data: d:\data\ 2 | [INFO ] – The system is running baseline approaches... 3 | [INFO ] – UBRec: Baseline UBRec has been executed. 4 | [INFO ] – UBRec: F1 = .090, NDCG = .177, Utility_List_Student U(s, L) = .699, Utility_List_Instructor U(p, L) = .133, Loss1 = .225, Loss2 = .306 5 | [INFO ] – UBRec: Student expectations have been learned and stored to d:\data\expectations_student_learned_by_UBRec.csv 6 | [INFO ] – Rankp: Baseline Rankp has been executed. 7 | [INFO ] – Rankp: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss1 = .203, Loss2 = .252 8 | [INFO ] – ------------------------------------------------------------------------------ 9 | [INFO ] – Running MSRS by using eMOEA as the optimizer... 10 | [INFO ] – eMOEA: F1 = .108, NDCG = .193, Utility_List_Student U(s, L) = .811, Utility_List_Instructor U(p, L) = .224, Loss1 = .009, Loss2 = .057, Alpha = .673 11 | [INFO ] – Running MSRS by using NSGAII as the optimizer... 12 | [INFO ] – NSGAII: F1 = .100, NDCG = .170, Utility_List_Student U(s, L) = .832, Utility_List_Instructor U(p, L) = .202, Loss1 = .061, Loss2 = .149, Alpha = .793 13 | [INFO ] – Running MSRS by using NSGAIII as the optimizer... 14 | [INFO ] – NSGAIII: F1 = .141, NDCG = .228, Utility_List_Student U(s, L) = .841, Utility_List_Instructor U(p, L) = .157, Loss1 = .009, Loss2 = .079, Alpha = .753 15 | [INFO ] – Running MSRS by using MSOPS as the optimizer... 16 | [INFO ] – MSOPS: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .678, Utility_List_Instructor U(p, L) = .241, Loss1 = .198, Loss2 = .252, Alpha = .000 17 | [INFO ] – Running MSRS by using SMPSO as the optimizer... 18 | [INFO ] – SMPSO: F1 = .074, NDCG = .119, Utility_List_Student U(s, L) = .638, Utility_List_Instructor U(p, L) = .232, Loss1 = .186, Loss2 = .207, Alpha = .579 19 | [INFO ] – Running MSRS by using OMOPSO as the optimizer... 20 | [INFO ] – OMOPSO: F1 = .077, NDCG = .113, Utility_List_Student U(s, L) = .731, Utility_List_Instructor U(p, L) = .233, Loss1 = .143, Loss2 = .205, Alpha = .603 21 | [INFO ] – ------------------------------------------------------------------------------ 22 | [INFO ] – The best model was learned by eMOEA. Setting: expectation.learn=on 23 | [INFO ] – eMOEA: F1 = .108, NDCG = .193, Utility_List_Student U(s, L) = .811, Utility_List_Instructor U(p, L) = .224, Loss1 = .009, Loss2 = .057, Alpha = .673 24 | [INFO ] – eMOEA: Student expectations have been learned and stored to d:\data\expectations_student_learned_by_BestMOO.csv 25 | -------------------------------------------------------------------------------- /UBMSRS/data/Sample of Outputs_TwoStage.txt: -------------------------------------------------------------------------------- 1 | [INFO ] – Local path of your data: d:\data\ 2 | [INFO ] – The system is running baseline approaches... 3 | [INFO ] – UBRec: Baseline UBRec has been executed. 4 | [INFO ] – UBRec: F1 = .095, NDCG = .194, Utility_List_Student U(s, L) = .805, Utility_List_Instructor U(p, L) = .132, Loss1 = .159, Loss2 = .279 5 | [INFO ] – UBRec: Student expectations have been learned and stored to d:\data\expectations_student_learned_by_UBRec.csv 6 | [INFO ] – Rankp: Baseline Rankp has been executed. 7 | [INFO ] – Rankp: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss1 = .203, Loss2 = .252 8 | [INFO ] – ------------------------------------------------------------------------------ 9 | [INFO ] – Running MSRS by using eMOEA as the optimizer... 10 | [INFO ] – eMOEA: F1 = .074, NDCG = .119, Utility_List_Student U(s, L) = .704, Utility_List_Instructor U(p, L) = .226, Loss1 = .166, Loss2 = .221, Alpha = .639 11 | [INFO ] – Running MSRS by using NSGAII as the optimizer... 12 | [INFO ] – NSGAII: F1 = .074, NDCG = .119, Utility_List_Student U(s, L) = .704, Utility_List_Instructor U(p, L) = .226, Loss1 = .166, Loss2 = .221, Alpha = .639 13 | [INFO ] – Running MSRS by using NSGAIII as the optimizer... 14 | [INFO ] – NSGAIII: F1 = .078, NDCG = .134, Utility_List_Student U(s, L) = .710, Utility_List_Instructor U(p, L) = .212, Loss1 = .164, Loss2 = .222, Alpha = .739 15 | [INFO ] – Running MSRS by using MSOPS as the optimizer... 16 | [INFO ] – MSOPS: F1 = .061, NDCG = .089, Utility_List_Student U(s, L) = .667, Utility_List_Instructor U(p, L) = .241, Loss1 = .203, Loss2 = .252, Alpha = .000 17 | [INFO ] – Running MSRS by using SMPSO as the optimizer... 18 | [INFO ] – SMPSO: F1 = .073, NDCG = .118, Utility_List_Student U(s, L) = .704, Utility_List_Instructor U(p, L) = .226, Loss1 = .167, Loss2 = .222, Alpha = .638 19 | [INFO ] – Running MSRS by using OMOPSO as the optimizer... 20 | [INFO ] – OMOPSO: F1 = .074, NDCG = .119, Utility_List_Student U(s, L) = .704, Utility_List_Instructor U(p, L) = .226, Loss1 = .166, Loss2 = .221, Alpha = .639 21 | [INFO ] – ------------------------------------------------------------------------------ 22 | [INFO ] – The best model was learned by eMOEA. Setting: expectation.learn=off 23 | [INFO ] – eMOEA: F1 = .074, NDCG = .119, Utility_List_Student U(s, L) = .704, Utility_List_Instructor U(p, L) = .226, Loss1 = .166, Loss2 = .221, Alpha = .639 24 | -------------------------------------------------------------------------------- /UBMSRS/data/ratings_instructor.csv: -------------------------------------------------------------------------------- 1 | Item,Data,Ease 2 | 21,5,3 3 | 22,4,4 4 | 23,4,4 5 | 24,5,5 6 | 25,4,4 7 | 26,3,3 8 | 27,2,2 9 | 28,5,5 10 | 29,5,5 11 | 30,3,3 12 | 31,2,4 13 | 32,3,4 14 | 33,4,4 15 | 34,4,5 16 | 35,5,5 17 | 36,3,4 18 | 37,3,4 19 | 38,5,5 20 | 39,5,5 21 | 40,3.5,3.5 22 | 41,3,3 23 | 42,3.5,4 24 | 43,4,4 25 | 44,2.5,3 26 | 45,3,3 27 | 46,3.5,3.5 28 | 47,4,4.5 29 | 48,2,2.5 30 | 49,4,4 31 | 50,4,4 32 | 51,3.5,3.5 33 | 52,4,5 34 | 53,4,4 35 | 54,4,4 36 | 55,4,4 37 | 56,4,4 38 | 57,3,3 39 | 58,3.5,4 40 | 59,3,3.5 41 | 60,4,4 42 | 61,4,4 43 | 62,3.5,3.5 44 | 63,3,4 45 | 64,5,5 46 | 65,3,3.5 47 | 66,4,4 48 | 67,3.5,3.5 49 | 68,4,4 50 | 69,4,5 51 | 70,4,4 52 | -------------------------------------------------------------------------------- /UBMSRS/data/ratings_student_test.csv: -------------------------------------------------------------------------------- 1 | User, Item, Rating, App, Data, Ease 2 | 1000,23,1,2,2,2 3 | 1000,27,1,2,2,2 4 | 1001,29,5,4,5,4 5 | 1001,44,4,5,5,4 6 | 1002,23,2,3,3,3 7 | 1002,33,4,3,4,3 8 | 1002,36,4,3,4,4 9 | 1002,37,1,2,3,4 10 | 1002,37,2,4,5,5 11 | 1002,48,4,5,4,3 12 | 1002,54,3,3,3,4 13 | 1003,58,5,5,5,5 14 | 1003,67,5,5,5,5 15 | 1004,38,3,5,4,3 16 | 1004,58,1,2,3,1 17 | 1005,26,2,2,3,3 18 | 1005,26,5,5,4,3 19 | 1005,46,4,3,3,3 20 | 1005,49,4,4,3,3 21 | 1005,49,5,5,5,3 22 | 1005,50,5,5,5,4 23 | 1006,23,5,5,5,4 24 | 1006,28,2,1,2,1 25 | 1006,35,2,1,2,1 26 | 1006,40,5,5,5,4 27 | 1006,43,5,5,5,4 28 | 1007,25,5,5,4,4 29 | 1007,25,5,4,4,5 30 | 1007,26,4,5,4,4 31 | 1007,32,5,5,5,5 32 | 1007,46,5,4,4,4 33 | 1007,56,5,4,4,5 34 | 1007,65,5,4,4,5 35 | 1008,45,2,4,3,1 36 | 1008,65,4,4,4,5 37 | 1009,25,5,4,5,4 38 | 1009,28,4,4,5,4 39 | 1009,49,4,4,4,4 40 | 1010,23,1,3,2,3 41 | 1010,31,2,3,3,3 42 | 1011,30,5,4,4,3 43 | 1011,70,1,1,1,1 44 | 1012,50,2,4,4,4 45 | 1012,52,2,3,4,4 46 | 1013,21,5,4,4,4 47 | 1013,36,1,2,2,2 48 | 1013,36,1,2,2,2 49 | 1013,58,1,2,2,2 50 | 1014,25,5,2,3,2 51 | 1014,30,4,3,2,3 52 | 1014,41,5,3,3,3 53 | 1014,52,3,3,3,3 54 | 1015,24,2,2,3,5 55 | 1015,70,2,2,3,5 56 | 1016,22,1,1,1,1 57 | 1016,37,4,4,4,4 58 | 1017,22,1,1,1,1 59 | 1017,25,5,5,5,5 60 | 1017,26,1,1,1,1 61 | 1017,27,1,1,1,1 62 | 1017,38,3,3,3,3 63 | 1018,52,2,3,2,3 64 | 1018,68,1,3,1,2 65 | 1019,25,5,5,4,5 66 | 1019,58,5,5,5,5 67 | 1020,21,5,4,5,4 68 | 1020,29,3,2,5,5 69 | 1020,30,1,2,2,1 70 | 1020,36,5,5,5,5 71 | 1020,40,4,3,3,3 72 | 1020,47,4,3,3,3 73 | 1020,47,4,5,4,3 74 | 1020,61,4,5,5,4 75 | 1020,63,4,3,4,4 76 | 1020,66,4,5,4,3 77 | 1020,68,1,2,1,4 78 | 1020,70,2,3,3,4 79 | 1021,36,5,5,4,4 80 | 1021,36,5,5,4,4 81 | 1022,49,1,1,1,3 82 | 1022,55,2,3,3,3 83 | 1022,58,4,4,4,3 84 | 1023,30,5,5,5,5 85 | 1023,31,5,5,5,4 86 | 1023,40,5,5,5,5 87 | 1023,44,4,5,5,5 88 | 1023,55,5,4,5,4 89 | 1023,63,5,4,5,5 90 | 1023,70,5,5,5,4 91 | 1024,28,5,5,5,2 92 | 1024,29,4,3,3,3 93 | 1024,30,5,5,5,3 94 | 1024,33,3,4,5,2 95 | 1024,39,4,4,5,2 96 | 1024,41,5,5,5,3 97 | 1024,46,4,4,4,2 98 | 1024,49,4,4,4,2 99 | 1024,52,1,2,3,2 100 | 1024,55,4,3,3,4 101 | 1024,58,4,4,4,3 102 | 1024,59,1,2,2,2 103 | 1024,69,4,3,4,3 104 | 1024,70,5,5,5,4 105 | 1025,23,2,4,4,3 106 | 1025,69,4,3,4,4 107 | 1026,27,2,2,2,2 108 | 1026,30,4,5,4,4 109 | 1027,21,5,5,5,4 110 | 1027,22,2,4,4,3 111 | 1027,25,4,3,3,5 112 | 1027,49,4,4,4,4 113 | 1027,49,4,5,4,4 114 | 1027,53,2,2,2,2 115 | 1028,22,5,5,5,5 116 | 1028,34,5,5,5,5 117 | 1028,41,5,5,5,5 118 | 1028,56,5,5,5,5 119 | 1028,58,5,5,5,5 120 | 1029,21,4,4,4,4 121 | 1029,25,3,4,4,3 122 | 1029,26,4,4,4,4 123 | 1029,28,4,5,4,4 124 | 1029,29,2,5,3,2 125 | 1029,29,2,1,2,2 126 | 1029,58,4,4,4,4 127 | 1029,62,4,3,3,4 128 | 1031,21,3,3,3,3 129 | 1031,22,1,1,1,1 130 | 1031,33,1,1,1,1 131 | 1031,40,1,1,1,1 132 | 1032,32,5,5,5,4 133 | 1032,47,4,4,4,4 134 | 1032,67,2,2,3,2 135 | 1033,24,2,1,3,2 136 | 1033,31,5,5,5,4 137 | 1035,28,5,4,5,4 138 | 1035,49,1,2,1,1 139 | 1036,21,4,4,4,3 140 | 1036,21,4,4,4,3 141 | 1036,24,3,2,2,3 142 | 1036,31,2,1,1,2 143 | 1036,40,4,4,3,4 144 | 1036,44,2,2,2,1 145 | 1036,64,2,1,2,2 146 | 1037,25,4,5,3,4 147 | 1037,27,5,5,4,4 148 | 1038,36,4,4,3,4 149 | 1038,64,1,2,2,4 150 | 1039,21,5,5,5,3 151 | 1039,22,2,1,2,1 152 | 1040,21,5,5,5,5 153 | 1040,25,4,4,4,4 154 | 1040,27,2,4,4,5 155 | 1040,40,4,5,4,4 156 | 1042,34,1,4,4,4 157 | 1042,35,1,3,4,4 158 | 1042,40,4,3,3,4 159 | 1042,60,5,4,3,5 160 | 1043,59,5,5,4,4 161 | 1043,61,3,3,4,3 162 | 1044,21,5,4,4,4 163 | 1044,28,5,5,4,5 164 | 1044,32,4,5,4,4 165 | 1044,34,3,4,4,3 166 | 1044,42,3,3,3,3 167 | 1044,44,3,3,3,2 168 | 1045,21,4,3,4,3 169 | 1045,30,5,3,3,4 170 | 1045,55,2,3,3,3 171 | 1048,21,5,4,4,5 172 | 1048,22,1,2,1,1 173 | 1048,35,1,1,1,3 174 | 1048,44,5,5,1,1 175 | 1048,49,5,4,4,2 176 | 1048,56,4,4,2,2 177 | 1048,58,5,4,4,2 178 | 1048,59,5,4,4,2 179 | 1051,22,4,2,4,3 180 | 1051,23,1,4,3,2 181 | 1051,23,1,2,1,1 182 | 1051,24,2,2,4,3 183 | 1051,25,4,4,3,3 184 | 1051,30,5,5,4,4 185 | 1052,40,4,5,4,3 186 | 1052,62,1,2,1,1 187 | 1053,34,1,2,3,3 188 | 1053,40,3,4,4,3 189 | 1054,41,4,4,4,4 190 | 1054,49,4,4,3,3 191 | 1054,66,3,2,3,2 192 | 1055,25,5,5,5,3 193 | 1055,44,1,2,2,2 194 | 1056,49,2,4,4,5 195 | 1056,70,2,3,4,5 196 | 1058,29,2,3,3,3 197 | 1058,63,4,2,2,3 198 | 1060,27,2,2,2,2 199 | 1060,30,5,5,4,5 200 | 1060,30,5,5,4,5 201 | 1060,32,5,5,5,4 202 | 1060,49,5,5,5,5 203 | 1060,68,1,1,2,2 204 | 1061,31,5,5,5,4 205 | 1061,35,1,3,2,2 206 | 1062,21,3,4,5,3 207 | 1062,22,2,2,2,2 208 | 1062,26,2,2,2,2 209 | 1062,33,5,3,5,4 210 | 1063,22,5,5,3,3 211 | 1063,23,4,5,4,4 212 | 1063,24,4,4,4,4 213 | 1063,25,1,1,3,3 214 | 1063,37,5,5,4,4 215 | 1063,38,4,4,3,3 216 | 1063,45,1,1,3,3 217 | 1063,51,4,4,4,4 218 | 1063,52,1,1,3,2 219 | 1063,66,4,4,4,4 220 | 1063,67,4,4,3,3 221 | 1064,21,4,5,4,4 222 | 1064,48,2,2,3,3 223 | 1065,27,1,1,2,3 224 | 1065,42,1,1,2,3 225 | 1065,56,5,5,5,3 226 | 1066,25,1,1,1,4 227 | 1066,30,4,4,4,2 228 | 1067,30,4,5,3,2 229 | 1067,36,4,3,3,3 230 | 1067,49,5,5,3,3 231 | 1067,58,5,2,3,1 232 | 1067,63,5,5,2,5 233 | 1067,67,3,2,1,2 234 | 1067,69,4,2,1,2 235 | 1068,23,1,1,1,1 236 | 1068,31,2,3,2,2 237 | 1068,34,1,1,2,2 238 | 1068,64,1,1,1,1 239 | 1070,50,1,1,1,1 240 | 1070,50,1,1,1,1 241 | 1070,58,4,4,4,4 242 | 1070,58,4,4,4,4 243 | 1070,70,1,1,2,1 244 | 1073,21,5,4,5,3 245 | 1073,52,2,3,4,2 246 | 1076,41,5,4,4,4 247 | 1076,60,4,4,4,4 248 | 1077,36,5,5,4,4 249 | 1077,40,5,5,5,4 250 | 1077,47,5,5,4,4 251 | 1077,52,4,4,5,4 252 | 1077,52,5,5,5,5 253 | 1078,25,2,1,2,2 254 | 1078,47,1,2,2,2 255 | 1078,59,4,5,4,4 256 | 1078,69,2,2,2,2 257 | 1078,70,4,4,4,4 258 | 1079,61,4,5,4,4 259 | 1079,69,5,5,4,4 260 | 1080,30,1,1,1,1 261 | 1080,40,2,2,2,2 262 | 1082,35,2,3,3,2 263 | 1082,58,5,4,3,2 264 | 1082,61,5,4,3,2 265 | 1082,68,5,4,3,2 266 | 1082,68,4,4,4,3 267 | 1083,26,4,4,3,2 268 | 1083,64,1,1,1,4 269 | 1085,41,5,5,5,4 270 | 1085,45,2,3,3,2 271 | 1086,44,1,2,1,1 272 | 1086,50,4,4,4,4 273 | 1086,59,3,3,3,3 274 | 1087,50,2,2,1,2 275 | 1087,57,1,1,1,2 276 | 1087,59,4,4,4,4 277 | 1087,60,4,4,5,5 278 | 1089,24,1,1,1,1 279 | 1089,40,2,2,2,1 280 | 1089,43,2,1,1,1 281 | 1090,21,4,4,4,4 282 | 1090,24,2,3,3,2 283 | 1091,33,2,2,2,3 284 | 1091,35,5,3,3,4 285 | 1092,26,3,2,2,3 286 | 1092,43,4,4,4,4 287 | 1093,24,2,2,3,2 288 | 1093,40,4,5,4,4 289 | 1094,25,2,3,3,3 290 | 1094,70,5,5,4,4 291 | 1095,21,2,1,1,1 292 | 1095,69,4,4,4,5 293 | 1096,21,1,2,2,3 294 | 1096,26,4,4,4,4 295 | 1096,29,2,2,2,3 296 | 1096,30,5,4,4,3 297 | 1096,66,4,4,5,4 298 | 1097,30,1,2,2,5 299 | 1097,57,4,3,3,3 300 | 1097,63,5,5,5,4 301 | 1100,22,1,2,2,3 302 | 1100,56,4,4,4,4 303 | 1101,23,1,2,2,3 304 | 1101,60,5,5,5,5 305 | 1102,25,4,5,5,4 306 | 1102,38,1,1,3,2 307 | 1102,41,5,5,5,4 308 | 1102,51,2,1,2,1 309 | 1103,22,2,2,3,4 310 | 1103,30,5,5,5,4 311 | 1103,49,5,5,5,4 312 | 1103,64,1,1,2,3 313 | 1105,22,4,4,1,2 314 | 1105,23,5,4,4,2 315 | 1105,25,3,2,3,3 316 | 1105,25,4,3,2,3 317 | 1105,40,1,1,1,1 318 | 1105,54,4,4,3,2 319 | 1105,69,4,1,1,1 320 | 1106,30,5,5,5,5 321 | 1106,61,5,5,5,5 322 | 1106,68,3,3,3,3 323 | 1107,23,3,4,4,4 324 | 1107,25,4,5,5,4 325 | 1107,25,2,4,3,4 326 | 1107,37,2,3,3,4 327 | 1107,61,5,5,5,4 328 | 1107,63,5,5,5,4 329 | 1109,21,3,5,5,4 330 | 1109,28,3,3,3,3 331 | 1109,31,1,2,2,1 332 | 1109,36,1,2,2,4 333 | 1110,25,5,4,3,3 334 | 1110,38,4,3,3,4 335 | 1110,60,5,5,3,5 336 | 1110,67,4,4,3,4 337 | 1111,23,4,4,3,3 338 | 1111,24,1,2,3,3 339 | 1111,27,3,3,4,3 340 | 1111,31,4,5,3,4 341 | 1111,32,1,5,2,2 342 | 1111,39,4,2,4,4 343 | 1111,51,4,4,5,2 344 | 1111,56,2,3,3,2 345 | 1111,57,5,4,3,4 346 | 1111,59,5,3,4,4 347 | 1111,63,3,5,4,5 348 | 1113,21,5,3,5,5 349 | 1113,33,5,4,4,4 350 | 1113,35,5,3,5,5 351 | 1113,35,5,3,5,5 352 | 1113,50,5,3,4,4 353 | 1113,54,3,2,2,2 354 | 1113,56,5,4,5,5 355 | 1115,29,2,3,3,3 356 | 1115,44,2,3,4,3 357 | 1115,62,4,3,5,3 358 | 1115,62,4,4,4,4 359 | 1116,32,4,4,3,3 360 | 1116,49,5,5,4,4 361 | 1116,58,5,5,4,4 362 | 1116,60,4,4,5,4 363 | 1116,62,5,5,5,3 364 | 1116,68,1,2,2,2 365 | 1117,22,2,3,2,1 366 | 1117,27,1,2,2,1 367 | 1117,42,4,3,4,3 368 | 1117,47,3,3,3,3 369 | 1118,23,3,5,5,3 370 | 1118,27,3,4,4,4 371 | 1118,46,4,4,4,4 372 | 1118,60,4,4,5,5 373 | 1119,21,2,3,2,2 374 | 1119,27,2,2,2,2 375 | 1119,32,5,5,5,3 376 | 1119,38,3,5,5,3 377 | 1119,58,2,2,3,3 378 | 1120,24,4,4,5,3 379 | 1120,25,1,2,1,1 380 | 1121,22,1,2,1,1 381 | 1121,23,1,2,3,3 382 | 1121,24,2,2,3,3 383 | 1121,26,5,5,3,3 384 | 1121,65,4,5,4,3 385 | 1122,21,5,4,5,3 386 | 1122,41,2,3,3,4 387 | 1124,25,4,5,5,3 388 | 1124,35,3,4,4,3 389 | 1124,69,4,5,5,3 390 | 1125,24,1,1,2,3 391 | 1125,56,5,5,5,5 392 | 1127,27,1,1,5,3 393 | 1127,35,5,5,5,3 394 | 1127,40,4,5,3,3 395 | 1127,66,2,1,5,3 396 | 1128,24,2,2,2,2 397 | 1128,27,2,2,1,2 398 | 1128,31,2,3,4,3 399 | 1128,50,2,2,2,2 400 | 1129,21,4,4,4,4 401 | 1129,53,2,2,1,2 402 | 1129,63,3,2,1,2 403 | 1129,63,3,2,1,2 404 | 1129,65,4,4,3,4 405 | 1130,31,2,3,1,1 406 | 1130,38,5,5,5,5 407 | 1130,39,4,4,5,3 408 | 1130,54,4,4,4,3 409 | 1130,64,5,5,5,5 410 | 1130,70,1,4,4,1 411 | 1131,22,1,2,2,2 412 | 1131,28,5,3,5,4 413 | 1132,34,2,3,3,2 414 | 1132,39,2,3,3,2 415 | 1132,64,1,2,2,2 416 | 1133,58,2,1,3,2 417 | 1133,59,5,4,4,4 418 | 1134,21,5,5,4,5 419 | 1134,24,2,2,2,3 420 | 1134,31,2,2,2,3 421 | 1134,34,5,5,3,5 422 | 1134,49,5,5,3,4 423 | 1135,40,4,4,4,4 424 | 1135,40,4,4,4,4 425 | 1135,49,4,4,4,4 426 | 1135,64,2,2,3,3 427 | 1137,21,4,5,5,3 428 | 1137,22,2,3,2,4 429 | 1137,27,1,1,1,4 430 | 1137,32,3,3,4,3 431 | 1138,22,2,2,2,2 432 | 1138,34,2,2,2,2 433 | 1139,27,4,5,4,4 434 | 1139,42,2,2,2,1 435 | 1143,33,2,2,2,2 436 | 1143,40,4,4,4,4 437 | 1144,23,3,2,1,1 438 | 1144,28,4,5,3,3 439 | 1144,30,3,4,3,3 440 | 1144,34,3,3,3,3 441 | 1144,42,4,4,4,4 442 | 1144,49,5,5,5,4 443 | 1145,29,2,2,3,2 444 | 1145,33,5,5,5,4 445 | 1145,35,2,2,1,1 446 | 1145,49,5,4,4,3 447 | 1147,21,2,2,4,4 448 | 1147,27,1,2,3,3 449 | 1148,33,5,4,4,4 450 | 1148,34,4,5,5,5 451 | 1148,60,5,4,4,4 452 | 1148,61,2,2,2,1 453 | 1148,65,5,5,5,4 454 | 1149,49,5,4,5,5 455 | 1149,56,2,3,3,1 456 | 1150,40,3,1,1,1 457 | 1150,49,3,1,1,1 458 | 1152,25,4,5,4,4 459 | 1152,58,4,5,4,4 460 | 1153,22,5,4,5,4 461 | 1153,30,5,5,5,5 462 | 1153,32,5,5,5,5 463 | 1153,32,5,5,5,5 464 | 1153,40,5,5,5,4 465 | 1153,41,4,4,5,4 466 | 1153,49,5,5,5,4 467 | 1153,64,1,1,2,3 468 | 1153,68,2,2,2,2 469 | 1153,69,5,4,5,4 470 | 1154,28,4,5,4,5 471 | 1154,32,5,5,5,4 472 | 1154,33,5,5,5,5 473 | 1154,43,4,4,4,4 474 | 1154,69,5,5,5,5 475 | 1155,21,4,4,3,4 476 | 1155,30,5,3,3,5 477 | 1155,32,1,1,2,3 478 | 1155,42,1,2,2,2 479 | 1155,43,3,3,3,3 480 | 1155,43,3,3,3,3 481 | 1155,68,1,2,2,2 482 | 1156,23,4,5,3,4 483 | 1156,32,5,5,4,4 484 | 1157,23,2,3,2,2 485 | 1157,28,5,4,3,3 486 | 1157,37,1,1,2,2 487 | 1157,58,5,4,4,4 488 | 1158,25,4,4,3,4 489 | 1158,26,4,5,4,3 490 | 1158,29,2,2,2,2 491 | 1158,68,2,2,2,2 492 | 1159,26,4,5,4,4 493 | 1159,31,5,5,5,5 494 | 1159,42,2,2,2,2 495 | 1159,65,5,5,5,5 496 | 1160,24,1,2,2,2 497 | 1160,33,5,4,3,4 498 | 1160,39,4,4,4,3 499 | 1160,40,2,2,1,2 500 | 1161,36,2,4,4,4 501 | 1161,39,3,4,5,5 502 | 1161,42,4,4,4,3 503 | 1162,34,2,1,2,1 504 | 1162,38,4,3,3,3 505 | 1162,38,5,4,4,3 506 | 1162,38,5,4,4,3 507 | 1162,58,4,4,4,3 508 | 1162,61,4,4,4,3 509 | 1162,61,4,4,4,3 510 | 1162,63,5,5,5,4 511 | 1164,21,5,5,5,4 512 | 1164,26,4,4,5,5 513 | 1164,27,4,5,5,5 514 | 1164,43,5,5,5,3 515 | 1164,52,3,3,4,4 516 | 1164,59,4,4,5,3 517 | 1165,45,1,1,2,2 518 | 1165,60,4,3,3,4 519 | 1166,49,5,4,4,3 520 | 1166,60,4,3,3,3 521 | 1166,62,2,3,3,3 522 | 1168,28,1,2,3,3 523 | 1168,64,1,3,2,5 524 | 1168,66,5,5,5,2 525 | 1169,24,2,2,2,2 526 | 1169,49,4,4,4,4 527 | 1170,23,3,3,2,2 528 | 1170,38,2,2,2,3 529 | 1170,40,5,5,5,5 530 | 1170,62,5,5,5,5 531 | 1171,27,4,4,4,3 532 | 1171,61,4,4,4,3 533 | 1173,21,5,5,5,5 534 | 1173,25,3,4,4,3 535 | 1173,28,4,5,4,4 536 | 1173,34,2,2,2,3 537 | 1173,35,4,4,4,3 538 | 1173,37,4,3,3,4 539 | 1173,40,5,5,4,4 540 | 1173,44,1,1,1,2 541 | 1173,49,5,5,4,4 542 | 1173,53,1,1,1,2 543 | 1173,60,5,5,5,4 544 | 1173,61,4,4,4,4 545 | 1173,69,4,4,4,4 546 | 1174,21,4,4,3,3 547 | 1174,49,4,4,3,3 548 | 1175,24,1,1,1,1 549 | 1175,24,1,1,1,1 550 | 1175,25,1,1,1,1 551 | 1175,65,4,4,5,4 552 | 1175,66,5,5,4,4 553 | 1176,33,4,5,5,3 554 | 1176,48,1,2,2,4 555 | 1178,21,4,4,5,4 556 | 1178,34,1,2,1,3 557 | 1178,49,5,4,4,3 558 | 1178,60,5,4,4,3 559 | 1179,25,2,2,2,3 560 | 1179,45,5,4,4,4 561 | 1180,27,5,4,4,4 562 | 1180,27,1,2,2,2 563 | 1180,60,4,4,5,5 564 | 1180,65,4,4,5,5 565 | 1182,31,3,3,2,1 566 | 1182,49,5,5,4,4 567 | 1182,56,2,5,4,4 568 | 1182,69,5,4,3,4 569 | 1184,22,1,3,3,3 570 | 1184,25,5,5,4,3 571 | 1184,27,1,2,2,2 572 | 1184,33,2,2,2,1 573 | 1184,40,4,4,3,3 574 | 1184,47,4,4,3,4 575 | 1184,60,4,4,4,4 576 | 1184,67,3,3,3,4 577 | 1185,21,1,2,2,2 578 | 1185,30,5,5,5,5 579 | 1185,49,4,5,3,3 580 | 1186,31,1,2,1,2 581 | 1186,33,5,4,5,5 582 | 1186,39,1,2,2,2 583 | 1186,40,4,4,5,3 584 | 1186,56,4,5,3,5 585 | 1186,65,5,4,3,4 586 | 1186,69,5,5,4,5 587 | 1187,28,2,2,2,2 588 | 1187,32,2,2,2,2 589 | 1187,49,5,3,4,3 590 | 1187,70,1,2,3,3 591 | 1188,24,2,1,2,2 592 | 1188,31,5,4,5,5 593 | 1188,49,4,5,4,4 594 | 1189,45,3,4,5,5 595 | 1189,65,5,4,4,4 596 | 1190,21,5,5,5,5 597 | 1190,23,3,3,3,3 598 | 1190,29,3,3,3,3 599 | 1190,38,3,3,3,3 600 | 1191,25,4,4,3,4 601 | 1191,49,4,3,3,4 602 | 1192,22,1,1,1,1 603 | 1192,25,3,4,3,3 604 | 1192,27,2,2,2,2 605 | 1192,28,3,3,3,3 606 | 1192,33,4,4,5,5 607 | 1192,37,2,1,1,1 608 | 1192,41,3,3,3,3 609 | 1192,43,1,2,1,1 610 | 1192,46,5,3,2,3 611 | 1192,47,5,3,3,3 612 | 1192,59,1,1,1,1 613 | 1192,63,4,3,3,3 614 | 1192,64,1,1,1,1 615 | 1192,69,5,5,5,5 616 | 1193,54,5,5,5,4 617 | 1193,69,5,5,5,4 618 | 1195,32,5,5,4,4 619 | 1195,38,5,5,4,4 620 | 1196,21,5,4,4,3 621 | 1196,22,4,4,5,4 622 | 1196,23,5,4,4,4 623 | 1196,25,5,5,4,5 624 | 1196,27,2,3,3,3 625 | 1196,32,2,2,3,2 626 | 1196,34,2,4,4,4 627 | 1196,34,2,3,3,3 628 | 1197,22,1,3,3,3 629 | 1197,24,2,2,3,3 630 | 1199,25,2,3,4,2 631 | 1199,61,5,4,4,3 632 | 1201,46,5,4,5,5 633 | 1201,47,3,4,4,4 634 | 1201,49,2,2,1,1 635 | 1201,51,3,1,2,2 636 | 1202,21,4,3,3,3 637 | 1202,30,4,4,4,3 638 | 1202,36,2,2,2,3 639 | 1202,60,1,2,2,3 640 | 1202,64,2,3,2,3 641 | 1203,22,4,3,4,4 642 | 1203,63,2,5,5,5 643 | 1204,22,1,1,1,1 644 | 1204,23,2,1,2,3 645 | 1204,25,4,4,4,4 646 | 1204,57,5,5,4,4 647 | 1204,64,3,3,3,4 648 | 1204,64,1,1,1,2 649 | 1205,28,3,4,4,4 650 | 1205,49,5,4,4,3 651 | 1205,60,2,4,3,2 652 | 1205,68,4,3,3,3 653 | 1205,70,1,2,3,1 654 | 1206,32,4,4,4,5 655 | 1206,37,3,3,4,4 656 | 1206,38,4,4,4,5 657 | 1206,49,5,5,5,5 658 | 1206,59,5,5,5,5 659 | 1206,60,3,3,3,3 660 | 1207,24,3,4,3,4 661 | 1207,36,4,4,5,4 662 | 1207,56,5,5,4,4 663 | 1207,68,2,3,2,2 664 | 1208,24,1,1,2,3 665 | 1208,35,1,1,1,2 666 | 1208,37,4,4,3,3 667 | 1208,64,5,5,4,4 668 | 1210,23,4,4,3,3 669 | 1210,28,5,4,4,4 670 | 1210,31,1,2,1,2 671 | 1210,61,4,4,4,4 672 | 1211,22,3,3,2,3 673 | 1211,27,3,3,2,3 674 | 1211,37,3,3,2,3 675 | 1211,49,4,5,3,4 676 | 1211,68,3,3,2,3 677 | 1213,25,5,5,5,2 678 | 1213,26,5,3,4,3 679 | 1213,28,5,5,5,3 680 | 1213,42,5,5,5,2 681 | 1213,66,5,5,5,3 682 | 1215,24,2,4,4,4 683 | 1215,25,5,4,4,4 684 | 1215,29,2,4,3,3 685 | 1215,49,4,4,4,4 686 | 1215,51,5,5,5,3 687 | 1217,38,5,5,5,4 688 | 1217,56,4,4,4,4 689 | 1220,26,2,2,2,2 690 | 1220,56,4,3,3,2 691 | 1221,21,5,4,4,3 692 | 1221,22,1,1,1,1 693 | 1221,59,3,5,4,3 694 | 1222,28,3,3,3,3 695 | 1222,45,2,2,2,3 696 | 1223,42,3,4,4,4 697 | 1223,62,4,4,5,3 698 | 1224,22,4,4,4,4 699 | 1224,44,3,3,4,3 700 | 1225,24,2,2,3,3 701 | 1225,34,5,4,4,4 702 | 1226,22,3,2,2,2 703 | 1226,25,4,3,5,4 704 | 1226,29,1,3,3,2 705 | 1226,49,5,5,5,4 706 | 1227,25,5,4,5,4 707 | 1227,28,2,2,4,3 708 | 1227,30,1,1,4,4 709 | 1227,62,2,2,4,2 710 | 1227,63,4,5,5,2 711 | 1229,22,2,3,3,3 712 | 1229,58,4,4,5,3 713 | 1229,62,5,5,5,3 714 | 1230,21,4,4,3,3 715 | 1230,22,2,1,2,2 716 | 1230,29,2,1,2,2 717 | 1230,49,4,5,4,4 718 | 1230,69,4,5,5,4 719 | 1233,31,5,5,3,3 720 | 1233,37,5,5,5,5 721 | 1233,52,3,3,3,3 722 | 1233,67,3,3,3,3 723 | 1235,21,2,2,2,2 724 | 1235,30,1,2,1,1 725 | 1235,49,5,5,5,5 726 | 1235,66,5,5,5,4 727 | 1236,22,2,1,1,2 728 | 1236,26,4,5,4,4 729 | 1236,49,2,3,1,2 730 | 1236,64,3,3,4,3 731 | 1236,68,3,2,3,4 732 | 1237,51,3,5,1,5 733 | 1237,52,2,2,3,3 734 | 1238,51,2,2,3,2 735 | 1238,62,4,4,4,4 736 | 1239,21,2,2,2,3 737 | 1239,26,4,4,5,5 738 | 1240,59,4,4,4,4 739 | 1240,59,4,4,4,4 740 | 1241,32,5,5,4,5 741 | 1241,38,5,5,4,5 742 | 1241,39,3,5,4,5 743 | 1241,44,3,5,5,5 744 | 1241,47,5,5,5,4 745 | 1244,29,3,4,4,3 746 | 1244,30,4,3,3,3 747 | 1244,32,2,3,2,1 748 | 1245,57,4,4,4,4 749 | 1245,63,5,5,5,4 750 | 1246,21,5,5,3,2 751 | 1246,23,2,2,2,2 752 | 1246,34,5,4,3,2 753 | 1246,41,5,5,4,2 754 | 1248,27,2,2,2,1 755 | 1248,32,5,5,5,2 756 | 1248,49,5,5,5,3 757 | 1248,50,4,3,2,2 758 | 1248,60,4,4,4,3 759 | 1249,43,4,3,4,3 760 | 1249,62,2,3,3,3 761 | 1250,24,1,1,1,1 762 | 1250,27,1,1,2,1 763 | 1250,32,5,5,5,5 764 | 1250,58,5,5,5,5 765 | 1251,25,5,5,5,5 766 | 1251,29,1,2,2,2 767 | 1251,30,5,5,5,5 768 | 1253,22,2,2,3,2 769 | 1253,62,4,5,5,4 770 | 1254,21,4,4,4,4 771 | 1254,28,4,4,4,3 772 | 1254,51,3,3,3,3 773 | 1255,25,5,4,3,4 774 | 1255,70,4,5,3,4 775 | 1256,25,4,4,3,3 776 | 1256,29,2,3,2,4 777 | 1256,39,2,5,4,3 778 | 1256,40,5,5,5,3 779 | 1256,58,5,4,5,4 780 | 1256,61,5,5,5,5 781 | 1257,23,1,2,3,2 782 | 1257,31,4,4,4,4 783 | 1258,22,3,2,3,2 784 | 1258,28,5,5,5,4 785 | 1259,30,3,3,3,2 786 | 1259,32,5,4,3,3 787 | 1259,37,5,5,4,4 788 | 1259,52,5,5,5,5 789 | 1259,66,5,4,5,5 790 | 1259,69,4,4,4,4 791 | 1260,28,4,5,4,4 792 | 1260,30,5,5,4,4 793 | 1260,40,5,5,4,4 794 | 1260,44,4,4,4,4 795 | 1260,64,1,1,2,2 796 | 1260,69,4,4,4,4 797 | 1262,22,1,1,2,1 798 | 1262,49,5,5,5,4 799 | 1263,25,5,4,5,5 800 | 1263,32,5,5,3,4 801 | 1263,37,1,1,3,1 802 | 1263,40,5,4,2,5 803 | 1263,49,5,5,4,5 804 | 1263,58,1,2,3,2 805 | 1263,58,4,5,4,5 806 | 1263,69,5,5,4,5 807 | 1264,21,4,3,4,3 808 | 1264,30,4,5,3,3 809 | 1264,39,2,1,2,4 810 | 1265,25,4,3,4,4 811 | 1265,34,3,3,4,4 812 | 1265,39,5,5,4,5 813 | 1265,39,5,5,4,4 814 | 1265,40,2,4,4,4 815 | 1265,44,3,3,5,4 816 | 1265,51,3,2,2,1 817 | 1266,40,5,5,5,4 818 | 1266,53,1,2,2,2 819 | 1267,44,1,1,1,1 820 | 1267,70,5,5,5,3 821 | 1268,37,4,4,4,4 822 | 1268,64,2,2,2,3 823 | 1268,68,2,2,3,3 824 | 1268,68,2,2,3,3 825 | 1269,21,4,4,4,4 826 | 1269,54,2,2,2,2 827 | 1269,64,5,4,4,4 828 | 1269,64,4,4,5,4 829 | 1269,67,4,5,5,4 830 | 1269,69,4,5,4,4 831 | 1272,28,1,3,3,4 832 | 1272,38,4,4,3,3 833 | 1275,21,4,4,4,4 834 | 1275,23,1,1,3,2 835 | 1275,23,1,1,3,2 836 | 1275,57,2,2,3,2 837 | 1276,22,2,2,3,2 838 | 1276,34,5,5,5,3 839 | 1276,49,4,5,5,3 840 | 1277,49,2,3,3,3 841 | 1277,68,2,3,3,3 842 | 1278,21,4,4,4,5 843 | 1278,30,3,3,3,3 844 | 1278,32,5,4,4,4 845 | 1278,38,5,5,5,5 846 | 1278,40,5,5,5,5 847 | 1278,56,4,4,3,3 848 | 1278,63,3,4,3,3 849 | 1278,67,4,4,4,4 850 | 1280,31,1,1,1,2 851 | 1280,45,1,1,1,2 852 | 1281,25,1,1,1,1 853 | 1281,27,1,1,1,1 854 | 1281,36,1,1,1,1 855 | 1281,40,5,5,5,2 856 | 1281,41,1,1,2,1 857 | 1281,44,1,2,1,1 858 | 1281,46,4,3,4,2 859 | 1281,58,4,4,4,2 860 | 1282,21,4,5,4,3 861 | 1282,51,3,4,2,4 862 | 1282,61,3,4,3,5 863 | 1282,69,5,4,4,3 864 | 1283,27,1,1,3,1 865 | 1283,48,2,2,3,1 866 | 1284,26,4,4,4,4 867 | 1284,30,2,1,2,2 868 | 1285,32,5,5,5,4 869 | 1285,59,5,5,5,4 870 | 1287,31,2,2,2,2 871 | 1287,66,4,3,4,3 872 | 1288,23,1,2,3,2 873 | 1288,49,5,4,5,4 874 | 1289,68,1,2,3,4 875 | 1289,69,5,5,5,4 876 | 1293,33,4,5,4,3 877 | 1293,33,4,5,4,3 878 | 1293,49,1,3,1,2 879 | 1293,58,1,3,1,2 880 | 1293,69,4,4,4,3 881 | 1295,22,4,4,4,4 882 | 1295,31,3,3,3,3 883 | 1297,49,5,5,5,5 884 | 1297,69,2,2,2,2 885 | 1298,40,1,3,3,3 886 | 1298,46,5,5,4,3 887 | 1299,25,4,5,5,5 888 | 1299,34,2,2,1,1 889 | 1300,25,5,5,4,3 890 | 1300,39,1,1,1,1 891 | 1300,53,1,1,1,1 892 | 1300,53,1,1,1,1 893 | 1301,27,1,1,3,3 894 | 1301,31,2,2,2,4 895 | 1301,40,5,5,4,4 896 | 1301,40,5,4,4,5 897 | 1302,25,2,2,2,2 898 | 1302,32,2,3,2,3 899 | 1304,30,5,5,3,3 900 | 1304,55,1,3,1,1 901 | 1305,22,2,3,4,5 902 | 1305,34,5,5,5,2 903 | 1306,22,2,3,3,3 904 | 1306,41,5,4,4,3 905 | 1307,21,5,4,4,4 906 | 1307,49,4,4,4,4 907 | 1308,54,2,2,3,4 908 | 1308,62,5,4,4,4 909 | 1309,25,5,5,4,4 910 | 1309,53,2,3,3,3 911 | 1309,60,5,5,5,5 912 | 1310,29,2,2,3,2 913 | 1310,49,4,4,4,4 914 | 1311,36,1,3,3,3 915 | 1311,38,1,3,3,3 916 | 1312,32,4,4,5,5 917 | 1312,50,2,3,3,3 918 | 1312,53,2,1,3,3 919 | 1312,54,1,2,3,3 920 | 1312,55,5,4,5,5 921 | 1312,60,5,2,4,3 922 | 1312,62,5,5,5,5 923 | 1313,28,3,5,5,4 924 | 1313,32,4,3,3,3 925 | 1314,22,3,4,4,3 926 | 1314,64,1,2,2,3 927 | 1315,25,4,4,5,3 928 | 1315,30,4,4,4,3 929 | 1316,36,4,4,4,3 930 | 1316,48,3,3,3,2 931 | 1317,21,4,4,5,4 932 | 1317,42,3,4,4,3 933 | 1317,59,1,3,4,3 934 | 1317,68,4,3,4,4 935 | 1318,21,4,4,4,3 936 | 1318,23,4,3,4,3 937 | 1318,27,2,3,2,3 938 | 1318,29,4,4,4,3 939 | 1318,32,5,3,2,4 940 | 1318,56,5,4,3,3 941 | 1318,57,4,3,3,4 942 | 1319,44,5,4,2,2 943 | 1319,49,1,1,1,1 944 | 1319,54,4,4,2,4 945 | 1319,66,4,2,2,1 946 | 1320,32,4,4,5,4 947 | 1320,39,2,2,4,3 948 | 1321,61,2,2,2,3 949 | 1321,68,1,2,2,3 950 | 1322,23,5,5,3,3 951 | 1322,33,5,5,3,3 952 | 1322,45,3,4,4,3 953 | 1322,55,3,4,4,3 954 | 1322,62,5,2,3,3 955 | 1323,31,4,4,4,4 956 | 1323,44,2,3,2,3 957 | 1323,49,5,4,4,4 958 | 1324,29,2,3,2,2 959 | 1324,30,5,5,4,3 960 | 1325,49,4,3,3,2 961 | 1325,69,2,3,3,3 962 | 1326,31,5,3,4,3 963 | 1326,36,2,3,2,2 964 | 1326,37,5,4,5,3 965 | 1326,49,4,4,4,3 966 | 1327,35,4,4,4,3 967 | 1327,56,5,5,4,2 968 | 1328,24,3,4,4,3 969 | 1328,25,5,5,3,3 970 | 1328,30,5,4,4,4 971 | 1328,49,4,4,4,4 972 | 1329,30,3,4,3,3 973 | 1329,33,3,4,4,3 974 | 1330,26,2,4,3,4 975 | 1330,30,5,5,4,3 976 | 1330,52,1,3,2,4 977 | 1330,56,4,4,5,2 978 | 1330,57,1,4,3,3 979 | 1330,62,1,2,2,4 980 | 1330,69,5,4,4,3 981 | 1330,70,2,3,3,3 982 | 1331,25,4,4,3,4 983 | 1331,38,3,3,3,3 984 | 1331,67,5,5,4,4 985 | -------------------------------------------------------------------------------- /UBMSRS/jar/UBMSRS.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/jar/UBMSRS.jar -------------------------------------------------------------------------------- /UBMSRS/jar/setting.conf: -------------------------------------------------------------------------------- 1 | 2 | # setup the path or the folder 3 | # just put all the data in this folder 4 | data.path.wins=d:\\data\\ 5 | data.path.lins=/home/user/ 6 | 7 | # turn on learning student expectations 8 | # the demo will load expectations if you turn off learning 9 | expectation.learn=on 10 | 11 | # the demo will run baseline approaches first if you turn on it 12 | runbaseline=on 13 | 14 | # to run baseline Rankp, we need such an expectation file, default file is "expectations_student_learned_by_UBRec_NDCG_0.214.csv" 15 | # you can also load other expectation file you learned by UBRec or MOO 16 | expectation.filename=expectations_student_learned_by_UBRec_NDCG_0.214.csv 17 | 18 | # how many items the system will recommend 19 | topN=10 20 | 21 | # maximal number of evaluations 22 | maxeval=1000 23 | 24 | # max metrics from UBRec and Rankp 25 | # they are used to calculate the loss 26 | maxf1=0.107 27 | maxndcg=0.214 28 | maxutil_instructor=0.241 29 | maxutil_student=0.745 -------------------------------------------------------------------------------- /UBMSRS/lib/JMetal-4.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/JMetal-4.3.jar -------------------------------------------------------------------------------- /UBMSRS/lib/MOEAFramework-2.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/MOEAFramework-2.12.jar -------------------------------------------------------------------------------- /UBMSRS/lib/commons-cli-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/commons-cli-1.2.jar -------------------------------------------------------------------------------- /UBMSRS/lib/commons-codec-1.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/commons-codec-1.8.jar -------------------------------------------------------------------------------- /UBMSRS/lib/commons-lang3-3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/commons-lang3-3.1.jar -------------------------------------------------------------------------------- /UBMSRS/lib/commons-math3-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/commons-math3-3.4.1.jar -------------------------------------------------------------------------------- /UBMSRS/lib/guava-15.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/guava-15.0.jar -------------------------------------------------------------------------------- /UBMSRS/lib/happy.coding.utils-1.2.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/happy.coding.utils-1.2.5.jar -------------------------------------------------------------------------------- /UBMSRS/lib/jcommon-1.0.20.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/jcommon-1.0.20.jar -------------------------------------------------------------------------------- /UBMSRS/lib/jfreechart-1.0.15.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/jfreechart-1.0.15.jar -------------------------------------------------------------------------------- /UBMSRS/lib/log4j-1.2.16.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/log4j-1.2.16.jar -------------------------------------------------------------------------------- /UBMSRS/lib/rsyntaxtextarea.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/rsyntaxtextarea.jar -------------------------------------------------------------------------------- /UBMSRS/lib/slf4j-api-1.6.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/slf4j-api-1.6.4.jar -------------------------------------------------------------------------------- /UBMSRS/lib/slf4j-log4j12-1.6.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irecsys/Tutorial_MSRS/ec2ebb7837348c1b7c00ffdba95778709ada2c46/UBMSRS/lib/slf4j-log4j12-1.6.4.jar -------------------------------------------------------------------------------- /UBMSRS/logs/demo.log: -------------------------------------------------------------------------------- 1 | INFO - 2020-02-25 16:21:25.766; happy.coding.io.Logs; Local path of your data: d:\data\ 2 | INFO - 2020-02-25 16:21:25.771; happy.coding.io.Logs; The system is running baseline approaches... 3 | -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/baseline/Rankp.java: -------------------------------------------------------------------------------- 1 | package msrs.baseline; 2 | 3 | 4 | import com.google.common.collect.ArrayListMultimap; 5 | import com.google.common.collect.HashMultimap; 6 | import com.google.common.collect.TreeMultimap; 7 | import com.google.common.collect.Ordering; 8 | import com.google.common.collect.Multimap; 9 | import com.google.common.primitives.Doubles; 10 | 11 | import happy.coding.io.FileConfiger; 12 | import happy.coding.io.FileIO; 13 | import happy.coding.io.Logs; 14 | import happy.coding.math.Maths; 15 | import happy.coding.math.Sims; 16 | import happy.coding.math.Stats; 17 | import msrs.demo.Config; 18 | import happy.coding.math.Measures; 19 | import happy.coding.io.Lists; 20 | 21 | import java.io.BufferedReader; 22 | import java.io.BufferedWriter; 23 | import java.io.File; 24 | import java.text.DecimalFormat; 25 | import java.util.*; 26 | import org.apache.commons.lang3.ArrayUtils; 27 | import org.apache.commons.math3.ml.distance.EuclideanDistance; 28 | import org.moeaframework.core.variable.EncodingUtils; 29 | 30 | import java.util.AbstractMap.SimpleImmutableEntry; 31 | 32 | /** 33 | * 34 | * Rank the items and produce top-N recommendations by considering the utility of the items from the perspective of instructors ONLY. 35 | * 36 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 37 | */ 38 | 39 | 40 | public class Rankp { 41 | 42 | String path=""; 43 | Config conf; 44 | String fileRatings_instructor="ratings_instructor.csv"; 45 | String fileRatings_candidates="ratings_student_candidates.csv"; 46 | String fileRatings_test="ratings_student_test.csv"; 47 | String fileExpectations_student; 48 | 49 | int numRecs; // top-N recommendations, N = 5 by default 50 | // student and instructor expectations 51 | HashMap> exp_students = new HashMap<>(); 52 | double[] exp_instructor; 53 | DecimalFormat df = new DecimalFormat("#.000"); 54 | 55 | // ratings 56 | HashMap> ratings_instructor = new HashMap<>(); 57 | HashMap> ratings_candidates= new HashMap<>(); 58 | 59 | HashMap utilities_instructor; 60 | EuclideanDistance dist=new EuclideanDistance(); 61 | HashMultimap truth; 62 | HashMultimap candidates; 63 | 64 | // metrics or objectives 65 | double utility_topN_students=0; 66 | double utility_topN_instructors=0; 67 | double utility_topN_diff=0; 68 | double f1_topN=0; 69 | double ndcg_topN=0; 70 | 71 | public Rankp(Config conf) { 72 | try 73 | { 74 | this.conf=conf; 75 | this.path=conf.getPath(); 76 | this.numRecs=conf.getNumRec(); 77 | this.fileExpectations_student=conf.getExpectationFilename(); 78 | 79 | // load student expectations from external file 80 | LoadExpectations_Student(fileExpectations_student); 81 | // set instructor expectations 82 | exp_instructor = new double[] {4.0, 4.0}; 83 | 84 | // load ratings by instructors 85 | LoadRatings_Instructor(); 86 | // load ratings with candidates 87 | LoadRatings_Candidates(); 88 | 89 | 90 | }catch(Exception e) { 91 | e.printStackTrace(); 92 | } 93 | } 94 | 95 | public void run() throws Exception { 96 | 97 | // calculate utility of the items in view of instructors 98 | calUtility_Instructor(); 99 | // collect Truth from the test set for the purpose of evaluations 100 | collectTruth(); 101 | // run recommendations and calculate metrics 102 | recommend(); 103 | 104 | // output information 105 | double[] loss=this.getLoss(); 106 | Logs.info("Rankp: Baseline Rankp has been executed."); 107 | Logs.info("Rankp: F1 = "+df.format(this.f1_topN) 108 | +", NDCG = "+df.format(this.ndcg_topN)+", Utility_List_Student U(s, L) = "+df.format(this.utility_topN_students) 109 | + ", Utility_List_Instructor U(p, L) = "+df.format(this.utility_topN_instructors) 110 | +", Loss1 = "+df.format(loss[0]) 111 | +", Loss2 = "+df.format(loss[1])); 112 | } 113 | 114 | protected double[] getLoss() { 115 | // maximal metrics from baseline approaches 116 | Double maxF1 = conf.getMaxF1(); // from UBRec 117 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 118 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 119 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 120 | 121 | double loss1 = ( (max_util_instructor - this.utility_topN_instructors)/max_util_instructor 122 | + (max_util_student - this.utility_topN_students)/max_util_student 123 | + ( (maxF1 - this.f1_topN)/maxF1 + (maxNDCG - this.ndcg_topN)/maxNDCG )/2 )/3; 124 | double loss2 = ( (max_util_instructor - this.utility_topN_instructors)/max_util_instructor 125 | + ( (maxF1 - this.f1_topN)/maxF1 + (maxNDCG - this.ndcg_topN)/maxNDCG )/2 )/2; 126 | 127 | double[] loss = new double[] {loss1, loss2}; 128 | return loss; 129 | } 130 | 131 | public double getUtility_students(){ 132 | return this.utility_topN_students; 133 | } 134 | 135 | public double getUtility_instructors() { 136 | return this.utility_topN_instructors; 137 | } 138 | 139 | public double getUtility_diffs() { 140 | return this.utility_topN_diff; 141 | } 142 | 143 | public double getF1() { 144 | return this.f1_topN; 145 | } 146 | 147 | public double getNDCG() { 148 | return this.ndcg_topN; 149 | } 150 | 151 | protected void LoadRatings_Instructor() throws Exception { 152 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 153 | String line = br.readLine(); 154 | 155 | while ((line = br.readLine()) != null) { 156 | String[] strs = line.split(","); 157 | String item = strs[0].trim(); 158 | this.ratings_instructor.put(item, 159 | new ArrayList(Arrays.asList( 160 | new Double(strs[1]), 161 | new Double(strs[2])) 162 | ) 163 | ); 164 | } 165 | br.close(); 166 | } 167 | 168 | protected void LoadRatings_Candidates() throws Exception { 169 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 170 | String line = br.readLine(); 171 | candidates=HashMultimap.create(); 172 | 173 | while ((line = br.readLine()) != null) { 174 | String[] strs = line.split(","); 175 | String student = strs[0].trim(); 176 | String item = strs[1].trim(); 177 | candidates.put(student, item); 178 | this.ratings_candidates.put(student+","+item, 179 | new ArrayList(Arrays.asList( 180 | new Double(strs[3]), 181 | new Double(strs[4]), 182 | new Double(strs[5])) 183 | ) 184 | ); 185 | } 186 | br.close(); 187 | } 188 | 189 | protected void LoadExpectations_Student(String fileExpectations_student) throws Exception { 190 | BufferedReader br=FileIO.getReader(path+fileExpectations_student); 191 | String line=br.readLine(); 192 | while((line=br.readLine())!=null) { 193 | String[] strs=line.split(","); 194 | this.exp_students.put(strs[0].trim(), 195 | new ArrayList(Arrays.asList( 196 | Double.parseDouble(strs[1].trim()), 197 | Double.parseDouble(strs[2].trim()), 198 | Double.parseDouble(strs[3].trim())))); 199 | } 200 | } 201 | 202 | protected void calUtility_Instructor() throws Exception{ 203 | utilities_instructor=new HashMap<>(); 204 | for(String item:ratings_instructor.keySet()) { 205 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 206 | double distance = dist.compute(this.exp_instructor, rates); 207 | // convert distance to dissimilarity: a normalization process 208 | // note that utility in view of instructor is the dissimilarity 209 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 210 | utilities_instructor.put(item, dissim); 211 | } 212 | } 213 | 214 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 215 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 216 | } 217 | 218 | protected void collectTruth() throws Exception 219 | { 220 | truth=HashMultimap.create(); 221 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 222 | String line=br.readLine(); 223 | while((line=br.readLine())!=null){ 224 | String[] strs=line.split(","); 225 | String user=strs[0].trim(); 226 | String item=strs[1].trim(); 227 | double rate=new Double(strs[2].trim()); 228 | if(rate>3) // this is an relevant item 229 | { 230 | truth.put(user, item); 231 | } 232 | } 233 | br.close(); 234 | } 235 | 236 | private void recommend() throws Exception { 237 | 238 | HashMap> rankingScore_student_item=new HashMap<>(); 239 | 240 | int count=0; 241 | for(String student:candidates.keySet()) { 242 | 243 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 244 | List> itemScores = new ArrayList<>(); 245 | HashMap utilities_student=new HashMap(); 246 | 247 | // for each candidate item to be recommended 248 | // calculate ranking score of the items for each student 249 | for(String candidateItem:candidates.get(student)) { 250 | 251 | String key=student+","+candidateItem; 252 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 253 | double distance = dist.compute(exp_student, rate_student); 254 | // convert distance to normalized similarity 255 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 256 | double itemUtility_instructor = this.utilities_instructor.get(candidateItem); 257 | // rank by the item utility in view of instructor only 258 | double score = itemUtility_instructor; 259 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 260 | utilities_student.put(candidateItem, itemUtility_student); 261 | } 262 | 263 | // produce top-N recommendations 264 | double utility_list_student=0; 265 | double utility_list_instructor=0; 266 | double utility_list_diff=0; 267 | 268 | Lists.sortList(itemScores, true); 269 | List rankedItems = new ArrayList<>(); 270 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 271 | : itemScores.subList(0, numRecs); 272 | for (Map.Entry kv : recomd) { 273 | String item = kv.getKey(); 274 | rankedItems.add(item); 275 | utility_list_student += utilities_student.get(item); 276 | utility_list_instructor += utilities_instructor.get(item); 277 | } 278 | 279 | // calculate metrics 280 | Set relevantItems = truth.get(student); 281 | List correctItems = new ArrayList<>(); 282 | correctItems.addAll(relevantItems); 283 | 284 | 285 | if(correctItems.size()>0 && rankedItems.size()>0) { 286 | ++count; 287 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 288 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 289 | double f1 = 0; 290 | if((precision+recall)!=0) 291 | f1 = 2*precision*recall/(precision+recall); 292 | double ndcg = Measures.nDCG(rankedItems, correctItems); 293 | utility_list_student/=rankedItems.size(); 294 | utility_list_instructor/=rankedItems.size(); 295 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 296 | 297 | utility_topN_students+=utility_list_student; 298 | utility_topN_instructors+=utility_list_instructor; 299 | utility_topN_diff+=utility_list_diff; 300 | f1_topN+=f1; 301 | ndcg_topN+=ndcg; 302 | } 303 | 304 | } 305 | // calculate the final metrics over all students 306 | utility_topN_students/=count; 307 | utility_topN_instructors/=count; 308 | utility_topN_diff/=count; 309 | f1_topN/=count; 310 | ndcg_topN/=count; 311 | } 312 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/baseline/ubrec/RunUBRec.java: -------------------------------------------------------------------------------- 1 | package msrs.baseline.ubrec; 2 | 3 | import happy.coding.io.FileConfiger; 4 | import happy.coding.io.FileIO; 5 | import happy.coding.io.LineConfiger; 6 | import happy.coding.io.Logs; 7 | import msrs.demo.Config; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.BufferedWriter; 11 | import java.text.DecimalFormat; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | 17 | import org.moeaframework.Executor; 18 | import org.moeaframework.core.NondominatedPopulation; 19 | import org.moeaframework.core.Solution; 20 | import org.moeaframework.core.variable.EncodingUtils; 21 | 22 | import com.google.common.collect.ArrayListMultimap; 23 | 24 | public class RunUBRec { 25 | String path=null; 26 | DecimalFormat df = new DecimalFormat("#.000"); 27 | protected Config conf; 28 | 29 | public RunUBRec(Config conf) { 30 | this.conf=conf; 31 | this.path=conf.getPath(); 32 | } 33 | 34 | public void execute() throws Exception { 35 | 36 | String[] algorithms = {"OMOPSO"}; // you can select other optimizers too! 37 | 38 | Solution bestSolution = null; 39 | double bestNDCG = -1; 40 | 41 | for (String algorithm : algorithms) { 42 | NondominatedPopulation rst = new Executor() 43 | .withProblemClass(UBRecProblem.class, conf) 44 | .withMaxEvaluations(conf.getMaxEval()) // maximal function evaluations 45 | .withAlgorithm(algorithm) 46 | .distributeOnAllCores() 47 | .run(); 48 | 49 | 50 | for (Solution sol : rst) { 51 | double ndcg=Math.abs(Math.abs(sol.getObjective(0))); 52 | if(ndcg>bestNDCG) { 53 | bestNDCG=ndcg; 54 | bestSolution=sol; 55 | } 56 | } 57 | } 58 | 59 | // output the learned student expectations into external files 60 | double[] array=EncodingUtils.getReal(bestSolution); 61 | BufferedWriter bw=FileIO.getWriter(path+"expectations_student_learned_by_UBRec.csv"); 62 | bw.write("User, App, Data, Ease\n"); 63 | bw.flush(); 64 | 65 | int uid=999; 66 | for(int i=0;i> exp_students = new HashMap<>(); 49 | double[] exp_instructor; 50 | 51 | // ratings 52 | HashMap> ratings_instructor = new HashMap<>(); 53 | HashMap> ratings_candidates= new HashMap<>(); 54 | 55 | HashMap utilities_instructor; 56 | EuclideanDistance dist=new EuclideanDistance(); 57 | HashMultimap truth; 58 | HashMultimap candidates; 59 | 60 | // metrics or objectives 61 | double utility_topN_students=0; 62 | double utility_topN_instructors=0; 63 | double utility_topN_diff=0; 64 | double f1_topN=0; 65 | double ndcg_topN=0; 66 | 67 | public UBRec(Config conf, double[] parameters) { 68 | try 69 | { 70 | this.conf=conf; 71 | this.path=conf.getPath(); 72 | this.numRecs=conf.getNumRec(); 73 | // load student expectations from the parameters 74 | LoadExpectations_Student(parameters); 75 | // set instructor expectations 76 | exp_instructor = new double[] {4.0, 4.0}; 77 | 78 | 79 | }catch(Exception e) { 80 | e.printStackTrace(); 81 | } 82 | } 83 | 84 | public void run() throws Exception { 85 | 86 | // load ratings by instructors 87 | LoadRatings_Instructor(); 88 | // load ratings with candidates 89 | LoadRatings_Candidates(); 90 | 91 | // calculate utility of the items in view of instructors 92 | calUtility_Instructor(); 93 | // collect Truth from the test set for the purpose of evaluations 94 | collectTruth(); 95 | // run recommendations and calculate metrics 96 | recommend(); 97 | } 98 | 99 | protected double[] getLoss() { 100 | // maximal metrics from baseline approaches 101 | Double maxF1 = conf.getMaxF1(); // from UBRec 102 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 103 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 104 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 105 | 106 | double loss1 = ( (max_util_instructor - this.utility_topN_instructors)/max_util_instructor 107 | + (max_util_student - this.utility_topN_students)/max_util_student 108 | + ( (maxF1 - this.f1_topN)/maxF1 + (maxNDCG - this.ndcg_topN)/maxNDCG )/2 )/3; 109 | double loss2 = ( (max_util_instructor - this.utility_topN_instructors)/max_util_instructor 110 | + ( (maxF1 - this.f1_topN)/maxF1 + (maxNDCG - this.ndcg_topN)/maxNDCG )/2 )/2; 111 | 112 | double[] loss = new double[] {loss1, loss2}; 113 | return loss; 114 | } 115 | 116 | public double getUtility_students(){ 117 | return this.utility_topN_students; 118 | } 119 | 120 | public double getUtility_instructors() { 121 | return this.utility_topN_instructors; 122 | } 123 | 124 | public double getUtility_diffs() { 125 | return this.utility_topN_diff; 126 | } 127 | 128 | public double getF1() { 129 | return this.f1_topN; 130 | } 131 | 132 | public double getNDCG() { 133 | return this.ndcg_topN; 134 | } 135 | 136 | protected void LoadRatings_Instructor() throws Exception { 137 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 138 | String line = br.readLine(); 139 | 140 | while ((line = br.readLine()) != null) { 141 | String[] strs = line.split(","); 142 | String item = strs[0].trim(); 143 | this.ratings_instructor.put(item, 144 | new ArrayList(Arrays.asList( 145 | new Double(strs[1]), 146 | new Double(strs[2])) 147 | ) 148 | ); 149 | } 150 | br.close(); 151 | } 152 | 153 | protected void LoadRatings_Candidates() throws Exception { 154 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 155 | String line = br.readLine(); 156 | candidates=HashMultimap.create(); 157 | 158 | while ((line = br.readLine()) != null) { 159 | String[] strs = line.split(","); 160 | String student = strs[0].trim(); 161 | String item = strs[1].trim(); 162 | candidates.put(student, item); 163 | this.ratings_candidates.put(student+","+item, 164 | new ArrayList(Arrays.asList( 165 | new Double(strs[3]), 166 | new Double(strs[4]), 167 | new Double(strs[5])) 168 | ) 169 | ); 170 | } 171 | br.close(); 172 | } 173 | 174 | protected void LoadExpectations_Student(double[] parameters) throws Exception { 175 | // we have UserID: 1000 to 1331, 332 users 176 | int student = 1000; 177 | for (int i = 0; i < parameters.length; ++i) { 178 | this.exp_students.put(""+student, 179 | new ArrayList(Arrays.asList( 180 | parameters[i], 181 | parameters[++i], 182 | parameters[++i]))); 183 | student++; 184 | } 185 | } 186 | 187 | protected void calUtility_Instructor() throws Exception{ 188 | utilities_instructor=new HashMap<>(); 189 | for(String item:ratings_instructor.keySet()) { 190 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 191 | double distance = dist.compute(this.exp_instructor, rates); 192 | // convert distance to dissimilarity: a normalization process 193 | // note that utility in view of instructor is the dissimilarity 194 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 195 | utilities_instructor.put(item, dissim); 196 | } 197 | } 198 | 199 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 200 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 201 | } 202 | 203 | protected void collectTruth() throws Exception 204 | { 205 | truth=HashMultimap.create(); 206 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 207 | String line=br.readLine(); 208 | while((line=br.readLine())!=null){ 209 | String[] strs=line.split(","); 210 | String user=strs[0].trim(); 211 | String item=strs[1].trim(); 212 | double rate=new Double(strs[2].trim()); 213 | if(rate>3) // this is an relevant item 214 | { 215 | truth.put(user, item); 216 | } 217 | } 218 | br.close(); 219 | } 220 | 221 | private void recommend() throws Exception { 222 | 223 | HashMap> rankingScore_student_item=new HashMap<>(); 224 | 225 | int count=0; 226 | if(candidates.size()==0) 227 | { 228 | Logs.error("No candidates"); 229 | return; 230 | } 231 | 232 | for(String student:candidates.keySet()) { 233 | 234 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 235 | List> itemScores = new ArrayList<>(); 236 | HashMap utilities_student=new HashMap(); 237 | 238 | // for each candidate item to be recommended 239 | // calculate ranking score of the items for each student 240 | for(String candidateItem:candidates.get(student)) { 241 | 242 | String key=student+","+candidateItem; 243 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 244 | double distance = dist.compute(exp_student, rate_student); 245 | // convert distance to normalized similarity 246 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 247 | // use the item utility in view of the students to rank the items 248 | double score = itemUtility_student; 249 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 250 | utilities_student.put(candidateItem, itemUtility_student); 251 | } 252 | 253 | // produce top-N recommendations 254 | double utility_list_student=0; 255 | double utility_list_instructor=0; 256 | double utility_list_diff=0; 257 | 258 | Lists.sortList(itemScores, true); 259 | List rankedItems = new ArrayList<>(); 260 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 261 | : itemScores.subList(0, numRecs); 262 | for (Map.Entry kv : recomd) { 263 | String item = kv.getKey(); 264 | rankedItems.add(item); 265 | utility_list_student += utilities_student.get(item); 266 | utility_list_instructor += utilities_instructor.get(item); 267 | } 268 | 269 | // calculate metrics 270 | Set relevantItems = truth.get(student); 271 | List correctItems = new ArrayList<>(); 272 | correctItems.addAll(relevantItems); 273 | 274 | 275 | if(correctItems.size()>0 && rankedItems.size()>0) { 276 | ++count; 277 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 278 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 279 | double f1 = 0; 280 | if((precision+recall)!=0) 281 | f1 = 2*precision*recall/(precision+recall); 282 | double ndcg = Measures.nDCG(rankedItems, correctItems); 283 | utility_list_student/=rankedItems.size(); 284 | utility_list_instructor/=rankedItems.size(); 285 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 286 | 287 | utility_topN_students+=utility_list_student; 288 | utility_topN_instructors+=utility_list_instructor; 289 | utility_topN_diff+=utility_list_diff; 290 | f1_topN+=f1; 291 | ndcg_topN+=ndcg; 292 | } 293 | 294 | } 295 | // calculate the final metrics over all students 296 | utility_topN_students/=count; 297 | utility_topN_instructors/=count; 298 | utility_topN_diff/=count; 299 | f1_topN/=count; 300 | ndcg_topN/=count; 301 | } 302 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/baseline/ubrec/UBRecProblem.java: -------------------------------------------------------------------------------- 1 | package msrs.baseline.ubrec; 2 | 3 | import org.moeaframework.core.Solution; 4 | import org.moeaframework.core.variable.EncodingUtils; 5 | import org.moeaframework.problem.AbstractProblem; 6 | 7 | import happy.coding.io.FileIO; 8 | import msrs.demo.Config; 9 | 10 | import java.io.BufferedReader; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | 14 | 15 | /** 16 | * 17 | * We learn user expectations by maximizing NDCG 18 | * 19 | * Yong Zheng. "Utility-Based Multi-Criteria Recommender Systems", Proceedings of the 34th ACM SIGAPP Symposium on Applied Computing (ACM SAC), Limassol, Cyprus, April, 2019 20 | * 21 | */ 22 | 23 | 24 | 25 | public class UBRecProblem extends AbstractProblem { 26 | 27 | Config conf; 28 | 29 | public UBRecProblem(Config conf) { 30 | 31 | // super(a, b); 32 | // a = # of parameters to be learned 33 | // b = # of objectives, we use NDCG as the single objective 34 | 35 | // we have UserID: 1000 to 1331, 332 users 36 | // student expectation is a 3-dimension vector 37 | // a = 332*3 = 996 38 | 39 | super(996,1); 40 | this.conf=conf; 41 | } 42 | 43 | @Override 44 | public void evaluate(Solution solution){ 45 | try { 46 | UBRec rec=new UBRec(conf, EncodingUtils.getReal(solution)); 47 | rec.run(); 48 | solution.setObjective(0, (-1.0)*rec.getNDCG()); // maximize 49 | solution.setAttribute("F1", rec.getF1()); 50 | solution.setAttribute("UtilityStudent", rec.getUtility_students()); 51 | solution.setAttribute("UtilityInstructor", rec.getUtility_instructors()); 52 | double[] loss=rec.getLoss(); 53 | solution.setAttribute("Loss1", loss[0]); 54 | solution.setAttribute("Loss2", loss[1]); 55 | }catch(Exception e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | 60 | @Override 61 | public Solution newSolution() { 62 | Solution solution =new Solution(996,1); 63 | for(int i=0;i<996;++i) 64 | solution.setVariable(i, EncodingUtils.newReal(1.0, 5.0)); // rating scale is 1 to 5 65 | 66 | // set additional attributes if necessary 67 | Map attributes = new HashMap(); 68 | attributes.put("F1", 0.0); 69 | attributes.put("UtilityStudent", 0.0); 70 | attributes.put("UtilityInstructor", 0.0); 71 | attributes.put("Loss", 0.0); 72 | solution.addAttributes(attributes); 73 | return solution; 74 | } 75 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/demo/Config.java: -------------------------------------------------------------------------------- 1 | package msrs.demo; 2 | 3 | public class Config { 4 | 5 | protected String path; 6 | protected String fileExpectations_student; 7 | protected int N; 8 | protected int maxEval; 9 | 10 | protected double maxf1; 11 | protected double maxndcg; 12 | protected double maxutil_instructor; 13 | protected double maxutil_student; 14 | 15 | public Config() { 16 | 17 | } 18 | 19 | public Config(String folder, String filename, int n, int maxEval, 20 | double maxf1, double maxndcg, double maxutil_instructor, double maxutil_student) { 21 | this.path=folder; 22 | this.fileExpectations_student=filename; 23 | this.N=n; 24 | this.maxEval=maxEval; 25 | this.maxf1=maxf1; 26 | this.maxndcg=maxndcg; 27 | this.maxutil_instructor=maxutil_instructor; 28 | this.maxutil_student=maxutil_student; 29 | } 30 | 31 | public String getPath() { 32 | return this.path; 33 | } 34 | 35 | public String getExpectationFilename() { 36 | return this.fileExpectations_student; 37 | } 38 | 39 | public int getNumRec() { 40 | return this.N; 41 | } 42 | 43 | public int getMaxEval() { 44 | return this.maxEval; 45 | } 46 | 47 | public double getMaxF1() { 48 | return this.maxf1; 49 | } 50 | 51 | public double getMaxNDCG() { 52 | return this.maxndcg; 53 | } 54 | 55 | public double getMaxUtil_Instructor() { 56 | return this.maxutil_instructor; 57 | } 58 | 59 | public double getMaxUtil_Student() { 60 | return this.maxutil_student; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/demo/UBMSRS.java: -------------------------------------------------------------------------------- 1 | package msrs.demo; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | import org.apache.log4j.PropertyConfigurator; 7 | 8 | import happy.coding.io.FileConfiger; 9 | import happy.coding.io.FileIO; 10 | import happy.coding.io.LineConfiger; 11 | import happy.coding.io.Logs; 12 | import msrs.baseline.Rankp; 13 | import msrs.baseline.ubrec.RunUBRec; 14 | import msrs.twostage.RunEduProblem_TwoStage; 15 | import msrs.onestage.RunEduProblem_OneStage; 16 | 17 | 18 | public class UBMSRS { 19 | 20 | protected List configFiles=null; 21 | protected String path; 22 | protected Config conf; 23 | 24 | 25 | public static void main(String[] args) throws Exception { 26 | try { 27 | new UBMSRS().execute(args); 28 | 29 | } catch (Exception e) { 30 | e.printStackTrace(); 31 | } 32 | } 33 | 34 | 35 | protected void execute(String[] args) throws Exception { 36 | 37 | cmdLine(args); 38 | 39 | FileConfiger cf = new FileConfiger(configFiles.get(0)); 40 | path=cf.getPath("data.path").trim(); 41 | Logs.info("Local path of your data: "+this.path); 42 | 43 | String filename_expectation = cf.getString("expectation.filename").trim(); 44 | int topN=cf.getInt("topN"); 45 | int max=cf.getInt("maxeval"); 46 | 47 | double maxf1=cf.getDouble("maxf1"); 48 | double maxndcg=cf.getDouble("maxndcg"); 49 | double maxutil_p=cf.getDouble("maxutil_instructor"); 50 | double maxutil_s=cf.getDouble("maxutil_student"); 51 | 52 | conf=new Config(path, filename_expectation, topN, max, maxf1, maxndcg, maxutil_p, maxutil_s); 53 | 54 | // Run baseline 55 | String runbaseline=cf.getString("runbaseline").toLowerCase().trim(); 56 | if(runbaseline.equals("on")) { 57 | Logs.info("The system is running baseline approaches..."); 58 | RunBaseline(conf); 59 | } 60 | 61 | // Run multi-stakeholder recommendations 62 | String learn=cf.getString("expectation.learn").toLowerCase().trim(); 63 | if(learn.equals("on")) { 64 | // learn user expectations and Alpha 65 | RunEduProblem_OneStage moo_learn=new RunEduProblem_OneStage(conf); 66 | moo_learn.execute(); 67 | }else if(learn.equals("off")){ 68 | // load user expectations and learn Alpha only 69 | RunEduProblem_TwoStage moo_learn=new RunEduProblem_TwoStage(conf); 70 | moo_learn.execute(); 71 | }else { 72 | Logs.error("expectation.learn: incorrect setting"); 73 | } 74 | } 75 | 76 | 77 | protected void cmdLine(String[] args) throws Exception { 78 | 79 | if (args == null || args.length < 1) { 80 | if (configFiles == null) 81 | configFiles = Arrays.asList("setting.conf"); 82 | return; 83 | } 84 | 85 | LineConfiger paramOptions = new LineConfiger(args); 86 | configFiles = paramOptions.contains("-c") ? paramOptions.getOptions("-c") : Arrays.asList("setting.conf"); 87 | } 88 | 89 | protected void RunBaseline(Config conf) throws Exception{ 90 | RunUBRec ubrec = new RunUBRec(conf); 91 | ubrec.execute(); 92 | 93 | Rankp rankp = new Rankp(conf); 94 | rankp.run(); 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/learnalpha/EduProblem.java: -------------------------------------------------------------------------------- 1 | package msrs.learnalpha; 2 | 3 | import org.moeaframework.core.Solution; 4 | import org.moeaframework.core.variable.EncodingUtils; 5 | import org.moeaframework.problem.AbstractProblem; 6 | 7 | import happy.coding.io.FileIO; 8 | import msrs.demo.Config; 9 | 10 | import java.io.BufferedReader; 11 | 12 | public class EduProblem extends AbstractProblem { 13 | 14 | String path=null; 15 | Config conf; 16 | 17 | public EduProblem(Config conf) { 18 | 19 | // super(a, b); 20 | // a = # of parameters to be learned 21 | // b = # of objectives 22 | 23 | // in this example, we only learn the Alpha 24 | 25 | super(1,5); 26 | this.conf=conf; 27 | this.path=conf.getPath(); 28 | } 29 | 30 | @Override 31 | public void evaluate(Solution solution){ 32 | try { 33 | EduRec rec=new EduRec(conf, EncodingUtils.getReal(solution)); 34 | rec.run(); 35 | solution.setObjective(0, (-1.0)*rec.getUtility_students()); // maximize 36 | solution.setObjective(1, (-1.0)*rec.getUtility_instructors()); // maximize 37 | solution.setObjective(2, rec.getUtility_diffs()); //minimize 38 | solution.setObjective(3, (-1.0)*rec.getF1()); // maximize 39 | solution.setObjective(4, (-1.0)*rec.getNDCG()); // maximize 40 | }catch(Exception e) { 41 | e.printStackTrace(); 42 | } 43 | } 44 | 45 | @Override 46 | public Solution newSolution() { 47 | Solution solution =new Solution(1,5); 48 | solution.setVariable(0, EncodingUtils.newReal(0.0,1.0)); // scale of the Alpha is 0 to 1 49 | return solution; 50 | } 51 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/learnalpha/EduRec.java: -------------------------------------------------------------------------------- 1 | package msrs.learnalpha; 2 | 3 | import com.google.common.collect.ArrayListMultimap; 4 | import com.google.common.collect.HashMultimap; 5 | import com.google.common.collect.TreeMultimap; 6 | import com.google.common.collect.Ordering; 7 | import com.google.common.collect.Multimap; 8 | import com.google.common.primitives.Doubles; 9 | 10 | import happy.coding.io.FileConfiger; 11 | import happy.coding.io.FileIO; 12 | import happy.coding.io.Logs; 13 | import happy.coding.math.Maths; 14 | import happy.coding.math.Sims; 15 | import happy.coding.math.Stats; 16 | import msrs.demo.Config; 17 | import happy.coding.math.Measures; 18 | import happy.coding.io.Lists; 19 | 20 | import java.io.BufferedReader; 21 | import java.io.BufferedWriter; 22 | import java.io.File; 23 | import java.util.*; 24 | import org.apache.commons.lang3.ArrayUtils; 25 | import org.apache.commons.math3.ml.distance.EuclideanDistance; 26 | import java.util.AbstractMap.SimpleImmutableEntry; 27 | 28 | 29 | /** 30 | * 31 | * Utility-based multi-stakeholder recommendations: the basic solution 32 | * Note: this implementation learns Alpha only 33 | * Note: this implementation did not deal with the issue of over-/under- expectations 34 | * 35 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 36 | * 37 | */ 38 | 39 | 40 | public class EduRec { 41 | 42 | String path=""; 43 | Config conf; 44 | String fileRatings_instructor="ratings_instructor.csv"; 45 | String fileRatings_candidates="ratings_student_candidates.csv"; 46 | String fileRatings_test="ratings_student_test.csv"; 47 | String fileExpectations_student; 48 | 49 | int numRecs; // top-N recommendations 50 | double alpha=1.0; 51 | // student and instructor expectations 52 | HashMap> exp_students = new HashMap<>(); 53 | double[] exp_instructor; 54 | 55 | // ratings 56 | HashMap> ratings_instructor = new HashMap<>(); 57 | HashMap> ratings_candidates= new HashMap<>(); 58 | 59 | HashMap utilities_instructor; 60 | EuclideanDistance dist=new EuclideanDistance(); 61 | HashMultimap truth; 62 | HashMultimap candidates; 63 | 64 | // metrics or objectives 65 | double utility_topN_students=0; 66 | double utility_topN_instructors=0; 67 | double utility_topN_diff=0; 68 | double f1_topN=0; 69 | double ndcg_topN=0; 70 | 71 | public EduRec(Config conf, double[] parameters) { 72 | try 73 | { 74 | this.conf=conf; 75 | this.path=conf.getPath(); 76 | this.numRecs=conf.getNumRec(); 77 | this.fileExpectations_student=conf.getExpectationFilename(); 78 | 79 | this.alpha=parameters[0]; 80 | // load student expectations from the parameters 81 | LoadExpectations_Student(fileExpectations_student); 82 | // set instructor expectations 83 | exp_instructor = new double[] {4.0, 4.0}; 84 | 85 | // load ratings by instructors 86 | LoadRatings_Instructor(); 87 | // load ratings with candidates 88 | LoadRatings_Candidates(); 89 | 90 | 91 | }catch(Exception e) { 92 | e.printStackTrace(); 93 | } 94 | } 95 | 96 | public void run() throws Exception { 97 | 98 | // calculate utility of the items in view of instructors 99 | calUtility_Instructor(); 100 | // collect Truth from the test set for the purpose of evaluations 101 | collectTruth(); 102 | // run recommendations and calculate metrics 103 | recommend(); 104 | } 105 | 106 | public double getUtility_students(){ 107 | return this.utility_topN_students; 108 | } 109 | 110 | public double getUtility_instructors() { 111 | return this.utility_topN_instructors; 112 | } 113 | 114 | public double getUtility_diffs() { 115 | return this.utility_topN_diff; 116 | } 117 | 118 | public double getF1() { 119 | return this.f1_topN; 120 | } 121 | 122 | public double getNDCG() { 123 | return this.ndcg_topN; 124 | } 125 | 126 | protected void LoadRatings_Instructor() throws Exception { 127 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 128 | String line = br.readLine(); 129 | 130 | while ((line = br.readLine()) != null) { 131 | String[] strs = line.split(","); 132 | String item = strs[0].trim(); 133 | this.ratings_instructor.put(item, 134 | new ArrayList(Arrays.asList( 135 | new Double(strs[1]), 136 | new Double(strs[2])) 137 | ) 138 | ); 139 | } 140 | br.close(); 141 | } 142 | 143 | protected void LoadRatings_Candidates() throws Exception { 144 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 145 | String line = br.readLine(); 146 | candidates=HashMultimap.create(); 147 | 148 | while ((line = br.readLine()) != null) { 149 | String[] strs = line.split(","); 150 | String student = strs[0].trim(); 151 | String item = strs[1].trim(); 152 | candidates.put(student, item); 153 | this.ratings_candidates.put(student+","+item, 154 | new ArrayList(Arrays.asList( 155 | new Double(strs[3]), 156 | new Double(strs[4]), 157 | new Double(strs[5])) 158 | ) 159 | ); 160 | } 161 | br.close(); 162 | } 163 | 164 | protected void LoadExpectations_Student(String fileExpectations_student) throws Exception { 165 | BufferedReader br=FileIO.getReader(path+fileExpectations_student); 166 | String line=br.readLine(); 167 | while((line=br.readLine())!=null) { 168 | String[] strs=line.split(","); 169 | this.exp_students.put(strs[0].trim(), 170 | new ArrayList(Arrays.asList( 171 | Double.parseDouble(strs[1].trim()), 172 | Double.parseDouble(strs[2].trim()), 173 | Double.parseDouble(strs[3].trim())))); 174 | } 175 | } 176 | 177 | protected void calUtility_Instructor() throws Exception{ 178 | utilities_instructor=new HashMap<>(); 179 | for(String item:ratings_instructor.keySet()) { 180 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 181 | double distance = dist.compute(this.exp_instructor, rates); 182 | // convert distance to dissimilarity: a normalization process 183 | // note that utility in view of instructor is the dissimilarity 184 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 185 | utilities_instructor.put(item, dissim); 186 | } 187 | } 188 | 189 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 190 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 191 | } 192 | 193 | protected void collectTruth() throws Exception 194 | { 195 | truth=HashMultimap.create(); 196 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 197 | String line=br.readLine(); 198 | while((line=br.readLine())!=null){ 199 | String[] strs=line.split(","); 200 | String user=strs[0].trim(); 201 | String item=strs[1].trim(); 202 | double rate=new Double(strs[2].trim()); 203 | if(rate>3) // this is an relevant item 204 | { 205 | truth.put(user, item); 206 | } 207 | } 208 | br.close(); 209 | } 210 | 211 | private void recommend() throws Exception { 212 | 213 | HashMap> rankingScore_student_item=new HashMap<>(); 214 | 215 | int count=0; 216 | for(String student:candidates.keySet()) { 217 | 218 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 219 | List> itemScores = new ArrayList<>(); 220 | HashMap utilities_student=new HashMap(); 221 | 222 | // for each candidate item to be recommended 223 | // calculate ranking score of the items for each student 224 | for(String candidateItem:candidates.get(student)) { 225 | 226 | String key=student+","+candidateItem; 227 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 228 | double distance = dist.compute(exp_student, rate_student); 229 | // convert distance to normalized similarity 230 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 231 | double itemUtility_instructor = this.utilities_instructor.get(candidateItem); 232 | double score = this.alpha*itemUtility_student + (1 - this.alpha)*itemUtility_instructor; 233 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 234 | utilities_student.put(candidateItem, itemUtility_student); 235 | } 236 | 237 | // produce top-N recommendations 238 | double utility_list_student=0; 239 | double utility_list_instructor=0; 240 | double utility_list_diff=0; 241 | 242 | Lists.sortList(itemScores, true); 243 | List rankedItems = new ArrayList<>(); 244 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 245 | : itemScores.subList(0, numRecs); 246 | for (Map.Entry kv : recomd) { 247 | String item = kv.getKey(); 248 | rankedItems.add(item); 249 | utility_list_student += utilities_student.get(item); 250 | utility_list_instructor += utilities_instructor.get(item); 251 | } 252 | 253 | // calculate metrics 254 | Set relevantItems = truth.get(student); 255 | List correctItems = new ArrayList<>(); 256 | correctItems.addAll(relevantItems); 257 | 258 | 259 | if(correctItems.size()>0 && rankedItems.size()>0) { 260 | ++count; 261 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 262 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 263 | double f1 = 0; 264 | if((precision+recall)!=0) 265 | f1 = 2*precision*recall/(precision+recall); 266 | double ndcg = Measures.nDCG(rankedItems, correctItems); 267 | utility_list_student/=rankedItems.size(); 268 | utility_list_instructor/=rankedItems.size(); 269 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 270 | 271 | utility_topN_students+=utility_list_student; 272 | utility_topN_instructors+=utility_list_instructor; 273 | utility_topN_diff+=utility_list_diff; 274 | f1_topN+=f1; 275 | ndcg_topN+=ndcg; 276 | } 277 | 278 | } 279 | // calculate the final metrics over all students 280 | utility_topN_students/=count; 281 | utility_topN_instructors/=count; 282 | utility_topN_diff/=count; 283 | f1_topN/=count; 284 | ndcg_topN/=count; 285 | } 286 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/learnalpha/RunEduProblem_LearnAlpha.java: -------------------------------------------------------------------------------- 1 | package msrs.learnalpha; 2 | 3 | import happy.coding.io.FileConfiger; 4 | import happy.coding.io.FileIO; 5 | import happy.coding.io.LineConfiger; 6 | import happy.coding.io.Logs; 7 | import msrs.demo.Config; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.BufferedWriter; 11 | import java.text.DecimalFormat; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | 17 | import org.moeaframework.Executor; 18 | import org.moeaframework.core.NondominatedPopulation; 19 | import org.moeaframework.core.Solution; 20 | import org.moeaframework.core.variable.EncodingUtils; 21 | 22 | import com.google.common.collect.ArrayListMultimap; 23 | 24 | 25 | /** 26 | * 27 | * Utility-based multi-stakeholder recommendations: the basic solution 28 | * Note: this implementation learns Alpha only 29 | * Note: this implementation did not deal with the issue of over-/under- expectations 30 | * 31 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 32 | * 33 | */ 34 | 35 | 36 | 37 | public class RunEduProblem_LearnAlpha { 38 | String path=null; 39 | Config conf; 40 | DecimalFormat df = new DecimalFormat("#.000"); 41 | 42 | public RunEduProblem_LearnAlpha(Config conf) { 43 | this.conf=conf; 44 | this.path=conf.getPath(); 45 | } 46 | 47 | public void execute() throws Exception { 48 | 49 | String[] algorithms = {"eMOEA", "NSGAII","NSGAIII", "MSOPS", "SMPSO", "OMOPSO"}; 50 | 51 | // maximal metrics from baseline approaches 52 | Double maxF1 = conf.getMaxF1(); // from UBRec 53 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 54 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 55 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 56 | 57 | Solution globalBest = null; 58 | double globalMinLoss=999; 59 | String globalBestAlgorithm=null; 60 | 61 | Logs.info("------------------------------------------------------------------------------"); 62 | 63 | for (String algorithm : algorithms) { 64 | Logs.info("Running MSRS by using "+algorithm+" as the optimizer..."); 65 | NondominatedPopulation rst = new Executor() 66 | .withProblemClass(EduProblem.class, conf) 67 | .withMaxEvaluations(conf.getMaxEval()) // maximal function evaluations 68 | .withAlgorithm(algorithm) 69 | .distributeOnAllCores() 70 | .run(); 71 | 72 | Solution localBest=null; 73 | double localMinLoss = 999; 74 | 75 | for (Solution sol : rst) { 76 | double f1=Math.abs(Math.abs(sol.getObjective(3))); 77 | double ndcg=Math.abs(sol.getObjective(4)); 78 | double util_student=Math.abs(sol.getObjective(0)); 79 | double util_instructor=Math.abs(sol.getObjective(1)); 80 | 81 | double loss = ( (max_util_instructor - util_instructor)/max_util_instructor 82 | + (max_util_student - util_student)/max_util_student 83 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/3; 84 | 85 | if(loss> exp_students = new HashMap<>(); 53 | double[] exp_instructor; 54 | 55 | // ratings 56 | HashMap> ratings_instructor = new HashMap<>(); 57 | HashMap> ratings_candidates= new HashMap<>(); 58 | 59 | HashMap utilities_instructor; 60 | EuclideanDistance dist=new EuclideanDistance(); 61 | HashMultimap truth; 62 | HashMultimap candidates; 63 | 64 | // metrics or objectives 65 | double utility_topN_students=0; 66 | double utility_topN_instructors=0; 67 | double utility_topN_diff=0; 68 | double f1_topN=0; 69 | double ndcg_topN=0; 70 | 71 | public EduRec(Config conf, double[] parameters) { 72 | try 73 | { 74 | this.conf=conf; 75 | this.path=conf.getPath(); 76 | this.numRecs=conf.getNumRec(); 77 | 78 | this.alpha=parameters[0]; 79 | // load student expectations from the parameters 80 | LoadExpectations_Student(parameters); 81 | // set instructor expectations 82 | exp_instructor = new double[] {4.0, 4.0}; 83 | 84 | // load ratings by instructors 85 | LoadRatings_Instructor(); 86 | // load ratings with candidates 87 | LoadRatings_Candidates(); 88 | 89 | 90 | }catch(Exception e) { 91 | e.printStackTrace(); 92 | } 93 | } 94 | 95 | public void run() throws Exception { 96 | 97 | // calculate utility of the items in view of instructors 98 | calUtility_Instructor(); 99 | // collect Truth from the test set for the purpose of evaluations 100 | collectTruth(); 101 | // run recommendations and calculate metrics 102 | recommend(); 103 | } 104 | 105 | public double getUtility_students(){ 106 | return this.utility_topN_students; 107 | } 108 | 109 | public double getUtility_instructors() { 110 | return this.utility_topN_instructors; 111 | } 112 | 113 | public double getUtility_diffs() { 114 | return this.utility_topN_diff; 115 | } 116 | 117 | public double getF1() { 118 | return this.f1_topN; 119 | } 120 | 121 | public double getNDCG() { 122 | return this.ndcg_topN; 123 | } 124 | 125 | protected void LoadRatings_Instructor() throws Exception { 126 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 127 | String line = br.readLine(); 128 | 129 | while ((line = br.readLine()) != null) { 130 | String[] strs = line.split(","); 131 | String item = strs[0].trim(); 132 | this.ratings_instructor.put(item, 133 | new ArrayList(Arrays.asList( 134 | new Double(strs[1]), 135 | new Double(strs[2])) 136 | ) 137 | ); 138 | } 139 | br.close(); 140 | } 141 | 142 | protected void LoadRatings_Candidates() throws Exception { 143 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 144 | String line = br.readLine(); 145 | candidates=HashMultimap.create(); 146 | 147 | while ((line = br.readLine()) != null) { 148 | String[] strs = line.split(","); 149 | String student = strs[0].trim(); 150 | String item = strs[1].trim(); 151 | candidates.put(student, item); 152 | this.ratings_candidates.put(student+","+item, 153 | new ArrayList(Arrays.asList( 154 | new Double(strs[3]), 155 | new Double(strs[4]), 156 | new Double(strs[5])) 157 | ) 158 | ); 159 | } 160 | br.close(); 161 | } 162 | 163 | protected void LoadExpectations_Student(double[] parameters) throws Exception { 164 | // we have UserID: 1000 to 1331, 332 users 165 | int student = 1000; 166 | for (int i = 1; i < parameters.length; ++i) { 167 | this.exp_students.put(""+student, 168 | new ArrayList(Arrays.asList( 169 | parameters[i], 170 | parameters[++i], 171 | parameters[++i]))); 172 | student++; 173 | } 174 | } 175 | 176 | protected void calUtility_Instructor() throws Exception{ 177 | utilities_instructor=new HashMap<>(); 178 | for(String item:ratings_instructor.keySet()) { 179 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 180 | double distance = dist.compute(this.exp_instructor, rates); 181 | // convert distance to dissimilarity: a normalization process 182 | // note that utility in view of instructor is the dissimilarity 183 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 184 | utilities_instructor.put(item, dissim); 185 | } 186 | } 187 | 188 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 189 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 190 | } 191 | 192 | protected void collectTruth() throws Exception 193 | { 194 | truth=HashMultimap.create(); 195 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 196 | String line=br.readLine(); 197 | while((line=br.readLine())!=null){ 198 | String[] strs=line.split(","); 199 | String user=strs[0].trim(); 200 | String item=strs[1].trim(); 201 | double rate=new Double(strs[2].trim()); 202 | if(rate>3) // this is an relevant item 203 | { 204 | truth.put(user, item); 205 | } 206 | } 207 | br.close(); 208 | } 209 | 210 | private void recommend() throws Exception { 211 | 212 | HashMap> rankingScore_student_item=new HashMap<>(); 213 | 214 | int count=0; 215 | for(String student:candidates.keySet()) { 216 | 217 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 218 | List> itemScores = new ArrayList<>(); 219 | HashMap utilities_student=new HashMap(); 220 | 221 | // for each candidate item to be recommended 222 | // calculate ranking score of the items for each student 223 | for(String candidateItem:candidates.get(student)) { 224 | 225 | String key=student+","+candidateItem; 226 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 227 | double distance = dist.compute(exp_student, rate_student); 228 | // convert distance to normalized similarity 229 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 230 | double itemUtility_instructor = this.utilities_instructor.get(candidateItem); 231 | double score = this.alpha*itemUtility_student + (1 - this.alpha)*itemUtility_instructor; 232 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 233 | utilities_student.put(candidateItem, itemUtility_student); 234 | } 235 | 236 | // produce top-N recommendations 237 | double utility_list_student=0; 238 | double utility_list_instructor=0; 239 | double utility_list_diff=0; 240 | 241 | Lists.sortList(itemScores, true); 242 | List rankedItems = new ArrayList<>(); 243 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 244 | : itemScores.subList(0, numRecs); 245 | for (Map.Entry kv : recomd) { 246 | String item = kv.getKey(); 247 | rankedItems.add(item); 248 | utility_list_student += utilities_student.get(item); 249 | utility_list_instructor += utilities_instructor.get(item); 250 | } 251 | 252 | // calculate metrics 253 | Set relevantItems = truth.get(student); 254 | List correctItems = new ArrayList<>(); 255 | correctItems.addAll(relevantItems); 256 | 257 | 258 | if(correctItems.size()>0 && rankedItems.size()>0) { 259 | ++count; 260 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 261 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 262 | double f1 = 0; 263 | if((precision+recall)!=0) 264 | f1 = 2*precision*recall/(precision+recall); 265 | double ndcg = Measures.nDCG(rankedItems, correctItems); 266 | utility_list_student/=rankedItems.size(); 267 | utility_list_instructor/=rankedItems.size(); 268 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 269 | 270 | utility_topN_students+=utility_list_student; 271 | utility_topN_instructors+=utility_list_instructor; 272 | utility_topN_diff+=utility_list_diff; 273 | f1_topN+=f1; 274 | ndcg_topN+=ndcg; 275 | } 276 | 277 | } 278 | // calculate the final metrics over all students 279 | utility_topN_students/=count; 280 | utility_topN_instructors/=count; 281 | utility_topN_diff/=count; 282 | f1_topN/=count; 283 | ndcg_topN/=count; 284 | } 285 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/learnexpectations/RunEduProblem_LearnExpectations.java: -------------------------------------------------------------------------------- 1 | package msrs.learnexpectations; 2 | 3 | import happy.coding.io.FileConfiger; 4 | import happy.coding.io.FileIO; 5 | import happy.coding.io.LineConfiger; 6 | import happy.coding.io.Logs; 7 | import msrs.demo.Config; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.BufferedWriter; 11 | import java.text.DecimalFormat; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | 17 | import org.moeaframework.Executor; 18 | import org.moeaframework.core.NondominatedPopulation; 19 | import org.moeaframework.core.Solution; 20 | import org.moeaframework.core.variable.EncodingUtils; 21 | 22 | import com.google.common.collect.ArrayListMultimap; 23 | 24 | 25 | /** 26 | * 27 | * Utility-based multi-stakeholder recommendations: the basic solution 28 | * Note: this implementation learns both Alpha and student expectations 29 | * Note: this implementation did not deal with the issue of over-/under- expectations 30 | * 31 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 32 | * 33 | */ 34 | 35 | 36 | public class RunEduProblem_LearnExpectations { 37 | String path=null; 38 | Config conf; 39 | DecimalFormat df = new DecimalFormat("#.000"); 40 | 41 | public RunEduProblem_LearnExpectations(Config conf) { 42 | this.conf=conf; 43 | this.path=conf.getPath(); 44 | } 45 | 46 | public void execute() throws Exception { 47 | 48 | String[] algorithms = {"eMOEA", "NSGAII","NSGAIII", "MSOPS", "SMPSO", "OMOPSO"}; 49 | 50 | 51 | // maximal metrics from baseline approaches 52 | Double maxF1 = conf.getMaxF1(); // from UBRec 53 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 54 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 55 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 56 | 57 | Solution globalBest = null; 58 | double globalMinLoss=999; 59 | String globalBestAlgorithm=null; 60 | 61 | Logs.info("------------------------------------------------------------------------------"); 62 | 63 | for (String algorithm : algorithms) { 64 | Logs.info("Running MSRS by using "+algorithm+" as the optimizer..."); 65 | NondominatedPopulation rst = new Executor() 66 | .withProblemClass(EduProblem.class, conf) 67 | .withMaxEvaluations(conf.getMaxEval()) // maximal function evaluations 68 | .withAlgorithm(algorithm) 69 | .distributeOnAllCores() 70 | .run(); 71 | 72 | Solution localBest=null; 73 | double localMinLoss = 999; 74 | 75 | for (Solution sol : rst) { 76 | double f1=Math.abs(Math.abs(sol.getObjective(3))); 77 | double ndcg=Math.abs(sol.getObjective(4)); 78 | double util_student=Math.abs(sol.getObjective(0)); 79 | double util_instructor=Math.abs(sol.getObjective(1)); 80 | 81 | double loss = ( (max_util_instructor - util_instructor)/max_util_instructor 82 | + (max_util_student - util_student)/max_util_student 83 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/3; 84 | 85 | if(loss> exp_students = new HashMap<>(); 53 | double[] exp_instructor; 54 | 55 | // ratings 56 | HashMap> ratings_instructor = new HashMap<>(); 57 | HashMap> ratings_candidates= new HashMap<>(); 58 | 59 | HashMap utilities_instructor; 60 | EuclideanDistance dist=new EuclideanDistance(); 61 | HashMultimap truth; 62 | HashMultimap candidates; 63 | 64 | // metrics or objectives 65 | double utility_topN_students=0; 66 | double utility_topN_instructors=0; 67 | double utility_topN_diff=0; 68 | double f1_topN=0; 69 | double ndcg_topN=0; 70 | 71 | public EduRec(Config conf, double[] parameters) { 72 | try 73 | { 74 | this.conf=conf; 75 | this.path=conf.getPath(); 76 | this.numRecs=conf.getNumRec(); 77 | 78 | this.alpha=parameters[0]; 79 | // load student expectations from the parameters 80 | LoadExpectations_Student(parameters); 81 | // set instructor expectations 82 | exp_instructor = new double[] {4.0, 4.0}; 83 | 84 | // load ratings by instructors 85 | LoadRatings_Instructor(); 86 | // load ratings with candidates 87 | LoadRatings_Candidates(); 88 | 89 | 90 | }catch(Exception e) { 91 | e.printStackTrace(); 92 | } 93 | } 94 | 95 | public void run() throws Exception { 96 | 97 | // calculate utility of the items in view of instructors 98 | calUtility_Instructor(); 99 | // collect Truth from the test set for the purpose of evaluations 100 | collectTruth(); 101 | // run recommendations and calculate metrics 102 | recommend(); 103 | } 104 | 105 | public double getUtility_students(){ 106 | return this.utility_topN_students; 107 | } 108 | 109 | public double getUtility_instructors() { 110 | return this.utility_topN_instructors; 111 | } 112 | 113 | public double getUtility_diffs() { 114 | return this.utility_topN_diff; 115 | } 116 | 117 | public double getF1() { 118 | return this.f1_topN; 119 | } 120 | 121 | public double getNDCG() { 122 | return this.ndcg_topN; 123 | } 124 | 125 | protected void LoadRatings_Instructor() throws Exception { 126 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 127 | String line = br.readLine(); 128 | 129 | while ((line = br.readLine()) != null) { 130 | String[] strs = line.split(","); 131 | String item = strs[0].trim(); 132 | this.ratings_instructor.put(item, 133 | new ArrayList(Arrays.asList( 134 | new Double(strs[1]), 135 | new Double(strs[2])) 136 | ) 137 | ); 138 | } 139 | br.close(); 140 | } 141 | 142 | protected void LoadRatings_Candidates() throws Exception { 143 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 144 | String line = br.readLine(); 145 | candidates=HashMultimap.create(); 146 | 147 | while ((line = br.readLine()) != null) { 148 | String[] strs = line.split(","); 149 | String student = strs[0].trim(); 150 | String item = strs[1].trim(); 151 | candidates.put(student, item); 152 | this.ratings_candidates.put(student+","+item, 153 | new ArrayList(Arrays.asList( 154 | new Double(strs[3]), 155 | new Double(strs[4]), 156 | new Double(strs[5])) 157 | ) 158 | ); 159 | } 160 | br.close(); 161 | } 162 | 163 | protected void LoadExpectations_Student(double[] parameters) throws Exception { 164 | // we have UserID: 1000 to 1331, 332 users 165 | int student = 1000; 166 | for (int i = 1; i < parameters.length; ++i) { 167 | this.exp_students.put(""+student, 168 | new ArrayList(Arrays.asList( 169 | parameters[i], 170 | parameters[++i], 171 | parameters[++i]))); 172 | student++; 173 | } 174 | } 175 | 176 | protected void calUtility_Instructor() throws Exception{ 177 | utilities_instructor=new HashMap<>(); 178 | for(String item:ratings_instructor.keySet()) { 179 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 180 | double distance = dist.compute(this.exp_instructor, rates); 181 | // convert distance to dissimilarity: a normalization process 182 | // note that utility in view of instructor is the dissimilarity 183 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 184 | utilities_instructor.put(item, dissim); 185 | } 186 | } 187 | 188 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 189 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 190 | } 191 | 192 | protected void collectTruth() throws Exception 193 | { 194 | truth=HashMultimap.create(); 195 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 196 | String line=br.readLine(); 197 | while((line=br.readLine())!=null){ 198 | String[] strs=line.split(","); 199 | String user=strs[0].trim(); 200 | String item=strs[1].trim(); 201 | double rate=new Double(strs[2].trim()); 202 | if(rate>3) // this is an relevant item 203 | { 204 | truth.put(user, item); 205 | } 206 | } 207 | br.close(); 208 | } 209 | 210 | private void recommend() throws Exception { 211 | 212 | HashMap> rankingScore_student_item=new HashMap<>(); 213 | 214 | int count=0; 215 | for(String student:candidates.keySet()) { 216 | 217 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 218 | List> itemScores = new ArrayList<>(); 219 | HashMap utilities_student=new HashMap(); 220 | 221 | // for each candidate item to be recommended 222 | // calculate ranking score of the items for each student 223 | for(String candidateItem:candidates.get(student)) { 224 | 225 | String key=student+","+candidateItem; 226 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 227 | double distance = dist.compute(exp_student, rate_student); 228 | // convert distance to normalized similarity 229 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 230 | double itemUtility_instructor = this.utilities_instructor.get(candidateItem); 231 | double score = this.alpha*itemUtility_student + (1 - this.alpha)*itemUtility_instructor; 232 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 233 | utilities_student.put(candidateItem, itemUtility_student); 234 | } 235 | 236 | // produce top-N recommendations 237 | double utility_list_student=0; 238 | double utility_list_instructor=0; 239 | double utility_list_diff=0; 240 | 241 | Lists.sortList(itemScores, true); 242 | List rankedItems = new ArrayList<>(); 243 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 244 | : itemScores.subList(0, numRecs); 245 | for (Map.Entry kv : recomd) { 246 | String item = kv.getKey(); 247 | rankedItems.add(item); 248 | utility_list_student += utilities_student.get(item); 249 | utility_list_instructor += utilities_instructor.get(item); 250 | } 251 | 252 | // calculate metrics 253 | Set relevantItems = truth.get(student); 254 | List correctItems = new ArrayList<>(); 255 | correctItems.addAll(relevantItems); 256 | 257 | 258 | if(correctItems.size()>0 && rankedItems.size()>0) { 259 | ++count; 260 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 261 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 262 | double f1 = 0; 263 | if((precision+recall)!=0) 264 | f1 = 2*precision*recall/(precision+recall); 265 | double ndcg = Measures.nDCG(rankedItems, correctItems); 266 | utility_list_student/=rankedItems.size(); 267 | utility_list_instructor/=rankedItems.size(); 268 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 269 | 270 | utility_topN_students+=utility_list_student; 271 | utility_topN_instructors+=utility_list_instructor; 272 | utility_topN_diff+=utility_list_diff; 273 | f1_topN+=f1; 274 | ndcg_topN+=ndcg; 275 | } 276 | 277 | } 278 | // calculate the final metrics over all students 279 | utility_topN_students/=count; 280 | utility_topN_instructors/=count; 281 | utility_topN_diff/=count; 282 | f1_topN/=count; 283 | ndcg_topN/=count; 284 | } 285 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/onestage/RunEduProblem_OneStage.java: -------------------------------------------------------------------------------- 1 | package msrs.onestage; 2 | 3 | import happy.coding.io.FileConfiger; 4 | import happy.coding.io.FileIO; 5 | import happy.coding.io.LineConfiger; 6 | import happy.coding.io.Logs; 7 | import msrs.demo.Config; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.BufferedWriter; 11 | import java.text.DecimalFormat; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | 17 | import org.moeaframework.Executor; 18 | import org.moeaframework.core.NondominatedPopulation; 19 | import org.moeaframework.core.Solution; 20 | import org.moeaframework.core.variable.EncodingUtils; 21 | 22 | import com.google.common.collect.ArrayListMultimap; 23 | 24 | 25 | /** 26 | * 27 | * Utility-based multi-stakeholder recommendations: the basic solution 28 | * Note: this implementation learns both Alpha and student expectations 29 | * Note: this implementation did not deal with the issue of over-/under- expectations 30 | * 31 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 32 | * 33 | */ 34 | 35 | 36 | public class RunEduProblem_OneStage { 37 | String path=null; 38 | Config conf; 39 | DecimalFormat df = new DecimalFormat("#.000"); 40 | 41 | public RunEduProblem_OneStage(Config conf) { 42 | this.conf=conf; 43 | this.path=conf.getPath(); 44 | } 45 | 46 | public void execute() throws Exception { 47 | 48 | String[] algorithms = {"eMOEA", "NSGAII","NSGAIII", "MSOPS", "SMPSO", "OMOPSO"}; 49 | 50 | 51 | // maximal metrics from baseline approaches 52 | Double maxF1 = conf.getMaxF1(); // from UBRec 53 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 54 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 55 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 56 | 57 | Solution globalBest = null; 58 | double globalMinLoss1=999, globalMinLoss2=999; 59 | String globalBestAlgorithm=null; 60 | 61 | Logs.info("------------------------------------------------------------------------------"); 62 | 63 | for (String algorithm : algorithms) { 64 | Logs.info("Running MSRS by using "+algorithm+" as the optimizer..."); 65 | NondominatedPopulation rst = new Executor() 66 | .withProblemClass(EduProblem.class, conf) 67 | .withMaxEvaluations(conf.getMaxEval()) // maximal function evaluations 68 | .withAlgorithm(algorithm) 69 | .distributeOnAllCores() 70 | .run(); 71 | 72 | Solution localBest=null; 73 | double localMinLoss1 = 999, localMinLoss2 = 999; 74 | 75 | for (Solution sol : rst) { 76 | double f1=Math.abs(Math.abs(sol.getObjective(3))); 77 | double ndcg=Math.abs(sol.getObjective(4)); 78 | double util_student=Math.abs(sol.getObjective(0)); 79 | double util_instructor=Math.abs(sol.getObjective(1)); 80 | 81 | double loss1 = ( (max_util_instructor - util_instructor)/max_util_instructor 82 | + (max_util_student - util_student)/max_util_student 83 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/3; 84 | double loss2 = ( (max_util_instructor - util_instructor)/max_util_instructor 85 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/2; 86 | 87 | if(loss2> exp_students = new HashMap<>(); 53 | double[] exp_instructor; 54 | 55 | // ratings 56 | HashMap> ratings_instructor = new HashMap<>(); 57 | HashMap> ratings_candidates= new HashMap<>(); 58 | 59 | HashMap utilities_instructor; 60 | EuclideanDistance dist=new EuclideanDistance(); 61 | HashMultimap truth; 62 | HashMultimap candidates; 63 | 64 | // metrics or objectives 65 | double utility_topN_students=0; 66 | double utility_topN_instructors=0; 67 | double utility_topN_diff=0; 68 | double f1_topN=0; 69 | double ndcg_topN=0; 70 | 71 | public EduRec(Config conf, double[] parameters) { 72 | try 73 | { 74 | this.conf=conf; 75 | this.path=conf.getPath(); 76 | this.numRecs=conf.getNumRec(); 77 | this.fileExpectations_student=conf.getExpectationFilename(); 78 | 79 | this.alpha=parameters[0]; 80 | // load student expectations from the parameters 81 | LoadExpectations_Student(fileExpectations_student); 82 | // set instructor expectations 83 | exp_instructor = new double[] {4.0, 4.0}; 84 | 85 | // load ratings by instructors 86 | LoadRatings_Instructor(); 87 | // load ratings with candidates 88 | LoadRatings_Candidates(); 89 | 90 | 91 | }catch(Exception e) { 92 | e.printStackTrace(); 93 | } 94 | } 95 | 96 | public void run() throws Exception { 97 | 98 | // calculate utility of the items in view of instructors 99 | calUtility_Instructor(); 100 | // collect Truth from the test set for the purpose of evaluations 101 | collectTruth(); 102 | // run recommendations and calculate metrics 103 | recommend(); 104 | } 105 | 106 | public double getUtility_students(){ 107 | return this.utility_topN_students; 108 | } 109 | 110 | public double getUtility_instructors() { 111 | return this.utility_topN_instructors; 112 | } 113 | 114 | public double getUtility_diffs() { 115 | return this.utility_topN_diff; 116 | } 117 | 118 | public double getF1() { 119 | return this.f1_topN; 120 | } 121 | 122 | public double getNDCG() { 123 | return this.ndcg_topN; 124 | } 125 | 126 | protected void LoadRatings_Instructor() throws Exception { 127 | BufferedReader br = FileIO.getReader(this.path + fileRatings_instructor); 128 | String line = br.readLine(); 129 | 130 | while ((line = br.readLine()) != null) { 131 | String[] strs = line.split(","); 132 | String item = strs[0].trim(); 133 | this.ratings_instructor.put(item, 134 | new ArrayList(Arrays.asList( 135 | new Double(strs[1]), 136 | new Double(strs[2])) 137 | ) 138 | ); 139 | } 140 | br.close(); 141 | } 142 | 143 | protected void LoadRatings_Candidates() throws Exception { 144 | BufferedReader br = FileIO.getReader(this.path + fileRatings_candidates); 145 | String line = br.readLine(); 146 | candidates=HashMultimap.create(); 147 | 148 | while ((line = br.readLine()) != null) { 149 | String[] strs = line.split(","); 150 | String student = strs[0].trim(); 151 | String item = strs[1].trim(); 152 | candidates.put(student, item); 153 | this.ratings_candidates.put(student+","+item, 154 | new ArrayList(Arrays.asList( 155 | new Double(strs[3]), 156 | new Double(strs[4]), 157 | new Double(strs[5])) 158 | ) 159 | ); 160 | } 161 | br.close(); 162 | } 163 | 164 | protected void LoadExpectations_Student(String fileExpectations_student) throws Exception { 165 | BufferedReader br=FileIO.getReader(path+fileExpectations_student); 166 | String line=br.readLine(); 167 | while((line=br.readLine())!=null) { 168 | String[] strs=line.split(","); 169 | this.exp_students.put(strs[0].trim(), 170 | new ArrayList(Arrays.asList( 171 | Double.parseDouble(strs[1].trim()), 172 | Double.parseDouble(strs[2].trim()), 173 | Double.parseDouble(strs[3].trim())))); 174 | } 175 | } 176 | 177 | protected void calUtility_Instructor() throws Exception{ 178 | utilities_instructor=new HashMap<>(); 179 | for(String item:ratings_instructor.keySet()) { 180 | double[] rates=Doubles.toArray(ratings_instructor.get(item)); 181 | double distance = dist.compute(this.exp_instructor, rates); 182 | // convert distance to dissimilarity: a normalization process 183 | // note that utility in view of instructor is the dissimilarity 184 | double dissim = minMaxNorm(distance, 0, Math.sqrt(50), 0, 1); 185 | utilities_instructor.put(item, dissim); 186 | } 187 | } 188 | 189 | protected double minMaxNorm(double x, double oldmin, double oldmax, double newmin, double newmax){ 190 | return newmin+(newmax-newmin)*(x-oldmin)/(oldmax-oldmin); 191 | } 192 | 193 | protected void collectTruth() throws Exception 194 | { 195 | truth=HashMultimap.create(); 196 | BufferedReader br = FileIO.getReader(path + fileRatings_test); 197 | String line=br.readLine(); 198 | while((line=br.readLine())!=null){ 199 | String[] strs=line.split(","); 200 | String user=strs[0].trim(); 201 | String item=strs[1].trim(); 202 | double rate=new Double(strs[2].trim()); 203 | if(rate>3) // this is an relevant item 204 | { 205 | truth.put(user, item); 206 | } 207 | } 208 | br.close(); 209 | } 210 | 211 | private void recommend() throws Exception { 212 | 213 | HashMap> rankingScore_student_item=new HashMap<>(); 214 | 215 | int count=0; 216 | for(String student:candidates.keySet()) { 217 | 218 | double[] exp_student = Doubles.toArray(exp_students.get(student)); 219 | List> itemScores = new ArrayList<>(); 220 | HashMap utilities_student=new HashMap(); 221 | 222 | // for each candidate item to be recommended 223 | // calculate ranking score of the items for each student 224 | for(String candidateItem:candidates.get(student)) { 225 | 226 | String key=student+","+candidateItem; 227 | double[] rate_student = Doubles.toArray(this.ratings_candidates.get(key)); 228 | double distance = dist.compute(exp_student, rate_student); 229 | // convert distance to normalized similarity 230 | double itemUtility_student = 1 - this.minMaxNorm(distance, 0, Math.sqrt(75), 0, 1); 231 | double itemUtility_instructor = this.utilities_instructor.get(candidateItem); 232 | double score = this.alpha*itemUtility_student + (1 - this.alpha)*itemUtility_instructor; 233 | itemScores.add(new SimpleImmutableEntry(candidateItem, score)); 234 | utilities_student.put(candidateItem, itemUtility_student); 235 | } 236 | 237 | // produce top-N recommendations 238 | double utility_list_student=0; 239 | double utility_list_instructor=0; 240 | double utility_list_diff=0; 241 | 242 | Lists.sortList(itemScores, true); 243 | List rankedItems = new ArrayList<>(); 244 | List> recomd = (numRecs <= 0 || itemScores.size() <= numRecs) ? itemScores 245 | : itemScores.subList(0, numRecs); 246 | for (Map.Entry kv : recomd) { 247 | String item = kv.getKey(); 248 | rankedItems.add(item); 249 | utility_list_student += utilities_student.get(item); 250 | utility_list_instructor += utilities_instructor.get(item); 251 | } 252 | 253 | // calculate metrics 254 | Set relevantItems = truth.get(student); 255 | List correctItems = new ArrayList<>(); 256 | correctItems.addAll(relevantItems); 257 | 258 | 259 | if(correctItems.size()>0 && rankedItems.size()>0) { 260 | ++count; 261 | double precision = Measures.PrecAt(rankedItems, correctItems, numRecs); 262 | double recall = Measures.RecallAt(rankedItems, correctItems, numRecs); 263 | double f1 = 0; 264 | if((precision+recall)!=0) 265 | f1 = 2*precision*recall/(precision+recall); 266 | double ndcg = Measures.nDCG(rankedItems, correctItems); 267 | utility_list_student/=rankedItems.size(); 268 | utility_list_instructor/=rankedItems.size(); 269 | utility_list_diff = Math.abs(utility_list_student - utility_list_instructor); 270 | 271 | utility_topN_students+=utility_list_student; 272 | utility_topN_instructors+=utility_list_instructor; 273 | utility_topN_diff+=utility_list_diff; 274 | f1_topN+=f1; 275 | ndcg_topN+=ndcg; 276 | } 277 | 278 | } 279 | // calculate the final metrics over all students 280 | utility_topN_students/=count; 281 | utility_topN_instructors/=count; 282 | utility_topN_diff/=count; 283 | f1_topN/=count; 284 | ndcg_topN/=count; 285 | } 286 | } -------------------------------------------------------------------------------- /UBMSRS/src/main/java/msrs/twostage/RunEduProblem_TwoStage.java: -------------------------------------------------------------------------------- 1 | package msrs.twostage; 2 | 3 | import happy.coding.io.FileConfiger; 4 | import happy.coding.io.FileIO; 5 | import happy.coding.io.LineConfiger; 6 | import happy.coding.io.Logs; 7 | import msrs.demo.Config; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.BufferedWriter; 11 | import java.text.DecimalFormat; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | 17 | import org.moeaframework.Executor; 18 | import org.moeaframework.core.NondominatedPopulation; 19 | import org.moeaframework.core.Solution; 20 | import org.moeaframework.core.variable.EncodingUtils; 21 | 22 | import com.google.common.collect.ArrayListMultimap; 23 | 24 | 25 | /** 26 | * 27 | * Utility-based multi-stakeholder recommendations: the basic solution 28 | * Note: this implementation learns Alpha only 29 | * Note: this implementation did not deal with the issue of over-/under- expectations 30 | * 31 | * Yong Zheng, Nastaran Ghane, Milad Sabouri. "Personalized Educational Learning with Multi-Stakeholder Optimizations", Adjunct Proceedings of the 27th ACM Conference on User Modeling, Adaptation and Personalization (ACM UMAP), Cyprus, June, 2019 32 | * 33 | */ 34 | 35 | 36 | 37 | public class RunEduProblem_TwoStage { 38 | String path=null; 39 | Config conf; 40 | DecimalFormat df = new DecimalFormat("#.000"); 41 | 42 | public RunEduProblem_TwoStage(Config conf) { 43 | this.conf=conf; 44 | this.path=conf.getPath(); 45 | } 46 | 47 | public void execute() throws Exception { 48 | 49 | String[] algorithms = {"eMOEA", "NSGAII","NSGAIII", "MSOPS", "SMPSO", "OMOPSO"}; 50 | 51 | // maximal metrics from baseline approaches 52 | Double maxF1 = conf.getMaxF1(); // from UBRec 53 | Double maxNDCG = conf.getMaxNDCG(); // from UBRec 54 | Double max_util_instructor = conf.getMaxUtil_Instructor(); // from Rankp 55 | Double max_util_student = conf.getMaxUtil_Student(); // from UBRec 56 | 57 | Solution globalBest = null; 58 | double globalMinLoss1=999, globalMinLoss2=999; 59 | String globalBestAlgorithm=null; 60 | 61 | Logs.info("------------------------------------------------------------------------------"); 62 | 63 | for (String algorithm : algorithms) { 64 | Logs.info("Running MSRS by using "+algorithm+" as the optimizer..."); 65 | NondominatedPopulation rst = new Executor() 66 | .withProblemClass(EduProblem.class, conf) 67 | .withMaxEvaluations(conf.getMaxEval()) // maximal function evaluations 68 | .withAlgorithm(algorithm) 69 | .distributeOnAllCores() 70 | .run(); 71 | 72 | Solution localBest=null; 73 | double localMinLoss1 = 999, localMinLoss2 = 999; 74 | 75 | for (Solution sol : rst) { 76 | double f1=Math.abs(Math.abs(sol.getObjective(3))); 77 | double ndcg=Math.abs(sol.getObjective(4)); 78 | double util_student=Math.abs(sol.getObjective(0)); 79 | double util_instructor=Math.abs(sol.getObjective(1)); 80 | 81 | double loss1 = ( (max_util_instructor - util_instructor)/max_util_instructor 82 | + (max_util_student - util_student)/max_util_student 83 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/3; 84 | double loss2 = ( (max_util_instructor - util_instructor)/max_util_instructor 85 | + ( (maxF1 - f1)/maxF1 + (maxNDCG - ndcg)/maxNDCG )/2 )/2; 86 | 87 | if(loss2