├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── datasets └── yes_reduced.csv ├── main.py ├── requirements.txt ├── sequeval ├── __init__.py ├── baseline │ ├── __init__.py │ ├── bigram.py │ ├── mostpopular.py │ ├── random.py │ └── unigram.py ├── builder.py ├── evaluator.py ├── indexlist.py ├── loader.py ├── profiler.py ├── recommender.py ├── similarity.py └── splitter.py ├── setup.py ├── static └── sequeval.js ├── templates └── index.html ├── tests ├── __init__.py ├── baseline │ ├── __init__.py │ ├── test_bigram.py │ ├── test_mostpopular.py │ ├── test_random.py │ └── test_unigram.py ├── test_builder.py ├── test_evaluator.py ├── test_indexlist.py ├── test_loader.py ├── test_profiler.py ├── test_recommender.py ├── test_similarity.py └── test_splitter.py └── webapp.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .cache 3 | .coverage 4 | .idea 5 | build/* 6 | datasets/* 7 | !datasets/yes_reduced.csv 8 | dist/* 9 | venv/* 10 | *.egg-info/* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: python 4 | 5 | python: 6 | - 3.6 7 | 8 | install: 9 | - pip install -r requirements.txt 10 | - pip install pytest 11 | - pip install pytest-cov 12 | 13 | script: 14 | - pytest tests 15 | - pytest --cov sequeval 16 | 17 | after_success: 18 | - pip install codecov 19 | - codecov -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Diego Monti 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sequeval 2 | 3 | [![Build Status](https://travis-ci.org/D2KLab/sequeval.svg?branch=master)](https://travis-ci.org/D2KLab/sequeval) 4 | [![codecov](https://codecov.io/gh/D2KLab/sequeval/branch/master/graph/badge.svg)](https://codecov.io/gh/D2KLab/sequeval) 5 | 6 | Sequeval is an offline evaluation framework for sequence-based recommender systems developed in Python. 7 | 8 | ## Architecture 9 | 10 | The package `sequeval` is composed of the following modules: 11 | 12 | - `loader.py`, which contains the code for reading the ratings from a file. The class *UIRTLoader* extends the abstract class *Loader* and it deals with a CSV file saved in a format similar to the one of [MovieLens](https://grouplens.org/datasets/movielens/); 13 | - `builder.py`, which contains the class *Builder* that creates a list of sequences from the ratings; 14 | - `profiler.py`, which contains the class *Profiler* that computes some statistics about the sequences; 15 | - `splitter.py`, which contains the abstract class *Splitter* and the concrete classes *RandomSplitter* and *TimestampSplitter*; 16 | - `recommender.py`, which contains the abstract class *Recommender* that needs to be implemented by any recommender relying on this framework; 17 | - `evaluator.py`, which contains the class *Evaluator* that includes the methods for computing the metrics during the evaluation phase; 18 | - `similarity.py`, which contains the abstract class *Similarity* and the concrete class *CosineSimilarity*; 19 | - `indexlist.py`, which contains the class *IndexList* that extends *MutableSequence*. 20 | 21 | The package `sequeval.baseline` includes the following baseline recommenders: 22 | 23 | - `mostpopular.py`, which contains the class *MostPopularRecommender*; 24 | - `random.py`, which contains the class *RandomRecommender*; 25 | - `unigram.py`, which contains the class *UnigramRecommender*; 26 | - `bigram.py`, which contains the class *BigramRecommender*. 27 | 28 | ## Dependencies 29 | 30 | Sequeval requires [numpy](http://www.numpy.org/), [pandas](http://pandas.pydata.org/), [pytimeparse](https://github.com/wroberts/pytimeparse), and [scipy](http://www.scipy.org/). 31 | 32 | ## Installation 33 | 34 | If you are interested in using sequeval in your own project, you can install it with `pip`: 35 | 36 | ```bash 37 | $ pip install git+https://github.com/D2KLab/sequeval.git 38 | ``` 39 | 40 | If you want to run the sample script `main.py` you need to first clone the repository and then install the requirements: 41 | 42 | ```bash 43 | $ pip install -r requirements.txt 44 | ``` 45 | 46 | ## Testing 47 | 48 | You can verify the results of sequeval unit tests by running `pytest tests`. You can also compute the coverage of the tests with `pytest --cov sequeval`. These commands require, respectively, [pytest](https://pytest.org/) and [pytest-cov](https://github.com/pytest-dev/pytest-cov). 49 | 50 | ## Usage 51 | 52 | You can try sequeval by running the script `python main.py`. For further information about the possible parameters, you can execute `python main.py -h`. 53 | 54 | If you want to try the toolkit with the sample Yes.com dataset, you can run the following command: 55 | 56 | ```bash 57 | python main.py --item 50 --delta "1000 s" datasets/yes_reduced.csv 58 | ``` 59 | 60 | If you want to try the web-based interface execute `python webapp.py` and then navigate to [http://localhost:5000](http://localhost:5000). 61 | 62 | The file `yes_reduced.csv` contains a random sample of the [Yes.com](http://web.archive.org/web/20170629232107/https://www.cs.cornell.edu/~shuochen/lme/data_page.html) dataset, reduced 10 times its original size. Please note that the Yes.com dataset was originally released under the terms of the [Creative Commons BY-NC-ND 3.0](http://creativecommons.org/licenses/by-nc-nd/3.0/) license. 63 | 64 | ## Publications 65 | 66 | - Monti D., Palumbo E., Rizzo G., Morisio M. (2018). Sequeval: A Framework to Assess and Benchmark Sequence-based Recommender Systems. In REVEAL 2018 Workshop on Offline Evaluation for Recommender Systems. https://arxiv.org/abs/1810.04956 67 | - Monti, D., Palumbo, E., Rizzo, G., Morisio, M. Sequeval: An Offline Evaluation Framework for Sequence-Based Recommender Systems. *Information*. 2019; 10(5):174. https://www.mdpi.com/2078-2489/10/5/174 68 | 69 | ## Team 70 | 71 | - Diego Monti 72 | - Enrico Palumbo 73 | - Giuseppe Rizzo 74 | - Maurizio Morisio 75 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import random 3 | 4 | import numpy as np 5 | 6 | import sequeval 7 | import sequeval.baseline as baseline 8 | 9 | 10 | def evaluation(compute, recommender, similarity): 11 | print("%10s\t" % recommender.name, end='') 12 | print("%10f\t" % compute.coverage(recommender), end='') 13 | print("%10f\t" % compute.precision(recommender), end='') 14 | print("%10f\t" % compute.ndpm(recommender), end='') 15 | print("%10f\t" % compute.diversity(recommender, similarity), end='') 16 | print("%10f\t" % compute.novelty(recommender), end='') 17 | print("%10f\t" % compute.serendipity(recommender), end='') 18 | print("%10f\t" % compute.confidence(recommender), end='') 19 | print("%10.2f" % compute.perplexity(recommender)) 20 | 21 | 22 | if __name__ == '__main__': 23 | parser = argparse.ArgumentParser(description='Sequeval: An offline evaluation framework for sequence-based RSs') 24 | 25 | parser.add_argument('file', type=str, help='file containing the ratings') 26 | parser.add_argument('--seed', type=int, default=None, help='seed for generating pseudo-random numbers') 27 | parser.add_argument('--user', type=int, default=0, help='minimum number of ratings per user') 28 | parser.add_argument('--item', type=int, default=0, help='minimum number of ratings per item') 29 | parser.add_argument('--delta', type=str, default='8 hours', help='time interval to create the sequences') 30 | parser.add_argument('--splitter', type=str, default='timestamp', help='dataset splitting protocol: ' 31 | 'random or timestamp') 32 | parser.add_argument('--ratio', type=float, default=0.2, help='percentage of sequences in the test set') 33 | parser.add_argument('--length', type=int, default=5, help='length of recommended sequences') 34 | 35 | args = parser.parse_args() 36 | 37 | print("\n# Parameters") 38 | print("File:", args.file) 39 | print("Seed:", args.seed) 40 | print("User ratings:", args.user) 41 | print("Item ratings:", args.item) 42 | print("Delta:", args.delta) 43 | print("Ratio:", args.ratio) 44 | print("Length:", args.length) 45 | 46 | # Set the random seed 47 | if args.seed is not None: 48 | random.seed(args.seed) 49 | np.random.seed(args.seed) 50 | 51 | loader = sequeval.UIRTLoader(user_ratings=args.user, item_ratings=args.item) 52 | ratings = loader.load(args.file) 53 | 54 | builder = sequeval.Builder(args.delta) 55 | sequences, items = builder.build(ratings) 56 | 57 | print("\n# Profiler") 58 | profiler = sequeval.Profiler(sequences) 59 | print("Users:", profiler.users()) 60 | print("Items:", profiler.items()) 61 | print("Ratings:", profiler.ratings()) 62 | print("Sequences:", profiler.sequences()) 63 | print("Sparsity:", profiler.sparsity()) 64 | print("Length:", profiler.sequence_length()) 65 | 66 | if args.splitter == 'random': 67 | print("\n# Random splitter") 68 | splitter = sequeval.RandomSplitter(args.ratio) 69 | elif args.splitter == 'timestamp': 70 | print("\n# Timestamp splitter") 71 | splitter = sequeval.TimestampSplitter(args.ratio) 72 | else: 73 | raise RuntimeError('Unknown splitter ' + args.splitter) 74 | training_set, test_set = splitter.split(sequences) 75 | print("Training set:", len(training_set)) 76 | print("Test set:", len(test_set)) 77 | 78 | print("\n# Evaluator") 79 | print("%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s" % 80 | ("Algorithm", "Coverage", "Precision", "nDPM", "Diversity", 81 | "Novelty", "Serendipity", "Confidence", "Perplexity")) 82 | evaluator = sequeval.Evaluator(training_set, test_set, items, args.length) 83 | cosine = sequeval.CosineSimilarity(training_set, items) 84 | 85 | most_popular = baseline.MostPopularRecommender(training_set, items) 86 | evaluation(evaluator, most_popular, cosine) 87 | 88 | random = baseline.RandomRecommender(training_set, items) 89 | evaluation(evaluator, random, cosine) 90 | 91 | unigram = baseline.UnigramRecommender(training_set, items) 92 | evaluation(evaluator, unigram, cosine) 93 | 94 | bigram = baseline.BigramRecommender(training_set, items) 95 | evaluation(evaluator, bigram, cosine) 96 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | pytimeparse 4 | scipy 5 | 6 | flask 7 | pytest 8 | pytest-cov -------------------------------------------------------------------------------- /sequeval/__init__.py: -------------------------------------------------------------------------------- 1 | from .builder import Builder 2 | from .builder import IndexList 3 | from .evaluator import Evaluator 4 | from .loader import UIRTLoader 5 | from .profiler import Profiler 6 | from .recommender import Recommender 7 | from .similarity import CosineSimilarity 8 | from .splitter import RandomSplitter 9 | from .splitter import TimestampSplitter 10 | -------------------------------------------------------------------------------- /sequeval/baseline/__init__.py: -------------------------------------------------------------------------------- 1 | from .bigram import BigramRecommender 2 | from .mostpopular import MostPopularRecommender 3 | from .random import RandomRecommender 4 | from .unigram import UnigramRecommender 5 | -------------------------------------------------------------------------------- /sequeval/baseline/bigram.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from ..recommender import Recommender 4 | 5 | 6 | class BigramRecommender(Recommender): 7 | name = 'Bigram' 8 | 9 | def __init__(self, training_set, items): 10 | super().__init__(training_set, items) 11 | 12 | # The matrix is initialized to 1.0 13 | self.bigrams = np.full((len(self.items), len(self.items)), 1.0, dtype=float) 14 | 15 | for sequence in self.training_set: 16 | previous_item_index = None 17 | 18 | for rating in sequence: 19 | item_index = self.items.index(rating[0]) 20 | 21 | if previous_item_index is not None: 22 | self.bigrams[previous_item_index, item_index] += 1 23 | 24 | previous_item_index = item_index 25 | 26 | # Probability normalization 27 | for row in self.bigrams: 28 | row /= row.sum() 29 | 30 | def predict(self, rating): 31 | item_index = self.items.index(rating[0]) 32 | return self.bigrams[item_index] 33 | 34 | def reset(self): 35 | pass 36 | -------------------------------------------------------------------------------- /sequeval/baseline/mostpopular.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from .unigram import UnigramRecommender 4 | 5 | 6 | class MostPopularRecommender(UnigramRecommender): 7 | name = 'Popular' 8 | 9 | def __init__(self, training_set, items): 10 | super().__init__(training_set, items) 11 | 12 | # Sort indexes in descending order 13 | self.sorted_weights = np.argsort(-self.weights) 14 | 15 | self.model = 0 16 | 17 | def predict(self, rating): 18 | prediction = np.full(len(self.items), 0.0, dtype=float) 19 | model_index = self.sorted_weights[self.model] 20 | prediction[model_index] = 1.0 21 | self.model += 1 22 | return prediction 23 | 24 | def reset(self): 25 | self.model = 0 26 | -------------------------------------------------------------------------------- /sequeval/baseline/random.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from ..recommender import Recommender 4 | 5 | 6 | class RandomRecommender(Recommender): 7 | name = 'Random' 8 | 9 | def predict(self, rating): 10 | return np.full(len(self.items), 1 / len(self.items), dtype=float) 11 | 12 | def reset(self): 13 | pass 14 | -------------------------------------------------------------------------------- /sequeval/baseline/unigram.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from ..recommender import Recommender 4 | 5 | 6 | class UnigramRecommender(Recommender): 7 | name = 'Unigram' 8 | 9 | def __init__(self, training_set, items): 10 | super().__init__(training_set, items) 11 | 12 | self.weights = np.full(len(self.items), 1.0, dtype=float) 13 | 14 | for sequence in training_set: 15 | for rating in sequence: 16 | item_index = self.items.index(rating[0]) 17 | self.weights[item_index] += 1 18 | 19 | self.weights /= self.weights.sum() 20 | 21 | def predict(self, rating): 22 | return self.weights 23 | 24 | def reset(self): 25 | pass 26 | -------------------------------------------------------------------------------- /sequeval/builder.py: -------------------------------------------------------------------------------- 1 | import pytimeparse 2 | 3 | from sequeval.indexlist import IndexList 4 | 5 | 6 | class Builder: 7 | 8 | def __init__(self, interval): 9 | """ 10 | :param interval: The time interval for creating the sequences. 11 | """ 12 | if type(interval) is str: 13 | self.interval = pytimeparse.parse(interval) 14 | else: 15 | self.interval = interval 16 | 17 | def build(self, ratings): 18 | """ 19 | Build a list of sequences and a list of unique items. 20 | Each sequence is a list of ratings. 21 | Each rating is a item, user, timestamp tuple. 22 | 23 | :param ratings: A list of ratings, ordered by user and timestamp. 24 | :return: A list of sequences, a list of unique items. 25 | """ 26 | sequences = [] 27 | 28 | s = None 29 | last_row = None 30 | 31 | # Each row is a rating 32 | for row in ratings: 33 | # This is the first rating available 34 | if last_row is None: 35 | last_row = row 36 | 37 | # We have found a new user 38 | elif last_row[1] != row[1]: 39 | last_row = row 40 | if s is not None: 41 | sequences.append(s) 42 | s = None 43 | 44 | # The new rating is part of the sequence 45 | elif row[2] < last_row[2] + self.interval: 46 | if s is None: 47 | s = [last_row] 48 | s.append(row) 49 | last_row = row 50 | 51 | # The sequence has ended 52 | else: 53 | if s is not None: 54 | sequences.append(s) 55 | s = None 56 | last_row = row 57 | 58 | # The last possible sequence 59 | if s is not None: 60 | sequences.append(s) 61 | 62 | # Create the list of items 63 | items = IndexList() 64 | 65 | for sequence in sequences: 66 | for rating in sequence: 67 | items.append(rating[0]) 68 | 69 | return sequences, items 70 | -------------------------------------------------------------------------------- /sequeval/evaluator.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import math 3 | 4 | import numpy as np 5 | 6 | import sequeval.baseline as baseline 7 | 8 | 9 | class Evaluator: 10 | 11 | def __init__(self, training_set, test_set, items, k): 12 | self.training_set = training_set 13 | self.test_set = test_set 14 | self.items = items 15 | self.k = k 16 | 17 | self.users = self._get_user_set(training_set + test_set) 18 | self.distribution = self._get_distribution(training_set, items) 19 | 20 | @staticmethod 21 | def _get_item_list(sequence): 22 | """ 23 | Convert a sequence in a list of items. 24 | 25 | :param sequence: A list of ratings. 26 | :return: A list of items. 27 | """ 28 | items = [] 29 | 30 | for rating in sequence: 31 | items.append(rating[0]) 32 | 33 | return items 34 | 35 | @staticmethod 36 | def _get_user_set(sequences): 37 | """ 38 | Create an ordered list of unique users from a list of sequences. 39 | 40 | :param sequences: A list of sequences. 41 | :return: An ordered list of unique users. 42 | """ 43 | users = [] 44 | 45 | for sequence in sequences: 46 | for rating in sequence: 47 | users.append(rating) 48 | 49 | return sorted(list(set(users))) 50 | 51 | @staticmethod 52 | def _get_distribution(sequences, items): 53 | """ 54 | Get an array representing the number of times the items 55 | appeared in a list of sequences. 56 | 57 | :param sequences: The list of sequences. 58 | :param items: The list of items. 59 | :return: An array representing the distribution. 60 | """ 61 | distribution = np.full(len(items), 0.0, dtype=float) 62 | 63 | for sequence in sequences: 64 | for rating in sequence: 65 | item_index = items.index(rating[0]) 66 | distribution[item_index] += 1 67 | 68 | distribution /= distribution.sum() 69 | 70 | return distribution 71 | 72 | def coverage(self, recommender): 73 | recommended_items = set() 74 | 75 | # For each sequence in the test set 76 | for sequence in self.test_set: 77 | recommended_sequence = recommender.recommend(sequence[0], self.k) 78 | 79 | for rating in recommended_sequence: 80 | recommended_items.add(rating[0]) 81 | 82 | coverage = len(recommended_items) / len(self.items) 83 | 84 | return coverage 85 | 86 | def precision(self, recommender): 87 | precision = np.full(len(self.test_set), 0.0, dtype=float) 88 | 89 | # For each sequence in the test set 90 | for sequence_index, sequence in enumerate(self.test_set): 91 | local_k = min(self.k, len(sequence) - 1) 92 | hit = 0 93 | 94 | recommended_sequence = recommender.recommend(sequence[0], self.k) 95 | reference_items = self._get_item_list(sequence[1:]) 96 | 97 | # For each rating in the recommended sequence 98 | for rating in recommended_sequence: 99 | # Check if the item is also in the reference path 100 | if rating[0] in reference_items: 101 | # Only the first time 102 | reference_items.remove(rating[0]) 103 | hit += 1 104 | 105 | precision[sequence_index] = hit / local_k 106 | 107 | return precision.mean() 108 | 109 | def ndpm(self, recommender): 110 | def count_item(item, item_list): 111 | return len(list(filter(lambda x: x == item, item_list))) 112 | 113 | ndpm = np.full(len(self.test_set), 0.0, dtype=float) 114 | 115 | for sequence_index, sequence in enumerate(self.test_set): 116 | recommended_sequence = recommender.recommend(sequence[0], self.k) 117 | recommended_items = self._get_item_list(recommended_sequence) 118 | reference_items = self._get_item_list(sequence[1:]) 119 | 120 | metric = 0 121 | worst_case = 0 122 | 123 | for items in itertools.combinations(recommended_items, 2): 124 | # If the two items are in the recommended sequence and they are unique 125 | if count_item(items[0], reference_items) == 1 and count_item(items[1], reference_items) == 1: 126 | index_i = reference_items.index(items[0]) 127 | index_j = reference_items.index(items[1]) 128 | 129 | # If the order is incorrect 130 | if index_j < index_i: 131 | metric += 2 132 | else: 133 | # If the order is correct 134 | metric += 0 135 | 136 | else: 137 | # If the order is irrelevant 138 | metric += 1 139 | 140 | worst_case += 2 141 | 142 | metric /= worst_case 143 | ndpm[sequence_index] = metric 144 | 145 | return ndpm.mean() 146 | 147 | def diversity(self, recommender, metric): 148 | diversity = np.full(len(self.test_set), 0.0, dtype=float) 149 | 150 | for sequence_index, sequence in enumerate(self.test_set): 151 | recommended_sequence = recommender.recommend(sequence[0], self.k) 152 | recommended_items = self._get_item_list(recommended_sequence) 153 | 154 | for items in itertools.combinations(recommended_items, 2): 155 | diversity[sequence_index] += (1 - metric.similarity(items[0], items[1])) 156 | 157 | diversity[sequence_index] /= self.k * (self.k - 1) * 0.5 158 | 159 | return diversity.mean() 160 | 161 | def novelty(self, recommender): 162 | novelty = np.full(len(self.test_set), 0.0, dtype=float) 163 | 164 | for sequence_index, sequence in enumerate(self.test_set): 165 | recommended_sequence = recommender.recommend(sequence[0], self.k) 166 | recommended_items = self._get_item_list(recommended_sequence) 167 | metric = 0 168 | 169 | for item in recommended_items: 170 | item_index = self.items.index(item) 171 | item_distribution = self.distribution[item_index] 172 | 173 | # log(0) = 0 by definition 174 | if item_distribution != 0: 175 | metric += math.log2(item_distribution) 176 | 177 | metric *= -1 * (1 / self.k) 178 | novelty[sequence_index] = metric 179 | 180 | return novelty.mean() 181 | 182 | def serendipity(self, recommender, primitive_k=None): 183 | primitive_recommender = baseline.MostPopularRecommender(self.training_set, self.items) 184 | primitive_sequence = primitive_recommender.recommend((0, 0, 0), self.k if primitive_k is None else primitive_k) 185 | primitive_items = self._get_item_list(primitive_sequence) 186 | 187 | serendipity = np.full(len(self.test_set), 0.0, dtype=float) 188 | 189 | for sequence_index, sequence in enumerate(self.test_set): 190 | local_k = min(self.k, len(sequence) - 1) 191 | hit = 0 192 | 193 | recommended_sequence = recommender.recommend(sequence[0], self.k) 194 | reference_items = self._get_item_list(sequence[1:]) 195 | 196 | # For each rating in the recommended sequence 197 | for rating in recommended_sequence: 198 | # Check if the item is also in the primitive sequence 199 | if rating[0] in primitive_items: 200 | continue 201 | 202 | # Check if the item is also in the reference path 203 | if rating[0] in reference_items: 204 | # Only the first time 205 | reference_items.remove(rating[0]) 206 | hit += 1 207 | 208 | serendipity[sequence_index] = hit / local_k 209 | 210 | return serendipity.mean() 211 | 212 | def confidence(self, recommender): 213 | confidence = np.full(len(self.test_set), 0.0, dtype=float) 214 | 215 | for sequence_index, sequence in enumerate(self.test_set): 216 | recommended_sequence = recommender.recommend(sequence[0], self.k) 217 | 218 | previous_rating = sequence[0] 219 | 220 | for rating in recommended_sequence: 221 | probability = recommender.predict_item(previous_rating, rating[0]) 222 | confidence[sequence_index] += probability 223 | previous_rating = rating 224 | 225 | recommender.reset() 226 | 227 | # Mean for each sequence 228 | confidence[sequence_index] /= self.k 229 | 230 | return confidence.mean() 231 | 232 | def perplexity(self, recommender): 233 | cross_entropy = np.full(len(self.test_set), 0.0, dtype=float) 234 | count_ratings = 0 235 | 236 | for sequence_index, sequence in enumerate(self.test_set): 237 | 238 | for rating_index, rating in enumerate(sequence): 239 | try: 240 | next_item = sequence[rating_index + 1][0] 241 | probability = recommender.predict_item(rating, next_item) 242 | 243 | if probability > 0: 244 | # noinspection PyUnresolvedReferences 245 | cross_entropy[sequence_index] -= np.log2(probability) 246 | else: 247 | cross_entropy[sequence_index] = math.inf 248 | 249 | count_ratings += 1 250 | 251 | except IndexError: 252 | # The sequence has ended 253 | pass 254 | 255 | recommender.reset() 256 | 257 | perplexity = 2 ** (cross_entropy.sum() / count_ratings) 258 | 259 | return perplexity 260 | -------------------------------------------------------------------------------- /sequeval/indexlist.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | 4 | class IndexList(collections.MutableSequence): 5 | 6 | def __init__(self): 7 | """ 8 | An IndexList is a list that efficiently stores the indexes associated to each value. 9 | Appending duplicate values is not allowed, in order to have unique indexes. 10 | """ 11 | self._list = [] 12 | self._index = {} 13 | 14 | def __setitem__(self, index, value): 15 | self._list.__setitem__(index, value) 16 | self._index[value] = index 17 | 18 | def __delitem__(self, index): 19 | # Not implemented 20 | pass 21 | 22 | def __getitem__(self, index): 23 | return self._list.__getitem__(index) 24 | 25 | def __len__(self): 26 | return self._list.__len__() 27 | 28 | def insert(self, index, value): 29 | self._list.insert(index, value) 30 | self._index[value] = self._list.index(value) 31 | 32 | def append(self, value): 33 | try: 34 | self._index[value] 35 | except KeyError: 36 | self._list.append(value) 37 | self._index[value] = len(self._list) - 1 38 | 39 | def index(self, value, start=0, stop=None): 40 | return self._index[value] 41 | -------------------------------------------------------------------------------- /sequeval/loader.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from abc import abstractmethod 3 | 4 | import pandas 5 | 6 | 7 | class Loader(ABC): 8 | 9 | @abstractmethod 10 | def load(self, file): 11 | """ 12 | Load a file containing ratings in a list of ratings. 13 | The list must be ordered by user and timestamp. 14 | 15 | :param file: The file path. 16 | :return: A list of ratings. 17 | :rtype: Iterable. 18 | """ 19 | pass 20 | 21 | 22 | class UIRTLoader(Loader): 23 | 24 | def __init__(self, user_ratings=0, item_ratings=0, threshold=None, skip=0): 25 | """ 26 | :param user_ratings: The minimum number of ratings per each user. 27 | :param item_ratings: The minimum number of ratings per each item. 28 | :param threshold: The threshold between positive and negative ratings. 29 | :param skip: The number of rows to skip when reading the file. 30 | """ 31 | self.user_ratings = user_ratings 32 | self.item_ratings = item_ratings 33 | self.threshold = threshold 34 | self.skip = skip 35 | 36 | def load(self, file): 37 | """ 38 | Load a CSV file with a MovieLens-like format. 39 | Only the ratings higher than the threshold will be considered. 40 | 41 | :param file: The file path. 42 | :return: A list of ratings. 43 | """ 44 | # Read the input file 45 | df_input = pandas.read_csv(file, names=['userId', 'itemId', 'rating', 'timestamp'], skiprows=self.skip) 46 | 47 | # Select the ratings higher than the threshold 48 | if self.threshold is not None: 49 | df_filtered = df_input.loc[df_input['rating'] >= self.threshold] 50 | else: 51 | df_filtered = df_input 52 | 53 | if self.user_ratings > 0: 54 | # Count the ratings per each user 55 | df_users_counter = df_filtered.groupby(['userId']).size().reset_index(name='counter') 56 | 57 | # Select the users with more ratings than the minimum value 58 | good_users = df_users_counter.loc[df_users_counter['counter'] >= self.user_ratings]['userId'] 59 | 60 | # Keep only the ratings associated with good users 61 | df_user_ratings = df_filtered.loc[df_filtered['userId'].isin(good_users)] 62 | else: 63 | df_user_ratings = df_filtered 64 | 65 | if self.item_ratings > 0: 66 | # Count the ratings per each item 67 | df_items_counter = df_user_ratings.groupby(['itemId']).size().reset_index(name='counter') 68 | 69 | # Select the items with more ratings than the minimum value 70 | good_items = df_items_counter.loc[df_items_counter['counter'] >= self.item_ratings]['itemId'] 71 | 72 | # Keep only the ratings associated with good items 73 | df_item_ratings = df_user_ratings.loc[df_user_ratings['itemId'].isin(good_items)] 74 | else: 75 | df_item_ratings = df_user_ratings 76 | 77 | # Sort the by user and timestamp 78 | df_sorted = df_item_ratings.sort_values(by=['userId', 'timestamp']) 79 | 80 | # Create a list of ratings 81 | ratings = [] 82 | for row in df_sorted.itertuples(): 83 | # itemId, userId, timestamp 84 | ratings.append((row[2], row[1], int(row[4]))) 85 | 86 | return ratings 87 | -------------------------------------------------------------------------------- /sequeval/profiler.py: -------------------------------------------------------------------------------- 1 | class Profiler: 2 | 3 | def __init__(self, sequences): 4 | users = [] 5 | items = [] 6 | ratings = [] 7 | 8 | for sequence in sequences: 9 | for rating in sequence: 10 | users.append(rating[1]) 11 | items.append(rating[0]) 12 | ratings.append(rating) 13 | 14 | self._sequences = sequences 15 | self._users = list(set(users)) 16 | self._items = list(set(items)) 17 | self._ratings = ratings 18 | 19 | def users(self): 20 | """ 21 | :return: The number of unique users. 22 | """ 23 | return len(self._users) 24 | 25 | def items(self): 26 | """ 27 | :return: The number of unique items. 28 | """ 29 | return len(self._items) 30 | 31 | def ratings(self): 32 | """ 33 | :return: The number of ratings. 34 | """ 35 | return len(self._ratings) 36 | 37 | def sequences(self): 38 | """ 39 | :return: The number of sequences. 40 | """ 41 | return len(self._sequences) 42 | 43 | def sparsity(self): 44 | """ 45 | Compute the sequence-item sparsity, that is the number of ratings 46 | divided by the number of sequences times the number of items. 47 | 48 | :return: The sequence-item sparsity. 49 | """ 50 | return self.ratings() / (self.sequences() * self.items()) 51 | 52 | def sequence_length(self): 53 | """ 54 | :return: The average length of a sequence. 55 | """ 56 | return self.ratings() / self.sequences() 57 | -------------------------------------------------------------------------------- /sequeval/recommender.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from abc import abstractmethod 3 | 4 | import numpy as np 5 | 6 | 7 | class Recommender(ABC): 8 | 9 | def __init__(self, training_set, items): 10 | self.training_set = training_set 11 | self.items = items 12 | # Dictionary to store the recommended sequences 13 | self.cache = {} 14 | 15 | def recommend(self, seed_rating, k): 16 | """ 17 | Generate a recommended sequence of length k from a seed rating. 18 | 19 | :param seed_rating: The seed rating. 20 | :param k: The length of the sequence. 21 | :return: A recommended sequence. 22 | """ 23 | try: 24 | # If this sequence has already been generated 25 | return list(self.cache[(seed_rating, k)]) 26 | except KeyError: 27 | pass 28 | 29 | current_rating = seed_rating 30 | sequence = [] 31 | 32 | for i in range(0, k): 33 | # The probabilities for the next item of the sequence 34 | prediction = self.predict(current_rating) 35 | # noinspection PyUnresolvedReferences 36 | item_index = np.random.multinomial(1, prediction).argmax() 37 | next_rating = (self.items[item_index], current_rating[1], current_rating[2] + 1) 38 | sequence.append(next_rating) 39 | current_rating = next_rating 40 | 41 | # After each sequence the model needs to be reset 42 | self.reset() 43 | 44 | # Save this sequence in the cache 45 | self.cache[(seed_rating, k)] = sequence 46 | 47 | return sequence 48 | 49 | def predict_item(self, rating, item): 50 | """ 51 | Given the current rating of the recommended sequence, predict 52 | the probability that the following rating will contain this item. 53 | 54 | :param rating: The current rating of the recommended sequence. 55 | :param item: The next item of the sequence. 56 | :return: A probability. 57 | """ 58 | prediction = self.predict(rating) 59 | return prediction[self.items.index(item)] 60 | 61 | @abstractmethod 62 | def predict(self, rating): 63 | """ 64 | Given the current rating of the recommended sequence, predict 65 | the probabilities for all the possible items of being in the next rating. 66 | 67 | :param rating: The current rating of the recommended sequence. 68 | :return: An array of probabilities. 69 | :rtype: Iterable. 70 | """ 71 | pass 72 | 73 | @abstractmethod 74 | def reset(self): 75 | """ 76 | Reset the internal model that represents the current sequence. 77 | """ 78 | pass 79 | -------------------------------------------------------------------------------- /sequeval/similarity.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from abc import abstractmethod 3 | 4 | import numpy as np 5 | import scipy.spatial.distance as distance 6 | 7 | 8 | class Similarity(ABC): 9 | 10 | def __init__(self, items): 11 | self.items = items 12 | 13 | @abstractmethod 14 | def similarity(self, item_i, item_j): 15 | """ 16 | Compute a similarity metric between two items. 17 | 18 | :param item_i: The first item. 19 | :param item_j: The second item. 20 | :return: A similarity value. 21 | """ 22 | pass 23 | 24 | 25 | class CosineSimilarity(Similarity): 26 | 27 | def __init__(self, training_set, items): 28 | super().__init__(items) 29 | 30 | # Create an item sequence matrix 31 | self.matrix = np.full((len(self.items), len(training_set)), 0) 32 | 33 | for sequence_index, sequence in enumerate(training_set): 34 | for rating in sequence: 35 | item_index = self.items.index(rating[0]) 36 | self.matrix[item_index, sequence_index] += 1 37 | 38 | self.cache = {} 39 | 40 | def similarity(self, item_i, item_j): 41 | """ 42 | Compute the cosine similarity between two items. 43 | If an item does not appear in any sequence the similarity is zero. 44 | 45 | :param item_i: The first item. 46 | :param item_j: The second item. 47 | :return: A cosine similarity value. 48 | """ 49 | index_i = self.items.index(item_i) 50 | index_j = self.items.index(item_j) 51 | 52 | if (index_i, index_j) in self.cache: 53 | return self.cache[(index_i, index_j)] 54 | 55 | array_i = self.matrix[index_i] 56 | array_j = self.matrix[index_j] 57 | 58 | if array_i.sum() == 0 or array_j.sum() == 0: 59 | self.cache[(index_i, index_j)] = 0 60 | else: 61 | self.cache[(index_i, index_j)] = 1 - distance.cosine(array_i, array_j) 62 | 63 | return self.cache[(index_i, index_j)] 64 | -------------------------------------------------------------------------------- /sequeval/splitter.py: -------------------------------------------------------------------------------- 1 | import random 2 | from abc import ABC 3 | from abc import abstractmethod 4 | 5 | 6 | class Splitter(ABC): 7 | 8 | def __init__(self, test_ratio): 9 | if test_ratio < 0 or test_ratio > 1: 10 | raise ValueError('Test ratio must be a number between 0 and 1') 11 | self.test_ratio = test_ratio 12 | 13 | @abstractmethod 14 | def split(self, sequences): 15 | pass 16 | 17 | 18 | class RandomSplitter(Splitter): 19 | 20 | def split(self, sequences): 21 | """ 22 | Perform a random splitting according to the test ratio. 23 | 24 | :param sequences: A list of sequences. 25 | :return: A list of training and a list of test sequences. 26 | """ 27 | training_set = [] 28 | test_set = [] 29 | 30 | # Randomize the sequences 31 | random_sequences = list(sequences) 32 | random.shuffle(random_sequences) 33 | 34 | # Target number of sequences in the training set 35 | target_training = len(random_sequences) * (1 - self.test_ratio) 36 | 37 | # Put the sequences in the test or training sets 38 | for counter, sequence in enumerate(random_sequences): 39 | if counter < target_training: 40 | training_set.append(sequence) 41 | else: 42 | test_set.append(sequence) 43 | 44 | return training_set, test_set 45 | 46 | 47 | class TimestampSplitter(Splitter): 48 | 49 | def split(self, sequences): 50 | """ 51 | Perform a timestamp splitting according to the test ratio. 52 | 53 | :param sequences: A list of sequences. 54 | :return: A list of training and a list of test sequences. 55 | """ 56 | training_set = [] 57 | test_set = [] 58 | 59 | # The sequences must be ordered by timestamp 60 | ordered_sequences = sorted(sequences, key=lambda item: item[0][2]) 61 | 62 | # Target number of sequences in the training set 63 | target_training = len(ordered_sequences) * (1 - self.test_ratio) 64 | 65 | # Put the sequences in the test or training sets 66 | for counter, sequence in enumerate(ordered_sequences): 67 | if counter < target_training: 68 | training_set.append(sequence) 69 | else: 70 | test_set.append(sequence) 71 | 72 | return training_set, test_set 73 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='sequeval', 5 | version='1.1.2', 6 | packages=['sequeval', 7 | 'sequeval.baseline'], 8 | url='https://github.com/D2KLab/sequeval', 9 | license='MIT', 10 | author='Diego Monti', 11 | author_email='diego.monti@polito.it', 12 | description='An offline evaluation framework for sequence-based recommender systems', 13 | install_requires=['numpy', 14 | 'pandas', 15 | 'pytimeparse', 16 | 'scipy'] 17 | ) 18 | -------------------------------------------------------------------------------- /static/sequeval.js: -------------------------------------------------------------------------------- 1 | $(function () { 2 | $('#run-evaluator').submit(function () { 3 | $('.btn').prop('disabled', true); 4 | $('#result').empty(); 5 | 6 | $.getJSON('/run', $(this).serialize(), function (json) { 7 | let profiler = json['profiler']; 8 | let result = $('#result'); 9 | let h3 = $('

').text('Profiler'); 10 | result.append(h3); 11 | let ul = $('
    '); 12 | result.append(ul); 13 | ul.append($('
  • ').text('Users: ' + profiler['users'])); 14 | ul.append($('
  • ').text('Items: ' + profiler['items'])); 15 | ul.append($('
  • ').text('Ratings: ' + profiler['ratings'])); 16 | ul.append($('
  • ').text('Sequences: ' + profiler['sequences'])); 17 | ul.append($('
  • ').text('Sparsity: ' + profiler['sparsity'])); 18 | ul.append($('
  • ').text('Length: ' + profiler['length'])); 19 | 20 | let splitter = json['splitter']; 21 | h3 = $('

    ').text('Splitter'); 22 | result.append(h3); 23 | ul = $('
      '); 24 | result.append(ul); 25 | ul.append($('
    • ').text('Training set: ' + splitter['training'])); 26 | ul.append($('
    • ').text('Test set: ' + splitter['test'])); 27 | 28 | let evaluator = json['evaluator']; 29 | h3 = $('

      ').text('Evaluator'); 30 | result.append(h3); 31 | let table = $('').addClass('table'); 32 | result.append(table); 33 | let tr = $(''); 34 | table.append(tr); 35 | tr.append($(''); 46 | table.append(tr); 47 | tr.append($('
      ').text('Algorithm')); 36 | tr.append($('').text('Coverage')); 37 | tr.append($('').text('Precision')); 38 | tr.append($('').text('nDPM')); 39 | tr.append($('').text('Diversity')); 40 | tr.append($('').text('Novelty')); 41 | tr.append($('').text('Serendipity')); 42 | tr.append($('').text('Confidence')); 43 | tr.append($('').text('Perplexity')); 44 | for (let i = 0; i < evaluator.length; i++) { 45 | let tr = $('
      ').text(evaluator[i][0])); 48 | for (let j = 1; j < evaluator[i].length; j++) { 49 | tr.append($('').text(evaluator[i][j])); 50 | } 51 | } 52 | 53 | $('.btn').prop('disabled', false); 54 | }); 55 | return false; 56 | }); 57 | }); -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | Sequeval 9 | 10 | 11 | 12 |
      13 |
      14 |

      Sequeval

      15 |
      16 |
      17 |
      18 |

      From this page it is possible to execute an experiment with the baseline recommenders 19 | and a down-sampled version of the Yes.com 20 | dataset.

      21 |
      22 |
      23 |
      24 |
      25 | 26 |
      27 |
      28 |
      29 |
      30 |
      31 | 32 | 34 | 35 | The minimum number of ratings for each user. 36 | 37 |
      38 |
      39 | 40 | 42 | 43 | The minimum number of ratings for each item. 44 | 45 |
      46 |
      47 | 48 |
      49 |
      50 | 51 | 55 | 56 | The splitting strategy. 57 | 58 |
      59 |
      60 | 61 |
      62 | 64 |
      65 | % 66 |
      67 |
      68 | 69 | The percentage of sequences included in the test set. 70 | 71 |
      72 |
      73 | 74 |
      75 |
      76 | 77 | 78 | 79 | The length of the recommended sequences. 80 | 81 |
      82 |
      83 | 84 |
      85 | 86 |
      87 |
      88 |
      89 | 90 |
      91 | 92 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/D2KLab/sequeval/71e1b40fa728492bef52a2ec0fb0353dce32602d/tests/__init__.py -------------------------------------------------------------------------------- /tests/baseline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/D2KLab/sequeval/71e1b40fa728492bef52a2ec0fb0353dce32602d/tests/baseline/__init__.py -------------------------------------------------------------------------------- /tests/baseline/test_bigram.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval.baseline as baseline 4 | from ..test_builder import sequences, items 5 | 6 | 7 | class BigramTestSuite(unittest.TestCase): 8 | 9 | def test_predict(self): 10 | bigram = baseline.BigramRecommender(sequences, items) 11 | 12 | # First item 13 | expected = [1 / 5, 3 / 5, 1 / 5] 14 | self.assertEqual(expected, bigram.predict((1, 1, 1)).tolist()) 15 | bigram.reset() 16 | 17 | # Second item 18 | expected = [2 / 5, 2 / 5, 1 / 5] 19 | self.assertEqual(expected, bigram.predict((2, 1, 1)).tolist()) 20 | bigram.reset() 21 | 22 | # Third item 23 | expected = [1 / 4, 2 / 4, 1 / 4] 24 | self.assertEqual(expected, bigram.predict((3, 1, 1)).tolist()) 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /tests/baseline/test_mostpopular.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | 5 | import sequeval.baseline as baseline 6 | from ..test_builder import sequences, items 7 | 8 | 9 | class MostPopularTestSuite(unittest.TestCase): 10 | 11 | def test_predict(self): 12 | most_popular = baseline.MostPopularRecommender(sequences, items) 13 | 14 | # First item 15 | expected = np.full(len(items), 0.0, dtype=float) 16 | expected[items.index(2)] = 1.0 17 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 18 | 19 | # Second item 20 | expected = np.full(len(items), 0.0, dtype=float) 21 | expected[items.index(1)] = 1.0 22 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 23 | 24 | # Third item 25 | expected = np.full(len(items), 0.0, dtype=float) 26 | expected[items.index(3)] = 1.0 27 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 28 | 29 | def test_reset(self): 30 | most_popular = baseline.MostPopularRecommender(sequences, items) 31 | 32 | # First item 33 | expected = np.full(len(items), 0.0, dtype=float) 34 | expected[items.index(2)] = 1.0 35 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 36 | 37 | # Second item 38 | expected = np.full(len(items), 0.0, dtype=float) 39 | expected[items.index(1)] = 1.0 40 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 41 | 42 | # Reset 43 | most_popular.reset() 44 | 45 | # First item 46 | expected = np.full(len(items), 0.0, dtype=float) 47 | expected[items.index(2)] = 1.0 48 | self.assertEqual(expected.tolist(), most_popular.predict(None).tolist()) 49 | 50 | 51 | if __name__ == '__main__': 52 | unittest.main() 53 | -------------------------------------------------------------------------------- /tests/baseline/test_random.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | 5 | import sequeval.baseline as baseline 6 | from ..test_builder import sequences, items 7 | 8 | 9 | class MostPopularTestSuite(unittest.TestCase): 10 | 11 | def test_predict(self): 12 | random = baseline.RandomRecommender(sequences, items) 13 | expected = np.full(len(items), 1 / 3, dtype=float) 14 | self.assertEqual(expected.tolist(), random.predict(None).tolist()) 15 | random.reset() 16 | self.assertEqual(expected.tolist(), random.predict(None).tolist()) 17 | 18 | 19 | if __name__ == '__main__': 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /tests/baseline/test_unigram.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval.baseline as baseline 4 | from ..test_builder import sequences, items 5 | 6 | 7 | class MostPopularTestSuite(unittest.TestCase): 8 | 9 | def test_predict(self): 10 | unigram = baseline.UnigramRecommender(sequences, items) 11 | expected = [4 / 11, 5 / 11, 2 / 11] 12 | self.assertEqual(expected, unigram.predict(None).tolist()) 13 | unigram.reset() 14 | self.assertEqual(expected, unigram.predict(None).tolist()) 15 | 16 | 17 | if __name__ == '__main__': 18 | unittest.main() 19 | -------------------------------------------------------------------------------- /tests/test_builder.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval 4 | from .test_loader import ratings 5 | 6 | sequences = [[(1, 1, 0), (2, 1, 5), (1, 1, 10)], 7 | [(3, 1, 30), (2, 1, 35)], 8 | [(1, 2, 30), (2, 2, 35), (2, 2, 40)]] 9 | 10 | items = [1, 2, 3] 11 | 12 | 13 | class BuilderTestSuite(unittest.TestCase): 14 | 15 | def test_builder(self): 16 | builder = sequeval.Builder(10) 17 | actual_sequences, actual_items = builder.build(ratings) 18 | self.assertEqual(sequences, actual_sequences) 19 | self.assertEqual(items, actual_items._list) 20 | 21 | def test_builder_parse(self): 22 | builder = sequeval.Builder('10 seconds') 23 | actual_sequences, actual_items = builder.build(ratings) 24 | self.assertEqual(sequences, actual_sequences) 25 | self.assertEqual(items, actual_items._list) 26 | 27 | 28 | if __name__ == '__main__': 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /tests/test_evaluator.py: -------------------------------------------------------------------------------- 1 | import math 2 | import unittest 3 | 4 | import sequeval 5 | import sequeval.baseline as baseline 6 | from .test_builder import sequences as training_set, items 7 | 8 | test_set = [[(1, 1, 0), (1, 1, 1), (2, 1, 2), (3, 1, 3)], 9 | [(1, 1, 0), (1, 1, 1), (2, 1, 2), (1, 1, 3)]] 10 | 11 | 12 | class EvaluatorTestSuite(unittest.TestCase): 13 | 14 | def test_coverage(self): 15 | recommender = baseline.MostPopularRecommender(training_set, items) 16 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 17 | self.assertEqual(3 / 3, evaluator.coverage(recommender)) 18 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 19 | self.assertEqual(2 / 3, evaluator.coverage(recommender)) 20 | 21 | def test_precision(self): 22 | recommender = baseline.MostPopularRecommender(training_set, items) 23 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 24 | self.assertAlmostEqual(5 / 6, evaluator.precision(recommender)) 25 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 26 | self.assertEqual(1.0, evaluator.precision(recommender)) 27 | 28 | def test_ndpm(self): 29 | recommender = baseline.MostPopularRecommender(training_set, items) 30 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 31 | self.assertAlmostEqual(5 / 12, evaluator.ndpm(recommender)) 32 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 33 | self.assertEqual(3 / 4, evaluator.ndpm(recommender)) 34 | 35 | def test_diversity(self): 36 | recommender = baseline.MostPopularRecommender(training_set, items) 37 | similarity = sequeval.CosineSimilarity(training_set, items) 38 | 39 | # Compute the possible diversities 40 | d1 = 1 - similarity.similarity(2, 1) 41 | d2 = 1 - similarity.similarity(2, 3) 42 | d3 = 1 - similarity.similarity(1, 3) 43 | 44 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 45 | self.assertAlmostEqual((d1 + d2 + d3) / 3, evaluator.diversity(recommender, similarity)) 46 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 47 | self.assertAlmostEqual(d1, evaluator.diversity(recommender, similarity)) 48 | 49 | def test_novelty(self): 50 | recommender = baseline.MostPopularRecommender(training_set, items) 51 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 52 | self.assertAlmostEqual((math.log2(0.375) + math.log2(0.5) + math.log2(0.125)) / -3, 53 | evaluator.novelty(recommender)) 54 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 55 | self.assertAlmostEqual((math.log2(0.375) + math.log2(0.5)) / -2, evaluator.novelty(recommender)) 56 | 57 | def test_serendipity(self): 58 | recommender = baseline.MostPopularRecommender(training_set, items) 59 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 60 | self.assertEqual(0.0, evaluator.serendipity(recommender)) 61 | self.assertAlmostEqual(1 / 6, evaluator.serendipity(recommender, primitive_k=2)) 62 | evaluator = sequeval.Evaluator(training_set, test_set, items, 2) 63 | self.assertEqual(0.0, evaluator.serendipity(recommender)) 64 | self.assertAlmostEqual(3 / 6, evaluator.serendipity(recommender, primitive_k=1)) 65 | 66 | def test_confidence(self): 67 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 68 | recommender = baseline.MostPopularRecommender(training_set, items) 69 | self.assertEqual(1.0, evaluator.confidence(recommender)) 70 | recommender = baseline.RandomRecommender(training_set, items) 71 | self.assertAlmostEqual(1 / 3, evaluator.confidence(recommender)) 72 | 73 | def test_perplexity(self): 74 | evaluator = sequeval.Evaluator(training_set, test_set, items, 3) 75 | recommender = baseline.MostPopularRecommender(training_set, items) 76 | self.assertAlmostEqual(math.inf, evaluator.perplexity(recommender)) 77 | recommender = baseline.RandomRecommender(training_set, items) 78 | self.assertAlmostEqual(3.0, evaluator.perplexity(recommender)) 79 | 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /tests/test_indexlist.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval 4 | 5 | 6 | class IndexListTestSuite(unittest.TestCase): 7 | 8 | def test_setitem(self): 9 | indexlist = sequeval.IndexList() 10 | indexlist.append(None) 11 | indexlist[0] = 'Zero' 12 | self.assertEqual(indexlist[0], 'Zero') 13 | self.assertEqual(indexlist.index('Zero'), 0) 14 | 15 | def test_append(self): 16 | indexlist = sequeval.IndexList() 17 | indexlist.append('Zero') 18 | indexlist.append('One') 19 | indexlist.append('Zero') 20 | self.assertEqual(indexlist[0], 'Zero') 21 | self.assertEqual(indexlist[1], 'One') 22 | self.assertEqual(indexlist.index('Zero'), 0) 23 | self.assertEqual(indexlist.index('One'), 1) 24 | 25 | def test_delete(self): 26 | indexlist = sequeval.IndexList() 27 | indexlist.append('Zero') 28 | del indexlist[0] 29 | self.assertEqual(indexlist[0], 'Zero') 30 | self.assertEqual(indexlist.index('Zero'), 0) 31 | 32 | def test_len(self): 33 | indexlist = sequeval.IndexList() 34 | indexlist.append('Zero') 35 | indexlist.append('One') 36 | indexlist.append('Zero') 37 | self.assertEqual(len(indexlist), 2) 38 | 39 | def test_insert(self): 40 | indexlist = sequeval.IndexList() 41 | indexlist.insert(1, 'Zero') 42 | self.assertEqual(indexlist[0], 'Zero') 43 | self.assertEqual(indexlist.index('Zero'), 0) 44 | 45 | 46 | if __name__ == '__main__': 47 | unittest.main() 48 | -------------------------------------------------------------------------------- /tests/test_loader.py: -------------------------------------------------------------------------------- 1 | import io 2 | import unittest 3 | 4 | import sequeval 5 | 6 | ratings = [(1, 1, 0), 7 | (2, 1, 5), 8 | (1, 1, 10), 9 | (3, 1, 30), 10 | (2, 1, 35), 11 | (1, 2, 5), 12 | (1, 2, 30), 13 | (2, 2, 35), 14 | (2, 2, 40)] 15 | 16 | 17 | class LoaderTestSuite(unittest.TestCase): 18 | 19 | def test_uirt(self): 20 | fake_input = "1,1,5,0\n1,2,5,5\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,5,40" 21 | loader = sequeval.UIRTLoader() 22 | self.assertEqual(ratings, loader.load(io.StringIO(fake_input))) 23 | 24 | def test_uirt_ordering(self): 25 | fake_input = "1,2,5,5\n1,1,5,0\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,5,40" 26 | loader = sequeval.UIRTLoader() 27 | self.assertEqual(ratings, loader.load(io.StringIO(fake_input))) 28 | 29 | def test_uirt_user_ratings(self): 30 | fake_input = "1,1,5,0\n1,2,5,5\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,5,40" 31 | loader = sequeval.UIRTLoader(user_ratings=5) 32 | self.assertEqual(ratings[:5], loader.load(io.StringIO(fake_input))) 33 | 34 | def test_uirt_item_ratings(self): 35 | fake_input = "1,1,5,0\n1,2,5,5\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,5,40" 36 | loader = sequeval.UIRTLoader(item_ratings=2) 37 | self.assertEqual(ratings[:3] + ratings[4:], loader.load(io.StringIO(fake_input))) 38 | 39 | def test_uirt_threshold(self): 40 | fake_input = "1,1,5,0\n1,2,5,5\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,2,40" 41 | loader = sequeval.UIRTLoader(threshold=3) 42 | self.assertEqual(ratings[0:8], loader.load(io.StringIO(fake_input))) 43 | 44 | def test_uirt_skip(self): 45 | fake_input = "1,1,5,0\n1,2,5,5\n1,1,5,10\n1,3,5,30\n1,2,5,35,\n2,1,5,5\n2,1,5,30\n2,2,5,35\n2,2,5,40" 46 | loader = sequeval.UIRTLoader(skip=1) 47 | self.assertEqual(ratings[1:], loader.load(io.StringIO(fake_input))) 48 | 49 | 50 | if __name__ == '__main__': 51 | unittest.main() 52 | -------------------------------------------------------------------------------- /tests/test_profiler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval 4 | from .test_builder import sequences 5 | 6 | 7 | class ProfilerTestSuite(unittest.TestCase): 8 | 9 | def test_users(self): 10 | profiler = sequeval.Profiler(sequences) 11 | self.assertEqual(2, profiler.users()) 12 | 13 | def test_items(self): 14 | profiler = sequeval.Profiler(sequences) 15 | self.assertEqual(3, profiler.items()) 16 | 17 | def test_ratings(self): 18 | profiler = sequeval.Profiler(sequences) 19 | self.assertEqual(8, profiler.ratings()) 20 | 21 | def test_sequences(self): 22 | profiler = sequeval.Profiler(sequences) 23 | self.assertEqual(3, profiler.sequences()) 24 | 25 | def test_sparsity(self): 26 | profiler = sequeval.Profiler(sequences) 27 | self.assertAlmostEqual(8 / 9, profiler.sparsity()) 28 | 29 | def test_sequence_length(self): 30 | profiler = sequeval.Profiler(sequences) 31 | self.assertAlmostEqual(8 / 3, profiler.sequence_length()) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /tests/test_recommender.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval.baseline as baseline 4 | from .test_builder import sequences, items 5 | 6 | 7 | class RecommenderTestSuite(unittest.TestCase): 8 | 9 | def test_recommend(self): 10 | recommender = baseline.MostPopularRecommender(sequences, items) 11 | expected = [(2, 1, 4), (1, 1, 5), (3, 1, 6)] 12 | self.assertEqual(expected, recommender.recommend((1, 1, 3), 3)) 13 | # Check that there is no memory 14 | self.assertEqual(expected, recommender.recommend((1, 1, 3), 3)) 15 | 16 | def test_predict_item(self): 17 | recommender = baseline.MostPopularRecommender(sequences, items) 18 | self.assertEqual(1.0, recommender.predict_item((1, 1, 3), 2)) 19 | self.assertEqual(1.0, recommender.predict_item((1, 1, 4), 1)) 20 | self.assertEqual(0.0, recommender.predict_item((1, 1, 5), 1)) 21 | 22 | 23 | if __name__ == '__main__': 24 | unittest.main() 25 | -------------------------------------------------------------------------------- /tests/test_similarity.py: -------------------------------------------------------------------------------- 1 | import math 2 | import unittest 3 | 4 | import sequeval 5 | from .test_builder import sequences as training_set, items 6 | 7 | 8 | class SimilarityTestSuite(unittest.TestCase): 9 | 10 | def test_cosine_similarity(self): 11 | cosine = sequeval.CosineSimilarity(training_set, items + [4]) 12 | self.assertAlmostEqual(4 / (math.sqrt(5) * math.sqrt(6)), cosine.similarity(1, 2)) 13 | self.assertAlmostEqual(0, cosine.similarity(1, 3)) 14 | self.assertAlmostEqual(1 / (math.sqrt(6)), cosine.similarity(2, 3)) 15 | self.assertAlmostEqual(0, cosine.similarity(1, 4)) 16 | 17 | 18 | if __name__ == '__main__': 19 | unittest.main() 20 | -------------------------------------------------------------------------------- /tests/test_splitter.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import sequeval 4 | 5 | sequences = [[(1, 1, 0), (2, 1, 5), (1, 1, 10)], 6 | [(3, 1, 30), (2, 1, 35)], 7 | [(1, 2, 25), (2, 2, 35), (2, 2, 40)]] 8 | 9 | 10 | class SplitterTestSuite(unittest.TestCase): 11 | 12 | def test_random_splitter(self): 13 | splitter = sequeval.RandomSplitter(0.4) 14 | training_set, test_test = splitter.split(sequences) 15 | self.assertEqual(2, len(training_set)) 16 | self.assertEqual(1, len(test_test)) 17 | 18 | def test_timestamp_splitter(self): 19 | splitter = sequeval.TimestampSplitter(0.4) 20 | training_set, test_test = splitter.split(sequences) 21 | self.assertEqual([sequences[0], sequences[2]], training_set) 22 | self.assertEqual([sequences[1]], test_test) 23 | 24 | 25 | if __name__ == '__main__': 26 | unittest.main() 27 | -------------------------------------------------------------------------------- /webapp.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, render_template, json, request 2 | 3 | import sequeval 4 | import sequeval.baseline as baseline 5 | 6 | app = Flask(__name__) 7 | 8 | 9 | @app.route("/") 10 | def main(): 11 | return render_template("index.html") 12 | 13 | 14 | def parse(value): 15 | if value == float('inf'): 16 | return 'Infinity' 17 | if isinstance(value, float): 18 | return str("%.4f" % round(value, 4)) 19 | return str(value) 20 | 21 | 22 | def evaluation(compute, recommender, similarity): 23 | return [recommender.name, 24 | parse(compute.coverage(recommender)), 25 | parse(compute.precision(recommender)), 26 | parse(compute.ndpm(recommender)), 27 | parse(compute.diversity(recommender, similarity)), 28 | parse(compute.novelty(recommender)), 29 | parse(compute.serendipity(recommender)), 30 | parse(compute.confidence(recommender)), 31 | parse(compute.perplexity(recommender))] 32 | 33 | 34 | @app.route("/run", methods=['GET']) 35 | def run(): 36 | _user_ratings = int(request.args.get('user-ratings')) 37 | _item_ratings = int(request.args.get('item-ratings')) 38 | _splitter = request.args.get('splitter') 39 | _ratio = float(request.args.get('ratio')) / 100 40 | _k = int(request.args.get('length')) 41 | 42 | loader = sequeval.UIRTLoader(user_ratings=_user_ratings, item_ratings=_item_ratings) 43 | ratings = loader.load('datasets/yes_reduced.csv') 44 | 45 | builder = sequeval.Builder('1000 s') 46 | sequences, items = builder.build(ratings) 47 | 48 | profiler = sequeval.Profiler(sequences) 49 | response = {'profiler': {'users': profiler.users(), 50 | 'items': profiler.items(), 51 | 'ratings': profiler.ratings(), 52 | 'sequences': profiler.sequences(), 53 | 'sparsity': parse(profiler.sparsity()), 54 | 'length': parse(profiler.sequence_length())}} 55 | 56 | if _splitter == 'random': 57 | splitter = sequeval.RandomSplitter(_ratio) 58 | else: 59 | splitter = sequeval.TimestampSplitter(_ratio) 60 | training_set, test_set = splitter.split(sequences) 61 | response['splitter'] = {'training': len(training_set), 62 | 'test': len(test_set)} 63 | 64 | evaluator = sequeval.Evaluator(training_set, test_set, items, _k) 65 | cosine = sequeval.CosineSimilarity(training_set, items) 66 | 67 | response['evaluator'] = [] 68 | 69 | most_popular = baseline.MostPopularRecommender(training_set, items) 70 | response['evaluator'].append(evaluation(evaluator, most_popular, cosine)) 71 | 72 | random = baseline.RandomRecommender(training_set, items) 73 | response['evaluator'].append(evaluation(evaluator, random, cosine)) 74 | 75 | unigram = baseline.UnigramRecommender(training_set, items) 76 | response['evaluator'].append(evaluation(evaluator, unigram, cosine)) 77 | 78 | bigram = baseline.BigramRecommender(training_set, items) 79 | response['evaluator'].append(evaluation(evaluator, bigram, cosine)) 80 | 81 | return json.dumps(response) 82 | 83 | 84 | if __name__ == "__main__": 85 | app.run() 86 | --------------------------------------------------------------------------------