├── web ├── core │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── flush_redis.py │ │ │ ├── prune_matches.py │ │ │ ├── add_admins.py │ │ │ ├── add_regions.py │ │ │ ├── queue_random_matches.py │ │ │ └── add_models.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0004_auto_20190607_1335.py │ │ ├── 0002_auto_20190606_0056.py │ │ ├── 0003_auto_20190606_1156.py │ │ └── 0001_initial.py │ ├── templatetags │ │ ├── __init__.py │ │ └── extra_humanize.py │ ├── static │ │ ├── riot.txt │ │ ├── lol_logo.png │ │ └── creme_logo.png │ ├── exceptions.py │ ├── apps.py │ ├── urls.py │ ├── forms.py │ ├── templates │ │ ├── match.html │ │ ├── base.html │ │ └── index.html │ ├── admin.py │ ├── api.py │ ├── views.py │ ├── models.py │ └── services.py ├── lolmd │ ├── __init__.py │ ├── settings │ │ ├── __init__.py │ │ ├── prod.py │ │ ├── dev.py │ │ └── base.py │ ├── wsgi.py │ └── urls.py ├── Dockerfile ├── requirements.txt ├── manage.py └── wait-for-it.sh ├── .gitignore ├── screenshots ├── home.png ├── match.png └── matches.png ├── nginx ├── Dockerfile └── sites-enabled │ └── lolmd ├── docker-compose.yml ├── docker-compose.dev.yml └── README.md /web/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/lolmd/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/core/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/core/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/core/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/lolmd/settings/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web/core/static/riot.txt: -------------------------------------------------------------------------------- 1 | 58e73844-075f-4cfa-bedd-226c41510120 2 | -------------------------------------------------------------------------------- /web/lolmd/settings/prod.py: -------------------------------------------------------------------------------- 1 | from .base import * 2 | 3 | 4 | DEBUG = False 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.ipynb 3 | .ipynb_checkpoints/ 4 | *.sqlite3 5 | .env 6 | -------------------------------------------------------------------------------- /web/core/exceptions.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import * 2 | from requests.exceptions import * 3 | -------------------------------------------------------------------------------- /screenshots/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/online-ml/lol-match-duration/HEAD/screenshots/home.png -------------------------------------------------------------------------------- /screenshots/match.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/online-ml/lol-match-duration/HEAD/screenshots/match.png -------------------------------------------------------------------------------- /screenshots/matches.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/online-ml/lol-match-duration/HEAD/screenshots/matches.png -------------------------------------------------------------------------------- /web/core/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class CoreConfig(AppConfig): 5 | name = 'core' 6 | -------------------------------------------------------------------------------- /web/core/static/lol_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/online-ml/lol-match-duration/HEAD/web/core/static/lol_logo.png -------------------------------------------------------------------------------- /web/core/static/creme_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/online-ml/lol-match-duration/HEAD/web/core/static/creme_logo.png -------------------------------------------------------------------------------- /nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tutum/nginx 2 | 3 | RUN rm /etc/nginx/sites-enabled/default 4 | 5 | COPY sites-enabled/ /etc/nginx/sites-enabled 6 | -------------------------------------------------------------------------------- /web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | ENV PYTHONUNBUFFERED 1 3 | COPY requirements.txt . 4 | RUN pip install -r requirements.txt --upgrade 5 | COPY . . 6 | -------------------------------------------------------------------------------- /web/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/creme-ml/creme 2 | django==2.2.10 3 | django-picklefield==2.0 4 | django-rq==2.1.0 5 | gunicorn==19.9.0 6 | psycopg2-binary==2.8.3 7 | redis==3.2.1 8 | requests==2.22.0 9 | rq==1.0 10 | rq-scheduler==0.9 11 | -------------------------------------------------------------------------------- /web/core/urls.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.urls import path 3 | 4 | from . import views 5 | 6 | urlpatterns = [ 7 | path('', views.index, name='index'), 8 | path('admin/', admin.site.urls, name='admin'), 9 | path('match//', views.match, name='match') 10 | ] 11 | -------------------------------------------------------------------------------- /web/lolmd/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for lolmd project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lolmd.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /web/lolmd/settings/dev.py: -------------------------------------------------------------------------------- 1 | from .base import * 2 | 3 | 4 | DEBUG = True 5 | 6 | LOGGING = { 7 | 'version': 1, 8 | 'disable_existing_loggers': False, 9 | 'handlers': { 10 | 'console': { 11 | 'class': 'logging.StreamHandler', 12 | }, 13 | }, 14 | 'loggers': { 15 | 'django.request': { 16 | 'handlers': ['console'], 17 | 'level': 'ERROR', 18 | 'propagate': False, 19 | }, 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /web/core/migrations/0004_auto_20190607_1335.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.7 on 2019-06-07 13:35 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('core', '0003_auto_20190606_1156'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='match', 15 | name='api_id', 16 | field=models.TextField(), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /nginx/sites-enabled/lolmd: -------------------------------------------------------------------------------- 1 | server { 2 | 3 | listen 80; 4 | server_name example.org; 5 | charset utf-8; 6 | 7 | location /static/ { 8 | autoindex on; 9 | alias /static/; 10 | } 11 | 12 | location /riot.txt { 13 | alias /static/riot.txt; 14 | } 15 | 16 | location / { 17 | proxy_pass http://web:8000; 18 | proxy_set_header Host $host; 19 | proxy_set_header X-Real-IP $remote_addr; 20 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /web/core/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | from . import models 4 | 5 | 6 | def get_region_choices(): 7 | return [(region.short_name, region.full_name) for region in models.Region.objects.all()] 8 | 9 | 10 | class AddMatchForm(forms.Form): 11 | 12 | summoner_name = forms.CharField(label='Summoner name', max_length=100) 13 | region = forms.ChoiceField(label='Region') 14 | 15 | def __init__(self, *args, **kwargs): 16 | super().__init__(*args, **kwargs) 17 | self.fields['region'].choices = get_region_choices() 18 | -------------------------------------------------------------------------------- /web/core/management/commands/flush_redis.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management import base 3 | import redis 4 | 5 | 6 | class Command(base.BaseCommand): 7 | 8 | def handle(self, *args, **options): 9 | 10 | print('Flushing redis') 11 | 12 | for conf in settings.RQ_QUEUES.values(): 13 | redis.StrictRedis( 14 | host=conf['HOST'], 15 | port=conf['PORT'], 16 | db=conf['DB'], 17 | password=conf['PASSWORD'] 18 | ).flushdb() 19 | -------------------------------------------------------------------------------- /web/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == '__main__': 6 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lolmd.settings.prod') 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError as exc: 10 | raise ImportError( 11 | "Couldn't import Django. Are you sure it's installed and " 12 | "available on your PYTHONPATH environment variable? Did you " 13 | "forget to activate a virtual environment?" 14 | ) from exc 15 | execute_from_command_line(sys.argv) 16 | -------------------------------------------------------------------------------- /web/core/migrations/0002_auto_20190606_0056.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.7 on 2019-06-06 00:56 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('core', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='match', 15 | name='duration', 16 | field=models.IntegerField(null=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='match', 20 | name='predicted_duration', 21 | field=models.IntegerField(null=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /web/core/management/commands/prune_matches.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from django.core.management import base 4 | from django.utils import timezone 5 | 6 | from core import models 7 | from core import services 8 | 9 | 10 | class Command(base.BaseCommand): 11 | 12 | def handle(self, *args, **options): 13 | 14 | print('Pruning unfinished matches') 15 | 16 | for match in models.Match.objects.exclude(duration__isnull=False): 17 | 18 | if timezone.now() - match.created_at > dt.timedelta(days=7): 19 | print(f'\tRemoving match {match.id}') 20 | match.delete() 21 | continue 22 | 23 | print(f'\tTrying to end match {match.id}') 24 | services.try_to_end_match(id=match.id) 25 | -------------------------------------------------------------------------------- /web/lolmd/urls.py: -------------------------------------------------------------------------------- 1 | """lolmd URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.1/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Classbased views 10 | 1. Add an import: from other_app.vie 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.urls import include, path 17 | 18 | urlpatterns = [ 19 | path('', include('core.urls')) 20 | ] 21 | -------------------------------------------------------------------------------- /web/core/migrations/0003_auto_20190606_1156.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.7 on 2019-06-06 11:56 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('core', '0002_auto_20190606_0056'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='match', 15 | name='api_id', 16 | field=models.TextField(unique=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='match', 20 | name='duration', 21 | field=models.PositiveSmallIntegerField(null=True), 22 | ), 23 | migrations.AlterField( 24 | model_name='match', 25 | name='predicted_duration', 26 | field=models.PositiveSmallIntegerField(null=True), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /web/core/management/commands/add_admins.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib import auth 3 | from django.core.management import base 4 | 5 | 6 | class Command(base.BaseCommand): 7 | 8 | def handle(self, *args, **options): 9 | 10 | print('Adding admin accounts') 11 | 12 | user_model = auth.get_user_model() 13 | 14 | for full_name, email in settings.ADMINS: 15 | first_name, last_name = full_name.split(' ') 16 | 17 | if user_model.objects.filter(email=email).exists(): 18 | print(f'\tAdmin account for {first_name} {last_name} ({email}) already exists') 19 | continue 20 | print(f'\tCreating admin account for {first_name} {last_name} ({email})') 21 | 22 | user_model.objects.create_superuser( 23 | email=email, 24 | username=email, 25 | password=settings.ADMIN_PASSWORD, 26 | ) 27 | -------------------------------------------------------------------------------- /web/core/templatetags/extra_humanize.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from django import template 4 | 5 | 6 | register = template.Library() 7 | 8 | 9 | @register.filter() 10 | def secondsduration(seconds): 11 | return dt.timedelta(seconds=seconds) 12 | 13 | 14 | @register.filter 15 | def naturalduration(timedelta): 16 | """Converts a timedelta to a detailed human readable string. 17 | 18 | This implementation is not generic at all, it's specific to our purposes. It's surprising that 19 | Django doesn't support this out of the box. 20 | 21 | """ 22 | 23 | hours = timedelta.seconds // 3600 24 | if hours: 25 | timedelta -= dt.timedelta(hours=hours) 26 | 27 | minutes = timedelta.seconds // 60 28 | if minutes: 29 | timedelta -= dt.timedelta(minutes=minutes) 30 | 31 | seconds = timedelta.seconds 32 | if seconds: 33 | timedelta -= dt.timedelta(seconds=seconds) 34 | 35 | human = '' 36 | 37 | if hours > 0: 38 | human += f'{hours} hour' + ('s' if hours > 1 else '') 39 | 40 | if minutes > 0: 41 | if human: 42 | human += ', ' 43 | human += f'{minutes} minute' + ('s' if minutes > 1 else '') 44 | 45 | if seconds > 0: 46 | if human: 47 | human += ', ' 48 | human += f'{seconds} second' + ('s' if seconds > 1 else '') 49 | 50 | return human 51 | -------------------------------------------------------------------------------- /web/core/management/commands/add_regions.py: -------------------------------------------------------------------------------- 1 | """ 2 | - https://developer.riotgames.com/regional-endpoints.html 3 | - https://support.riotgames.com/hc/en-us/articles/201751684-League-of-Legends-Servers 4 | """ 5 | import collections 6 | 7 | from django.core.management import base 8 | 9 | from core import models 10 | 11 | 12 | class Command(base.BaseCommand): 13 | 14 | def handle(self, *args, **options): 15 | 16 | print('Adding regions') 17 | 18 | Region = collections.namedtuple('Region', 'short_name full_name platform') 19 | 20 | regions = [ 21 | Region('BR', 'Brazil', 'BR1'), 22 | Region('EUNE', 'Europe Nordic & East', 'EUN1'), 23 | Region('EUW', 'Europe West', 'EUW1'), 24 | Region('JP', 'Japan', 'JP1'), 25 | Region('KR', 'Korea', 'KR'), 26 | Region('LAN', 'Latin America North', 'LA1'), 27 | Region('LAS', 'Latin America South', 'LA2'), 28 | Region('NA', 'North America', 'NA1'), 29 | Region('OCE', 'Oceania', 'OC1'), 30 | Region('TR', 'Turkey', 'TR1'), 31 | Region('RU', 'Russia', 'RU'), 32 | ] 33 | 34 | for region in regions: 35 | if models.Region.objects.filter(short_name=region.short_name).exists(): 36 | print(f'\t{region.short_name} has already been added') 37 | continue 38 | models.Region(**region._asdict()).save() 39 | print(f'\tAdded {region.short_name}') 40 | -------------------------------------------------------------------------------- /web/core/templates/match.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 | {% load extra_humanize %} 6 | 7 |
8 |

Match {{ match.api_id }}

9 |

10 | {{ match.region.full_name }} 11 | {{ match.mode }} 12 | {{ match.type }} 13 |

14 |
15 | 16 |
17 |

Predicted duration: {{ match.predicted_duration | secondsduration | naturalduration }}

18 | 19 | {% if match.has_ended %} 20 | 30% 21 |

True duration: {{ match.duration | secondsduration | naturalduration }}

22 | 15% 23 |

Absolute error: {{ err_duration | naturalduration }}

24 | 15% 25 | {% else %} 26 |

The match has not ended yet.

27 | {% endif %} 28 | 29 |
30 | 31 |

Data available at the start of the match

32 | 33 | 
{{ raw_info }}
34 | 35 | {% endblock %} 36 | -------------------------------------------------------------------------------- /web/core/admin.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | from django.contrib import admin 4 | from django.http import HttpResponse 5 | 6 | from . import models 7 | 8 | 9 | class ExportCSVMixin: 10 | 11 | def export_as_csv(self, request, queryset): 12 | 13 | meta = self.model._meta 14 | field_names = [field.name for field in meta.fields] 15 | 16 | response = HttpResponse(content_type='text/csv') 17 | response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta.verbose_name_plural) 18 | writer = csv.writer(response) 19 | 20 | writer.writerow(field_names) 21 | for obj in queryset: 22 | writer.writerow([getattr(obj, field) for field in field_names]) 23 | 24 | return response 25 | 26 | export_as_csv.short_description = 'Export selected' 27 | 28 | 29 | @admin.register(models.Match) 30 | class MatchAdmin(admin.ModelAdmin, ExportCSVMixin): 31 | 32 | list_display = ('id', 'api_id', 'rq_job_id', 'created_at', 'started_at', 'duration', 33 | 'predicted_duration', 'winning_team_id', 'region') 34 | list_display_link = ('id',) 35 | actions = ['export_as_csv'] 36 | 37 | 38 | @admin.register(models.Region) 39 | class RegionAdmin(admin.ModelAdmin): 40 | 41 | def n_queued(obj): 42 | return obj.match_set.exclude(duration__isnull=False).count() 43 | n_queued.short_description = 'Number of queued matches' 44 | 45 | def n_finished(obj): 46 | return obj.match_set.exclude(duration__isnull=True).count() 47 | n_finished.short_description = 'Number of finished matches' 48 | 49 | list_display = ('short_name', 'full_name', 'platform', n_queued, n_finished) 50 | list_per_page = 20 51 | 52 | 53 | admin.site.register(models.CremeModel) 54 | -------------------------------------------------------------------------------- /web/core/management/commands/queue_random_matches.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | import logging 3 | import random 4 | 5 | from django.core.management import base 6 | from django.utils import timezone 7 | import django_rq 8 | 9 | from core import exceptions 10 | from core import models 11 | from core import api 12 | from core import services 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def queue_random_match(max_attempts=20): 19 | 20 | # Shuffle regions 21 | regions = [r for r in models.Region.objects.all()] 22 | random.shuffle(regions) 23 | 24 | # Keep track of how many attempts are made 25 | n_attempts = 0 26 | 27 | for region in regions: 28 | 29 | # Fetch random matches for the current region 30 | matches = api.fetch_random_matches(region) 31 | 32 | for match in matches: 33 | 34 | # Try to process the match once per user, because sometimes fetching a particular 35 | # user bugs out 36 | for participant in match['participants']: 37 | 38 | # Stop when too many attempts have been made 39 | n_attempts += 1 40 | if n_attempts > max_attempts: 41 | logger.warning('Did not find any match to queue') 42 | return 43 | 44 | try: 45 | _, already_exists = services.queue_match( 46 | summoner_name=participant['summonerName'], 47 | region=region, 48 | raise_if_exists=True 49 | ) 50 | if already_exists: 51 | break 52 | return 53 | except exceptions.HTTPError: 54 | logger.error('HTTP error', exc_info=True) 55 | break 56 | 57 | logger.warning('Did not find any match to queue') 58 | 59 | 60 | class Command(base.BaseCommand): 61 | 62 | def handle(self, *args, **options): 63 | 64 | print('Started queuing random matches') 65 | 66 | django_rq.get_scheduler('default').schedule( 67 | scheduled_time=timezone.now() + dt.timedelta(seconds=60), 68 | interval=60, 69 | func=queue_random_match, 70 | repeat=None # None means forever 71 | ) 72 | -------------------------------------------------------------------------------- /web/core/api.py: -------------------------------------------------------------------------------- 1 | import random 2 | from urllib.parse import urljoin 3 | 4 | from django.conf import settings 5 | import requests 6 | 7 | from . import models 8 | 9 | 10 | __all__ = [ 11 | 'fetch_active_match_by_summoner_id', 12 | 'fetch_champion_mastery', 13 | 'fetch_match', 14 | 'fetch_random_matches', 15 | 'fetch_summoner_by_name', 16 | 'fetch_summoner_ranking', 17 | 'fetch_total_champion_mastery', 18 | ] 19 | 20 | 21 | def get_from_api(path, region): 22 | """Convenience method for making a GET request from Riot's API.""" 23 | 24 | if not isinstance(region, models.Region): 25 | region = models.Region.get(short_name=region) 26 | 27 | url = urljoin(f'https://{region.platform.lower()}.api.riotgames.com', path) 28 | 29 | req = requests.get(url, params={'api_key': settings.RIOT_API_KEY}) 30 | 31 | req.raise_for_status() 32 | 33 | return req.json() 34 | 35 | 36 | def fetch_summoner_by_name(summoner_name, region): 37 | """Fetchs summoner information using a name.""" 38 | path = f'/lol/summoner/v4/summoners/by-name/{summoner_name}' 39 | return get_from_api(path, region) 40 | 41 | 42 | def fetch_active_match_by_summoner_id(summoner_id, region): 43 | """Fetchs match information using a summoner ID.""" 44 | path = f'/lol/spectator/v4/active-games/by-summoner/{summoner_id}' 45 | return get_from_api(path, region) 46 | 47 | 48 | def fetch_match(match_api_id, region): 49 | """Fetch match information using a match ID.""" 50 | path = f'/lol/match/v4/matches/{match_api_id}' 51 | return get_from_api(path, region) 52 | 53 | 54 | def fetch_random_matches(region): 55 | """Fetch random matches.""" 56 | path = f'/lol/spectator/v4/featured-games' 57 | matches = get_from_api(path, region)['gameList'] 58 | random.shuffle(matches) 59 | return matches 60 | 61 | 62 | def fetch_champion_mastery(summoner_id, champion_id, region): 63 | """Fetch champion mastery information using a summoner ID.""" 64 | path = f'/lol/champion-mastery/v4/champion-masteries/by-summoner/{summoner_id}/by-champion/{champion_id}' 65 | data = get_from_api(path, region) 66 | data.pop('championId', None) 67 | data.pop('summonerId', None) 68 | return data 69 | 70 | 71 | def fetch_total_champion_mastery(summoner_id, region): 72 | """Fetch total champion mastery using a summoner ID.""" 73 | path = f'/lol/champion-mastery/v4/scores/by-summoner/{summoner_id}' 74 | return get_from_api(path, region) 75 | 76 | 77 | def fetch_summoner_ranking(summoner_id, region): 78 | """Fetch ranking information using a summoner ID.""" 79 | path = f'/lol/league/v4/entries/by-summoner/{summoner_id}' 80 | return get_from_api(path, region) 81 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | x-web-variables: &web-variables 4 | DJANGO_SETTINGS_MODULE: lolmd.settings.prod 5 | SECRET_KEY: ${SECRET_KEY} 6 | RIOT_API_KEY: ${RIOT_API_KEY} 7 | POSTGRES_HOST: postgres 8 | POSTGRES_PORT: 5432 9 | POSTGRES_NAME: lolmd 10 | POSTGRES_USER: ${POSTGRES_USER} 11 | POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} 12 | HEARTBEAT_RATE: 60 13 | REDIS_HOST: redis 14 | REDIS_PORT: 6379 15 | REDIS_PASSWORD: ${REDIS_PASSWORD} 16 | ADMIN_PASSWORD: ${ADMIN_PASSWORD} 17 | 18 | services: 19 | 20 | postgres: 21 | restart: always 22 | image: postgres:11.3 23 | environment: 24 | - POSTGRES_HOST=postgres 25 | - POSTGRES_DB=lolmd 26 | - POSTGRES_USER=${POSTGRES_USER} 27 | - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} 28 | - PGDATA=/var/lib/postgresql/data/ 29 | volumes: 30 | - postgres_data:/var/lib/postgresql/data/ 31 | ports: 32 | - "5432:5432" 33 | 34 | redis: 35 | image: redis:5.0.5 36 | environment: 37 | - REDIS_PASSWORD=${REDIS_PASSWORD:-redis} 38 | volumes: 39 | - redisdata:/data 40 | ports: 41 | - "6379:6379" 42 | command: > 43 | --requirepass $REDIS_PASSWORD 44 | 45 | web: 46 | restart: always 47 | build: ./web/ 48 | environment: *web-variables 49 | command: > 50 | bash -c "python manage.py makemigrations core 51 | && python manage.py migrate --noinput 52 | && python manage.py add_regions 53 | && python manage.py add_models 54 | && python manage.py add_admins 55 | && python manage.py prune_matches 56 | && python manage.py flush_redis 57 | && python manage.py collectstatic --noinput 58 | && python manage.py queue_random_matches 59 | && /usr/local/bin/gunicorn lolmd.wsgi:application -w 2 -b :8000" 60 | volumes: 61 | - /static:/static 62 | ports: 63 | - "8000" 64 | depends_on: 65 | - postgres 66 | - redis 67 | 68 | nginx: 69 | restart: always 70 | build: ./nginx/ 71 | ports: 72 | - "80:80" 73 | volumes: 74 | - /static:/static 75 | links: 76 | - web:web 77 | 78 | rqworker: 79 | restart: always 80 | build: ./web/ 81 | environment: *web-variables 82 | command: bash wait-for-it.sh web:8000 -t 0 -- python manage.py rqworker default 83 | depends_on: 84 | - web 85 | 86 | rqscheduler: 87 | restart: always 88 | build: ./web/ 89 | environment: *web-variables 90 | command: bash wait-for-it.sh web:8000 -t 0 -- python manage.py rqscheduler --interval 60 91 | depends_on: 92 | - web 93 | 94 | redis-commander: 95 | restart: always 96 | image: rediscommander/redis-commander 97 | ports: 98 | - "8082:8082" 99 | links: 100 | - redis:redis 101 | environment: 102 | - REDIS_HOST=redis 103 | - REDIS_PORT=6379 104 | - REDIS_PASSWORD=${REDIS_PASSWORD:-redis} 105 | - PORT=8082 106 | depends_on: 107 | - redis 108 | 109 | volumes: 110 | web-static: 111 | postgres_data: 112 | redisdata: 113 | -------------------------------------------------------------------------------- /web/core/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.7 on 2019-06-06 00:27 2 | 3 | import django.contrib.postgres.fields.jsonb 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | import picklefield.fields 7 | import uuid 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='CremeModel', 20 | fields=[ 21 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 22 | ('created_at', models.DateTimeField(auto_now_add=True)), 23 | ('updated_at', models.DateTimeField(auto_now=True)), 24 | ('name', models.TextField(unique=True)), 25 | ('pipeline', picklefield.fields.PickledObjectField(editable=False)), 26 | ], 27 | options={ 28 | 'verbose_name_plural': 'models', 29 | 'db_table': 't_creme_models', 30 | }, 31 | ), 32 | migrations.CreateModel( 33 | name='Match', 34 | fields=[ 35 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 36 | ('created_at', models.DateTimeField(auto_now_add=True)), 37 | ('updated_at', models.DateTimeField(auto_now=True)), 38 | ('api_id', models.PositiveIntegerField()), 39 | ('rq_job_id', models.UUIDField(null=True)), 40 | ('raw_info', django.contrib.postgres.fields.jsonb.JSONField(null=True)), 41 | ('started_at', models.DateTimeField()), 42 | ('duration', models.PositiveIntegerField(null=True)), 43 | ('predicted_duration', models.PositiveIntegerField(null=True)), 44 | ('winning_team_id', models.PositiveSmallIntegerField(null=True)), 45 | ('predicted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.CremeModel')), 46 | ], 47 | options={ 48 | 'verbose_name_plural': 'matches', 49 | 'db_table': 't_matches', 50 | }, 51 | ), 52 | migrations.CreateModel( 53 | name='Region', 54 | fields=[ 55 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 56 | ('created_at', models.DateTimeField(auto_now_add=True)), 57 | ('updated_at', models.DateTimeField(auto_now=True)), 58 | ('short_name', models.TextField()), 59 | ('full_name', models.TextField()), 60 | ('platform', models.TextField()), 61 | ], 62 | options={ 63 | 'verbose_name_plural': 'regions', 64 | 'db_table': 't_regions', 65 | }, 66 | ), 67 | migrations.AddField( 68 | model_name='match', 69 | name='region', 70 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Region'), 71 | ), 72 | ] 73 | -------------------------------------------------------------------------------- /web/core/views.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | import re 3 | 4 | from django import shortcuts 5 | from django.db.models import Avg, IntegerField, F, Func, ExpressionWrapper 6 | from django.contrib import messages 7 | from django.core import paginator 8 | 9 | from . import forms 10 | from . import models 11 | from . import services 12 | 13 | 14 | def index(request): 15 | 16 | if request.method == 'POST': 17 | 18 | form = forms.AddMatchForm(request.POST) 19 | 20 | if form.is_valid(): 21 | 22 | try: 23 | match_id, _ = services.queue_match( 24 | summoner_name=form.cleaned_data['summoner_name'], 25 | region=form.cleaned_data['region'] 26 | ) 27 | except Exception as exc: 28 | exc_str = re.sub(r'api_key=.+&', '', str(exc)) 29 | messages.error(request, f'{exc.__class__.__name__}: {exc_str}') 30 | return shortcuts.redirect('index') 31 | 32 | return shortcuts.redirect('match', match_id=match_id) 33 | 34 | else: 35 | messages.error(request, 'Bad input') 36 | return shortcuts.redirect('index') 37 | 38 | # Compute the mean average error (MAE) 39 | abs_error = ExpressionWrapper( 40 | Func(F('duration') - F('predicted_duration'), function='ABS'), 41 | output_field=IntegerField() 42 | ) 43 | mae = models.Match.objects.exclude(duration__isnull=True)\ 44 | .annotate(abs_error=abs_error)\ 45 | .aggregate(Avg('abs_error'))['abs_error__avg'] 46 | 47 | # Paginate the matches 48 | matches = models.Match.objects.prefetch_related('region').order_by('-created_at') 49 | matches_pag = paginator.Paginator(matches, 15) 50 | 51 | context = { 52 | 'n_queued': models.Match.objects.exclude(rq_job_id__isnull=True).count(), 53 | 'n_fit': models.Match.objects.exclude(rq_job_id__isnull=False).count(), 54 | 'mae': None if mae is None else dt.timedelta(seconds=mae), 55 | 'form': forms.AddMatchForm(), 56 | 'matches': matches_pag.get_page(request.GET.get('page')), 57 | } 58 | 59 | return shortcuts.render(request, 'index.html', context) 60 | 61 | 62 | def match(request, match_id): 63 | match = shortcuts.get_object_or_404(models.Match, pk=match_id) 64 | 65 | import json 66 | 67 | context = { 68 | 'match': match, 69 | 'raw_info': json.dumps(match.raw_info, indent=4) 70 | } 71 | 72 | if match.has_ended: 73 | 74 | duration = dt.timedelta(seconds=match.duration) 75 | pred_duration = dt.timedelta(seconds=match.predicted_duration) 76 | err_duration = abs(duration - pred_duration) 77 | max_duration = max(duration, pred_duration) + dt.timedelta(minutes=5) 78 | 79 | context = { 80 | **context, 81 | 'true_bar_percentage': int(100 * duration / max_duration), 82 | 'pred_bar_percentage': int(100 * pred_duration / max_duration), 83 | 'err_duration': err_duration, 84 | 'err_bar_percentage': int(100 * err_duration / max_duration) 85 | } 86 | 87 | return shortcuts.render(request, 'match.html', context) 88 | -------------------------------------------------------------------------------- /docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | x-web-variables: &web-variables 4 | DJANGO_SETTINGS_MODULE: lolmd.settings.dev 5 | SECRET_KEY: ${SECRET_KEY:-Keep_it_secret,_keep_it_safe} 6 | RIOT_API_KEY: ${RIOT_API_KEY} 7 | POSTGRES_HOST: postgres 8 | POSTGRES_PORT: 5432 9 | POSTGRES_NAME: lolmd 10 | POSTGRES_USER: ${POSTGRES_USER:-postgres} 11 | POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} 12 | HEARTBEAT_RATE: 60 13 | REDIS_HOST: redis 14 | REDIS_PORT: 6379 15 | REDIS_PASSWORD: ${REDIS_PASSWORD:-redis} 16 | ADMIN_PASSWORD: ${ADMIN_PASSWORD:-creme} 17 | 18 | services: 19 | 20 | postgres: 21 | restart: always 22 | image: postgres:11.3 23 | environment: 24 | - POSTGRES_HOST=postgres 25 | - POSTGRES_DB=lolmd 26 | - POSTGRES_USER=${POSTGRES_USER} 27 | - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} 28 | - PGDATA=/var/lib/postgresql/data/ 29 | volumes: 30 | - postgres_data:/var/lib/postgresql/data/ 31 | ports: 32 | - "5432:5432" 33 | 34 | redis: 35 | image: redis:5.0.5 36 | environment: 37 | - REDIS_PASSWORD=${REDIS_PASSWORD:-redis} 38 | volumes: 39 | - redisdata:/data 40 | ports: 41 | - "6379:6379" 42 | command: > 43 | --requirepass $REDIS_PASSWORD 44 | 45 | web: 46 | restart: always 47 | build: ./web/ 48 | environment: *web-variables 49 | command: > 50 | bash -c "python manage.py makemigrations core 51 | && python manage.py migrate --noinput 52 | && python manage.py flush --noinput 53 | && python manage.py flush_redis 54 | && python manage.py add_regions 55 | && python manage.py add_models 56 | && python manage.py add_admins 57 | && python manage.py prune_matches 58 | && python manage.py flush_redis 59 | && python manage.py collectstatic --noinput 60 | && python manage.py queue_random_matches 61 | && /usr/local/bin/gunicorn lolmd.wsgi:application -w 2 -b :8000" 62 | volumes: 63 | - /static:/static 64 | - .:/code 65 | ports: 66 | - "8000" 67 | depends_on: 68 | - postgres 69 | - redis 70 | 71 | nginx: 72 | restart: always 73 | build: ./nginx/ 74 | ports: 75 | - "80:80" 76 | volumes: 77 | - /static:/static 78 | links: 79 | - web:web 80 | 81 | rqworker: 82 | restart: always 83 | build: ./web/ 84 | environment: *web-variables 85 | command: bash wait-for-it.sh web:8000 -t 0 -- python manage.py rqworker default 86 | depends_on: 87 | - web 88 | 89 | rqscheduler: 90 | restart: always 91 | build: ./web/ 92 | environment: *web-variables 93 | command: bash wait-for-it.sh web:8000 -t 0 -- python manage.py rqscheduler --interval 60 94 | depends_on: 95 | - web 96 | 97 | redis-commander: 98 | restart: always 99 | image: rediscommander/redis-commander 100 | ports: 101 | - "8082:8082" 102 | links: 103 | - redis:redis 104 | environment: 105 | - REDIS_HOST=redis 106 | - REDIS_PORT=6379 107 | - REDIS_PASSWORD=${REDIS_PASSWORD:-redis} 108 | - PORT=8082 109 | depends_on: 110 | - redis 111 | 112 | volumes: 113 | web-static: 114 | postgres_data: 115 | redisdata: 116 | -------------------------------------------------------------------------------- /web/core/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.db import models 4 | from django.contrib.postgres.fields import JSONField 5 | from picklefield import fields as picklefield 6 | 7 | 8 | __all__ = [ 9 | 'CremeModel', 10 | 'Match', 11 | 'Region' 12 | ] 13 | 14 | 15 | class BaseModel(models.Model): 16 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 17 | created_at = models.DateTimeField(auto_now_add=True) 18 | updated_at = models.DateTimeField(auto_now=True) 19 | 20 | class Meta: 21 | abstract = True 22 | 23 | 24 | class CremeModel(BaseModel): 25 | name = models.TextField(unique=True) 26 | pipeline = picklefield.PickledObjectField() 27 | 28 | class Meta: 29 | db_table = 't_creme_models' 30 | verbose_name_plural = 'models' 31 | 32 | def fit_one(self, x, y): 33 | """Attempts to update the model and indicates if the update was successful or not.""" 34 | return CremeModel.objects\ 35 | .filter(id=self.id, updated_at=self.updated_at)\ 36 | .update(pipeline=self.pipeline.fit_one(x, y)) > 0 37 | 38 | def predict_one(self, x): 39 | return ( 40 | self.pipeline.predict_one(x), 41 | CremeModel.objects 42 | .filter(id=self.id, updated_at=self.updated_at) 43 | .update(pipeline=self.pipeline) > 0 44 | ) 45 | 46 | def __str__(self): 47 | return f'Model {self.name}' 48 | 49 | 50 | class Region(BaseModel): 51 | short_name = models.TextField() 52 | full_name = models.TextField() 53 | platform = models.TextField() 54 | 55 | class Meta: 56 | db_table = 't_regions' 57 | verbose_name_plural = 'regions' 58 | 59 | def __str__(self): 60 | return f'{self.full_name} ({self.short_name})' 61 | 62 | 63 | class Match(BaseModel): 64 | api_id = models.TextField() 65 | rq_job_id = models.UUIDField(null=True) 66 | raw_info = JSONField(null=True) 67 | started_at = models.DateTimeField() 68 | duration = models.PositiveSmallIntegerField(null=True) 69 | predicted_duration = models.PositiveSmallIntegerField(null=True) 70 | winning_team_id = models.PositiveSmallIntegerField(null=True) 71 | 72 | region = models.ForeignKey(Region, null=True, on_delete=models.SET_NULL) 73 | predicted_by = models.ForeignKey(CremeModel, null=True, on_delete=models.SET_NULL) 74 | 75 | @property 76 | def absolute_error(self): 77 | if self.duration and self.predicted_duration: 78 | return abs(self.duration - self.predicted_duration) 79 | return None 80 | 81 | @property 82 | def has_ended(self): 83 | return self.duration is not None 84 | 85 | @property 86 | def mode(self): 87 | if not self.raw_info: 88 | return None 89 | try: 90 | return self.raw_info['gameMode'] 91 | except KeyError: 92 | return None 93 | 94 | @property 95 | def type(self): 96 | if not self.raw_info: 97 | return None 98 | try: 99 | return self.raw_info['gameType'] 100 | except KeyError: 101 | return None 102 | 103 | class Meta: 104 | db_table = 't_matches' 105 | verbose_name_plural = 'matches' 106 | 107 | def __str__(self): 108 | return f'Match {self.api_id}' 109 | -------------------------------------------------------------------------------- /web/lolmd/settings/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import redis 4 | 5 | 6 | MODEL_NAME = 'v0' 7 | 8 | SECRET_KEY = os.environ['SECRET_KEY'] 9 | ADMIN_PASSWORD = os.environ['ADMIN_PASSWORD'] 10 | RIOT_API_KEY = os.environ['RIOT_API_KEY'] 11 | 12 | APP_DOMAIN = 'localhost:8000' 13 | 14 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 15 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 16 | 17 | ALLOWED_HOSTS = ['*'] 18 | 19 | CORS_ORIGIN_ALLOW_ALL = True 20 | 21 | HEARTBEAT_RATE = int(os.environ['HEARTBEAT_RATE']) 22 | 23 | # List of administrators 24 | ADMINS = ( 25 | ('Max Halford', 'maxhalford25@gmail.com'), 26 | ) 27 | 28 | # Application definition 29 | 30 | INSTALLED_APPS = [ 31 | 'django.contrib.admin', 32 | 'django.contrib.auth', 33 | 'django.contrib.contenttypes', 34 | 'django.contrib.sessions', 35 | 'django.contrib.messages', 36 | 'django.contrib.staticfiles', 37 | 'django.contrib.humanize', 38 | 39 | 'django_rq', 40 | 41 | 'core', 42 | ] 43 | 44 | MIDDLEWARE = [ 45 | 'django.middleware.security.SecurityMiddleware', 46 | 'django.contrib.sessions.middleware.SessionMiddleware', 47 | 'django.middleware.common.CommonMiddleware', 48 | 'django.middleware.csrf.CsrfViewMiddleware', 49 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware' 52 | ] 53 | 54 | ROOT_URLCONF = 'core.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'lolmd.wsgi.application' 73 | 74 | # Database configuration 75 | # https://docs.djangoproject.com/en/2.1/ref/databases/ 76 | DATABASES = { 77 | 'default': { 78 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 79 | 'HOST': os.environ['POSTGRES_HOST'], 80 | 'PORT': os.environ['POSTGRES_PORT'], 81 | 'NAME': os.environ['POSTGRES_NAME'], 82 | 'USER': os.environ['POSTGRES_USER'], 83 | 'PASSWORD': os.environ['POSTGRES_PASSWORD'] 84 | } 85 | } 86 | 87 | # Redis connection pool 88 | REDIS_POOL = redis.ConnectionPool( 89 | host=os.environ['REDIS_HOST'], 90 | port=os.environ['REDIS_PORT'], 91 | db=0, 92 | password=os.environ['REDIS_PASSWORD'], 93 | decode_responses=True 94 | ) 95 | 96 | # rq config 97 | # https://github.com/rq/django-rq 98 | 99 | RQ_QUEUES = { 100 | 'default': { 101 | 'HOST': os.environ['REDIS_HOST'], 102 | 'PORT': os.environ['REDIS_PORT'], 103 | 'DB': 0, 104 | 'PASSWORD': os.environ['REDIS_PASSWORD'], 105 | 'DEFAULT_TIMEOUT': 360, 106 | } 107 | } 108 | 109 | # Internationalization 110 | # https://docs.djangoproject.com/en/1.11/topics/i18n/ 111 | 112 | LANGUAGE_CODE = 'en-us' 113 | 114 | TIME_ZONE = 'UTC' 115 | 116 | USE_I18N = True 117 | 118 | USE_L10N = True 119 | 120 | USE_TZ = True 121 | 122 | 123 | # Static files (CSS, JavaScript, Images) 124 | # https://docs.djangoproject.com/en/1.11/howto/static-files/ 125 | 126 | STATIC_URL = '/static/' 127 | STATIC_ROOT = '/static/' 128 | -------------------------------------------------------------------------------- /web/core/management/commands/add_models.py: -------------------------------------------------------------------------------- 1 | import statistics 2 | 3 | import creme 4 | from creme import compose 5 | from creme import linear_model 6 | from creme import optim 7 | from creme import preprocessing 8 | from django.core.management import base 9 | 10 | from core import models 11 | 12 | 13 | def safe_mean(seq): 14 | try: 15 | return statistics.mean(seq) 16 | except statistics.StatisticsError: 17 | return 1 18 | 19 | 20 | def safe_ratio(a, b): 21 | return (a + 1) / (b + 1) 22 | 23 | 24 | def get_ranking(participant): 25 | 26 | tiers = { 27 | tier: i 28 | for i, tier in enumerate(('IRON', 'BRONZE', 'SILVER', 'GOLD', 29 | 'PLATINUM', 'DIAMOND', 'MASTER', 30 | 'GRANDMASTER', 'CHALLENGER')) 31 | } 32 | 33 | ranks = { 34 | tier: i 35 | for i, tier in enumerate(('IV', 'III', 'II', 'I')) 36 | } 37 | 38 | try: 39 | tier = participant['position'][0]['tier'] 40 | rank = participant['position'][0]['rank'] 41 | return tiers[tier] * len(tiers) + ranks[rank] 42 | except IndexError: 43 | return None 44 | 45 | 46 | def process_match(match): 47 | 48 | # Split the blue side from the red side 49 | blue_side = [p for p in match['participants'] if p['teamId'] == 100] 50 | red_side = [p for p in match['participants'] if p['teamId'] == 200] 51 | 52 | # Champion mastery points ratio 53 | blue_points = safe_mean([p['mastery']['championPoints'] for p in blue_side]) 54 | red_points = safe_mean([p['mastery']['championPoints'] for p in red_side]) 55 | champion_points_ratio = safe_ratio(max(blue_points, red_points), min(blue_points, red_points)) 56 | 57 | # Total mastery points ratio 58 | blue_points = safe_mean([p['total_mastery'] for p in blue_side]) 59 | red_points = safe_mean([p['total_mastery'] for p in red_side]) 60 | total_points_ratio = safe_ratio(max(blue_points, red_points), min(blue_points, red_points)) 61 | 62 | # Ranking ratio 63 | blue_rank = safe_mean(filter(None, [get_ranking(p) for p in blue_side])) 64 | red_rank = safe_mean(filter(None, [get_ranking(p) for p in red_side])) 65 | rank_ratio = safe_ratio(max(blue_rank, red_rank), min(blue_rank, red_rank)) 66 | 67 | return { 68 | 'mode': match['gameMode'], 69 | 'type': match['gameType'], 70 | 'champion_mastery_points_ratio': champion_points_ratio, 71 | 'total_mastery_points_ratio': total_points_ratio, 72 | 'rank_ratio': rank_ratio 73 | } 74 | 75 | 76 | MODELS = { 77 | 'v0': ( 78 | compose.FuncTransformer(process_match) | 79 | compose.TransformerUnion([ 80 | compose.Whitelister( 81 | 'champion_mastery_points_ratio', 82 | 'total_mastery_points_ratio', 83 | 'rank_ratio', 84 | ), 85 | preprocessing.OneHotEncoder('mode', sparse=False), 86 | preprocessing.OneHotEncoder('type', sparse=False) 87 | ]) | 88 | preprocessing.StandardScaler() | 89 | linear_model.LinearRegression(optim.VanillaSGD(0.005)) 90 | ) 91 | } 92 | 93 | 94 | class Command(base.BaseCommand): 95 | 96 | def handle(self, *args, **options): 97 | 98 | print(f'Adding models with creme version {creme.__version__}') 99 | 100 | for name, pipeline in MODELS.items(): 101 | 102 | if models.CremeModel.objects.filter(name=name).exists(): 103 | print(f'\t{name} has already been added') 104 | continue 105 | 106 | models.CremeModel(name=name, pipeline=pipeline).save() 107 | print(f'\tAdded {name}') 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # League of Legends match duration forecasting 2 | 3 | This is a simple project to demonstrate how `creme` may be used to build a "real-time" machine learning app. The idea is to predict the duration of LoL matches using information that is available at the start of the match. Once the match ends, the true duration is used to update the model. 4 | 5 | ## Screenshots 6 | 7 | ![home](screenshots/home.png) 8 | 9 | ![matches](screenshots/matches.png) 10 | 11 | ![matches](screenshots/match.png) 12 | 13 | ## Architecture 14 | 15 | ![architecture](screenshots/architecture.svg) 16 | 17 | The goal of this project is to demonstrate that online learning is easy to put in place. Indeed predicting and training are both done inside web requests. 18 | 19 | - The machine learning model is stored in [`core/management/commands/add_models.py`](core/management/commands/add_models.py) 20 | - Predictions happen in the `queue_match` function of [`core/services.py`](core/services.py) 21 | - Training happens in the `try_to_end_match` function of [`core/services.py`](core/services.py) 22 | - The average error is computed in the `index` function of [`core/views.py`](core/views.py) 23 | 24 | ## Usage 25 | 26 | ### Development 27 | 28 | Create an `.env` file with the following structure: 29 | 30 | ```sh 31 | RIOT_API_KEY=https://developer.riotgames.com/ 32 | ``` 33 | 34 | Next, create a local machine named `dev` and connect to it. 35 | 36 | ```sh 37 | >>> docker-machine create dev 38 | >>> eval "$(docker-machine env dev)" 39 | ``` 40 | 41 | Now you can build the stack. 42 | 43 | ```sh 44 | docker-compose build 45 | ``` 46 | 47 | You can then start the stack. 48 | 49 | ```sh 50 | docker-compose docker-compose.dev.yml up -d 51 | ``` 52 | 53 | You only have to build the stack once. However you have to rebuild it if you add or modify a service. You can now navigate to the following pages: 54 | 55 | - `localhost:8000` for the app 56 | - `localhost:8082` for [Redis Commander](http://joeferner.github.io/redis-commander/) 57 | 58 | Run `docker-compose down` to spin the stack down. 59 | 60 | :warning: If you want to delete absolutely everything then run the following command. 61 | 62 | ```sh 63 | docker container stop $(docker container ls -a -q) && docker system prune -a -f --volumes 64 | ``` 65 | 66 | ### Production 67 | 68 | Create an `.env` file with the following structure: 69 | 70 | ```sh 71 | SECRET_KEY=Keep_it_secret,_keep_it_safe 72 | RIOT_API_KEY=https://developer.riotgames.com 73 | POSTGRES_USER=postgres 74 | POSTGRES_PASSWORD=postgres 75 | REDIS_PASSWORD=redis 76 | ADMIN_PASSWORD=creme 77 | ``` 78 | 79 | Run the following command to create a DigitalOcean droplet named `prod`. Replace the variables as you wish (for example `$DIGITALOCEAN_SIZE` could be `s-1vcpu-1gb` and region could be `nyc3`). Run `docker-machine -h` for more details. 80 | 81 | ```sh 82 | >>> docker-machine create --driver digitalocean 83 | --digitalocean-access-token $DIGITALOCEAN_ACCESS_TOKEN 84 | --digitalocean-size $DIGITALOCEAN_SIZE 85 | --digitalocean-region $DIGITALOCEAN_REGION 86 | prod 87 | ``` 88 | 89 | You can now run `docker-machine ls` to see the instance you just created. Next run the following commands to deploy the app. 90 | 91 | ```sh 92 | >>> eval "$(docker-machine env prod)" 93 | >>> docker-compose build 94 | >>> docker-compose up -d 95 | ``` 96 | 97 | Finally run `docker-machine ip prod` to get the IP address of the production instance. If you want to check out the logs run `docker-compose logs --tail=1000`. 98 | 99 | For more information about deploying a Django app with Docker check out [this](https://realpython.com/django-development-with-docker-compose-and-machine/) down to earth post. 100 | -------------------------------------------------------------------------------- /web/wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Use this script to test if a given TCP host/port are available 3 | 4 | cmdname=$(basename $0) 5 | 6 | echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } 7 | 8 | usage() 9 | { 10 | cat << USAGE >&2 11 | Usage: 12 | $cmdname host:port [-s] [-t timeout] [-- command args] 13 | -h HOST | --host=HOST Host or IP under test 14 | -p PORT | --port=PORT TCP port under test 15 | Alternatively, you specify the host and port as host:port 16 | -s | --strict Only execute subcommand if the test succeeds 17 | -q | --quiet Don't output any status messages 18 | -t TIMEOUT | --timeout=TIMEOUT 19 | Timeout in seconds, zero for no timeout 20 | -- COMMAND ARGS Execute command with args after the test finishes 21 | USAGE 22 | exit 1 23 | } 24 | 25 | wait_for() 26 | { 27 | if [[ $TIMEOUT -gt 0 ]]; then 28 | echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" 29 | else 30 | echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" 31 | fi 32 | start_ts=$(date +%s) 33 | while : 34 | do 35 | if [[ $ISBUSY -eq 1 ]]; then 36 | nc -z $HOST $PORT 37 | result=$? 38 | else 39 | (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 40 | result=$? 41 | fi 42 | if [[ $result -eq 0 ]]; then 43 | end_ts=$(date +%s) 44 | echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" 45 | break 46 | fi 47 | sleep 1 48 | done 49 | return $result 50 | } 51 | 52 | wait_for_wrapper() 53 | { 54 | # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 55 | if [[ $QUIET -eq 1 ]]; then 56 | timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & 57 | else 58 | timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & 59 | fi 60 | PID=$! 61 | trap "kill -INT -$PID" INT 62 | wait $PID 63 | RESULT=$? 64 | if [[ $RESULT -ne 0 ]]; then 65 | echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" 66 | fi 67 | return $RESULT 68 | } 69 | 70 | # process arguments 71 | while [[ $# -gt 0 ]] 72 | do 73 | case "$1" in 74 | *:* ) 75 | hostport=(${1//:/ }) 76 | HOST=${hostport[0]} 77 | PORT=${hostport[1]} 78 | shift 1 79 | ;; 80 | --child) 81 | CHILD=1 82 | shift 1 83 | ;; 84 | -q | --quiet) 85 | QUIET=1 86 | shift 1 87 | ;; 88 | -s | --strict) 89 | STRICT=1 90 | shift 1 91 | ;; 92 | -h) 93 | HOST="$2" 94 | if [[ $HOST == "" ]]; then break; fi 95 | shift 2 96 | ;; 97 | --host=*) 98 | HOST="${1#*=}" 99 | shift 1 100 | ;; 101 | -p) 102 | PORT="$2" 103 | if [[ $PORT == "" ]]; then break; fi 104 | shift 2 105 | ;; 106 | --port=*) 107 | PORT="${1#*=}" 108 | shift 1 109 | ;; 110 | -t) 111 | TIMEOUT="$2" 112 | if [[ $TIMEOUT == "" ]]; then break; fi 113 | shift 2 114 | ;; 115 | --timeout=*) 116 | TIMEOUT="${1#*=}" 117 | shift 1 118 | ;; 119 | --) 120 | shift 121 | CLI=("$@") 122 | break 123 | ;; 124 | --help) 125 | usage 126 | ;; 127 | *) 128 | echoerr "Unknown argument: $1" 129 | usage 130 | ;; 131 | esac 132 | done 133 | 134 | if [[ "$HOST" == "" || "$PORT" == "" ]]; then 135 | echoerr "Error: you need to provide a host and port to test." 136 | usage 137 | fi 138 | 139 | TIMEOUT=${TIMEOUT:-15} 140 | STRICT=${STRICT:-0} 141 | CHILD=${CHILD:-0} 142 | QUIET=${QUIET:-0} 143 | 144 | # check to see if timeout is from busybox? 145 | # check to see if timeout is from busybox? 146 | TIMEOUT_PATH=$(realpath $(which timeout)) 147 | if [[ $TIMEOUT_PATH =~ "busybox" ]]; then 148 | ISBUSY=1 149 | BUSYTIMEFLAG="-t" 150 | else 151 | ISBUSY=0 152 | BUSYTIMEFLAG="" 153 | fi 154 | 155 | if [[ $CHILD -gt 0 ]]; then 156 | wait_for 157 | RESULT=$? 158 | exit $RESULT 159 | else 160 | if [[ $TIMEOUT -gt 0 ]]; then 161 | wait_for_wrapper 162 | RESULT=$? 163 | else 164 | wait_for 165 | RESULT=$? 166 | fi 167 | fi 168 | 169 | if [[ $CLI != "" ]]; then 170 | if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then 171 | echoerr "$cmdname: strict mode, refusing to execute subprocess" 172 | exit $RESULT 173 | fi 174 | exec "${CLI[@]}" 175 | else 176 | exit $RESULT 177 | fi 178 | -------------------------------------------------------------------------------- /web/core/services.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | import logging 3 | import pytz 4 | 5 | from django.conf import settings 6 | import django_rq 7 | 8 | from . import exceptions 9 | from . import models 10 | from . import api 11 | 12 | 13 | __all__ = [ 14 | 'queue_match', 15 | 'try_to_end_match' 16 | ] 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | def queue_match(summoner_name, region, raise_if_exists=False): 23 | """Queues a match and returns it's database ID.""" 24 | 25 | # Get the region 26 | if not isinstance(region, models.Region): 27 | region = models.Region.objects.get(short_name=region) 28 | 29 | # Fetch summoner information 30 | summoner = api.fetch_summoner_by_name(summoner_name, region) 31 | 32 | # Fetch current match information from the summoner's ID 33 | match_info = api.fetch_active_match_by_summoner_id(summoner['id'], region) 34 | 35 | # Check if the match has ended or not 36 | if 'gameDuration' in match_info: 37 | raise ValueError('The match has already ended') 38 | 39 | # Check if the match has already been inserted 40 | try: 41 | match = models.Match.objects.get(api_id=match_info['gameId'], region=region) 42 | return match.id, True 43 | except exceptions.ObjectDoesNotExist: 44 | pass 45 | 46 | # Add extra information 47 | for i, participant in enumerate(match_info['participants']): 48 | 49 | # Champion mastery for the current champion 50 | match_info['participants'][i]['mastery'] = api.fetch_champion_mastery( 51 | summoner_id=participant['summonerId'], 52 | champion_id=participant['championId'], 53 | region=region 54 | ) 55 | 56 | # Total champion mastery score, which is the sum of individual champion mastery levels 57 | match_info['participants'][i]['total_mastery'] = api.fetch_total_champion_mastery( 58 | summoner_id=participant['summonerId'], 59 | region=region 60 | ) 61 | 62 | # Current ranked position 63 | match_info['participants'][i]['position'] = api.fetch_summoner_ranking( 64 | summoner_id=participant['summonerId'], 65 | region=region 66 | ) 67 | 68 | # The game length is the current amount of time spent in the game, we don't need it 69 | match_info.pop('gameLength', None) 70 | match_info.pop('observers', None) 71 | match_info.pop('platformId', None) 72 | 73 | # Build the Match instance 74 | match = models.Match( 75 | api_id=str(match_info['gameId']), 76 | region=region, 77 | raw_info=match_info, 78 | started_at=pytz.utc.localize(dt.datetime.fromtimestamp(match_info['gameStartTime'] / 1000)) 79 | ) 80 | 81 | # Predict the match duration 82 | ok = False 83 | while not ok: 84 | duration, ok = models.CremeModel.objects.get(name=settings.MODEL_NAME)\ 85 | .predict_one(match.raw_info) 86 | if not ok: 87 | logger.warning('Optimistic logging failed when predicting') 88 | 89 | # Clamp the prediction because we know the min/max time of a match 90 | min_duration = dt.timedelta(minutes=10 if match.mode == 'ARAM' else 15).seconds 91 | max_duration = dt.timedelta(hours=3).seconds 92 | 93 | predicted_duration = min(max(duration, min_duration), max_duration) 94 | 95 | # Save the model to get an ID that we can give to RQ 96 | match.predicted_duration = int(predicted_duration) 97 | match.save() 98 | 99 | # Schedule a job that calls try_to_end_match until the game ends 100 | scheduler = django_rq.get_scheduler('default') 101 | job = scheduler.schedule( 102 | scheduled_time=match.started_at + dt.timedelta(minutes=15), 103 | interval=60, 104 | func=try_to_end_match, 105 | kwargs={'id': match.id}, 106 | repeat=None # None means forever 107 | ) 108 | match.rq_job_id = job.get_id() 109 | match.save() 110 | 111 | return match.id, False 112 | 113 | 114 | def try_to_end_match(id): 115 | 116 | match = models.Match.objects.get(id=id) 117 | region = match.region 118 | 119 | # Fetch the match information 120 | try: 121 | match_info = api.fetch_match(match_api_id=match.api_id, region=region) 122 | except exceptions.HTTPError: 123 | logger.error('HTTP error', exc_info=True) 124 | return 125 | 126 | # Get the duration in seconds 127 | duration = match_info.get('gameDuration') 128 | 129 | # Can't do anything if the game hasn't ended yet 130 | if duration is None: 131 | logger.info(f'Match {match.id} has not finished') 132 | return 133 | 134 | # Set the match's end time 135 | match.duration = int(duration) 136 | 137 | # Set the winning team ID 138 | if match_info['teams'][0]['win'] == 'Win': 139 | match.winning_team_id = match_info['teams'][0]['teamId'] 140 | else: 141 | match.winning_team_id = match_info['teams'][1]['teamId'] 142 | 143 | ok = False 144 | while not ok: 145 | ok = models.CremeModel.objects.get(name=settings.MODEL_NAME)\ 146 | .fit_one(match.raw_info, match.duration) 147 | if not ok: 148 | logger.warning('Optimistic logging failed when fitting') 149 | 150 | # Stop polling the match 151 | scheduler = django_rq.get_scheduler('default') 152 | scheduler.cancel(match.rq_job_id) 153 | match.rq_job_id = None 154 | match.save() 155 | -------------------------------------------------------------------------------- /web/core/templates/base.html: -------------------------------------------------------------------------------- 1 | {% load static from staticfiles %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 16 | 17 | 18 | 19 | LoL match duration forecasting 20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 |
32 |
33 |
34 |
35 | 36 | creme_logo 37 | 38 |
39 |
40 | 41 | lol_logo 42 | 43 |
44 |
45 |
46 |
47 |

48 | LoL match duration forecasting 49 |

50 |

51 | Using online machine learning 52 |

53 |
54 |
55 |
56 |
57 |
58 | 59 |
60 |
61 | {% if messages %} 62 |
63 |
64 |
65 |
66 |

Messages

67 |
68 |
69 |
    70 | {% for message in messages %} 71 |
  • {{ message }}
    • 72 | {% endfor %} 73 |
74 |
75 |
76 |
77 |
78 | {% endif %} 79 | {% block content %}{% endblock %} 80 |
81 |
82 | 83 | 84 | 85 |
86 |
87 |

88 | Powered by FOSS, including Django and Bulma 89 |

90 |

91 | Find us on GitHub 92 |

93 |
94 |
95 | 96 | 97 | 98 | 99 | 100 | 107 | -------------------------------------------------------------------------------- /web/core/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% load humanize %} 4 | {% load extra_humanize %} 5 | 6 | {% block content %} 7 | 8 |
9 | 12 |
13 | 14 |
15 | 16 | 36 | 37 | 51 | 52 | 64 | 65 |
66 |
67 |
68 |
69 |

Add a match

70 |
71 | {% csrf_token %} 72 | 73 | {{ form.non_field_errors }} 74 | 75 | {{ form.source.errors }} 76 | {{ form.source }} 77 | 78 |
79 | 80 |
81 | 92 |
93 |
94 | 95 |
96 | 97 |
98 |
99 | 112 |
113 |
114 | 115 |
116 |
117 |
118 | 119 |
120 | 121 |
122 |
123 | 124 |
125 |
126 | 127 |
128 |
129 |
130 |
131 |
132 | 133 |

Latest matches

134 | 135 |

You can click each box for more information.

136 | 137 |
138 |
139 | {% for match in matches %} 140 |
141 |
142 | 143 |
144 |
145 |

146 | Match {{ match.api_id }} 147 |

148 |
149 |
150 |
151 |
    152 | 153 |
  • Started {{ match.started_at|naturaltime }}
    • 154 |
  • Predicted duration is {{ match.predicted_duration | secondsduration | naturalduration }}
    • 155 | 156 | {% if match.duration %} 157 |
  • True duration is {{ match.duration | secondsduration | naturalduration }}
    • 158 | {% else %} 159 |
  • Still in progress
    • 160 | {% endif %} 161 | 162 | {% if match.absolute_error %} 163 |
  • Off by {{ match.absolute_error | secondsduration | naturalduration }}
    • 164 | {% endif %} 165 | 166 |
167 |
168 |
169 |
170 |

171 | {{ match.region.full_name }} 172 | {{ match.mode }} 173 | {{ match.type }} 174 |

175 |
176 |
177 |
178 | 179 |
180 | {% endfor %} 181 |
182 |
183 | 184 |
185 |
186 |
187 | {% if matches.has_previous %} 188 | Previous page 189 | {% endif %} 190 | {% if matches.has_next %} 191 | Next page 192 | {% endif %} 193 |
194 |
195 |
196 | 197 | 198 |
199 | Fetching data (slow) and making a prediction (fast) 200 |
201 | 202 | 228 | 229 | {% endblock %} 230 | 231 | 243 | --------------------------------------------------------------------------------