├── src ├── lambda │ ├── requirements.txt │ └── lambda_handler.py └── s3 │ └── index.html ├── .gitignore ├── terraform.tfvars.example ├── main.tf ├── variables.tf ├── terraform ├── lambda │ ├── variables.tf │ └── main.tf └── s3 │ └── main.tf └── README.md /src/lambda/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 -------------------------------------------------------------------------------- /src/s3/index.html: -------------------------------------------------------------------------------- 1 | brood war tournaments json 2 |
3 | starcraft 2 tournaments json -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .terraform/* 2 | terraform.tfstate* 3 | terraform.tfvars 4 | 5 | build/* 6 | !build/.gitignore 7 | !build/packages 8 | !build/packages/.gitignore 9 | -------------------------------------------------------------------------------- /terraform.tfvars.example: -------------------------------------------------------------------------------- 1 | aws_region = "eu-west-2" 2 | name = "twitch-events-banner" 3 | teb_version = "0.0.1" 4 | author = "tn#9673" 5 | update_rate = "rate(30 minutes)" 6 | domain_name = "example.com" -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "local" {} 3 | } 4 | 5 | provider "aws" { 6 | region = var.aws_region 7 | } 8 | 9 | module "bucket" { 10 | source = "./terraform/s3" 11 | domain_name = var.domain_name 12 | } 13 | 14 | module "lambda" { 15 | source = "./terraform/lambda" 16 | name = var.name 17 | author = var.author 18 | teb_version = var.teb_version 19 | domain_name = var.domain_name 20 | update_rate = var.update_rate 21 | bucket_arn = module.bucket.arn 22 | } -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | type = string 3 | } 4 | 5 | # used for lambda components and in user-agent for liquipedia api 6 | variable "name" { 7 | type = string 8 | } 9 | 10 | # used in user-agent for liquipedia api 11 | variable "author" { 12 | type = string 13 | } 14 | 15 | variable "teb_version" { 16 | type = string 17 | } 18 | 19 | # how often to check the liquipedia api 20 | # https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html 21 | variable "update_rate" { 22 | type = string 23 | } 24 | 25 | variable "domain_name" { 26 | type = string 27 | } -------------------------------------------------------------------------------- /terraform/lambda/variables.tf: -------------------------------------------------------------------------------- 1 | variable "bucket_arn" {} 2 | 3 | variable "domain_name" { 4 | type = string 5 | } 6 | 7 | variable "name" { 8 | type = string 9 | } 10 | 11 | variable "teb_version" { 12 | type = string 13 | } 14 | 15 | variable "author" { 16 | type = string 17 | } 18 | 19 | # https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html 20 | variable "update_rate" { 21 | type = string 22 | default = "rate(30 minutes)" 23 | } 24 | 25 | variable "source_dir" { 26 | type = string 27 | default = "./src/lambda" 28 | } 29 | 30 | variable "lambda_zip_path" { 31 | type = string 32 | default = "./build/lambda.zip" 33 | } 34 | 35 | variable "deps_dir" { 36 | type = string 37 | default = "./build/packages" 38 | } -------------------------------------------------------------------------------- /terraform/s3/main.tf: -------------------------------------------------------------------------------- 1 | variable "domain_name" { 2 | type = string 3 | } 4 | 5 | variable "content_dir" { 6 | type = string 7 | default = "./src/s3" 8 | } 9 | 10 | output "arn" { 11 | value = aws_s3_bucket.s3.arn 12 | } 13 | 14 | resource "aws_s3_bucket" "s3" { 15 | bucket = var.domain_name 16 | force_destroy = true 17 | acl = "public-read" 18 | 19 | website { 20 | index_document = "index.html" 21 | error_document = "error.html" 22 | } 23 | 24 | cors_rule { 25 | allowed_methods = ["GET"] 26 | allowed_origins = ["*"] 27 | } 28 | } 29 | 30 | resource "null_resource" "upload_to_s3" { 31 | provisioner "local-exec" { 32 | command = "aws s3 sync --acl public-read ${var.content_dir} s3://${aws_s3_bucket.s3.id}" 33 | } 34 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # twitch-events-banner-backend 2 | 3 | backend to grab upcoming events from liquipedia and populate an s3 static site 4 | 5 | [starcraft json](https://twitch.nydus.club/starcraft.json) [starcraft2 json](https://twitch.nydus.club/starcraft2.json) 6 | 7 | creates the following resources: 8 | - s3 static site using the contents of `src/s3` 9 | - lambda function to retrieve the upcoming tournaments from the liquipedia API, and generate json files in the s3 bucket 10 | - cloudwatch event rule to trigger the lambda function periodically 11 | 12 | 13 | ### notes 14 | 15 | the lambda uses the `Liquipedia:Tournaments` page to grab the upcoming events 16 | 17 | you can get a list of images on a page, but im not sure how to specify the infobox image or grab that reliably 18 | `https://liquipedia.net/starcraft2/api.php?action=query&prop=images&titles=ESL%20Pro%20Tour/2020/21/Masters/Fall/CN&format=json` 19 | 20 | 21 | ### deployment 22 | 23 | copy `terraform.tfvars.example` to `terraform.tfvars` and fill in your domain name etc 24 | 25 | use terraform to create the resources using `terraform init`, `terraform apply` 26 | 27 | create a CNAME dns record on pointing to `s3-website.[your aws region].amazonaws.com` 28 | -------------------------------------------------------------------------------- /src/lambda/lambda_handler.py: -------------------------------------------------------------------------------- 1 | import os, requests, re, json, time, boto3 2 | 3 | filename = "%s.json" 4 | base_url = "https://liquipedia.net/%s/api.php?action=query&prop=revisions&titles=Liquipedia:Tournaments&rvprop=content&format=json" 5 | page_url = "https://liquipedia.net/%s/%s" 6 | icon_url = "https://liquipedia.net/%s/Special:FilePath/%s" 7 | user_agent = "%s/%s; %s" % (os.getenv("NAME", "twitch-events-banner-prototype"), 8 | os.getenv("VERSION", "0.0.1"), 9 | os.getenv("AUTHOR", "@tnsc2")) 10 | 11 | def lambda_handler(event, context): 12 | for game in ["starcraft", "starcraft2"]: 13 | data = get_liquipedia_events(game) 14 | if data: 15 | put_into_s3(game, data) 16 | 17 | def put_into_s3(game, content): 18 | bucket_name = os.getenv("BUCKET_NAME") 19 | if not bucket_name: 20 | return 21 | 22 | s3 = boto3.resource("s3") 23 | s3.Bucket(bucket_name).put_object(Key=(filename % game), Body=json.dumps({ 24 | "events": content, 25 | "created": time.time() 26 | }), ContentType="application/json", ACL="public-read") 27 | 28 | def get_liquipedia_events(game): 29 | response = requests.get((base_url % game), headers = {"User-Agent": user_agent}) 30 | data = response.json() 31 | content = "" 32 | for key in data["query"]["pages"].keys(): 33 | for revision in data["query"]["pages"][key]["revisions"]: 34 | content = revision["*"] 35 | 36 | matches = re.findall(r"\*Upcoming((.|\n)*)\*Ongoing", content) 37 | if len(matches) != 1: 38 | return 39 | 40 | return parse_liquipedia_events(game, matches[0]) 41 | 42 | def parse_liquipedia_events(game, events): 43 | output = [] 44 | filtered_events = list(filter(None, events[0].split("\n"))) 45 | for event in filtered_events: 46 | match_data = { 47 | "link": "", 48 | "name": "", 49 | "start": "", 50 | "end": "", 51 | "icon": "", 52 | "icon_url": "", 53 | } 54 | for value in event.split(" | "): 55 | if value.startswith("**"): 56 | match_data["link"] = page_url % (game, value.replace("**", "")) 57 | elif value.startswith("startdate="): 58 | match_data["start"] = value.replace("startdate=", "") 59 | elif value.startswith("enddate="): 60 | match_data["end"] = value.replace("enddate=", "") 61 | elif value.startswith("icon=") and value != "icon=": 62 | match_data["icon"] = value.replace("icon=", "") 63 | elif value.startswith("iconfile=") and value != "iconfile=": 64 | match_data["icon_url"] = icon_url % (game, value.replace("iconfile=", "")) 65 | elif value != "icon=" and value != "iconfile=": 66 | match_data["name"] = value 67 | output.append(match_data) 68 | return output 69 | 70 | if __name__ == "__main__": 71 | lambda_handler({}, {}) -------------------------------------------------------------------------------- /terraform/lambda/main.tf: -------------------------------------------------------------------------------- 1 | # build the package for deployment 2 | resource "null_resource" "package_deps" { 3 | triggers = { 4 | reqs_changed = filebase64sha256("${var.source_dir}/requirements.txt") 5 | } 6 | 7 | provisioner "local-exec" { 8 | command = "pip install -r ${var.source_dir}/requirements.txt -t ${var.deps_dir}" 9 | } 10 | } 11 | 12 | resource "null_resource" "package_src" { 13 | triggers = { 14 | reqs_changed = filebase64sha256("${var.source_dir}/lambda_handler.py") 15 | } 16 | 17 | provisioner "local-exec" { 18 | command = "cp ${var.source_dir}/* ${var.deps_dir}" 19 | } 20 | 21 | depends_on = [null_resource.package_deps] 22 | } 23 | 24 | data "null_data_source" "wait_for_packaging" { 25 | inputs = { 26 | lambda_exporter_id = null_resource.package_src.id 27 | source_dir = var.deps_dir 28 | } 29 | } 30 | 31 | data "archive_file" "lambda_zip" { 32 | type = "zip" 33 | source_dir = data.null_data_source.wait_for_packaging.outputs["source_dir"] 34 | output_path = var.lambda_zip_path 35 | } 36 | 37 | # the lambda 38 | resource "aws_lambda_function" "lambda" { 39 | filename = var.lambda_zip_path 40 | function_name = "${var.name}_lambda" 41 | role = aws_iam_role.role.arn 42 | handler = "lambda_handler.lambda_handler" 43 | runtime = "python3.7" 44 | source_code_hash = filebase64sha256(var.lambda_zip_path) 45 | timeout = 5 46 | environment { 47 | variables = { 48 | NAME = var.name 49 | AUTHOR = var.author 50 | VERSION = var.teb_version 51 | BUCKET_NAME = var.domain_name 52 | } 53 | } 54 | depends_on = [data.archive_file.lambda_zip] 55 | } 56 | 57 | # event rule 58 | resource "aws_cloudwatch_event_rule" "event_rule" { 59 | name = "${var.name}_event_rule" 60 | schedule_expression = var.update_rate 61 | } 62 | 63 | resource "aws_cloudwatch_event_target" "event_target" { 64 | rule = aws_cloudwatch_event_rule.event_rule.name 65 | arn = aws_lambda_function.lambda.arn 66 | } 67 | 68 | # permissions 69 | resource "aws_lambda_permission" "event_permission" { 70 | statement_id = "${var.name}_permission" 71 | action = "lambda:InvokeFunction" 72 | function_name = aws_lambda_function.lambda.function_name 73 | principal = "events.amazonaws.com" 74 | source_arn = aws_cloudwatch_event_rule.event_rule.arn 75 | } 76 | 77 | resource "aws_iam_role" "role" { 78 | name = "${var.name}_lambda_role" 79 | 80 | assume_role_policy = <