├── .python-version
├── Procfile
├── static
├── css
│ ├── overrides.css
│ ├── c3.min.css
│ └── c3.css
├── favicon.ico
├── apple-touch-icon-precomposed.png
├── sse.tag
├── fetch-table.tag
├── parser-chart.tag
├── fetcher-chart.tag
└── js
│ ├── route.min.js
│ ├── history.min.js
│ ├── eventsource.min.js
│ ├── riot.min.js
│ ├── zepto.min.js
│ ├── riot+compiler.min.js
│ └── moment.min.js
├── screenshots
└── 2017-12-10.png
├── ENV
├── Dockerfile
├── requirements.txt
├── repl.py
├── Makefile
├── LICENSE
├── legacy-sqlite
├── fetch.hy
└── fetch.py
├── .gitignore
├── compose-swarm.yml
├── scheduler.py
├── docker-compose.yml
├── import.py
├── config.py
├── common.py
├── views
└── layout.tpl
├── README.md
├── langkit.py
├── web.py
├── fetcher.py
├── parser.py
└── metrics.py
/.python-version:
--------------------------------------------------------------------------------
1 | anaconda3-5.2.0
2 |
--------------------------------------------------------------------------------
/Procfile:
--------------------------------------------------------------------------------
1 | fetcher: python fetcher.py
2 | parser: python parser.py
3 |
--------------------------------------------------------------------------------
/static/css/overrides.css:
--------------------------------------------------------------------------------
1 | .c3-chart-arcs-title {
2 | font-size: 2em;
3 | }
4 |
--------------------------------------------------------------------------------
/static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rcarmo/newsfeed-corpus/HEAD/static/favicon.ico
--------------------------------------------------------------------------------
/screenshots/2017-12-10.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rcarmo/newsfeed-corpus/HEAD/screenshots/2017-12-10.png
--------------------------------------------------------------------------------
/static/apple-touch-icon-precomposed.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rcarmo/newsfeed-corpus/HEAD/static/apple-touch-icon-precomposed.png
--------------------------------------------------------------------------------
/ENV:
--------------------------------------------------------------------------------
1 | PYTHON_VERSION=3
2 | MONGO_SERVER=db:27017
3 | REDIS_SERVER=redis:6379
4 | HTTP_PORT=8000
5 | DATABASE_NAME=feeds
6 | PORT=4656
7 | FETCH_INTERVAL=1800
8 | CHECK_INTERVAL=3600
9 | DEBUG=True
10 | PROFILER=False
11 | PYTHONIOENCODING=UTF_8:replace
12 | LC_ALL=en_US.UTF-8
13 | LANG=en_US.UTF-8
14 | TIMEZONE=Europe/Lisbon
15 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM rcarmo/ubuntu-python:3.7-onbuild-amd64
2 |
3 | # Install nltk data we need
4 | RUN python -m nltk.downloader -d /usr/local/share/nltk_data stopwords punkt rslp averaged_perceptron_tagger
5 |
6 | # Bake code into containers rather than use the Compose mount point
7 | #ADD . /app
8 | WORKDIR /app
9 | EXPOSE 8000
10 |
--------------------------------------------------------------------------------
/static/sse.tag:
--------------------------------------------------------------------------------
1 |
2 |
16 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | aiocache>=0.11.1
2 | aiodns>=3.0.0
3 | aiohttp>=3.7.4
4 | aioredis>=2.0.0
5 | aiotasks>=1.0.0
6 | aiozmq>=0.9.0
7 | beautifulsoup4>=4.9.3
8 | bpython>=0.21
9 | feedparser>=6.0.8
10 | gensim>=4.0.1
11 | hiredis>=2.0.0
12 | langdetect>=1.0.9
13 | mako>=1.1.4
14 | motor>=2.5.0
15 | msgpack-python>=0.5.6
16 | nltk>=3.6.2
17 | numpy>=1.21.2
18 | pattern3>=3.0.0
19 | pycallgraph>=1.0.1
20 | pymongo[tls]>=3.12.0
21 | sanic>=21.6.2
22 | stop-words>=2018.7.23
23 | ujson>=4.1.0
24 | uvloop>=0.16.0
--------------------------------------------------------------------------------
/repl.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """ REPL for database inspection """
4 |
5 | from config import (CHECK_INTERVAL, DATABASE_NAME, FETCH_INTERVAL,
6 | MAX_CONCURRENT_REQUESTS, MONGO_SERVER, log)
7 | from datetime import datetime, timedelta
8 | from pymongo import MongoClient
9 | from code import interact
10 | from bpython import embed
11 |
12 | def main():
13 | client = MongoClient(MONGO_SERVER)
14 | db = client[DATABASE_NAME]
15 |
16 | embed(locals_=locals())
17 |
18 | if __name__ == '__main__':
19 | main()
20 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | CHECK_INTERVAL?=5
2 | FETCH_INTERVAL?=15
3 | MONGO_SERVER?=localhost:27017
4 | DATABASE_NAME?=feeds
5 | PORT?=8000
6 |
7 | # Permanent local overrides
8 | -include .env
9 |
10 | # Run the stack locally
11 | serve:
12 | docker-compose up --build
13 |
14 | # Build the base container when we update dependencies
15 | build:
16 | docker-compose build
17 |
18 | restart-web:
19 | docker-compose stop web
20 | docker-compose build web
21 | docker-compose up -d --no-deps web
22 |
23 | repl:
24 | docker-compose exec web python repl.py
25 |
26 | nuke-database:
27 | docker-compose stop db
28 | docker-compose build db
29 | docker-compose up -d --no-deps db
30 |
31 | # Install deps locally for REPL
32 | host-deps:
33 | pip install -U -r requirements.txt
34 |
35 | clean:
36 | -rm -f *.pyc
37 | -docker rm -v $$(docker ps -a -q -f status=exited)
38 | -docker rmi $$(docker images -q -f dangling=true)
39 | -docker rmi $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '$(IMAGE_NAME)')
40 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2016 Rui Carmo
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/legacy-sqlite/fetch.hy:
--------------------------------------------------------------------------------
1 | (import [lxml [etree]]
2 | [tinydb [TinyDB]]
3 | [asyncio [coroutine get-event-loop as-completed Semaphore]]
4 | [aiohttp [request]]
5 | [tqdm [tqdm]]
6 | [aiohttp.connector [TCPConnector]])
7 |
8 | (def db (TinyDB "feeds.json"))
9 | (def semaphore (Semaphore 5))
10 |
11 |
12 | (with-decorator coroutine
13 | (defn fetch-one [feed]
14 | (print feed)
15 | (let [[response (yield-from (apply request ["GET" (str (.get feed "url"))] {"compress" true}))]]
16 | (yield-from (.text response)))))
17 |
18 |
19 | (with-decorator coroutine
20 | (defn print-status [feed]
21 | (print (yield-from (fetch-one feed)))))
22 |
23 |
24 | (with-decorator coroutine
25 | (defn show-progress [jobs]
26 | (for [j (apply tqdm [(as-completed jobs)] {"total" (len jobs)})]
27 | (yield-from j))))
28 |
29 |
30 | (defn feeds-from-opml [filename]
31 | (let [[tree (.parse etree filename)]
32 | [feeds (.xpath tree "//outline")]]
33 | (for [feed feeds]
34 | (yield {"title"(.get feed "title")
35 | "url" (.get feed "xmlUrl")}))))
36 |
37 | (defn main []
38 | (let [[loop (get-event-loop)]
39 | [pool (TCPConnector)]
40 | [feeds (feeds-from-opml "feeds.opml")]
41 | [jobs (list (map print-status feeds))]]
42 | (.run-until-complete loop (show-progress jobs))))
43 |
44 |
45 | (main)
46 |
--------------------------------------------------------------------------------
/static/fetch-table.tag:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | | {time} |
7 | {status} |
8 | {url} |
9 |
10 |
11 |
12 |
13 |
14 | info_outline
15 | Nothing being fetched at the moment
16 |
17 |
38 |
39 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 |
27 | # PyInstaller
28 | # Usually these files are written by a python script from a template
29 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 |
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 |
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 |
48 | # Translations
49 | *.mo
50 | *.pot
51 |
52 | # Django stuff:
53 | *.log
54 | local_settings.py
55 |
56 | # Flask stuff:
57 | instance/
58 | .webassets-cache
59 |
60 | # Scrapy stuff:
61 | .scrapy
62 |
63 | # Sphinx documentation
64 | docs/_build/
65 |
66 | # PyBuilder
67 | target/
68 |
69 | # IPython Notebook
70 | .ipynb_checkpoints
71 |
72 | # pyenv
73 | .python-version
74 |
75 | # celery beat schedule file
76 | celerybeat-schedule
77 |
78 | # dotenv
79 | .env
80 |
81 | # virtualenv
82 | venv/
83 | ENV/
84 |
85 | # Spyder project settings
86 | .spyderproject
87 |
88 | # Rope project settings
89 | .ropeproject
90 |
91 | *.opml
92 |
93 | # VS Code
94 | .vscode
95 |
96 | # AppleTalk junk
97 | .AppleDouble
98 |
--------------------------------------------------------------------------------
/compose-swarm.yml:
--------------------------------------------------------------------------------
1 | version: '2'
2 | services:
3 | redis:
4 | image: redis:3.2.8
5 | ports:
6 | - "6379:6379"
7 | logging:
8 | driver: "none"
9 | environment:
10 | - "constraint:node.role==master"
11 |
12 | db:
13 | image: mongo:3.4
14 | ports:
15 | - "27017:27017"
16 | logging:
17 | driver: "none"
18 | environment:
19 | - "constraint:node.role==master"
20 |
21 | importer:
22 | build: .
23 | env_file:
24 | - ENV
25 | environment:
26 | - "constraint:node.role==worker"
27 | volumes:
28 | - .:/app
29 | links:
30 | - db
31 | - redis
32 | working_dir: /app
33 | command: python -u import.py
34 |
35 | scheduler:
36 | build: .
37 | env_file:
38 | - ENV
39 | environment:
40 | - "constraint:node.role==worker"
41 | volumes:
42 | - .:/app
43 | links:
44 | - db
45 | - redis
46 | working_dir: /app
47 | command: python -u scheduler.py
48 |
49 | fetcher:
50 | build: .
51 | env_file:
52 | - ENV
53 | environment:
54 | - "constraint:node.role==worker"
55 | volumes:
56 | - .:/app
57 | links:
58 | - db
59 | - redis
60 | working_dir: /app
61 | command: python -u fetcher.py
62 |
63 | parser:
64 | build: .
65 | env_file:
66 | - ENV
67 | environment:
68 | - "constraint:node.role==worker"
69 | volumes:
70 | - .:/app
71 | links:
72 | - db
73 | - redis
74 | working_dir: /app
75 | command: python -u parser.py
76 |
77 | web:
78 | build: .
79 | ports:
80 | - "8000:8000"
81 | env_file:
82 | - ENV
83 | environment:
84 | - "constraint:node.role==master"
85 | volumes:
86 | - .:/app
87 | links:
88 | - db
89 | - redis
90 | working_dir: /app
91 | command: python -u web.py
--------------------------------------------------------------------------------
/static/css/c3.min.css:
--------------------------------------------------------------------------------
1 | .c3 svg{font:10px sans-serif;-webkit-tap-highlight-color:transparent}.c3 line,.c3 path{fill:none;stroke:#000}.c3 text{-webkit-user-select:none;-moz-user-select:none;user-select:none}.c3-bars path,.c3-event-rect,.c3-legend-item-tile,.c3-xgrid-focus,.c3-ygrid{shape-rendering:crispEdges}.c3-chart-arc path{stroke:#fff}.c3-chart-arc text{fill:#fff;font-size:13px}.c3-grid line{stroke:#aaa}.c3-grid text{fill:#aaa}.c3-xgrid,.c3-ygrid{stroke-dasharray:3 3}.c3-text.c3-empty{fill:grey;font-size:2em}.c3-line{stroke-width:1px}.c3-circle._expanded_{stroke-width:1px;stroke:#fff}.c3-selected-circle{fill:#fff;stroke-width:2px}.c3-bar{stroke-width:0}.c3-bar._expanded_{fill-opacity:1;fill-opacity:.75}.c3-target.c3-focused{opacity:1}.c3-target.c3-focused path.c3-line,.c3-target.c3-focused path.c3-step{stroke-width:2px}.c3-target.c3-defocused{opacity:.3!important}.c3-region{fill:#4682b4;fill-opacity:.1}.c3-brush .extent{fill-opacity:.1}.c3-legend-item{font-size:12px}.c3-legend-item-hidden{opacity:.15}.c3-legend-background{opacity:.75;fill:#fff;stroke:#d3d3d3;stroke-width:1}.c3-title{font:14px sans-serif}.c3-tooltip-container{z-index:10}.c3-tooltip{border-collapse:collapse;border-spacing:0;background-color:#fff;empty-cells:show;-webkit-box-shadow:7px 7px 12px -9px #777;-moz-box-shadow:7px 7px 12px -9px #777;box-shadow:7px 7px 12px -9px #777;opacity:.9}.c3-tooltip tr{border:1px solid #ccc}.c3-tooltip th{background-color:#aaa;font-size:14px;padding:2px 5px;text-align:left;color:#fff}.c3-tooltip td{font-size:13px;padding:3px 6px;background-color:#fff;border-left:1px dotted #999}.c3-tooltip td>span{display:inline-block;width:10px;height:10px;margin-right:6px}.c3-tooltip td.value{text-align:right}.c3-area{stroke-width:0;opacity:.2}.c3-chart-arcs-title{dominant-baseline:middle;font-size:1.3em}.c3-chart-arcs .c3-chart-arcs-background{fill:#e0e0e0;stroke:none}.c3-chart-arcs .c3-chart-arcs-gauge-unit{fill:#000;font-size:16px}.c3-chart-arcs .c3-chart-arcs-gauge-max{fill:#777}.c3-chart-arcs .c3-chart-arcs-gauge-min{fill:#777}.c3-chart-arc .c3-gauge-value{fill:#000}.c3-chart-arc.c3-target g path{opacity:1}.c3-chart-arc.c3-target.c3-focused g path{opacity:1}
--------------------------------------------------------------------------------
/legacy-sqlite/fetch.py:
--------------------------------------------------------------------------------
1 | from time import time
2 | from asyncio import get_event_loop, Semaphore, gather, ensure_future
3 | from aiohttp import ClientSession
4 | from xml.etree import ElementTree
5 | from playhouse.sqlite_ext import SqliteExtDatabase
6 | from playhouse.kv import JSONKeyStore
7 |
8 | async def fetch_one(session, feed, db):
9 | url = feed.value.get("url")
10 | print("Fetching {0}".format(url))
11 | try:
12 | async with session.get(url) as response:
13 | text = await response.text()
14 | feed.value.update({"text": text, "last_status": response.status, "last_fetched": time()})
15 | feed.save()
16 | return feed, response.status
17 | except Exception as e:
18 | print(e)
19 | feed.value.update({"last_status": 0, "last_fetched": time()})
20 | feed.save()
21 | return feed, 0
22 | pass
23 |
24 | async def throttle(sem, session, feed, db):
25 | async with sem:
26 | res = await fetch_one(session, feed, db)
27 | print(res[0].value["url"],res[1])
28 |
29 | async def fetcher(db):
30 | sem = Semaphore(100)
31 | tasks = []
32 | async with ClientSession() as session:
33 | for feed in db.model.select().where(db.model.key.startswith("feed:")):
34 | task = ensure_future(throttle(sem, session, feed, db))
35 | tasks.append(task)
36 |
37 | responses = gather(*tasks)
38 | await responses
39 |
40 | def feeds_from_opml(filename):
41 | tree = ElementTree.parse(filename)
42 | for feed in tree.findall('.//outline'):
43 | if feed.get('xmlUrl'):
44 | yield {'title': feed.get('title'),
45 | 'url': feed.get('xmlUrl')}
46 |
47 | def update_database(db, filename):
48 | for feed in feeds_from_opml(filename):
49 | if not "feed:" + feed['url'] in db:
50 | feed['last_fetched'] = 0
51 | db["feed:" + feed['url']] = feed
52 |
53 | if __name__ == '__main__':
54 | db = JSONKeyStore(database=SqliteExtDatabase('/tmp/feeds.db'))
55 | update_database(db,'feeds.opml')
56 | loop = get_event_loop()
57 | tasks = ensure_future(fetcher(db))
58 | loop.run_until_complete(tasks)
59 |
--------------------------------------------------------------------------------
/scheduler.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """ Feed enumerator """
4 |
5 | from asyncio import ensure_future, get_event_loop, set_event_loop_policy, sleep
6 | from config import (CHECK_INTERVAL, DATABASE_NAME, FETCH_INTERVAL,
7 | MONGO_SERVER, log)
8 | from datetime import datetime, timedelta
9 |
10 | from common import REDIS_NAMESPACE, connect_redis, enqueue
11 | from motor.motor_asyncio import AsyncIOMotorClient
12 | from uvloop import EventLoopPolicy
13 |
14 |
15 | async def scan_feeds(db):
16 | """Enumerate all feeds and queue them for fetching"""
17 |
18 | # let importer run first while we're testing
19 | await sleep(5)
20 |
21 | log.info("Beginning run.")
22 |
23 | while True:
24 | threshold = datetime.now() - timedelta(seconds=FETCH_INTERVAL)
25 | log.debug(threshold)
26 | queue = await connect_redis()
27 | log.info("Scanning feed list.")
28 | log.debug("Starting loop")
29 | count = 0
30 | async for feed in db.feeds.find({'last_fetched': {'$lt': threshold}}):
31 | count = count + 1
32 | url = feed['url']
33 | log.debug("Checking %d: %s", count, url)
34 | log.debug("Queueing %s", url)
35 | await enqueue(queue, "fetcher", {
36 | "_id": feed['_id'],
37 | "scheduled_at": datetime.now()
38 | })
39 | if not (count % 10):
40 | await queue.hset(REDIS_NAMESPACE + 'status', 'feed_count', count)
41 | await queue.hset(REDIS_NAMESPACE + 'status', 'feed_count', count)
42 | queue.close()
43 | await queue.wait_closed()
44 | log.info("Run complete, sleeping %ds...", CHECK_INTERVAL)
45 | await sleep(CHECK_INTERVAL)
46 |
47 |
48 | def main():
49 | """Setup event loop and start coroutines"""
50 |
51 | set_event_loop_policy(EventLoopPolicy())
52 | loop = get_event_loop()
53 |
54 | client = AsyncIOMotorClient(MONGO_SERVER)
55 | database = client[DATABASE_NAME]
56 |
57 | ensure_future(scan_feeds(database))
58 | try:
59 | loop.run_forever()
60 | finally:
61 | loop.close()
62 |
63 |
64 | if __name__ == '__main__':
65 | main()
66 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '2'
2 | services:
3 | redis:
4 | image: redis:3.2.8
5 | ports:
6 | - "6379:6379"
7 | logging:
8 | driver: "none"
9 | environment:
10 | - "constraint:node.role==master"
11 |
12 | db:
13 | image: mongo:3.4
14 | ports:
15 | - "27017:27017"
16 | logging:
17 | driver: "none"
18 | environment:
19 | - "constraint:node.role==master"
20 |
21 | importer:
22 | build: .
23 | env_file:
24 | - ENV
25 | environment:
26 | - "constraint:node.role==worker"
27 | volumes:
28 | - .:/app
29 | links:
30 | - db
31 | - redis
32 | working_dir: /app
33 | command: python -u import.py
34 |
35 | scheduler:
36 | build: .
37 | env_file:
38 | - ENV
39 | environment:
40 | - "constraint:node.role==worker"
41 | volumes:
42 | - .:/app
43 | links:
44 | - db
45 | - redis
46 | working_dir: /app
47 | command: python -u scheduler.py
48 |
49 | fetcher:
50 | build: .
51 | env_file:
52 | - ENV
53 | environment:
54 | - "constraint:node.role==worker"
55 | volumes:
56 | - .:/app
57 | links:
58 | - db
59 | - redis
60 | working_dir: /app
61 | command: python -u fetcher.py
62 |
63 | parser:
64 | build: .
65 | env_file:
66 | - ENV
67 | environment:
68 | - "constraint:node.role==worker"
69 | volumes:
70 | - .:/app
71 | links:
72 | - db
73 | - redis
74 | working_dir: /app
75 | command: python -u parser.py
76 |
77 | metrics:
78 | build: .
79 | ports:
80 | - "9213:8000"
81 | env_file:
82 | - ENV
83 | environment:
84 | - "constraint:node.role==master"
85 | volumes:
86 | - .:/app
87 | links:
88 | - db
89 | - redis
90 | working_dir: /app
91 | command: python -u metrics.py
92 |
93 | web:
94 | build: .
95 | ports:
96 | - "8000:8000"
97 | env_file:
98 | - ENV
99 | environment:
100 | - "constraint:node.role==master"
101 | volumes:
102 | - .:/app
103 | links:
104 | - db
105 | - redis
106 | working_dir: /app
107 | command: python -u web.py
108 |
--------------------------------------------------------------------------------
/import.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """OPML importer"""
4 |
5 | from asyncio import (Semaphore, ensure_future, gather, get_event_loop,
6 | set_event_loop_policy)
7 | from config import (DATABASE_NAME, DATE_FORMAT, FETCH_INTERVAL, MONGO_SERVER,
8 | log)
9 | from datetime import datetime
10 | from time import sleep, strftime, time
11 | from xml.etree import ElementTree
12 |
13 | from aiohttp import ClientSession
14 | from common import REDIS_NAMESPACE, connect_redis, safe_id
15 | from motor.motor_asyncio import AsyncIOMotorClient
16 | from pymongo import ASCENDING, DESCENDING
17 | from pymongo.errors import DuplicateKeyError
18 | from uvloop import EventLoopPolicy
19 |
20 |
21 | def feeds_from_opml(filename):
22 | """Extract useful attributes from OPML"""
23 |
24 | tree = ElementTree.parse(filename)
25 | for feed in tree.findall('.//outline'):
26 | if feed.get('xmlUrl'):
27 | yield {'title': feed.get('title'),
28 | 'url': feed.get('xmlUrl')}
29 |
30 |
31 | async def update_database(db, filename):
32 | """Create indexes and import feeds"""
33 |
34 | entries = db.entries
35 | await db.entries.create_index([("date", DESCENDING)])
36 | await db.entries.create_index([("url", ASCENDING)])
37 | feeds = db.feeds
38 | await db.feeds.create_index([("url", ASCENDING)])
39 | # TODO: turn this into a bulk upsert
40 | for feed in feeds_from_opml(filename):
41 | if not await feeds.find_one({'url': feed['url']}):
42 | log.debug("Inserting %s" % feed)
43 | feed = dict(feed, **{
44 | '_id': safe_id(feed['url']),
45 | 'created': datetime.now(),
46 | 'last_fetched': datetime(1970, 1, 1)
47 | })
48 | try:
49 | await feeds.insert_one(feed)
50 | except DuplicateKeyError as e:
51 | log.debug(e)
52 | redis = await connect_redis()
53 | await redis.hset(REDIS_NAMESPACE + 'status', 'feed_count', await db.feeds.count_documents({}))
54 |
55 |
56 | if __name__ == '__main__':
57 |
58 | set_event_loop_policy(EventLoopPolicy())
59 | loop = get_event_loop()
60 |
61 | c = AsyncIOMotorClient(MONGO_SERVER)
62 | db = c[DATABASE_NAME]
63 | try:
64 | loop.run_until_complete(update_database(db,'feeds.opml'))
65 | finally:
66 | loop.close()
67 |
--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
1 | #!/bin/env python3
2 |
3 | """ Configuration """
4 |
5 | from cProfile import Profile
6 | from logging import getLogger
7 | from logging.config import dictConfig
8 | from os import environ
9 |
10 | FETCH_INTERVAL = int(environ.get('FETCH_INTERVAL', 3600))
11 | CHECK_INTERVAL = int(environ.get('CHECK_INTERVAL', 900))
12 | METRICS_INTERVAL = int(environ.get('METRICS_INTERVAL', 5))
13 | HTTP_PORT = int(environ.get('HTTP_PORT', 8000))
14 | BIND_ADDRESS = environ.get('BIND_ADDRESS','0.0.0.0')
15 | DEBUG = environ.get('DEBUG','False').lower() == 'true'
16 | MAX_CONCURRENT_REQUESTS = int(environ.get('MAX_CONCURRENT_REQUESTS', 100))
17 | MONGO_SERVER = environ.get('MONGO_SERVER', 'localhost:27017')
18 | REDIS_SERVER = environ.get('REDIS_SERVER', 'localhost:6379')
19 | REDIS_NAMESPACE = environ.get('REDIS_NAMESPACE', 'newspipe:')
20 | DATABASE_NAME = environ.get('DATABASE_NAME', 'feeds')
21 | DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
22 | PROFILER = environ.get('PROFILER','False').lower() == "true"
23 |
24 | if PROFILER:
25 | profiler = Profile()
26 | profiler.enable()
27 | else:
28 | profiler = None
29 |
30 | async def get_profile():
31 | """Return a profile dump"""
32 |
33 | global profiler
34 | log.info(profiler)
35 |
36 | if profiler:
37 | profiler.create_stats()
38 | return profiler.stats
39 | return None
40 |
41 |
42 | dictConfig({
43 | "version": 1,
44 | "formatters": {
45 | "http": {
46 | "format" : "localhost - - [%(asctime)s] %(process)d %(levelname)s %(message)s",
47 | "datefmt": "%Y/%m/%d %H:%M:%S"
48 | },
49 | "service": {
50 | "format" : "%(asctime)s %(levelname)s %(message)s",
51 | "datefmt": "%Y-%m-%d %H:%M:%S"
52 | }
53 | },
54 | "handlers": {
55 | "console": {
56 | "class" : "logging.StreamHandler",
57 | "formatter": "service",
58 | "level" : "DEBUG",
59 | "stream" : "ext://sys.stdout"
60 | }
61 | },
62 | "loggers": {
63 | "sanic.static": {
64 | "level" : "INFO",
65 | "handlers": ["console"]
66 | }
67 | },
68 | "root": {
69 | "level" : "DEBUG" if DEBUG else "INFO",
70 | "handlers": ["console"]
71 | }
72 | })
73 |
74 | log = getLogger()
75 |
76 | log.info("Configuration loaded.")
77 | #for k in sorted(environ.keys()):
78 | # log.debug("{}={}".format(k,environ[k]))
79 |
--------------------------------------------------------------------------------
/static/parser-chart.tag:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
77 |
78 |
--------------------------------------------------------------------------------
/common.py:
--------------------------------------------------------------------------------
1 | #!/bin/env python3
2 |
3 | """ Common functions """
4 |
5 | from asyncio import get_event_loop, sleep
6 | from config import REDIS_NAMESPACE, REDIS_SERVER, log
7 | from hashlib import sha1
8 | from json import dumps, loads
9 | from time import time
10 | from urllib.parse import urlparse
11 | from uuid import uuid4
12 |
13 | from aioredis import create_redis
14 | from bson import json_util
15 |
16 |
17 | def retry(attempts, on_exception, interval=0):
18 | def wrap(func):
19 | def f_retry(*args, **kwargs):
20 | for i in range(attempts):
21 | try:
22 | return func(*args, **kwargs)
23 | except on_exception as e:
24 | log.debug("retry %d for %s(%s, %s), waiting %d" % (i, func,args,kwargs, interval))
25 | sleep(interval)
26 | continue
27 | return f_retry
28 | return wrap
29 |
30 |
31 | def safe_id(url):
32 | """Build a CosmosDB-safe and URL-safe ID that is still palatable to humans"""
33 | fragments = urlparse(url)
34 | safe = fragments.netloc + fragments.path.replace('/', '_').replace('+', '-')
35 | if fragments.params or fragments.query:
36 | # Add a short hash to distinguish between feeds from same site
37 | safe += sha1(bytes(url, 'utf-8')).hexdigest()[6]
38 | return safe.rstrip('_-')
39 |
40 | async def connect_redis(loop=None):
41 | """Connect to a Redis server"""
42 | if not loop:
43 | loop = get_event_loop()
44 |
45 | parts = REDIS_SERVER.split(",")
46 | address = tuple(parts[0].split(":"))
47 | rest = parts[1:]
48 | types = {'db': int, 'password': str, 'ssl': bool}
49 | params = {'loop':loop}
50 | for param in rest:
51 | try:
52 | name, value = param.split('=',1)
53 | if name in types:
54 | params[name] = types[name](value)
55 | except ValueError:
56 | log.warn("Could not parse %s" % param)
57 | next
58 | return await create_redis(address, **params)
59 |
60 | async def enqueue(server, queue_name, data):
61 | """Enqueue an object in a given redis queue"""
62 | return await server.rpush(REDIS_NAMESPACE + queue_name, dumps(data, default=json_util.default))
63 |
64 | async def dequeue(server, queue_name):
65 | """Blocking dequeue from Redis"""
66 | _, data = await server.blpop(REDIS_NAMESPACE + queue_name, 0)
67 | return loads(data, object_hook=json_util.object_hook)
68 |
69 | async def publish(server, topic_name, data):
70 | """Publish data"""
71 | _ = await server.publish_json(topic_name, data)
72 |
73 | async def subscribe(server, topic_name):
74 | """Subscribe to topic data"""
75 | chan = await server.subscribe(topic_name)
76 | return chan
77 |
78 | async def unsubscribe(server, topic_name):
79 | """Unsubscribe from topic data"""
80 | _ = await server.unsubscribe(topic_name)
--------------------------------------------------------------------------------
/static/fetcher-chart.tag:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
81 |
82 |
--------------------------------------------------------------------------------
/views/layout.tpl:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Monitor
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
20 |
21 |
22 |
34 |
35 |
36 |
37 |
38 |
39 |
40 | Feeds
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 | Fetch Status
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
Parsed Entries
59 |
60 |
61 |
62 |
63 |
64 |
65 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
94 |
95 |
96 |
--------------------------------------------------------------------------------
/static/js/route.min.js:
--------------------------------------------------------------------------------
1 | var route=function(){"use strict";var e=function(e){e=e||{};var t={},r=Array.prototype.slice;Object.defineProperties(e,{on:{value:function(r,n){if(typeof n=="function"){(t[r]=t[r]||[]).push(n)}return e},enumerable:false,writable:false,configurable:false},off:{value:function(r,n){if(r=="*"&&!n){t={}}else{if(n){var i=t[r];for(var a=0,f;f=i&&i[a];++a){if(f==n){i.splice(a--,1)}}}else{delete t[r]}}return e},enumerable:false,writable:false,configurable:false},one:{value:function(t,r){function n(){e.off(t,n);r.apply(e,arguments)}return e.on(t,n)},enumerable:false,writable:false,configurable:false},trigger:{value:function(n){var i=arguments;var a=arguments.length-1,f=new Array(a),u,o,l;for(l=0;l
4 |
5 | ## What
6 |
7 | An ongoing attempt at tying together various ML techniques for trending topic and sentiment analysis, coupled with some experimental Python `async` coding, a distributed architecture, EventSource and lots of Docker goodness.
8 |
9 | ## Why
10 |
11 | I needed a readily available corpus for doing text analytics and sentiment analysis, so I decided to make one from my RSS feeds.
12 |
13 | Things escalated quickly from there on several fronts:
14 |
15 | * I decided I wanted this to be fully distributed, so I split the logic into several worker processes who coordinate via [Redis][redis] queues, orchestrated (and deployed) using `docker-compose`
16 | * I decided to take a ride on the bleeding edge and refactored everything to use `asyncio/uvloop` (as well as [Sanic][sanic] for the web front-end)
17 | * Rather than just consuming cognitive APIs, I also decided to implement a few NLP processing techniques (I started with a RAKE keyword extractor, and am now looking at NLTK-based tagging)
18 | * Instead of using React, I went with RiotJS, largely because I wanted to be able to deploy new visual components without a build step.
19 | * I also started using this as a "complex" Docker/Kubernetes demo, which meant some flashy features (like a graphical dashboard) started taking precedence.
20 |
21 | ## How
22 |
23 | This was originally the "dumb" part of the pipeline -- the corpus was fed into [Azure ML][aml] and the [Cognitive Services APIs][csa] for the nice stuff, so this started out mostly focusing fetching, parsing and filing away feeds.
24 |
25 | It's now a rather more complex beast than I originally bargained for. Besides acting as a technology demonstrator for a number of things (including odds and ends like how to bundle NLTK datasets inside Docker) it is currently pushing the envelope on [my Python Docker containers][ap], which now feature Python 3.6.3 atop Ubuntu LTS.
26 |
27 | ## ToDo
28 |
29 | * [ ] move to [billboard.js](https://github.com/naver/billboard.js)
30 | * [ ] Add `auth0` support
31 | * [ ] Move to Cosmos DB
32 |
33 | ### Architecture
34 |
35 | * [x] `import.py` is a one-shot OPML importer (you should place your own `feeds.opml` in the root directory)
36 | * [ ] `metrics.py` keeps tabs on various stats and pushes them out every few seconds
37 | * [x] `scheduler.py` iterates through the database and queues feeds for fetching
38 | * [x] `fetcher.py` fetches feeds and stores them on DocumentDB/MongoDB
39 | * [x] `parser.py` parses updated feeds into separate items and performs:
40 | [x] language detection
41 | [x] keyword extraction (using `langkit.py`)
42 | [ ] basic sentiment analysis
43 | * [ ] `cortana.py` (WIP) will do topic detection and sentiment analysis
44 | * [x] `web.py` provides a simple web front-end for live status updates via SSE.
45 |
46 | Processes are written to leverage `asyncio/uvloop` and interact via [Redis][redis] (previously they interacted via [ZeroMQ][0mq], but I'm already playing around with deploying this on [Swarm and an Azure VM scaleset][swarm]).
47 |
48 | A Docker compose file is supplied for running the entire stack locally - you can tweak it up to version `3` and get things running on Swarm if you manually push the images to a private registry first, but I'll automate that once things are a little more stable.
49 |
50 | [0mq]: https://github.com/aio-libs/aiozmq
51 | [csa]: https://www.microsoft.com/cognitive-services
52 | [aml]: https://studio.azureml.net
53 | [ap]: https://github.com/rcarmo/alpine-python
54 | [swarm]: https://github.com/rcarmo/azure-docker-swarm-cluster
55 | [sanic]: http://sanic.readthedocs.io
56 | [piku]: https://github.com/rcarmo/piku
57 | [redis]: http://redis.io
58 |
--------------------------------------------------------------------------------
/static/css/c3.css:
--------------------------------------------------------------------------------
1 | /*-- Chart --*/
2 | .c3 svg {
3 | font: 10px sans-serif;
4 | -webkit-tap-highlight-color: transparent; }
5 |
6 | .c3 path, .c3 line {
7 | fill: none;
8 | stroke: #000; }
9 |
10 | .c3 text {
11 | -webkit-user-select: none;
12 | -moz-user-select: none;
13 | user-select: none; }
14 |
15 | .c3-legend-item-tile,
16 | .c3-xgrid-focus,
17 | .c3-ygrid,
18 | .c3-event-rect,
19 | .c3-bars path {
20 | shape-rendering: crispEdges; }
21 |
22 | .c3-chart-arc path {
23 | stroke: #fff; }
24 |
25 | .c3-chart-arc text {
26 | fill: #fff;
27 | font-size: 13px; }
28 |
29 | /*-- Axis --*/
30 | /*-- Grid --*/
31 | .c3-grid line {
32 | stroke: #aaa; }
33 |
34 | .c3-grid text {
35 | fill: #aaa; }
36 |
37 | .c3-xgrid, .c3-ygrid {
38 | stroke-dasharray: 3 3; }
39 |
40 | /*-- Text on Chart --*/
41 | .c3-text.c3-empty {
42 | fill: #808080;
43 | font-size: 2em; }
44 |
45 | /*-- Line --*/
46 | .c3-line {
47 | stroke-width: 1px; }
48 |
49 | /*-- Point --*/
50 | .c3-circle._expanded_ {
51 | stroke-width: 1px;
52 | stroke: white; }
53 |
54 | .c3-selected-circle {
55 | fill: white;
56 | stroke-width: 2px; }
57 |
58 | /*-- Bar --*/
59 | .c3-bar {
60 | stroke-width: 0; }
61 |
62 | .c3-bar._expanded_ {
63 | fill-opacity: 1;
64 | fill-opacity: 0.75; }
65 |
66 | /*-- Focus --*/
67 | .c3-target.c3-focused {
68 | opacity: 1; }
69 |
70 | .c3-target.c3-focused path.c3-line, .c3-target.c3-focused path.c3-step {
71 | stroke-width: 2px; }
72 |
73 | .c3-target.c3-defocused {
74 | opacity: 0.3 !important; }
75 |
76 | /*-- Region --*/
77 | .c3-region {
78 | fill: steelblue;
79 | fill-opacity: .1; }
80 |
81 | /*-- Brush --*/
82 | .c3-brush .extent {
83 | fill-opacity: .1; }
84 |
85 | /*-- Select - Drag --*/
86 | /*-- Legend --*/
87 | .c3-legend-item {
88 | font-size: 12px; }
89 |
90 | .c3-legend-item-hidden {
91 | opacity: 0.15; }
92 |
93 | .c3-legend-background {
94 | opacity: 0.75;
95 | fill: white;
96 | stroke: lightgray;
97 | stroke-width: 1; }
98 |
99 | /*-- Title --*/
100 | .c3-title {
101 | font: 14px sans-serif; }
102 |
103 | /*-- Tooltip --*/
104 | .c3-tooltip-container {
105 | z-index: 10; }
106 |
107 | .c3-tooltip {
108 | border-collapse: collapse;
109 | border-spacing: 0;
110 | background-color: #fff;
111 | empty-cells: show;
112 | -webkit-box-shadow: 7px 7px 12px -9px #777777;
113 | -moz-box-shadow: 7px 7px 12px -9px #777777;
114 | box-shadow: 7px 7px 12px -9px #777777;
115 | opacity: 0.9; }
116 |
117 | .c3-tooltip tr {
118 | border: 1px solid #CCC; }
119 |
120 | .c3-tooltip th {
121 | background-color: #aaa;
122 | font-size: 14px;
123 | padding: 2px 5px;
124 | text-align: left;
125 | color: #FFF; }
126 |
127 | .c3-tooltip td {
128 | font-size: 13px;
129 | padding: 3px 6px;
130 | background-color: #fff;
131 | border-left: 1px dotted #999; }
132 |
133 | .c3-tooltip td > span {
134 | display: inline-block;
135 | width: 10px;
136 | height: 10px;
137 | margin-right: 6px; }
138 |
139 | .c3-tooltip td.value {
140 | text-align: right; }
141 |
142 | /*-- Area --*/
143 | .c3-area {
144 | stroke-width: 0;
145 | opacity: 0.2; }
146 |
147 | /*-- Arc --*/
148 | .c3-chart-arcs-title {
149 | dominant-baseline: middle;
150 | font-size: 1.3em; }
151 |
152 | .c3-chart-arcs .c3-chart-arcs-background {
153 | fill: #e0e0e0;
154 | stroke: none; }
155 |
156 | .c3-chart-arcs .c3-chart-arcs-gauge-unit {
157 | fill: #000;
158 | font-size: 16px; }
159 |
160 | .c3-chart-arcs .c3-chart-arcs-gauge-max {
161 | fill: #777; }
162 |
163 | .c3-chart-arcs .c3-chart-arcs-gauge-min {
164 | fill: #777; }
165 |
166 | .c3-chart-arc .c3-gauge-value {
167 | fill: #000;
168 | /* font-size: 28px !important;*/ }
169 |
170 | .c3-chart-arc.c3-target g path {
171 | opacity: 1; }
172 |
173 | .c3-chart-arc.c3-target.c3-focused g path {
174 | opacity: 1; }
175 |
--------------------------------------------------------------------------------
/langkit.py:
--------------------------------------------------------------------------------
1 | #!/bin/env python3
2 | # Rui Carmo, 2017
3 | # Miscellaneous helpers for NLTK
4 |
5 | from operator import itemgetter
6 | from nltk import FreqDist
7 | from nltk.corpus import stopwords
8 | from nltk.tokenize import sent_tokenize, word_tokenize, RegexpTokenizer
9 | from nltk.stem.porter import PorterStemmer
10 | from nltk.stem import RSLPStemmer
11 | from sys import maxunicode
12 | from unicodedata import category
13 | from logging import getLogger
14 | from traceback import format_exc
15 |
16 | log = getLogger()
17 |
18 | STOPWORDS = {'en': stopwords.words('english'),
19 | 'pt': stopwords.words('portuguese')}
20 |
21 | STEMMERS = {'en': PorterStemmer(),
22 | 'pt': RSLPStemmer()}
23 |
24 | # RAKE extractor - requires python -m nltk.downloader stopwords punkt
25 |
26 | # Build a full unicode punctuation dictionary based on glyph category
27 | # (strings.punctuation doesn't cut it)
28 | PUNCTUATION = dict.fromkeys([i for i in range(maxunicode) if category(chr(i)).startswith('P')])
29 |
30 | def _extract_phrases(sentences, language="english"):
31 | """Extract phrases from a list of sentences"""
32 |
33 | def is_punctuation(word):
34 | return len(word) == 1 and ord(word) in PUNCTUATION
35 |
36 | lang_stopwords = set(stopwords.words(language))
37 |
38 | phrase_list = []
39 | for sentence in sentences:
40 | # NOTE: word_tokenize can't quote cope with rich quotes,
41 | # so we'll need to clean up after it deals with punctuation
42 | words = map(lambda x: "|" if x in lang_stopwords else x, word_tokenize(sentence.lower(), language))
43 | phrase = []
44 | for word in words:
45 | if word == "|" or is_punctuation(word):
46 | if len(phrase) > 0:
47 | phrase_list.append(phrase)
48 | phrase = []
49 | else:
50 | phrase.append(word.translate(PUNCTUATION)) # remove unicode quotes
51 | return phrase_list
52 |
53 |
54 | def _score_words(phrase_list):
55 | """Score words based on frequency"""
56 |
57 | def is_numeric(word):
58 | # NOTE: this is a quick and dirty way to cope with multi-digit figures
59 | # but will be confused by currency
60 | try:
61 | int(word.replace(',', '').replace('.', ''))
62 | return True
63 | except ValueError:
64 | return False
65 |
66 | word_freq = FreqDist()
67 | word_degree = FreqDist()
68 |
69 | for phrase in phrase_list:
70 | degree = len(list(filter(lambda x: not is_numeric(x), phrase))) - 1
71 | for word in phrase:
72 | word_freq[word] += 1
73 | word_degree[word] += degree
74 |
75 | for word in word_freq.keys():
76 | word_degree[word] = word_degree[word] + word_freq[word] # itself
77 |
78 | # word score = deg(w) / freq(w)
79 | word_scores = {}
80 | for word in word_freq.keys():
81 | word_scores[word] = word_degree[word] / word_freq[word]
82 | return word_scores
83 |
84 |
85 | def _score_phrases(phrase_list, word_scores):
86 | """Score a phrase by tallying individual word scores"""
87 |
88 | phrase_scores = {}
89 | for phrase in phrase_list:
90 | phrase_score = 0
91 | # cumulative score of words
92 | for word in phrase:
93 | phrase_score += word_scores[word]
94 | phrase_scores[" ".join(phrase)] = phrase_score
95 | return phrase_scores
96 |
97 |
98 | def extract_keywords(text, language="en", scores=False):
99 | """RAKE extractor"""
100 |
101 | try:
102 | lang = {"en": "english",
103 | "pt": "portuguese"}[language]
104 | except KeyError:
105 | log.error(format_exc())
106 | return
107 |
108 | sentences = sent_tokenize(text, lang)
109 | phrase_list = _extract_phrases(sentences, lang)
110 | word_scores = _score_words(phrase_list)
111 | phrase_scores = _score_phrases(phrase_list, word_scores)
112 | sorted_scores = sorted(phrase_scores.items(), key=itemgetter(1), reverse=True)
113 | if scores:
114 | return sorted_scores
115 | else:
116 | return list(map(lambda x: x[0], sorted_scores))
117 |
118 |
119 | def tokenize(plaintext, language):
120 | """tokenize into stemmed tokens"""
121 |
122 | try:
123 | stop_words = STOPWORDS[language]
124 | stemmer = STEMMERS[language]
125 | except KeyError:
126 | log.error(format_exc())
127 | return
128 |
129 | # Tokenize, remove stop words and stem
130 | tokenizer = RegexpTokenizer(r'\w+')
131 | tokens = [stemmer.stem(i) for i in tokenizer.tokenize(plaintext.lower()) if not i in stop_words]
132 | return tokens
--------------------------------------------------------------------------------
/web.py:
--------------------------------------------------------------------------------
1 | #!/bin/env python3
2 |
3 | """ Web server """
4 |
5 | from config import (BIND_ADDRESS, DATABASE_NAME, DEBUG, HTTP_PORT,
6 | MONGO_SERVER, log)
7 | from datetime import datetime, timedelta
8 | from functools import lru_cache
9 | from multiprocessing import cpu_count
10 | from traceback import format_exc
11 |
12 | from aiocache import SimpleMemoryCache, cached
13 | from common import (REDIS_NAMESPACE, connect_redis, dequeue, subscribe,
14 | unsubscribe)
15 | from mako.template import Template
16 | from metrics import database_entries, database_feeds, tree_split
17 | from motor.motor_asyncio import AsyncIOMotorClient
18 | from sanic import Sanic
19 | from sanic.exceptions import FileNotFound, NotFound, RequestTimeout
20 | from sanic.response import html, json, stream, text
21 | from sanic.server import HttpProtocol
22 | from ujson import dumps
23 |
24 | app = Sanic(__name__)
25 | layout = Template(filename='views/layout.tpl')
26 |
27 | redis = None
28 | db = None
29 |
30 |
31 | @app.route('/', methods=['GET'])
32 | async def homepage(req):
33 | """Main page"""
34 | return html(layout.render(timestr=datetime.now().strftime("%H:%M:%S.%f")))
35 |
36 |
37 | @app.route('/test', methods=['GET'])
38 | async def get_name(req):
39 | """Endpoint for front-end load testing using wrk.
40 | Reference measurement: 25K requests/s on 4 cores of a 2.9GHz i5"""
41 | return text("test")
42 |
43 |
44 | @app.route('/events')
45 | async def sse(request):
46 | async def streaming_fn(response):
47 | i = 1
48 | [ch] = await subscribe(redis, 'ui')
49 | log.debug("%s subscribed to UI events" % request.ip)
50 | while (await ch.wait_message()):
51 | msg = await ch.get_json()
52 | s = ''
53 | if 'event' in msg:
54 | s = s + 'event: ' + msg['event'] + '\r\n'
55 | s = s + 'data: ' + dumps(msg) + '\r\n\r\n'
56 | try:
57 | response.write(s.encode())
58 | i += 1
59 | except Exception:
60 | log.error(format_exc())
61 | await unsubscribe(redis, 'ui')
62 | log.debug("%s unsubscribed from UI events" % request.ip)
63 | break
64 | return stream(streaming_fn, content_type='text/event-stream')
65 |
66 |
67 | @app.route('/status', methods=['GET'])
68 | async def get_status(req):
69 | """Status endpoint for the web UI - will expose all counters."""
70 |
71 | return json({
72 | "feed_count": await redis.hget(REDIS_NAMESPACE + 'status', 'feed_count'),
73 | "item_count": await redis.hget(REDIS_NAMESPACE + 'status', 'item_count')
74 | })
75 |
76 |
77 | @app.route('/stats/feeds', methods=['GET'])
78 | async def handler(req):
79 | return json(tree_split(await database_feeds(db), drop_last=1)['database']['feeds'])
80 |
81 | @app.route('/stats/entries', methods=['GET'])
82 | async def handler(req):
83 | return json(tree_split(await database_entries(db), drop_last=1)['database']['entries'])
84 |
85 |
86 | @app.route('/feeds/', methods=['GET'])
87 | @app.route('/feeds//', methods=['GET'])
88 | @cached(ttl=20)
89 | async def get_feeds(req, order, last_id=None):
90 | """Paged navigation of feeds - experimental, using aiocache to lessen database hits.
91 | Right now this clocks in at 10K requests/s when using only 2 cores on my i5 Mac."""
92 |
93 | limit = 50
94 | fields = {'_id': 1, 'title': 1, 'last_fetched': 1, 'last_status': 1}
95 | if last_id:
96 | data = await db.feeds.find({last_id < '_id'},
97 | fields).sort(order).limit(limit).to_list(limit)
98 | else:
99 | data = await db.feeds.find({},
100 | fields).sort(order).limit(limit).to_list(limit)
101 | return json(data)
102 |
103 |
104 | # Add a route-specific timeout for the SSE handler
105 | class CustomHttpProtocol(HttpProtocol):
106 | def on_message_complete(self):
107 | if self.url == b'/events':
108 | self.request_timeout = 1000
109 | super().on_message_complete()
110 |
111 |
112 | # Map static assets
113 | app.static('/', './static')
114 |
115 | @app.listener('after_server_start')
116 | async def init_connections(sanic, loop):
117 | """Bind the database and Redis client to Sanic's event loop."""
118 |
119 | global redis, db
120 | redis = await connect_redis()
121 | motor = AsyncIOMotorClient(MONGO_SERVER, io_loop=loop)
122 | db = motor[DATABASE_NAME]
123 |
124 |
125 | if __name__ == '__main__':
126 | log.debug("Beginning run.")
127 | app.run(host=BIND_ADDRESS, port=HTTP_PORT, workers=cpu_count(), debug=DEBUG, protocol=CustomHttpProtocol)
128 |
--------------------------------------------------------------------------------
/fetcher.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """ Feed fetcher """
4 |
5 | from asyncio import (Semaphore, ensure_future, gather, get_event_loop,
6 | set_event_loop_policy, sleep)
7 | from config import (CHECK_INTERVAL, DATABASE_NAME, FETCH_INTERVAL,
8 | MAX_CONCURRENT_REQUESTS, MONGO_SERVER, log)
9 | from datetime import datetime, timedelta
10 | from hashlib import sha1
11 | from traceback import format_exc
12 |
13 | from aiohttp import ClientSession, TCPConnector
14 | from common import connect_redis, dequeue, enqueue, publish, retry
15 | from motor.motor_asyncio import AsyncIOMotorClient
16 | from pymongo.errors import OperationFailure
17 | from uvloop import EventLoopPolicy
18 |
19 | @retry(10, OperationFailure, 3)
20 | async def do_update(collection, *args, **kwargs):
21 | collection.update_one(*args, **kwargs)
22 |
23 | async def fetch_one(session, feed, client, database, queue):
24 | """Fetch a single feed"""
25 | url = feed['url']
26 | checksum = feed.get('checksum', None)
27 | changed = False
28 | headers = {}
29 |
30 | await publish(queue, 'ui', {'event': 'fetch_one', 'url':url})
31 | log.debug("Fetching %s", url)
32 |
33 | if 'etag' in feed:
34 | headers['etag'] = feed['etag']
35 | if 'last_modified' in feed:
36 | headers['if-modified-since'] = feed['last_modified']
37 |
38 | try:
39 | async with session.get(url, headers=headers) as response:
40 | text = await response.text()
41 | # TODO: check behavior for 301/302
42 | update = {
43 | 'last_status': response.status,
44 | 'last_fetched': datetime.now(),
45 | }
46 | await publish(queue, 'ui', {'event':'fetch_result',
47 | 'url':url,
48 | 'status': response.status})
49 | if response.status == 200:
50 | if 'checksum' not in feed or feed['checksum'] != checksum:
51 | changed = True
52 | update['raw'] = text
53 | update['checksum'] = sha1(text.encode('utf-8')).hexdigest()
54 |
55 | if 'etag' in response.headers:
56 | update['etag'] = response.headers['etag']
57 |
58 | if 'last-modified' in response.headers:
59 | update['last_modified'] = response.headers['last-modified']
60 |
61 | await do_update(database.feeds, {'url': url}, {'$set': update})
62 |
63 | if changed:
64 | await enqueue(queue, 'parser', {
65 | "_id": feed['_id'],
66 | "scheduled_at": datetime.now()
67 | })
68 | return feed, response.status
69 |
70 | except Exception:
71 | log.error(format_exc())
72 | await do_update(database.feeds, {'url': url},
73 | {'$set': {'last_status': 0,
74 | 'last_fetched': datetime.now()}})
75 | return feed, 0
76 |
77 |
78 | async def throttle(sem, session, feed, client, database, queue):
79 | """Throttle number of simultaneous requests"""
80 |
81 | async with sem:
82 | res = await fetch_one(session, feed, client, database, queue)
83 | log.info("%s: %d", res[0]['url'], res[1])
84 |
85 |
86 | async def fetcher(database):
87 | """Fetch all the feeds"""
88 |
89 | # disable certificate validation to cope with self-signed certificates in some feed back-ends
90 | client = ClientSession(connector=TCPConnector(verify_ssl=False))
91 | sem = Semaphore(MAX_CONCURRENT_REQUESTS)
92 |
93 | queue = await connect_redis()
94 | while True:
95 | log.info("Beginning run.")
96 | tasks = []
97 | threshold = datetime.now() - timedelta(seconds=FETCH_INTERVAL)
98 | async with ClientSession() as session:
99 | while True:
100 | try:
101 | job = await dequeue(queue, 'fetcher')
102 | feed = await database.feeds.find_one({'_id': job['_id']})
103 | last_fetched = feed.get('last_fetched', threshold)
104 | if last_fetched <= threshold:
105 | task = ensure_future(throttle(sem, session, feed, client, database, queue))
106 | tasks.append(task)
107 | except Exception:
108 | log.error(format_exc())
109 | break
110 | responses = gather(*tasks)
111 | await responses
112 | log.info("Run complete, sleeping %ds...", CHECK_INTERVAL)
113 | await sleep(CHECK_INTERVAL)
114 | queue.close()
115 | await queue.wait_closed()
116 |
117 |
118 | def main():
119 | """Setup coroutines and kickstart fetcher"""
120 | set_event_loop_policy(EventLoopPolicy())
121 |
122 | motor = AsyncIOMotorClient(MONGO_SERVER)
123 | database = motor[DATABASE_NAME]
124 |
125 | loop = get_event_loop()
126 | ensure_future(fetcher(database))
127 | try:
128 | loop.run_forever()
129 | finally:
130 | loop.close()
131 |
132 |
133 | if __name__ == '__main__':
134 | main()
135 |
--------------------------------------------------------------------------------
/parser.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """ Feed parser """
4 |
5 | from asyncio import get_event_loop, set_event_loop_policy
6 | from config import (DATABASE_NAME, MONGO_SERVER, REDIS_NAMESPACE, get_profile,
7 | log)
8 | from datetime import datetime
9 | from hashlib import sha1
10 | from time import mktime
11 | from traceback import format_exc
12 |
13 | from bs4 import BeautifulSoup
14 | from common import connect_redis, dequeue, enqueue, safe_id, publish
15 | from feedparser import parse as feed_parse
16 | from gensim import corpora, models
17 | from langdetect import detect
18 | from langkit import extract_keywords, tokenize
19 | from motor.motor_asyncio import AsyncIOMotorClient
20 | from uvloop import EventLoopPolicy
21 |
22 |
23 | def get_entry_content(entry):
24 | """Select the best content from an entry"""
25 |
26 | candidates = entry.get('content', [])
27 | if 'summary_detail' in entry:
28 | candidates.append(entry.summary_detail)
29 | for candidate in candidates:
30 | if hasattr(candidate, 'type'): # speedparser doesn't set this
31 | if 'html' in candidate.type:
32 | return candidate.value
33 | if candidates:
34 | try:
35 | return candidates[0].value
36 | except AttributeError: # speedparser does this differently
37 | return candidates[0]['value']
38 | return ''
39 |
40 |
41 | def get_entry_date(entry):
42 | """Select the best timestamp for an entry"""
43 |
44 | for header in ['modified', 'issued', 'created']:
45 | when = entry.get(header+'_parsed', None)
46 | if when:
47 | return datetime.fromtimestamp(mktime(when))
48 | return datetime.now()
49 |
50 |
51 | def get_entry_id(entry):
52 | """Get a useful id from a feed entry"""
53 |
54 | if 'id' in entry and entry.id:
55 | if isinstance(entry.id, dict):
56 | return entry.id.values()[0]
57 | return entry.id
58 |
59 | content = get_entry_content(entry)
60 | if content:
61 | return sha1(content.encode('utf-8')).hexdigest()
62 | if 'link' in entry:
63 | return entry.link
64 | if 'title' in entry:
65 | return sha1(entry.title.encode('utf-8')).hexdigest()
66 |
67 |
68 | def get_plaintext(html):
69 | """Scrub out tags and extract plaintext"""
70 |
71 | soup = BeautifulSoup(html)
72 | for script in soup(["script", "style"]):
73 | script.extract()
74 | return soup.get_text()
75 |
76 |
77 | def lda(tokens):
78 | # Perform Latent Dirchelet Allocation
79 | dictionary = corpora.Dictionary(tokens)
80 | corpus = [dictionary.doc2bow(token) for token in tokens]
81 | lda_model = gensim.models.ldamodel.LdaModel(corpus, num_topics=3, id2word=dictionary, passes=20)
82 | return lda_model
83 |
84 |
85 | async def parse(database, feed, redis):
86 | """Parse a feed into its constituent entries"""
87 |
88 | result = feed_parse(feed['raw'])
89 | if not len(result.entries):
90 | log.info('%s: No valid entries', feed['_id'])
91 | return
92 | else:
93 | log.info('%s: %d entries', feed['_id'], len(result.entries))
94 | # TODO: turn this into a bulk insert
95 | for entry in result.entries:
96 | log.debug(entry.link)
97 | when = get_entry_date(entry)
98 | body = get_entry_content(entry)
99 | plaintext = entry.title + " " + get_plaintext(body)
100 | lang = detect(plaintext)
101 |
102 | try:
103 | keywords = extract_keywords(plaintext, lang, scores=True)[:10]
104 | tokens = list(set(tokenize(plaintext, lang)))
105 | except (KeyError, TypeError):
106 | keywords = None
107 | tokens = None
108 | await publish(redis, 'ui', {'event':'new_entry', 'url':entry.link})
109 | await database.entries.update_one({'_id': safe_id(entry.link)},
110 | {'$set': {"date": when,
111 | "title": entry.title,
112 | "body": body,
113 | "plaintext": plaintext,
114 | "lang": lang,
115 | "keywords": keywords,
116 | "tokens": tokens,
117 | "url": entry.link}},
118 | upsert=True)
119 |
120 |
121 | async def item_handler(database):
122 | """Break down feeds into individual items"""
123 |
124 | redis = await connect_redis()
125 | log.info("Beginning run.")
126 | while True:
127 | try:
128 | job = await(dequeue(redis, 'parser'))
129 | log.debug(job)
130 | feed = await database.feeds.find_one({'_id': job['_id']})
131 | if feed:
132 | await parse(database, feed, redis)
133 | except Exception:
134 | log.error(format_exc())
135 | except KeyboardInterrupt:
136 | break
137 | await redis.hset(REDIS_NAMESPACE + 'status', 'item_count', await database.items.count())
138 | redis.close()
139 | await redis.wait_closed()
140 |
141 |
142 | def main():
143 | """Main loop"""
144 |
145 | set_event_loop_policy(EventLoopPolicy())
146 | conn = AsyncIOMotorClient(MONGO_SERVER)
147 | database = conn[DATABASE_NAME]
148 | loop = get_event_loop()
149 | try:
150 | loop.run_until_complete(item_handler(database))
151 | finally:
152 | loop.close()
153 |
154 | if __name__ == '__main__':
155 | main()
156 |
--------------------------------------------------------------------------------
/metrics.py:
--------------------------------------------------------------------------------
1 | #!/bin/env python3
2 |
3 | """ Metrics agent """
4 |
5 | from asyncio import ensure_future, sleep
6 | from config import (BIND_ADDRESS, DATABASE_NAME, DEBUG, HTTP_PORT,
7 | METRICS_INTERVAL, MONGO_SERVER, log)
8 | from copy import deepcopy
9 | from datetime import datetime, timedelta
10 | from functools import lru_cache, reduce
11 | from multiprocessing import cpu_count
12 | from time import time
13 |
14 | from aiocache import SimpleMemoryCache, cached
15 | from common import REDIS_NAMESPACE, connect_redis, dequeue, publish, subscribe
16 | from mako.template import Template
17 | from motor.motor_asyncio import AsyncIOMotorClient
18 | from sanic import Sanic
19 | from sanic.exceptions import FileNotFound, NotFound
20 | from sanic.response import html, json, stream, text
21 | from sanic.server import HttpProtocol
22 | from ujson import dumps
23 |
24 | app = Sanic(__name__)
25 | redis = None
26 | db = None
27 |
28 | # Metrics in Prometheus format
29 | metrics = {
30 | "database_feeds_total": None,
31 | "database_feeds_status_error_total": None,
32 | "database_feeds_status_unreachable_total": None,
33 | "database_feeds_status_inaccessible_total": None,
34 | "database_feeds_status_redirected_total": None,
35 | "database_feeds_status_fetched_total": None,
36 | "database_feeds_status_pending_total": None,
37 | "database_entries_total": None,
38 | "database_entries_language_en": None,
39 | "database_entries_language_pt": None,
40 | "database_entries_language_other": None,
41 | "fetcher_requests_total": None,
42 | }
43 |
44 | @app.route('/', methods=['GET'])
45 | async def homepage(req):
46 | """Main page"""
47 | return ""
48 |
49 |
50 | @app.route('/status', methods=['GET'])
51 | async def get_status(req):
52 | """Status endpoint for the web UI - will expose all counters."""
53 |
54 | return json({
55 | "feed_count": await redis.hget(REDIS_NAMESPACE + 'status', 'feed_count'),
56 | "item_count": await redis.hget(REDIS_NAMESPACE + 'status', 'item_count')
57 | })
58 |
59 |
60 | async def database_feeds(db):
61 | """Get database metrics pertaining to feeds"""
62 |
63 | metrics = {
64 | "database_feeds_count_total": await db.feeds.count(),
65 | "database_feeds_status_error_total": 0,
66 | "database_feeds_status_unreachable_total": 0,
67 | "database_feeds_status_inaccessible_total": 0,
68 | "database_feeds_status_redirected_total": 0,
69 | "database_feeds_status_fetched_total": 0,
70 | "database_feeds_status_pending_total": 0,
71 | }
72 | cursor = db.feeds.aggregate([{"$group": {"_id": "$last_status", "count": {"$sum": 1}}},
73 | {"$sort":{"count":-1}}])
74 | counts = {i['_id']: i['count'] async for i in cursor}
75 | for k,v in counts.items():
76 | if k==None:
77 | metrics['database_feeds_status_pending_total'] += v
78 | elif k==0:
79 | metrics['database_feeds_status_unreachable_total'] += v
80 | elif 0 < k < 300:
81 | metrics['database_feeds_status_fetched_total'] += v
82 | elif 300 <= k < 400:
83 | metrics['database_feeds_status_redirected_total'] += v
84 | elif 400 <= k < 500:
85 | metrics['database_feeds_status_inaccessible_total'] += v
86 | else:
87 | metrics['database_feeds_status_error_total'] += v
88 | return metrics
89 |
90 |
91 | async def database_entries(db):
92 | """Get database metrics pertaining to entries"""
93 |
94 | metrics = {
95 | "database_entries_count_total": await db.entries.count(),
96 | "database_entries_lang_en_total": 0,
97 | "database_entries_lang_pt_total": 0,
98 | "database_entries_lang_other_total": 0
99 | }
100 | cursor = db.entries.aggregate([{"$group": {"_id": "$lang", "count": {"$sum": 1}}},
101 | {"$sort":{"count":-1}}])
102 | counts = {i['_id']: i['count'] async for i in cursor}
103 | for k, v in counts.items():
104 | if k in ['en', 'pt']:
105 | metrics['database_entries_lang_' + k + '_total'] += v
106 | else:
107 | metrics['database_entries_lang_other_total'] += v
108 | return metrics
109 |
110 |
111 |
112 | @app.route('/stats/post_times', methods=['GET'])
113 | async def handler(req):
114 | # TODO: this aggregation is broken
115 | cursor = db.entries.aggregate([{"$match":{"date":{"$gte": datetime.now() - timedelta(days=7), "$lt": datetime.now()}}},
116 | {"$group":{"_id": {"lang":"$lang", "hour": { "$hour": "$date"}},"count":{"$sum": "$count"}}},
117 | {"$sort":{"hour":1}}])
118 |
119 |
120 | def tree_split(flat, drop_last=0):
121 |
122 | def merge(a, b):
123 | if not isinstance(b, dict):
124 | return b
125 | for k, v in b.items():
126 | if k in a and isinstance(a[k], dict):
127 | merge(a[k], v)
128 | elif v:
129 | a[k] = deepcopy(v)
130 | return a
131 |
132 | segments = []
133 | for k, v in flat.items():
134 | parts = k.split('_')[:-drop_last]
135 | seg = {parts[-1:][0]: v}
136 | for part in reversed(parts[:-1]):
137 | seg = {part: seg}
138 | segments.append(seg)
139 |
140 | return reduce(merge, segments)
141 |
142 |
143 | async def monitor_loop():
144 | global redis, db, metrics
145 | while True:
146 | log.debug("updating metrics")
147 | metrics.update(await database_feeds(db))
148 | metrics.update(await database_entries(db))
149 | tree = tree_split(metrics, drop_last=1)
150 | #await redis.mset('metrics:timestamp', time(), *{'metrics:' + k:v for k,v in metrics.items() if v})
151 | await publish(redis, 'ui', {'event': 'metrics_feeds', 'data': tree['database']['feeds']})
152 | await publish(redis, 'ui', {'event': 'metrics_entries', 'data': tree['database']['entries']})
153 | await sleep(METRICS_INTERVAL)
154 | redis.close()
155 | await redis.wait_closed()
156 |
157 |
158 | @app.listener('after_server_start')
159 | async def init_connections(sanic, loop):
160 | """Bind the database and Redis client to Sanic's event loop."""
161 |
162 | global redis, db
163 | redis = await connect_redis()
164 | motor = AsyncIOMotorClient(MONGO_SERVER, io_loop=loop)
165 | db = motor[DATABASE_NAME]
166 | log.debug("adding task")
167 | app.add_task(loop.create_task(monitor_loop()))
168 |
169 |
170 | if __name__ == '__main__':
171 | log.debug("Beginning run.")
172 | app.run(host=BIND_ADDRESS, port=HTTP_PORT, debug=DEBUG)
173 |
--------------------------------------------------------------------------------
/static/js/history.min.js:
--------------------------------------------------------------------------------
1 | /*!
2 | * History API JavaScript Library v4.2.4
3 | *
4 | * Support: IE8+, FF3+, Opera 9+, Safari, Chrome and other
5 | *
6 | * Copyright 2011-2015, Dmitrii Pakhtinov ( spb.piksel@gmail.com )
7 | *
8 | * http://spb-piksel.ru/
9 | *
10 | * Dual licensed under the MIT and GPL licenses:
11 | * http://www.opensource.org/licenses/mit-license.php
12 | * http://www.gnu.org/licenses/gpl.html
13 | *
14 | * Update: 2015-10-16 22:16
15 | */
16 | (function(m){if("function"===typeof define&&define.amd){var w="[history"+(new Date).getTime()+"]",j=requirejs.onError;m.toString=function(){return w};requirejs.onError=function(m){-1===m.message.indexOf(w)&&j.call(requirejs,m)};define([],m)}if("object"===typeof exports&&"undefined"!==typeof module)module.exports=m();else return m()})(function(){var h=!0,i=null,p=!1;function m(a,b){var c=e.history!==n;c&&(e.history=n);a.apply(n,b);c&&(e.history=k)}function w(){}function j(a,b,c){if(a!=i&&""!==a&&!b){var b=j(),d=g.getElementsByTagName("base")[0];
17 | !c&&d&&d.getAttribute("href")&&(d.href=d.href,b=j(d.href,i,h));c=b.d;d=b.h;a=""+a;a=/^(?:\w+\:)?\/\//.test(a)?0===a.indexOf("/")?d+a:a:d+"//"+b.g+(0===a.indexOf("/")?a:0===a.indexOf("?")?c+a:0===a.indexOf("#")?c+b.e+a:c.replace(/[^\/]+$/g,"")+a)}else if(a=b?a:f.href,!o||c)a=a.replace(/^[^#]*/,"")||"#",a=f.protocol.replace(/:.*$|$/,":")+"//"+f.host+l.basepath+a.replace(RegExp("^#[/]?(?:"+l.type+")?"),"");N.href=a;var a=/(?:(\w+\:))?(?:\/\/(?:[^@]*@)?([^\/:\?#]+)(?::([0-9]+))?)?([^\?#]*)(?:(\?[^#]+)|\?)?(?:(#.*))?/.exec(N.href),
18 | b=a[2]+(a[3]?":"+a[3]:""),c=a[4]||"/",d=a[5]||"",e="#"===a[6]?"":a[6]||"",k=c+d+e,m=c.replace(RegExp("^"+l.basepath,"i"),l.type)+d;return{b:a[1]+"//"+b+k,h:a[1],g:b,i:a[2],k:a[3]||"",d:c,e:d,a:e,c:k,j:m,f:m+e}}function Z(){var a;try{a=e.sessionStorage,a.setItem(A+"t","1"),a.removeItem(A+"t")}catch(b){a={getItem:function(a){a=g.cookie.split(a+"=");return 1e;e+=1){var f=a.charCodeAt(e);(f==="\n".charCodeAt(0)||f==="\r".charCodeAt(0))&&(this.charOffset=e+1)}this.offset=d;var g=a.slice(c,this.charOffset);this.onProgressCallback.call(this.thisArg,g)}},c.prototype.onFinish=function(){this.onProgress(),3===this.state&&(this.state=4,0!==this.timeout&&(m(this.timeout),this.timeout=0),this.onFinishCallback.call(this.thisArg))},c.prototype.onReadyStateChange=function(){void 0!=this.xhr&&(4===this.xhr.readyState?0===this.xhr.status?this.onFinish():this.onFinish():3===this.xhr.readyState?this.onProgress():2===this.xhr.readyState)},c.prototype.onTimeout2=function(){this.timeout=0;var b=/^data\:([^,]*?)(base64)?,([\S]*)$/.exec(this.url),c=b[1],d="base64"===b[2]?a.atob(b[3]):decodeURIComponent(b[3]);1===this.state&&(this.state=2,this.onStartCallback.call(this.thisArg,200,"OK",c)),(2===this.state||3===this.state)&&(this.state=3,this.onProgressCallback.call(this.thisArg,d)),3===this.state&&(this.state=4,this.onFinishCallback.call(this.thisArg))},c.prototype.onTimeout1=function(){this.timeout=0,this.open(this.url,this.withCredentials)},c.prototype.onTimeout0=function(){var a=this;this.timeout=l(function(){a.onTimeout0()},500),3===this.xhr.readyState&&this.onProgress()},c.prototype.handleEvent=function(a){"load"===a.type?this.onFinish():"error"===a.type?this.onFinish():"abort"===a.type?this.onFinish():"progress"===a.type?this.onProgress():"readystatechange"===a.type&&this.onReadyStateChange()},c.prototype.open=function(b,c){0!==this.timeout&&(m(this.timeout),this.timeout=0),this.url=b,this.withCredentials=c,this.state=1,this.charOffset=0,this.offset=0;var d=this,e=/^data\:([^,]*?)(?:;base64)?,[\S]*$/.exec(b);if(void 0!=e)return void(this.timeout=l(function(){d.onTimeout2()},0));if((!("ontimeout"in this.xhr)||"sendAsBinary"in this.xhr||"mozAnon"in this.xhr)&&void 0!=a.document&&void 0!=a.document.readyState&&"complete"!==a.document.readyState)return void(this.timeout=l(function(){d.onTimeout1()},4));this.xhr.onload=function(a){d.handleEvent({type:"load"})},this.xhr.onerror=function(){d.handleEvent({type:"error"})},this.xhr.onabort=function(){d.handleEvent({type:"abort"})},this.xhr.onprogress=function(){d.handleEvent({type:"progress"})},this.xhr.onreadystatechange=function(){d.handleEvent({type:"readystatechange"})},this.xhr.open("GET",b,!0),this.xhr.withCredentials=c,this.xhr.responseType="text","setRequestHeader"in this.xhr&&this.xhr.setRequestHeader("Accept","text/event-stream");try{this.xhr.send(void 0)}catch(f){throw f}"readyState"in this.xhr&&void 0!=a.opera&&(this.timeout=l(function(){d.onTimeout0()},0))},c.prototype.cancel=function(){0!==this.state&&4!==this.state&&(this.state=4,this.xhr.onload=n,this.xhr.onerror=n,this.xhr.onabort=n,this.xhr.onprogress=n,this.xhr.onreadystatechange=n,this.xhr.abort(),0!==this.timeout&&(m(this.timeout),this.timeout=0),this.onFinishCallback.call(this.thisArg)),this.state=0},d.prototype.get=function(a){return this._data[a+"~"]},d.prototype.set=function(a,b){this._data[a+"~"]=b},d.prototype["delete"]=function(a){delete this._data[a+"~"]},e.prototype.dispatchEvent=function(a){a.target=this;var b=a.type.toString(),c=this._listeners,d=c.get(b);if(void 0!=d)for(var e=d.length,g=void 0,h=0;e>h;h+=1){g=d[h];try{"function"==typeof g.handleEvent?g.handleEvent(a):g.call(this,a)}catch(i){f(i)}}},e.prototype.addEventListener=function(a,b){a=a.toString();var c=this._listeners,d=c.get(a);void 0==d&&(d=[],c.set(a,d));for(var e=d.length;e>=0;e-=1)if(d[e]===b)return;d.push(b)},e.prototype.removeEventListener=function(a,b){a=a.toString();var c=this._listeners,d=c.get(a);if(void 0!=d){for(var e=d.length,f=[],g=0;e>g;g+=1)d[g]!==b&&f.push(d[g]);0===f.length?c["delete"](a):c.set(a,f)}},h.prototype=g.prototype;var o=a.XMLHttpRequest,p=a.XDomainRequest,q=void 0!=o&&void 0!=(new o).withCredentials,r=q||void 0!=o&&void 0==p?o:p,s=-1,t=0,u=1,v=2,w=3,x=4,y=5,z=6,A=7,B=/^text\/event\-stream;?(\s*charset\=utf\-8)?$/i,C=1e3,D=18e6,E=function(a,b){var c=a;return c!==c&&(c=b),C>c?C:c>D?D:c},F=function(a,b,c){try{"function"==typeof b&&b.call(a,c)}catch(d){f(d)}};j.prototype.onStart=function(a,b,c){if(this.currentState===t)if(void 0==c&&(c=""),200===a&&B.test(c)){this.currentState=u,this.wasActivity=!0,this.retry=this.initialRetry,this.readyState=u,this.es.readyState=u;var d=new g("open");this.es.dispatchEvent(d),F(this.es,this.es.onopen,d)}else if(0!==a){var e="";e=200!==a?"EventSource's response has a status "+a+" "+b.replace(/\s+/g," ")+" that is not 200. Aborting the connection.":"EventSource's response has a Content-Type specifying an unsupported type: "+c.replace(/\s+/g," ")+". Aborting the connection.",f(new Error(e)),this.close();var d=new g("error");this.es.dispatchEvent(d),F(this.es,this.es.onerror,d)}},j.prototype.onProgress=function(a){if(this.currentState===u){var b=a.length;0!==b&&(this.wasActivity=!0);for(var c=0;b>c;c+=1){var d=a.charCodeAt(c);if(this.state===w&&d==="\n".charCodeAt(0))this.state=x;else if(this.state===w&&(this.state=x),d==="\r".charCodeAt(0)||d==="\n".charCodeAt(0)){if(this.state!==x){this.state===y&&(this.valueStart=c+1);var e=a.slice(this.fieldStart,this.valueStart-1),f=a.slice(this.valueStart+(this.valueStart16*this.initialRetry&&(this.retry=16*this.initialRetry),this.retry>D&&(this.retry=D);var a=this;this.timeout=l(function(){a.onTimeout()},this.retry),this.retry=2*this.retry+1,this.readyState=t,this.es.readyState=t;var b=new g("error");this.es.dispatchEvent(b),F(this.es,this.es.onerror,b)}},j.prototype.onTimeout=function(){if(this.timeout=0,this.currentState===s){this.wasActivity=!1;var a=this;this.timeout=l(function(){a.onTimeout()},this.heartbeatTimeout),this.currentState=t,this.dataBuffer.length=0,this.eventTypeBuffer="",this.lastEventIdBuffer=this.lastEventId,this.fieldStart=0,this.valueStart=0,this.state=x;var b=this.url.slice(0,5);b="data:"!==b&&"blob:"!==b?this.url+((-1===this.url.indexOf("?",0)?"?":"&")+"lastEventId="+encodeURIComponent(this.lastEventId)+"&r="+(Math.random()+1).toString().slice(2)):this.url;try{this.transport.open(b,this.withCredentials)}catch(c){throw this.close(),c}}else if(this.wasActivity){this.wasActivity=!1;var a=this;this.timeout=l(function(){a.onTimeout()},this.heartbeatTimeout)}else f(new Error("No activity within "+this.heartbeatTimeout+" milliseconds. Reconnecting.")),this.transport.cancel()},j.prototype.close=function(){this.currentState=v,this.transport.cancel(),0!==this.timeout&&(m(this.timeout),this.timeout=0),this.readyState=v,this.es.readyState=v},k.prototype=e.prototype,i.prototype=new k,i.prototype.close=function(){this._internal.close()},k.call(i),q&&(i.prototype.withCredentials=void 0);var G=function(){return void 0!=a.EventSource&&"withCredentials"in a.EventSource.prototype};void 0!=r&&(void 0==a.EventSource||q&&!G())&&(a.NativeEventSource=a.EventSource,a.EventSource=i)}("undefined"!=typeof window?window:this);
7 |
--------------------------------------------------------------------------------
/static/js/riot.min.js:
--------------------------------------------------------------------------------
1 | /* Riot v3.3.1, @license MIT */
2 | !function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e(t.riot=t.riot||{})}(this,function(t){"use strict";function e(t){return Dt.test(t)}function n(t){return Gt.test(t)}function r(t){return typeof t===It}function i(t){return t&&typeof t===St}function o(t){return typeof t===kt}function a(t){return typeof t===Rt}function s(t){return o(t)||null===t||""===t}function u(t){return Array.isArray(t)||t instanceof Array}function l(t,e){var n=Object.getOwnPropertyDescriptor(t,e);return o(t[e])||n&&n.writable}function c(t){return Ut.test(t)}function p(t,e){return(e||document).querySelectorAll(t)}function f(t,e){return(e||document).querySelector(t)}function h(){return document.createDocumentFragment()}function d(){return document.createTextNode("")}function g(t,e){return e?document.createElementNS("http://www.w3.org/2000/svg","svg"):document.createElement(t)}function m(t){if(t.outerHTML)return t.outerHTML;var e=g("div");return e.appendChild(t.cloneNode(!0)),e.innerHTML}function v(t,e){if(o(t.innerHTML)){var n=(new DOMParser).parseFromString(e,"application/xml"),r=t.ownerDocument.importNode(n.documentElement,!0);t.appendChild(r)}else t.innerHTML=e}function y(t,e){t.removeAttribute(e)}function x(t,e){return t.getAttribute(e)}function b(t,e,n){var r=Pt.exec(e);r&&r[1]?t.setAttributeNS(Ht,r[1],n):t.setAttribute(e,n)}function _(t,e,n){t.insertBefore(e,n.parentNode&&n)}function w(t,e){if(t)for(var n;n=zt.exec(t);)e(n[1].toLowerCase(),n[2]||n[3]||n[4])}function N(t,e,n){if(t){var r,i=e(t,n);if(i===!1)return;for(t=t.firstChild;t;)r=t.nextSibling,N(t,e,i),t=r}}function C(t,e){for(var n,r=t?t.length:0,i=0;ir;)n--,P.apply(e[n],[e,n])}function P(t,e){t.splice(e,1),this.unmount(),ft(this.parent,this,this.__.tagName,!0)}function $(t){var e=this;C(Object.keys(this.tags),function(n){var r=e.tags[n];u(r)?C(r,function(e){ot.apply(e,[n,t])}):ot.apply(r,[n,t])})}function B(t,e,n){n?vt.apply(this,[t,e]):_(t,this.root,e.root)}function V(t,e,n){n?mt.apply(this,[t,e]):_(t,this.root,e.root)}function U(t,e){e?mt.call(this,t):t.appendChild(this.root)}function D(t,e,n){y(t,Lt);var r,i=typeof x(t,Mt)!==Rt||y(t,Mt),o=lt(t),s=wt[o],l=t.parentNode,c=d(),p=rt(t),f=x(t,Et),g=[],m=[],v=!0,b=!wt[o],_="VIRTUAL"===t.tagName;return n=te.loopKeys(n),n.isLoop=!0,f&&y(t,Et),l.insertBefore(c,t),l.removeChild(t),n.update=function(){var l=te(n.val,e),d=h(),y=!u(l)&&!a(l),x=c.parentNode;y?(r=l||!1,l=r?Object.keys(l).map(function(t){return I(n,l[t],t)}):[]):r=!1,f&&(l=l.filter(function(t,r){return n.key&&!y?!!te(f,I(n,t,r,e)):!!te(f,M(Object.create(e),t))})),C(l,function(a,u){var c=i&&typeof a===St&&!r,f=m.indexOf(a),h=f===-1,y=!h&&c?f:u,w=g[y],N=u>=m.length,C=c&&h||!c&&!w;a=!r&&n.key?I(n,a,u):a,C?(w=new nt(s,{parent:e,isLoop:v,isAnonymous:b,tagName:o,root:t.cloneNode(b),item:a,index:u},t.innerHTML),w.mount(),N?U.apply(w,[d||x,_]):V.apply(w,[x,g[u],_]),N||m.splice(u,0,a),g.splice(u,0,w),p&&pt(e.tags,o,w,!0)):y!==u&&c&&(O(l,m[y])?(B.apply(w,[x,g[u],_]),g.splice(u,0,g.splice(y,1)[0]),m.splice(u,0,m.splice(y,1)[0])):(P.apply(g[u],[g,u]),m.splice(u,1)),n.pos&&(w[n.pos]=u),!p&&w.tags&&$.call(w,u)),w.__.item=a,w.__.index=u,w.__.parent=e,C||w.update(a)}),H(l,g),m=l.slice(),x.insertBefore(d,c)},n.unmount=function(){C(g,function(t){t.unmount()})},n}function z(t,e,n){var r=this,i={parent:{children:e}};return N(t,function(e,i){var o,a,s,u=e.nodeType,l=i.parent;if(!n&&e===t)return{parent:l};if(3===u&&"STYLE"!==e.parentNode.tagName&&te.hasExpr(e.nodeValue)&&l.children.push({dom:e,expr:e.nodeValue}),1!==u)return i;var c="VIRTUAL"===e.tagName;if(o=x(e,Lt))return c&&b(e,"loopVirtual",!0),l.children.push(D(e,r,o)),!1;if(o=x(e,Et))return l.children.push(Object.create(oe).init(e,r,o)),!1;if((a=x(e,Tt))&&te.hasExpr(a))return l.children.push({isRtag:!0,expr:a,dom:e,attrs:[].slice.call(e.attributes)}),!1;if(s=rt(e),c&&(x(e,"virtualized")&&e.parentElement.removeChild(e),s||x(e,"virtualized")||x(e,"loopVirtual")||(s={tmpl:e.outerHTML})),s&&(e!==t||n)){if(!c||x(e,Tt)){var p={root:e,parent:r,hasImpl:!0};return l.children.push(at(s,p,e.innerHTML,r)),!1}b(e,"virtualized",!0);var f=new nt({tmpl:e.outerHTML},{root:e,parent:r},e.innerHTML);l.children.push(f)}return F.apply(r,[e,e.attributes,function(t,e){e&&l.children.push(e)}]),{parent:l}},i),{tree:i,root:t}}function F(t,e,r){var i=this;C(e,function(e){var o,a=e.name,s=n(a);O(Ot,a)?o=Object.create(ae).init(t,i,a,e.value):te.hasExpr(e.value)&&(o={dom:t,expr:e.value,attr:e.name,bool:s}),r(e,o)})}function G(t,e,n){var r="o"===n[0],i=r?"select>":"table>";if(t.innerHTML="<"+i+e.trim()+""+i,i=t.firstChild,r)i.selectedIndex=-1;else{var o=pe[n];o&&1===i.childElementCount&&(i=f(o,i))}return i}function K(t,e){if(!se.test(t))return t;var n={};return e=e&&e.replace(le,function(t,e,r){return n[e]=n[e]||r,""}).trim(),t.replace(ce,function(t,e,r){return n[e]||r||""}).replace(ue,function(t,n){return e||n||""})}function q(t,n,r){var i=t&&t.match(/^\s*<([-\w]+)/),o=i&&i[1].toLowerCase(),a=g(he,r&&e(o));return t=K(t,n),fe.test(o)?a=G(a,t,o):v(a,t),a}function Z(t,e){var n=this,r=n.name,i=n.tmpl,o=n.css,a=n.attrs,s=n.onCreate;return wt[r]||(Q(r,i,o,a,s),wt[r].class=this.constructor),dt(t,r,e,this),o&&Yt.inject(),this}function Q(t,e,n,i,o){return r(i)&&(o=i,/^[\w\-]+\s?=/.test(n)?(i=n,n=""):i=""),n&&(r(n)?o=n:Yt.add(n)),t=t.toLowerCase(),wt[t]={name:t,tmpl:e,attrs:i,fn:o},t}function W(t,e,n,r,i){n&&Yt.add(n,t);var o=!!wt[t];return wt[t]={name:t,tmpl:e,attrs:r,fn:i},o&&_e.hotReloader&&_e.hotReloader(t),t}function X(t,e,n){function r(t){if(t.tagName){var i=x(t,Tt);e&&i!==e&&(i=e,b(t,Tt,e));var a=dt(t,i||t.tagName.toLowerCase(),n);a&&o.push(a)}else t.length&&C(t,r)}var o=[];Yt.inject(),i(e)&&(n=e,e=0);var s,u;if(a(t)?(t="*"===t?u=yt():t+yt(t.split(/, */)),s=t?p(t):[]):s=t,"*"===e){if(e=u||yt(),s.tagName)s=p(e,s);else{var l=[];C(s,function(t){return l.push(p(e,t))}),s=l}e=0}return r(s),o}function Y(t,e,n){if(i(t))return void Y("__unnamed_"+me++,t,!0);var a=n?ge:de;if(!e){if(o(a[t]))throw new Error("Unregistered mixin: "+t);return a[t]}a[t]=r(e)?M(e.prototype,a[t]||{})&&e:M(a[t]||{},e)}function J(){return C(_t,function(t){return t.update()})}function tt(t){delete wt[t]}function et(t,e,n,r,i){if(!t||!n){var o=!n&&t?this:e||this;C(i,function(t){t.expr&&k.call(o,[t.expr]),r[T(t.name)]=t.expr?t.expr.value:t.value})}}function nt(t,e,n){void 0===t&&(t={}),void 0===e&&(e={});var i,o=M({},e.opts),s=e.parent,u=e.isLoop,l=!!e.isAnonymous,c=re.skipAnonymousTags&&l,p=ct(e.item),f=e.index,h=[],d=[],g=[],m=e.root,v=e.tagName||lt(m),x="virtual"===v,_=[];c||ee(this),t.name&&m._tag&&m._tag.unmount(!0),this.isMounted=!1,L(this,"__",{isAnonymous:l,instAttrs:h,innerHTML:n,tagName:v,index:f,isLoop:u,virts:[],tail:null,head:null,parent:null,item:null}),L(this,"_riot_id",++ye),L(this,"root",m),M(this,{opts:o},p),L(this,"parent",s||null),L(this,"tags",{}),L(this,"refs",{}),i=u&&l?m:q(t.tmpl,n,u),L(this,"update",function(t){var e={},n=this.isMounted&&!c;return et.apply(this,[u,s,l,e,h]),this.isMounted&&r(this.shouldUpdate)&&!this.shouldUpdate(t,e)?this:(t=ct(t),u&&l&&it.apply(this,[this.parent,_]),M(this,t),M(o,e),n&&this.trigger("update",t),k.call(this,g),n&&this.trigger("updated"),this)}.bind(this)),L(this,"mixin",function(){var t=this;return C(arguments,function(e){var n,i,o=[];e=a(e)?Y(e):e,n=r(e)?new e:e;var s=Object.getPrototypeOf(n);do o=o.concat(Object.getOwnPropertyNames(i||n));while(i=Object.getPrototypeOf(i||n));C(o,function(e){if("init"!==e){var i=Object.getOwnPropertyDescriptor(n,e)||Object.getOwnPropertyDescriptor(s,e),o=i&&(i.get||i.set);!t.hasOwnProperty(e)&&o?Object.defineProperty(t,e,i):t[e]=r(n[e])?n[e].bind(t):n[e]}}),n.init&&n.init.bind(t)()}),this}.bind(this)),L(this,"mount",function(){var e=this,n=this.__.parent;m._tag=this,F.apply(s,[m,m.attributes,function(t,n){!l&&ae.isPrototypeOf(n)&&(n.tag=e),t.expr=n,h.push(t)}]),d=[],w(t.attrs,function(t,e){d.push({name:t,value:e})}),F.apply(this,[m,d,function(t,e){e?g.push(e):b(m,t.name,t.value)}]),n&&l&&it.apply(this,[n,_]),et.apply(this,[u,s,l,o,h]);var r=Y(Nt);if(r&&!c)for(var a in r)r.hasOwnProperty(a)&&e.mixin(r[a]);if(t.fn&&t.fn.call(this,o),c||this.trigger("before-mount"),z.apply(this,[i,g,l]),this.update(p),!l)for(;i.firstChild;)m.appendChild(i.firstChild);if(L(this,"root",m),L(this,"isMounted",!0),!c){if(this.parent){var f=st(this.parent);f.one(f.isMounted?"updated":"mount",function(){e.trigger("mount")})}else this.trigger("mount");return this}}.bind(this)),L(this,"unmount",function(e){var n,r=this,i=this.root,o=i.parentNode,a=_t.indexOf(this);if(c||this.trigger("before-unmount"),w(t.attrs,function(t){E(t,Ct)&&(t=t.slice(Ct.length)),y(m,t)}),a!==-1&&_t.splice(a,1),o||x){if(s)n=st(s),x?Object.keys(this.tags).forEach(function(t){ft(n.tags,t,r.tags[t])}):(ft(n.tags,v,this),s!==n&&ft(s.tags,v,this));else for(;i.firstChild;)i.removeChild(i.firstChild);o&&(e?y(o,Tt):o.removeChild(i))}return this.__.virts&&C(this.__.virts,function(t){t.parentNode&&t.parentNode.removeChild(t)}),ut(g),C(h,function(t){return t.expr&&t.expr.unmount&&t.expr.unmount()}),this.__.onUnmount&&this.__.onUnmount(),c||(this.trigger("unmount"),this.off("*")),L(this,"isMounted",!1),delete this.root._tag,this}.bind(this))}function rt(t){return t.tagName&&wt[x(t,Tt)||x(t,Tt)||t.tagName.toLowerCase()]}function it(t,e){var n=this;C(Object.keys(t),function(r){var i=!c(r)&&O(e,r);(o(n[r])||i)&&(i||e.push(r),n[r]=t[r])})}function ot(t,e){var n,r=this.parent;r&&(n=r.tags[t],u(n)?n.splice(e,0,n.splice(n.indexOf(this),1)[0]):pt(r.tags,t,this))}function at(t,e,n,r){var i=new nt(t,e,n),o=e.tagName||lt(e.root,!0),a=st(r);return L(i,"parent",a),i.__.parent=r,pt(a.tags,o,i),a!==r&&pt(r.tags,o,i),e.root.innerHTML="",i}function st(t){for(var e=t;e.__.isAnonymous&&e.parent;)e=e.parent;return e}function ut(t){C(t,function(t){t instanceof nt?t.unmount(!0):t.unmount&&t.unmount()})}function lt(t,e){var n=rt(t),r=!e&&x(t,Tt);return r&&!te.hasExpr(r)?r:n?n.name:t.tagName.toLowerCase()}function ct(t){if(!(t instanceof nt||t&&r(t.trigger)))return t;var e={};for(var n in t)Ut.test(n)||(e[n]=t[n]);return e}function pt(t,e,n,r,i){var a=t[e],s=u(a),l=!o(i);if(!a||a!==n)if(!a&&r)t[e]=[n];else if(a)if(s){var c=a.indexOf(n);if(c===i)return;c!==-1&&a.splice(c,1),l?a.splice(i,0,n):a.push(n)}else t[e]=[a,n];else t[e]=n}function ft(t,e,n,r){if(u(t[e])){var i=t[e].indexOf(n);i!==-1&&t[e].splice(i,1),t[e].length?1!==t[e].length||r||(t[e]=t[e][0]):delete t[e]}else delete t[e]}function ht(t){for(;t;){if(t.inStub)return!0;t=t.parentNode}return!1}function dt(t,e,n,r){var i=wt[e],o=wt[e].class,a=r||(o?Object.create(o.prototype):{}),s=t._innerHTML=t._innerHTML||t.innerHTML;t.innerHTML="";var u={root:t,opts:n};return n&&n.parent&&(u.parent=n.parent),i&&t&&nt.apply(a,[i,u,s]),a&&a.mount&&(a.mount(!0),O(_t,a)||_t.push(a)),a}function gt(t,e){var n=h();mt.call(t,n),e.parentNode.replaceChild(n,e)}function mt(t,e){var n,r,i=this,o=d(),a=d(),s=h();for(this.__.head=this.root.insertBefore(o,this.root.firstChild),this.__.tail=this.root.appendChild(a),r=this.__.head;r;)n=r.nextSibling,s.appendChild(r),i.__.virts.push(r),r=n;e?t.insertBefore(s,e.__.head):t.appendChild(s)}function vt(t,e){for(var n,r=this,i=this.__.head,o=h();i;)if(n=i.nextSibling,o.appendChild(i),i=n,i===r.__.tail){o.appendChild(i),t.insertBefore(o,e.__.head);break}}function yt(t){if(!t){var e=Object.keys(wt);return e+yt(e)}return t.filter(function(t){return!/[^-\w]/.test(t)}).reduce(function(t,e){var n=e.trim().toLowerCase();return t+",["+Tt+'="'+n+'"]'},"")}var xt,bt,_t=[],wt={},Nt="__global_mixin",Ct="riot-",Ot=["ref","data-ref"],Tt="data-is",Et="if",Lt="each",Mt="no-reorder",jt="show",At="hide",Rt="string",St="object",kt="undefined",It="function",Ht="http://www.w3.org/1999/xlink",Pt=/^xlink:(\w+)/,$t=typeof window===kt?void 0:window,Bt=/^(?:t(?:body|head|foot|[rhd])|caption|col(?:group)?|opt(?:ion|group))$/,Vt=/^(?:t(?:body|head|foot|[rhd])|caption|col(?:group)?)$/,Ut=/^(?:_(?:item|id|parent)|update|root|(?:un)?mount|mixin|is(?:Mounted|Loop)|tags|refs|parent|opts|trigger|o(?:n|ff|ne))$/,Dt=/^(altGlyph|animate(?:Color)?|circle|clipPath|defs|ellipse|fe(?:Blend|ColorMatrix|ComponentTransfer|Composite|ConvolveMatrix|DiffuseLighting|DisplacementMap|Flood|GaussianBlur|Image|Merge|Morphology|Offset|SpecularLighting|Tile|Turbulence)|filter|font|foreignObject|g(?:lyph)?(?:Ref)?|image|line(?:arGradient)?|ma(?:rker|sk)|missing-glyph|path|pattern|poly(?:gon|line)|radialGradient|rect|stop|svg|switch|symbol|text(?:Path)?|tref|tspan|use)$/,zt=/([-\w]+) ?= ?(?:"([^"]*)|'([^']*)|({[^}]*}))/g,Ft={viewbox:"viewBox"},Gt=/^(?:disabled|checked|readonly|required|allowfullscreen|auto(?:focus|play)|compact|controls|default|formnovalidate|hidden|ismap|itemscope|loop|multiple|muted|no(?:resize|shade|validate|wrap)?|open|reversed|seamless|selected|sortable|truespeed|typemustmatch)$/,Kt=0|($t&&$t.document||{}).documentMode,qt=Object.freeze({isSVGTag:e,isBoolAttr:n,isFunction:r,isObject:i,isUndefined:o,isString:a,isBlank:s,isArray:u,isWritable:l,isReservedName:c}),Zt=Object.freeze({$$:p,$:f,createFrag:h,createDOMPlaceholder:d,mkEl:g,getOuterHTML:m,setInnerHTML:v,remAttr:y,getAttr:x,setAttr:b,safeInsert:_,walkAttrs:w,walkNodes:N}),Qt={},Wt=[],Xt=!1;$t&&(xt=function(){var t=g("style");b(t,"type","text/css");var e=f("style[type=riot]");return e?(e.id&&(t.id=e.id),e.parentNode.replaceChild(t,e)):document.getElementsByTagName("head")[0].appendChild(t),t}(),bt=xt.styleSheet);var Yt={styleNode:xt,add:function(t,e){e?Qt[e]=t:Wt.push(t),Xt=!0},inject:function(){if($t&&Xt){Xt=!1;var t=Object.keys(Qt).map(function(t){return Qt[t]}).concat(Wt).join("\n");bt?bt.cssText=t:xt.innerHTML=t}}},Jt=function(t){function e(t){return t}function n(t,e){return e||(e=x),new RegExp(t.source.replace(/{/g,e[2]).replace(/}/g,e[3]),t.global?l:"")}function r(t){if(t===m)return v;var e=t.split(" ");if(2!==e.length||h.test(t))throw new Error('Unsupported brackets "'+t+'"');return e=e.concat(t.replace(d,"\\").split(" ")),e[4]=n(e[1].length>1?/{[\S\s]*?}/:v[4],e),e[5]=n(t.length>3?/\\({|})/g:v[5],e),e[6]=n(v[6],e),e[7]=RegExp("\\\\("+e[3]+")|([[({])|("+e[3]+")|"+f,l),e[8]=t,e}function i(t){return t instanceof RegExp?s(t):x[t]}function o(t){(t||(t=m))!==x[8]&&(x=r(t),s=t===m?e:n,x[9]=s(v[9])),y=t}function a(t){var e;t=t||{},e=t.brackets,Object.defineProperty(t,"brackets",{set:o,get:function(){return y},enumerable:!0}),u=t,o(e)}var s,u,l="g",c=/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//g,p=/"[^"\\]*(?:\\[\S\s][^"\\]*)*"|'[^'\\]*(?:\\[\S\s][^'\\]*)*'/g,f=p.source+"|"+/(?:\breturn\s+|(?:[$\w\)\]]|\+\+|--)\s*(\/)(?![*\/]))/.source+"|"+/\/(?=[^*\/])[^[\/\\]*(?:(?:\[(?:\\.|[^\]\\]*)*\]|\\.)[^[\/\\]*)*?(\/)[gim]*/.source,h=RegExp("[\\x00-\\x1F<>a-zA-Z0-9'\",;\\\\]"),d=/(?=[[\]()*+?.^$|])/g,g={"(":RegExp("([()])|"+f,l),"[":RegExp("([[\\]])|"+f,l),"{":RegExp("([{}])|"+f,l)},m="{ }",v=["{","}","{","}",/{[^}]*}/,/\\([{}])/g,/\\({)|{/g,RegExp("\\\\(})|([[({])|(})|"+f,l),m,/^\s*{\^?\s*([$\w]+)(?:\s*,\s*(\S+))?\s+in\s+(\S.*)\s*}/,/(^|[^\\]){=[\S\s]*?}/],y=t,x=[];return i.split=function(t,e,n){function r(t){e||a?l.push(t&&t.replace(n[5],"$1")):l.push(t)}function i(t,e,n){var r,i=g[e];for(i.lastIndex=n,n=1;(r=i.exec(t))&&(!r[1]||(r[1]===e?++n:--n)););return n?t.length:i.lastIndex}n||(n=x);var o,a,s,u,l=[],c=n[6];for(a=s=c.lastIndex=0;o=c.exec(t);){if(u=o.index,a){if(o[2]){c.lastIndex=i(t,o[2],c.lastIndex);continue}if(!o[3])continue}o[1]||(r(t.slice(s,u)),s=c.lastIndex,c=n[6+(a^=1)],c.lastIndex=s)}return t&&s tag",e.riotData.tagName),console.error(e))}function n(t){var e=r(t);return"try{return "!==e.slice(0,11)&&(e="return "+e),new Function("E",e+";")}function r(t){var e,n=[],r=Jt.split(t.replace(c,'"'),1);if(r.length>2||r[0]){var o,a,s=[];for(o=a=0;o2&&!e?s+(n.push(t)-1)+"~":t}).replace(/\s+/g," ").trim().replace(/\ ?([[\({},?\.:])\ ?/g,"$1")){for(var i,a=[],c=0;t&&(i=t.match(u))&&!i.index;){var p,h,d=/,|([[{(])|$/g;for(t=RegExp.rightContext,p=i[2]?n[i[2]].slice(1,-1).trim().replace(/\s+/g," "):i[1];h=(i=d.exec(t))[1];)r(h,d);h=t.slice(0,i.index),t=RegExp.rightContext,a[c++]=o(h,1,p)}t=c?c>1?"["+a.join(",")+'].join(" ").trim()':a[0]:o(t,e)}return t}function o(t,e,n){var r;return t=t.replace(d,function(t,e,n,i,o){return n&&(i=r?0:i+t.length,"this"!==n&&"global"!==n&&"window"!==n?(t=e+'("'+n+h+n,i&&(r="."===(o=o[i])||"("===o||"["===o)):i&&(r=!g.test(o.slice(i)))),t}),r&&(t="try{return "+t+"}catch(e){E(e,this)}"),n?t=(r?"function(){"+t+"}.call(this)":"("+t+")")+'?"'+n+'":""':e&&(t="function(v){"+(r?t.replace("return ","v="):"v=("+t+")")+';return v||v===0?v:""}.call(this)'),t}var a={};t.hasExpr=Jt.hasExpr,t.loopKeys=Jt.loopKeys,t.clearCache=function(){a={}},t.errorHandler=null;var s=String.fromCharCode(8279),u=/^(?:(-?[_A-Za-z\xA0-\xFF][-\w\xA0-\xFF]*)|\u2057(\d+)~):/,l=RegExp(Jt.S_QBLOCKS,"g"),c=/\u2057/g,p=/\u2057(\d+)~/g,f={"(":/[()]/g,"[":/[[\]]/g,"{":/[{}]/g},h='"in this?this:'+("object"!=typeof window?"global":"window")+").",d=/[,{][\$\w]+(?=:)|(^ *|[^$\w\.{])(?!(?:typeof|true|false|null|undefined|in|instanceof|is(?:Finite|NaN)|void|NaN|new|Date|RegExp|Math)(?![$\w]))([$_A-Za-z][$\w]*)/g,g=/^(?=(\.[$\w]+))\1(?:[^.[(]|$)/;return t.version=Jt.version="v3.0.2",t}(),ee=function(t){t=t||{};var e={},n=Array.prototype.slice;return Object.defineProperties(t,{on:{value:function(n,r){return"function"==typeof r&&(e[n]=e[n]||[]).push(r),t},enumerable:!1,writable:!1,configurable:!1},off:{value:function(n,r){if("*"!=n||r)if(r)for(var i,o=e[n],a=0;i=o&&o[a];++a)i==r&&o.splice(a--,1);else delete e[n];else e={};return t},enumerable:!1,writable:!1,configurable:!1},one:{value:function(e,n){function r(){t.off(e,r),n.apply(t,arguments)}return t.on(e,r)},enumerable:!1,writable:!1,configurable:!1},trigger:{value:function(r){var i,o,a,s=arguments,u=arguments.length-1,l=new Array(u);for(a=0;a|>([\S\s]*?)<\/yield\s*>|>)/gi,le=/]*)['"]\s*>([\S\s]*?)<\/yield\s*>/gi,ce=/|>([\S\s]*?)<\/yield\s*>)/gi,pe={tr:"tbody",th:"tr",td:"tr",col:"colgroup"},fe=Kt&&Kt<10?Bt:Vt,he="div",de={},ge=de[Nt]={},me=0,ve=Object.freeze({Tag:Z,tag:Q,tag2:W,mount:X,mixin:Y,update:J,unregister:tt}),ye=0,xe=Object.freeze({getTag:rt,inheritFrom:it,moveChildTag:ot,initChildTag:at,getImmediateCustomParentTag:st,unmountAll:ut,getTagName:lt,cleanUpData:ct,arrayishAdd:pt,arrayishRemove:ft,isInStub:ht,mountTo:dt,makeReplaceVirtual:gt,makeVirtual:mt,moveVirtual:vt,selectTags:yt}),be=re,_e={tmpl:te,brackets:Jt,styleManager:Yt,vdom:_t,styleNode:Yt.styleNode,dom:Zt,check:qt,misc:ne,tags:xe},we=Z,Ne=Q,Ce=W,Oe=X,Te=Y,Ee=J,Le=tt,Me=ee,je=M({},ve,{observable:ee,settings:be,util:_e});t.settings=be,t.util=_e,t.Tag=we,t.tag=Ne,t.tag2=Ce,t.mount=Oe,t.mixin=Te,t.update=Ee,t.unregister=Le,t.observable=Me,t.default=je,Object.defineProperty(t,"__esModule",{value:!0})});
3 |
--------------------------------------------------------------------------------
/static/js/zepto.min.js:
--------------------------------------------------------------------------------
1 | /* Zepto v1.2.0 - zepto event ajax form ie - zeptojs.com/license */
2 | !function(t,e){"function"==typeof define&&define.amd?define(function(){return e(t)}):e(t)}(this,function(t){var e=function(){function $(t){return null==t?String(t):S[C.call(t)]||"object"}function F(t){return"function"==$(t)}function k(t){return null!=t&&t==t.window}function M(t){return null!=t&&t.nodeType==t.DOCUMENT_NODE}function R(t){return"object"==$(t)}function Z(t){return R(t)&&!k(t)&&Object.getPrototypeOf(t)==Object.prototype}function z(t){var e=!!t&&"length"in t&&t.length,n=r.type(t);return"function"!=n&&!k(t)&&("array"==n||0===e||"number"==typeof e&&e>0&&e-1 in t)}function q(t){return a.call(t,function(t){return null!=t})}function H(t){return t.length>0?r.fn.concat.apply([],t):t}function I(t){return t.replace(/::/g,"/").replace(/([A-Z]+)([A-Z][a-z])/g,"$1_$2").replace(/([a-z\d])([A-Z])/g,"$1_$2").replace(/_/g,"-").toLowerCase()}function V(t){return t in l?l[t]:l[t]=new RegExp("(^|\\s)"+t+"(\\s|$)")}function _(t,e){return"number"!=typeof e||h[I(t)]?e:e+"px"}function B(t){var e,n;return c[t]||(e=f.createElement(t),f.body.appendChild(e),n=getComputedStyle(e,"").getPropertyValue("display"),e.parentNode.removeChild(e),"none"==n&&(n="block"),c[t]=n),c[t]}function U(t){return"children"in t?u.call(t.children):r.map(t.childNodes,function(t){return 1==t.nodeType?t:void 0})}function X(t,e){var n,r=t?t.length:0;for(n=0;r>n;n++)this[n]=t[n];this.length=r,this.selector=e||""}function J(t,r,i){for(n in r)i&&(Z(r[n])||L(r[n]))?(Z(r[n])&&!Z(t[n])&&(t[n]={}),L(r[n])&&!L(t[n])&&(t[n]=[]),J(t[n],r[n],i)):r[n]!==e&&(t[n]=r[n])}function W(t,e){return null==e?r(t):r(t).filter(e)}function Y(t,e,n,r){return F(e)?e.call(t,n,r):e}function G(t,e,n){null==n?t.removeAttribute(e):t.setAttribute(e,n)}function K(t,n){var r=t.className||"",i=r&&r.baseVal!==e;return n===e?i?r.baseVal:r:void(i?r.baseVal=n:t.className=n)}function Q(t){try{return t?"true"==t||("false"==t?!1:"null"==t?null:+t+""==t?+t:/^[\[\{]/.test(t)?r.parseJSON(t):t):t}catch(e){return t}}function tt(t,e){e(t);for(var n=0,r=t.childNodes.length;r>n;n++)tt(t.childNodes[n],e)}var e,n,r,i,O,P,o=[],s=o.concat,a=o.filter,u=o.slice,f=t.document,c={},l={},h={"column-count":1,columns:1,"font-weight":1,"line-height":1,opacity:1,"z-index":1,zoom:1},p=/^\s*<(\w+|!)[^>]*>/,d=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,m=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,g=/^(?:body|html)$/i,v=/([A-Z])/g,y=["val","css","html","text","data","width","height","offset"],x=["after","prepend","before","append"],b=f.createElement("table"),E=f.createElement("tr"),j={tr:f.createElement("tbody"),tbody:b,thead:b,tfoot:b,td:E,th:E,"*":f.createElement("div")},w=/complete|loaded|interactive/,T=/^[\w-]*$/,S={},C=S.toString,N={},A=f.createElement("div"),D={tabindex:"tabIndex",readonly:"readOnly","for":"htmlFor","class":"className",maxlength:"maxLength",cellspacing:"cellSpacing",cellpadding:"cellPadding",rowspan:"rowSpan",colspan:"colSpan",usemap:"useMap",frameborder:"frameBorder",contenteditable:"contentEditable"},L=Array.isArray||function(t){return t instanceof Array};return N.matches=function(t,e){if(!e||!t||1!==t.nodeType)return!1;var n=t.matches||t.webkitMatchesSelector||t.mozMatchesSelector||t.oMatchesSelector||t.matchesSelector;if(n)return n.call(t,e);var r,i=t.parentNode,o=!i;return o&&(i=A).appendChild(t),r=~N.qsa(i,e).indexOf(t),o&&A.removeChild(t),r},O=function(t){return t.replace(/-+(.)?/g,function(t,e){return e?e.toUpperCase():""})},P=function(t){return a.call(t,function(e,n){return t.indexOf(e)==n})},N.fragment=function(t,n,i){var o,s,a;return d.test(t)&&(o=r(f.createElement(RegExp.$1))),o||(t.replace&&(t=t.replace(m,"<$1>$2>")),n===e&&(n=p.test(t)&&RegExp.$1),n in j||(n="*"),a=j[n],a.innerHTML=""+t,o=r.each(u.call(a.childNodes),function(){a.removeChild(this)})),Z(i)&&(s=r(o),r.each(i,function(t,e){y.indexOf(t)>-1?s[t](e):s.attr(t,e)})),o},N.Z=function(t,e){return new X(t,e)},N.isZ=function(t){return t instanceof N.Z},N.init=function(t,n){var i;if(!t)return N.Z();if("string"==typeof t)if(t=t.trim(),"<"==t[0]&&p.test(t))i=N.fragment(t,RegExp.$1,n),t=null;else{if(n!==e)return r(n).find(t);i=N.qsa(f,t)}else{if(F(t))return r(f).ready(t);if(N.isZ(t))return t;if(L(t))i=q(t);else if(R(t))i=[t],t=null;else if(p.test(t))i=N.fragment(t.trim(),RegExp.$1,n),t=null;else{if(n!==e)return r(n).find(t);i=N.qsa(f,t)}}return N.Z(i,t)},r=function(t,e){return N.init(t,e)},r.extend=function(t){var e,n=u.call(arguments,1);return"boolean"==typeof t&&(e=t,t=n.shift()),n.forEach(function(n){J(t,n,e)}),t},N.qsa=function(t,e){var n,r="#"==e[0],i=!r&&"."==e[0],o=r||i?e.slice(1):e,s=T.test(o);return t.getElementById&&s&&r?(n=t.getElementById(o))?[n]:[]:1!==t.nodeType&&9!==t.nodeType&&11!==t.nodeType?[]:u.call(s&&!r&&t.getElementsByClassName?i?t.getElementsByClassName(o):t.getElementsByTagName(e):t.querySelectorAll(e))},r.contains=f.documentElement.contains?function(t,e){return t!==e&&t.contains(e)}:function(t,e){for(;e&&(e=e.parentNode);)if(e===t)return!0;return!1},r.type=$,r.isFunction=F,r.isWindow=k,r.isArray=L,r.isPlainObject=Z,r.isEmptyObject=function(t){var e;for(e in t)return!1;return!0},r.isNumeric=function(t){var e=Number(t),n=typeof t;return null!=t&&"boolean"!=n&&("string"!=n||t.length)&&!isNaN(e)&&isFinite(e)||!1},r.inArray=function(t,e,n){return o.indexOf.call(e,t,n)},r.camelCase=O,r.trim=function(t){return null==t?"":String.prototype.trim.call(t)},r.uuid=0,r.support={},r.expr={},r.noop=function(){},r.map=function(t,e){var n,i,o,r=[];if(z(t))for(i=0;i=0?t:t+this.length]},toArray:function(){return this.get()},size:function(){return this.length},remove:function(){return this.each(function(){null!=this.parentNode&&this.parentNode.removeChild(this)})},each:function(t){return o.every.call(this,function(e,n){return t.call(e,n,e)!==!1}),this},filter:function(t){return F(t)?this.not(this.not(t)):r(a.call(this,function(e){return N.matches(e,t)}))},add:function(t,e){return r(P(this.concat(r(t,e))))},is:function(t){return this.length>0&&N.matches(this[0],t)},not:function(t){var n=[];if(F(t)&&t.call!==e)this.each(function(e){t.call(this,e)||n.push(this)});else{var i="string"==typeof t?this.filter(t):z(t)&&F(t.item)?u.call(t):r(t);this.forEach(function(t){i.indexOf(t)<0&&n.push(t)})}return r(n)},has:function(t){return this.filter(function(){return R(t)?r.contains(this,t):r(this).find(t).size()})},eq:function(t){return-1===t?this.slice(t):this.slice(t,+t+1)},first:function(){var t=this[0];return t&&!R(t)?t:r(t)},last:function(){var t=this[this.length-1];return t&&!R(t)?t:r(t)},find:function(t){var e,n=this;return e=t?"object"==typeof t?r(t).filter(function(){var t=this;return o.some.call(n,function(e){return r.contains(e,t)})}):1==this.length?r(N.qsa(this[0],t)):this.map(function(){return N.qsa(this,t)}):r()},closest:function(t,e){var n=[],i="object"==typeof t&&r(t);return this.each(function(r,o){for(;o&&!(i?i.indexOf(o)>=0:N.matches(o,t));)o=o!==e&&!M(o)&&o.parentNode;o&&n.indexOf(o)<0&&n.push(o)}),r(n)},parents:function(t){for(var e=[],n=this;n.length>0;)n=r.map(n,function(t){return(t=t.parentNode)&&!M(t)&&e.indexOf(t)<0?(e.push(t),t):void 0});return W(e,t)},parent:function(t){return W(P(this.pluck("parentNode")),t)},children:function(t){return W(this.map(function(){return U(this)}),t)},contents:function(){return this.map(function(){return this.contentDocument||u.call(this.childNodes)})},siblings:function(t){return W(this.map(function(t,e){return a.call(U(e.parentNode),function(t){return t!==e})}),t)},empty:function(){return this.each(function(){this.innerHTML=""})},pluck:function(t){return r.map(this,function(e){return e[t]})},show:function(){return this.each(function(){"none"==this.style.display&&(this.style.display=""),"none"==getComputedStyle(this,"").getPropertyValue("display")&&(this.style.display=B(this.nodeName))})},replaceWith:function(t){return this.before(t).remove()},wrap:function(t){var e=F(t);if(this[0]&&!e)var n=r(t).get(0),i=n.parentNode||this.length>1;return this.each(function(o){r(this).wrapAll(e?t.call(this,o):i?n.cloneNode(!0):n)})},wrapAll:function(t){if(this[0]){r(this[0]).before(t=r(t));for(var e;(e=t.children()).length;)t=e.first();r(t).append(this)}return this},wrapInner:function(t){var e=F(t);return this.each(function(n){var i=r(this),o=i.contents(),s=e?t.call(this,n):t;o.length?o.wrapAll(s):i.append(s)})},unwrap:function(){return this.parent().each(function(){r(this).replaceWith(r(this).children())}),this},clone:function(){return this.map(function(){return this.cloneNode(!0)})},hide:function(){return this.css("display","none")},toggle:function(t){return this.each(function(){var n=r(this);(t===e?"none"==n.css("display"):t)?n.show():n.hide()})},prev:function(t){return r(this.pluck("previousElementSibling")).filter(t||"*")},next:function(t){return r(this.pluck("nextElementSibling")).filter(t||"*")},html:function(t){return 0 in arguments?this.each(function(e){var n=this.innerHTML;r(this).empty().append(Y(this,t,e,n))}):0 in this?this[0].innerHTML:null},text:function(t){return 0 in arguments?this.each(function(e){var n=Y(this,t,e,this.textContent);this.textContent=null==n?"":""+n}):0 in this?this.pluck("textContent").join(""):null},attr:function(t,r){var i;return"string"!=typeof t||1 in arguments?this.each(function(e){if(1===this.nodeType)if(R(t))for(n in t)G(this,n,t[n]);else G(this,t,Y(this,r,e,this.getAttribute(t)))}):0 in this&&1==this[0].nodeType&&null!=(i=this[0].getAttribute(t))?i:e},removeAttr:function(t){return this.each(function(){1===this.nodeType&&t.split(" ").forEach(function(t){G(this,t)},this)})},prop:function(t,e){return t=D[t]||t,1 in arguments?this.each(function(n){this[t]=Y(this,e,n,this[t])}):this[0]&&this[0][t]},removeProp:function(t){return t=D[t]||t,this.each(function(){delete this[t]})},data:function(t,n){var r="data-"+t.replace(v,"-$1").toLowerCase(),i=1 in arguments?this.attr(r,n):this.attr(r);return null!==i?Q(i):e},val:function(t){return 0 in arguments?(null==t&&(t=""),this.each(function(e){this.value=Y(this,t,e,this.value)})):this[0]&&(this[0].multiple?r(this[0]).find("option").filter(function(){return this.selected}).pluck("value"):this[0].value)},offset:function(e){if(e)return this.each(function(t){var n=r(this),i=Y(this,e,t,n.offset()),o=n.offsetParent().offset(),s={top:i.top-o.top,left:i.left-o.left};"static"==n.css("position")&&(s.position="relative"),n.css(s)});if(!this.length)return null;if(f.documentElement!==this[0]&&!r.contains(f.documentElement,this[0]))return{top:0,left:0};var n=this[0].getBoundingClientRect();return{left:n.left+t.pageXOffset,top:n.top+t.pageYOffset,width:Math.round(n.width),height:Math.round(n.height)}},css:function(t,e){if(arguments.length<2){var i=this[0];if("string"==typeof t){if(!i)return;return i.style[O(t)]||getComputedStyle(i,"").getPropertyValue(t)}if(L(t)){if(!i)return;var o={},s=getComputedStyle(i,"");return r.each(t,function(t,e){o[e]=i.style[O(e)]||s.getPropertyValue(e)}),o}}var a="";if("string"==$(t))e||0===e?a=I(t)+":"+_(t,e):this.each(function(){this.style.removeProperty(I(t))});else for(n in t)t[n]||0===t[n]?a+=I(n)+":"+_(n,t[n])+";":this.each(function(){this.style.removeProperty(I(n))});return this.each(function(){this.style.cssText+=";"+a})},index:function(t){return t?this.indexOf(r(t)[0]):this.parent().children().indexOf(this[0])},hasClass:function(t){return t?o.some.call(this,function(t){return this.test(K(t))},V(t)):!1},addClass:function(t){return t?this.each(function(e){if("className"in this){i=[];var n=K(this),o=Y(this,t,e,n);o.split(/\s+/g).forEach(function(t){r(this).hasClass(t)||i.push(t)},this),i.length&&K(this,n+(n?" ":"")+i.join(" "))}}):this},removeClass:function(t){return this.each(function(n){if("className"in this){if(t===e)return K(this,"");i=K(this),Y(this,t,n,i).split(/\s+/g).forEach(function(t){i=i.replace(V(t)," ")}),K(this,i.trim())}})},toggleClass:function(t,n){return t?this.each(function(i){var o=r(this),s=Y(this,t,i,K(this));s.split(/\s+/g).forEach(function(t){(n===e?!o.hasClass(t):n)?o.addClass(t):o.removeClass(t)})}):this},scrollTop:function(t){if(this.length){var n="scrollTop"in this[0];return t===e?n?this[0].scrollTop:this[0].pageYOffset:this.each(n?function(){this.scrollTop=t}:function(){this.scrollTo(this.scrollX,t)})}},scrollLeft:function(t){if(this.length){var n="scrollLeft"in this[0];return t===e?n?this[0].scrollLeft:this[0].pageXOffset:this.each(n?function(){this.scrollLeft=t}:function(){this.scrollTo(t,this.scrollY)})}},position:function(){if(this.length){var t=this[0],e=this.offsetParent(),n=this.offset(),i=g.test(e[0].nodeName)?{top:0,left:0}:e.offset();return n.top-=parseFloat(r(t).css("margin-top"))||0,n.left-=parseFloat(r(t).css("margin-left"))||0,i.top+=parseFloat(r(e[0]).css("border-top-width"))||0,i.left+=parseFloat(r(e[0]).css("border-left-width"))||0,{top:n.top-i.top,left:n.left-i.left}}},offsetParent:function(){return this.map(function(){for(var t=this.offsetParent||f.body;t&&!g.test(t.nodeName)&&"static"==r(t).css("position");)t=t.offsetParent;return t})}},r.fn.detach=r.fn.remove,["width","height"].forEach(function(t){var n=t.replace(/./,function(t){return t[0].toUpperCase()});r.fn[t]=function(i){var o,s=this[0];return i===e?k(s)?s["inner"+n]:M(s)?s.documentElement["scroll"+n]:(o=this.offset())&&o[t]:this.each(function(e){s=r(this),s.css(t,Y(this,i,e,s[t]()))})}}),x.forEach(function(n,i){var o=i%2;r.fn[n]=function(){var n,a,s=r.map(arguments,function(t){var i=[];return n=$(t),"array"==n?(t.forEach(function(t){return t.nodeType!==e?i.push(t):r.zepto.isZ(t)?i=i.concat(t.get()):void(i=i.concat(N.fragment(t)))}),i):"object"==n||null==t?t:N.fragment(t)}),u=this.length>1;return s.length<1?this:this.each(function(e,n){a=o?n:n.parentNode,n=0==i?n.nextSibling:1==i?n.firstChild:2==i?n:null;var c=r.contains(f.documentElement,a);s.forEach(function(e){if(u)e=e.cloneNode(!0);else if(!a)return r(e).remove();a.insertBefore(e,n),c&&tt(e,function(e){if(!(null==e.nodeName||"SCRIPT"!==e.nodeName.toUpperCase()||e.type&&"text/javascript"!==e.type||e.src)){var n=e.ownerDocument?e.ownerDocument.defaultView:t;n.eval.call(n,e.innerHTML)}})})})},r.fn[o?n+"To":"insert"+(i?"Before":"After")]=function(t){return r(t)[n](this),this}}),N.Z.prototype=X.prototype=r.fn,N.uniq=P,N.deserializeValue=Q,r.zepto=N,r}();return t.Zepto=e,void 0===t.$&&(t.$=e),function(e){function h(t){return t._zid||(t._zid=n++)}function p(t,e,n,r){if(e=d(e),e.ns)var i=m(e.ns);return(a[h(t)]||[]).filter(function(t){return t&&(!e.e||t.e==e.e)&&(!e.ns||i.test(t.ns))&&(!n||h(t.fn)===h(n))&&(!r||t.sel==r)})}function d(t){var e=(""+t).split(".");return{e:e[0],ns:e.slice(1).sort().join(" ")}}function m(t){return new RegExp("(?:^| )"+t.replace(" "," .* ?")+"(?: |$)")}function g(t,e){return t.del&&!f&&t.e in c||!!e}function v(t){return l[t]||f&&c[t]||t}function y(t,n,i,o,s,u,f){var c=h(t),p=a[c]||(a[c]=[]);n.split(/\s/).forEach(function(n){if("ready"==n)return e(document).ready(i);var a=d(n);a.fn=i,a.sel=s,a.e in l&&(i=function(t){var n=t.relatedTarget;return!n||n!==this&&!e.contains(this,n)?a.fn.apply(this,arguments):void 0}),a.del=u;var c=u||i;a.proxy=function(e){if(e=T(e),!e.isImmediatePropagationStopped()){e.data=o;var n=c.apply(t,e._args==r?[e]:[e].concat(e._args));return n===!1&&(e.preventDefault(),e.stopPropagation()),n}},a.i=p.length,p.push(a),"addEventListener"in t&&t.addEventListener(v(a.e),a.proxy,g(a,f))})}function x(t,e,n,r,i){var o=h(t);(e||"").split(/\s/).forEach(function(e){p(t,e,n,r).forEach(function(e){delete a[o][e.i],"removeEventListener"in t&&t.removeEventListener(v(e.e),e.proxy,g(e,i))})})}function T(t,n){return(n||!t.isDefaultPrevented)&&(n||(n=t),e.each(w,function(e,r){var i=n[e];t[e]=function(){return this[r]=b,i&&i.apply(n,arguments)},t[r]=E}),t.timeStamp||(t.timeStamp=Date.now()),(n.defaultPrevented!==r?n.defaultPrevented:"returnValue"in n?n.returnValue===!1:n.getPreventDefault&&n.getPreventDefault())&&(t.isDefaultPrevented=b)),t}function S(t){var e,n={originalEvent:t};for(e in t)j.test(e)||t[e]===r||(n[e]=t[e]);return T(n,t)}var r,n=1,i=Array.prototype.slice,o=e.isFunction,s=function(t){return"string"==typeof t},a={},u={},f="onfocusin"in t,c={focus:"focusin",blur:"focusout"},l={mouseenter:"mouseover",mouseleave:"mouseout"};u.click=u.mousedown=u.mouseup=u.mousemove="MouseEvents",e.event={add:y,remove:x},e.proxy=function(t,n){var r=2 in arguments&&i.call(arguments,2);if(o(t)){var a=function(){return t.apply(n,r?r.concat(i.call(arguments)):arguments)};return a._zid=h(t),a}if(s(n))return r?(r.unshift(t[n],t),e.proxy.apply(null,r)):e.proxy(t[n],t);throw new TypeError("expected function")},e.fn.bind=function(t,e,n){return this.on(t,e,n)},e.fn.unbind=function(t,e){return this.off(t,e)},e.fn.one=function(t,e,n,r){return this.on(t,e,n,r,1)};var b=function(){return!0},E=function(){return!1},j=/^([A-Z]|returnValue$|layer[XY]$|webkitMovement[XY]$)/,w={preventDefault:"isDefaultPrevented",stopImmediatePropagation:"isImmediatePropagationStopped",stopPropagation:"isPropagationStopped"};e.fn.delegate=function(t,e,n){return this.on(e,t,n)},e.fn.undelegate=function(t,e,n){return this.off(e,t,n)},e.fn.live=function(t,n){return e(document.body).delegate(this.selector,t,n),this},e.fn.die=function(t,n){return e(document.body).undelegate(this.selector,t,n),this},e.fn.on=function(t,n,a,u,f){var c,l,h=this;return t&&!s(t)?(e.each(t,function(t,e){h.on(t,n,a,e,f)}),h):(s(n)||o(u)||u===!1||(u=a,a=n,n=r),(u===r||a===!1)&&(u=a,a=r),u===!1&&(u=E),h.each(function(r,o){f&&(c=function(t){return x(o,t.type,u),u.apply(this,arguments)}),n&&(l=function(t){var r,s=e(t.target).closest(n,o).get(0);return s&&s!==o?(r=e.extend(S(t),{currentTarget:s,liveFired:o}),(c||u).apply(s,[r].concat(i.call(arguments,1)))):void 0}),y(o,t,u,a,n,l||c)}))},e.fn.off=function(t,n,i){var a=this;return t&&!s(t)?(e.each(t,function(t,e){a.off(t,n,e)}),a):(s(n)||o(i)||i===!1||(i=n,n=r),i===!1&&(i=E),a.each(function(){x(this,t,i,n)}))},e.fn.trigger=function(t,n){return t=s(t)||e.isPlainObject(t)?e.Event(t):T(t),t._args=n,this.each(function(){t.type in c&&"function"==typeof this[t.type]?this[t.type]():"dispatchEvent"in this?this.dispatchEvent(t):e(this).triggerHandler(t,n)})},e.fn.triggerHandler=function(t,n){var r,i;return this.each(function(o,a){r=S(s(t)?e.Event(t):t),r._args=n,r.target=a,e.each(p(a,t.type||t),function(t,e){return i=e.proxy(r),r.isImmediatePropagationStopped()?!1:void 0})}),i},"focusin focusout focus blur load resize scroll unload click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select keydown keypress keyup error".split(" ").forEach(function(t){e.fn[t]=function(e){return 0 in arguments?this.bind(t,e):this.trigger(t)}}),e.Event=function(t,e){s(t)||(e=t,t=e.type);var n=document.createEvent(u[t]||"Events"),r=!0;if(e)for(var i in e)"bubbles"==i?r=!!e[i]:n[i]=e[i];return n.initEvent(t,r,!0),T(n)}}(e),function(e){function p(t,n,r){var i=e.Event(n);return e(t).trigger(i,r),!i.isDefaultPrevented()}function d(t,e,n,i){return t.global?p(e||r,n,i):void 0}function m(t){t.global&&0===e.active++&&d(t,null,"ajaxStart")}function g(t){t.global&&!--e.active&&d(t,null,"ajaxStop")}function v(t,e){var n=e.context;return e.beforeSend.call(n,t,e)===!1||d(e,n,"ajaxBeforeSend",[t,e])===!1?!1:void d(e,n,"ajaxSend",[t,e])}function y(t,e,n,r){var i=n.context,o="success";n.success.call(i,t,o,e),r&&r.resolveWith(i,[t,o,e]),d(n,i,"ajaxSuccess",[e,n,t]),b(o,e,n)}function x(t,e,n,r,i){var o=r.context;r.error.call(o,n,e,t),i&&i.rejectWith(o,[n,e,t]),d(r,o,"ajaxError",[n,r,t||e]),b(e,n,r)}function b(t,e,n){var r=n.context;n.complete.call(r,e,t),d(n,r,"ajaxComplete",[e,n]),g(n)}function E(t,e,n){if(n.dataFilter==j)return t;var r=n.context;return n.dataFilter.call(r,t,e)}function j(){}function w(t){return t&&(t=t.split(";",2)[0]),t&&(t==c?"html":t==f?"json":a.test(t)?"script":u.test(t)&&"xml")||"text"}function T(t,e){return""==e?t:(t+"&"+e).replace(/[&?]{1,2}/,"?")}function S(t){t.processData&&t.data&&"string"!=e.type(t.data)&&(t.data=e.param(t.data,t.traditional)),!t.data||t.type&&"GET"!=t.type.toUpperCase()&&"jsonp"!=t.dataType||(t.url=T(t.url,t.data),t.data=void 0)}function C(t,n,r,i){return e.isFunction(n)&&(i=r,r=n,n=void 0),e.isFunction(r)||(i=r,r=void 0),{url:t,data:n,success:r,dataType:i}}function O(t,n,r,i){var o,s=e.isArray(n),a=e.isPlainObject(n);e.each(n,function(n,u){o=e.type(u),i&&(n=r?i:i+"["+(a||"object"==o||"array"==o?n:"")+"]"),!i&&s?t.add(u.name,u.value):"array"==o||!r&&"object"==o?O(t,u,r,n):t.add(n,u)})}var i,o,n=+new Date,r=t.document,s=/