├── predictive ├── __init__.py ├── tests.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── import_vocabulary.py │ │ └── analyze_articles.py ├── migrations │ ├── __init__.py │ ├── 0002_delete_vocabularyrelation.py │ ├── 0004_auto_20180114_1420.py │ ├── 0005_auto_20180205_1051.py │ ├── 0001_initial.py │ └── 0003_auto_20180113_2319.py ├── apps.py ├── urls.py ├── serializers.py ├── cache_keys.py ├── models.py ├── utils.py └── views.py ├── resource_collector ├── tests.py ├── views.py ├── __init__.py ├── crawler │ ├── __init__.py │ ├── multithread_crawler.py │ ├── voa.py │ └── utils.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── crawl_todays_article.py │ │ └── init_data.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── apps.py └── models.py ├── demo.gif ├── predictive_text ├── __init__.py ├── urls.py ├── wsgi.py └── settings.py ├── requirements.txt ├── manage.py ├── README.md ├── .gitignore └── api.apib /predictive/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /predictive/tests.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/tests.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/views.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /predictive/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /predictive/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/crawler/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /predictive/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resource_collector/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JeffyLu/predictive-text/HEAD/demo.gif -------------------------------------------------------------------------------- /predictive_text/__init__.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | pymysql.install_as_MySQLdb() 3 | -------------------------------------------------------------------------------- /predictive/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class PredictiveConfig(AppConfig): 5 | name = 'predictive' 6 | -------------------------------------------------------------------------------- /resource_collector/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ResourceCollectorConfig(AppConfig): 5 | name = 'resource_collector' 6 | -------------------------------------------------------------------------------- /predictive_text/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url, include 2 | 3 | urlpatterns = [ 4 | url(r'^api/v1/predictive/', include('predictive.urls')) 5 | ] 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.11 2 | PyMySQL==0.7.11 3 | requests==2.18.4 4 | django-crontab==0.7.1 5 | djangorestframework==3.7.7 6 | django-redis==4.8.0 7 | django-cors-headers==2.2.0 8 | -------------------------------------------------------------------------------- /resource_collector/management/commands/crawl_todays_article.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from resource_collector.crawler.voa import run_crawler 3 | 4 | 5 | class Command(BaseCommand): 6 | 7 | def handle(self, *args, **options): 8 | run_crawler(crawl_today=True) 9 | -------------------------------------------------------------------------------- /resource_collector/management/commands/init_data.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from resource_collector.crawler.voa import run_crawler 3 | 4 | 5 | class Command(BaseCommand): 6 | 7 | def add_arguments(self, parser): 8 | parser.add_argument('-c', '--cpu', dest='cpu', type=int) 9 | 10 | def handle(self, *args, **options): 11 | cpu = options.get('cpu') or 4 12 | run_crawler(cpu=cpu) 13 | -------------------------------------------------------------------------------- /predictive/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url, include 2 | from rest_framework import routers 3 | from predictive import views 4 | 5 | 6 | router = routers.DefaultRouter() 7 | router.register(r'vocabularies', views.VocabularyViewSet, 8 | base_name='invitations') 9 | router.register(r'phrases', views.PhraseViewSet, 10 | base_name='phrases') 11 | 12 | urlpatterns = [ 13 | url(r'^', include(router.urls)) 14 | ] 15 | -------------------------------------------------------------------------------- /predictive/migrations/0002_delete_vocabularyrelation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-01-13 22:22 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('predictive', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.DeleteModel( 16 | name='VocabularyRelation', 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /predictive_text/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for predictive_text project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "predictive_text.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /predictive/serializers.py: -------------------------------------------------------------------------------- 1 | from predictive.models import Vocabulary, VocabularyRelation 2 | from rest_framework.serializers import ModelSerializer 3 | 4 | 5 | class VocabularySerializer(ModelSerializer): 6 | 7 | class Meta: 8 | model = Vocabulary 9 | fields = ['id', 'word', 'frequency'] 10 | 11 | 12 | class PhraseSerializer(ModelSerializer): 13 | 14 | class Meta: 15 | model = VocabularyRelation.get_sharding_model(1) 16 | fields = ['id', 'word', 'next_word', 'frequency'] 17 | -------------------------------------------------------------------------------- /predictive/cache_keys.py: -------------------------------------------------------------------------------- 1 | def apply_prefix(f): 2 | def wrapper(*args, **kwargs): 3 | name = f.__name__.lstrip('key_of_') 4 | key = '{}.{}'.format(name, f(*args, **kwargs)) 5 | return key 6 | return wrapper 7 | 8 | 9 | @apply_prefix 10 | def key_of_vocabulary(word): 11 | return word 12 | 13 | 14 | @apply_prefix 15 | def key_of_relation(vocab_id, next_vocab_id): 16 | return '{}.{}'.format(vocab_id, next_vocab_id) 17 | 18 | 19 | @apply_prefix 20 | def key_of_vocabulary_queryset(prefix): 21 | return prefix 22 | 23 | 24 | @apply_prefix 25 | def key_of_phrase_queryset(word): 26 | return word 27 | 28 | 29 | @apply_prefix 30 | def key_of_user_vocabulary(ip): 31 | return ip 32 | 33 | 34 | @apply_prefix 35 | def key_of_user_relation(ip): 36 | return ip 37 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "predictive_text.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /resource_collector/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Article(models.Model): 5 | 6 | source_url = models.CharField(max_length=255) 7 | source_url_hash = models.CharField(max_length=32, unique=True) 8 | is_used = models.BooleanField(default=False) 9 | created_at = models.DateTimeField(auto_now_add=True) 10 | updated_at = models.DateTimeField(auto_now=True) 11 | 12 | @property 13 | def article_content(self): 14 | try: 15 | ac = ArticleContent.objects.get(article_id=self.pk) 16 | except ArticleContent.DoesNotExist: 17 | return '' 18 | return ac.content 19 | 20 | 21 | class ArticleContent(models.Model): 22 | 23 | article_id = models.BigIntegerField(unique=True) 24 | content = models.TextField() 25 | created_at = models.DateTimeField(auto_now_add=True) 26 | updated_at = models.DateTimeField(auto_now=True) 27 | -------------------------------------------------------------------------------- /predictive/management/commands/import_vocabulary.py: -------------------------------------------------------------------------------- 1 | from predictive.models import Vocabulary 2 | from django.db import IntegrityError 3 | from django.core.management.base import BaseCommand 4 | 5 | 6 | class Command(BaseCommand): 7 | 8 | def add_arguments(self, parser): 9 | parser.add_argument('-f', '--filename', dest='filename') 10 | 11 | def handle(self, *args, **options): 12 | filename = options.get('filename', '').strip() 13 | if not filename: 14 | print('{} not found') 15 | return 16 | with open(filename, 'r') as f: 17 | imported = 0 18 | for line in f: 19 | word = line.strip() 20 | if not word or len(word) >= 32: 21 | continue 22 | try: 23 | Vocabulary.objects.create(word=word) 24 | imported += 1 25 | except IntegrityError: 26 | pass 27 | print('total imported: {}'.format(imported)) 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # predictive-text 2 | 3 | - - - 4 | 5 | ### 环境依赖 6 | - MySQL 7 | - Redis 8 | - Python3 9 | 10 | 11 | ### 使用方法 12 | 在[settings.py](predictive_text/settings.py)中修改数据库和缓存的配置信息，然后装pip包。 13 | ``` 14 | $ pip3 install -r requirements.txt 15 | ``` 16 | 17 | 下载英文词汇[words.txt](https://github.com/dwyl/english-words/blob/master/words.txt)并初始化数据。 18 | ``` 19 | $ python3 manage.py migrate 20 | $ python3 manage.py init_data -c [num of cup] 21 | $ python3 manage.py import_vocabulary -f [path to words.txt] 22 | ``` 23 | 24 | 注册定时任务，两个任务分别用来更新每日新闻和分析一定数量的文章，从而提高补全或提示的准确性。 25 | ``` 26 | $ python3 manage.py crontab add 27 | ``` 28 | 29 | 运行。 30 | ``` 31 | $ python3 manage.py runserver 32 | ``` 33 | 34 | 35 | ### API文档 36 | [文档](api.apib)是基于APIBlueprint语法写的，可以使用[aglio](https://github.com/danielgtaylor/aglio)来渲染。 37 | ``` 38 | $ aglio -i api.apib -s 39 | ``` 40 | 访问```localhost:3000```效果如下： 41 | ![效果](https://user-images.githubusercontent.com/16357973/36898984-19157f6a-1e58-11e8-9f6d-58a18c51428f.png) 42 | 43 | 44 | ### 系统效果 45 | - [前端源码](https://github.com/Hsuer/predictive-text-web) 46 | ![运行效果](demo.gif) 47 | -------------------------------------------------------------------------------- /predictive/migrations/0004_auto_20180114_1420.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-01-14 14:20 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('predictive', '0003_auto_20180113_2319'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='vocabularyrelation_0', 17 | name='vocab_id', 18 | field=models.BigIntegerField(), 19 | ), 20 | migrations.AlterField( 21 | model_name='vocabularyrelation_1', 22 | name='vocab_id', 23 | field=models.BigIntegerField(), 24 | ), 25 | migrations.AlterField( 26 | model_name='vocabularyrelation_2', 27 | name='vocab_id', 28 | field=models.BigIntegerField(), 29 | ), 30 | migrations.AlterField( 31 | model_name='vocabularyrelation_3', 32 | name='vocab_id', 33 | field=models.BigIntegerField(), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /predictive/migrations/0005_auto_20180205_1051.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-02-05 10:51 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('predictive', '0004_auto_20180114_1420'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name='vocabulary', 17 | options={'ordering': ['-frequency', 'word']}, 18 | ), 19 | migrations.AlterModelOptions( 20 | name='vocabularyrelation_0', 21 | options={'ordering': ['-frequency']}, 22 | ), 23 | migrations.AlterModelOptions( 24 | name='vocabularyrelation_1', 25 | options={'ordering': ['-frequency']}, 26 | ), 27 | migrations.AlterModelOptions( 28 | name='vocabularyrelation_2', 29 | options={'ordering': ['-frequency']}, 30 | ), 31 | migrations.AlterModelOptions( 32 | name='vocabularyrelation_3', 33 | options={'ordering': ['-frequency']}, 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /predictive/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-01-10 22:30 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Vocabulary', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('word', models.CharField(max_length=32, unique=True)), 21 | ('frequency', models.IntegerField(default=0)), 22 | ('created_at', models.DateTimeField(auto_now_add=True)), 23 | ('updated_at', models.DateTimeField(auto_now=True)), 24 | ], 25 | ), 26 | migrations.CreateModel( 27 | name='VocabularyRelation', 28 | fields=[ 29 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 30 | ('vocab_id', models.BigIntegerField(db_index=True)), 31 | ('next_vocab_id', models.BigIntegerField()), 32 | ('frequency', models.IntegerField(default=0)), 33 | ('created_at', models.DateTimeField(auto_now_add=True)), 34 | ('updated_at', models.DateTimeField(auto_now=True)), 35 | ], 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /resource_collector/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-01-07 03:39 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Article', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('source_url', models.CharField(max_length=255)), 21 | ('source_url_hash', models.CharField(max_length=32, unique=True)), 22 | ('is_used', models.BooleanField(default=False)), 23 | ('created_at', models.DateTimeField(auto_now_add=True)), 24 | ('updated_at', models.DateTimeField(auto_now=True)), 25 | ], 26 | ), 27 | migrations.CreateModel( 28 | name='ArticleContent', 29 | fields=[ 30 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 31 | ('article_id', models.BigIntegerField(unique=True)), 32 | ('content', models.TextField()), 33 | ('created_at', models.DateTimeField(auto_now_add=True)), 34 | ('updated_at', models.DateTimeField(auto_now=True)), 35 | ], 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | .DS_Store 3 | *.swp 4 | *~ 5 | *.sqlite3 6 | *.db 7 | data/VOA/ 8 | data/words.txt 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *,cover 54 | .hypothesis/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # IPython Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # dotenv 87 | .env 88 | 89 | # virtualenv 90 | venv/ 91 | ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | -------------------------------------------------------------------------------- /resource_collector/crawler/multithread_crawler.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from django.db.utils import InternalError 3 | from hashlib import md5 4 | from resource_collector.models import Article, ArticleContent 5 | 6 | 7 | class MultiThreadCrawler(threading.Thread): 8 | 9 | TASK_TYPE_ARTICLE = 0 10 | TASK_TYPE_ARCHIVER = 1 11 | 12 | def __init__(self, crawler, task_queue): 13 | super(MultiThreadCrawler, self).__init__() 14 | self.task_queue = task_queue 15 | self.crawler = crawler 16 | self.error_task = set() 17 | 18 | def save_article(self, url, content): 19 | url_hash = md5(url.encode('utf-8')).hexdigest() 20 | try: 21 | article, _ = Article.objects.get_or_create( 22 | source_url_hash=url_hash, defaults={'source_url': url}) 23 | ArticleContent.objects.update_or_create( 24 | article_id=article.pk, defaults={'content': content}) 25 | return True 26 | except InternalError as e: 27 | print('db internal error: {}'.format(str(e))) 28 | if url_hash not in self.error_task: 29 | self.task_queue.put((url, self.TASK_TYPE_ARTICLE)) 30 | self.error_task.add(url_hash) 31 | return False 32 | 33 | def run(self): 34 | while not self.task_queue.empty(): 35 | task, task_type = self.task_queue.get() 36 | if task_type == self.TASK_TYPE_ARTICLE: 37 | content = self.crawler.get_article_content_by_url(task) 38 | if not self.save_article(task, content): 39 | continue 40 | elif task_type == self.TASK_TYPE_ARCHIVER: 41 | urls = self.crawler.get_archiver_article_urls(task) 42 | for url in urls: 43 | self.task_queue.put((url, self.TASK_TYPE_ARTICLE)) 44 | self.task_queue.task_done() 45 | 46 | print('remain: {}, current: {}'.format( 47 | self.task_queue.qsize(), task)) 48 | -------------------------------------------------------------------------------- /api.apib: -------------------------------------------------------------------------------- 1 | FORMAT: v1 2 | HOST: http://localhost/api/v1/predictive 3 | 4 | # Group Predictive-Text 5 | 6 | ## Vocabulary Collection [/vocabularies/{?ipp,page,prefix,user_mode}] 7 | 8 | ### 根据前缀补全单词 [GET] 9 | + Parameters 10 | + ipp: 7 (number, optional) - 每页提示个数，默认7个 11 | + page: 1 (number, optional) - 当前页，默认1 12 | + prefix: `th` (string, required) - 单词前缀 13 | + user_mode: 1 (number, optional) - 开启用户个性化提示 14 | 15 | + Response 200 (application/json) 16 | + Attributes (VocabularyResponse) 17 | 18 | ## Vocabulary [/vocabularies/{id}/] 19 | 20 | ### 更新用户数据 [PUT] 21 | + Parameters 22 | + id: 1 (number, required) - 单词id 23 | 24 | + Response 200 (application/json) 25 | + Attributes (VocabularyRead) 26 | 27 | 28 | ## Phrase Collection [/phrases/{?ipp,page,word,user_mode}] 29 | 30 | ### 根据单词提示短语 [GET] 31 | + Parameters 32 | + ipp: 7 (number, optional) - 每页提示个数，默认7个 33 | + page: 1 (number, optional) - 当前页，默认1 34 | + word: `this` (string, required) - 单词 35 | + user_mode: 1 (number, optional) - 开启用户个性化提示 36 | 37 | + Response 200 (application/json) 38 | + Attributes (PhraseResponse) 39 | 40 | ## Phrase [/phrases/{id}/{?vocab_id}] 41 | 42 | ### 更新用户数据 [PUT] 43 | + Parameters 44 | + id: 1 (number, required) - 短语id 45 | 46 | + Request (application/json) 47 | + Attributes 48 | + vocab_id: 1 (number, required) - 短语的第一个单词id 49 | 50 | + Response 200 (application/json) 51 | + Attributes (PhraseRead) 52 | 53 | 54 | # Data Structures 55 | 56 | ## VocabularyRead 57 | + id: 1 (number) 58 | + word: `this` (string) - 单词 59 | + frequency: 1029 (number) - 频率 60 | 61 | ## PhraseRead 62 | + id: 1 (number) 63 | + word: `this` (string) - 单词 64 | + next_word: `is` (string) - 补全短语 65 | + frequency: 1029 (number) - 频率 66 | 67 | ## VocabularyResponse 68 | + ipp: 7 (number) 69 | + page: 1 (number) 70 | + total: 4 (number) 71 | + objects (array[VocabularyRead]) 72 | 73 | ## PhraseResponse 74 | + ipp: 7 (number) 75 | + page: 1 (number) 76 | + total: 4 (number) 77 | + objects (array[PhraseRead]) 78 | -------------------------------------------------------------------------------- /predictive/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Vocabulary(models.Model): 5 | 6 | word = models.CharField(max_length=32, unique=True) 7 | frequency = models.IntegerField(default=0) 8 | created_at = models.DateTimeField(auto_now_add=True) 9 | updated_at = models.DateTimeField(auto_now=True) 10 | 11 | def __str__(self): 12 | return '{}: {}'.format(self.word, self.frequency) 13 | 14 | class Meta: 15 | ordering = ['-frequency', 'word'] 16 | 17 | 18 | class VocabularyRelation(models.Model): 19 | 20 | vocab_id = models.BigIntegerField() 21 | next_vocab_id = models.BigIntegerField() 22 | frequency = models.IntegerField(default=0) 23 | created_at = models.DateTimeField(auto_now_add=True) 24 | updated_at = models.DateTimeField(auto_now=True) 25 | 26 | SHARDING_PIECE = 4 27 | SHARDING_MODEL = {} 28 | 29 | def __str__(self): 30 | return '{} {}: {}'.format(self.word, self.next_word, self.frequency) 31 | 32 | class Meta: 33 | unique_together = ['vocab_id', 'next_vocab_id'] 34 | ordering = ['-frequency'] 35 | abstract = True 36 | 37 | @property 38 | def word(self): 39 | vocab = Vocabulary.objects.filter(pk=self.vocab_id).first() 40 | return vocab.word if vocab else '' 41 | 42 | @property 43 | def next_word(self): 44 | vocab = Vocabulary.objects.filter(pk=self.next_vocab_id).first() 45 | return vocab.word if vocab else '' 46 | 47 | @classmethod 48 | def get_sharding_model(cls, sharding_key): 49 | piece = sharding_key % cls.SHARDING_PIECE 50 | 51 | class Meta: 52 | db_table = 'predictive_vocabularyrelation_{}'.format(piece) 53 | unique_together = ['vocab_id', 'next_vocab_id'] 54 | ordering = ['-frequency'] 55 | attrs = {'__module__': cls.__module__, 'Meta': Meta} 56 | class_name = 'VocabularyRelation_{}'.format(piece) 57 | if class_name not in cls.SHARDING_MODEL: 58 | cls.SHARDING_MODEL[class_name] = type(class_name, (cls, ), attrs) 59 | return cls.SHARDING_MODEL[class_name] 60 | 61 | @classmethod 62 | def count(cls): 63 | return sum(cls.get_sharding_model(i).objects.count() 64 | for i in range(cls.SHARDING_PIECE)) 65 | 66 | 67 | for i in range(VocabularyRelation.SHARDING_PIECE): 68 | VocabularyRelation.get_sharding_model(i) 69 | -------------------------------------------------------------------------------- /predictive_text/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 5 | 6 | SECRET_KEY = 'l+38ezh8br+y8_-#iphf7xv4a5rzg&wl$5%5p27c*lk6@0m-xo' 7 | 8 | DEBUG = True 9 | 10 | ALLOWED_HOSTS = ['jeffy.me', 'localhost', '127.0.0.1'] 11 | 12 | CORS_ORIGIN_ALLOW_ALL = True 13 | 14 | INSTALLED_APPS = [ 15 | 'django.contrib.auth', 16 | 'django.contrib.contenttypes', 17 | 'django.contrib.sessions', 18 | 'corsheaders', 19 | 'django_crontab', 20 | 'rest_framework', 21 | 'resource_collector', 22 | 'predictive', 23 | ] 24 | 25 | MIDDLEWARE = [ 26 | 'django.middleware.security.SecurityMiddleware', 27 | 'django.contrib.sessions.middleware.SessionMiddleware', 28 | 'corsheaders.middleware.CorsMiddleware', 29 | 'django.middleware.common.CommonMiddleware', 30 | 'django.middleware.csrf.CsrfViewMiddleware', 31 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 32 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 33 | ] 34 | 35 | REST_FRAMEWORK = { 36 | 'DEFAULT_PAGINATION_CLASS': 'predictive.utils.PageNumberPaginationExt', 37 | 'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer',), 38 | 'PAGE_SIZE': 10, 39 | } 40 | 41 | ROOT_URLCONF = 'predictive_text.urls' 42 | 43 | WSGI_APPLICATION = 'predictive_text.wsgi.application' 44 | 45 | DATABASES = { 46 | 'default': { 47 | 'ENGINE': 'django.db.backends.mysql', 48 | 'NAME': 'predictive_text', 49 | 'USER': 'root', 50 | 'PASSWORD': 'root123456', 51 | 'HOST': 'localhost', 52 | 'PORT': '3306', 53 | } 54 | } 55 | 56 | CACHES = { 57 | "default": { 58 | "BACKEND": "django_redis.cache.RedisCache", 59 | "LOCATION": "redis://127.0.0.1:6379/1", 60 | "KEY_PREFIX": "django.predictive_text.caches", 61 | "TIMEOUT": 60 * 60 * 24, 62 | "OPTIONS": { 63 | "CLIENT_CLASS": "django_redis.client.DefaultClient", 64 | "PICKLE_VERSION": -1, 65 | } 66 | } 67 | } 68 | 69 | CRONJOBS = [ 70 | ( 71 | '0 0 * * *', 72 | 'django.core.management.call_command', 73 | ['crawl_todays_article'], 74 | {}, 75 | '>> %s/tmp/cronjobs/crawl_todays_article.log' % BASE_DIR, 76 | ), 77 | ( 78 | '0 1 * * *', 79 | 'django.core.management.call_command', 80 | ['analyze_articles'], 81 | {}, 82 | '>> %s/tmp/cronjobs/analyze_articles.log' % BASE_DIR, 83 | ), 84 | ] 85 | 86 | LANGUAGE_CODE = 'zh-hans' 87 | 88 | TIME_ZONE = 'Asia/Shanghai' 89 | 90 | USE_I18N = True 91 | 92 | USE_L10N = True 93 | 94 | USE_TZ = False 95 | -------------------------------------------------------------------------------- /predictive/management/commands/analyze_articles.py: -------------------------------------------------------------------------------- 1 | import re 2 | import nltk 3 | from django.core.cache import cache 4 | from predictive.cache_keys import key_of_vocabulary, key_of_relation 5 | from collections import Counter 6 | from django.core.management.base import BaseCommand 7 | from resource_collector.models import Article 8 | from predictive.models import Vocabulary, VocabularyRelation 9 | 10 | 11 | class Command(BaseCommand): 12 | 13 | def add_arguments(self, parser): 14 | parser.add_argument('-a', '--articles', dest='articles', type=int) 15 | 16 | def analyze(self, article_content): 17 | sents = nltk.sent_tokenize(article_content) 18 | for sent in sents: 19 | sent = sent.lower() 20 | words = re.findall(r'[0-9a-zA-Z\']+', sent) 21 | relations = [(words[w-1], words[w]) for w in range(1, len(words))] 22 | yield words, relations 23 | 24 | def handle(self, *args, **options): 25 | task_num = options.get('articles') or 100 26 | vocab_counter = Counter() 27 | vocab_relation_counter = Counter() 28 | articles = Article.objects.filter(is_used=False)[:task_num] 29 | for article in articles: 30 | for words, relations in self.analyze(article.article_content): 31 | vocab_counter.update(words) 32 | vocab_relation_counter.update(relations) 33 | article.is_used = True 34 | article.save() 35 | 36 | vocab_id_dict = {} 37 | for k, v in vocab_counter.items(): 38 | key = key_of_vocabulary(k) 39 | vocab = cache.get(key) 40 | if not vocab: 41 | try: 42 | vocab = Vocabulary.objects.get(word=k) 43 | except Vocabulary.DoesNotExist: 44 | continue 45 | vocab.frequency = vocab.frequency + v 46 | vocab.save() 47 | cache.set(key, vocab) 48 | vocab_id_dict[k] = vocab.pk 49 | 50 | for k, v in vocab_relation_counter.items(): 51 | vocab_id = vocab_id_dict.get(k[0]) 52 | next_vocab_id = vocab_id_dict.get(k[1]) 53 | if not all([vocab_id, next_vocab_id]): 54 | continue 55 | key = key_of_relation(vocab_id, next_vocab_id) 56 | relation = cache.get(key) 57 | if not relation: 58 | model = VocabularyRelation.get_sharding_model(vocab_id) 59 | relation, _ = model.objects.update_or_create( 60 | vocab_id=vocab_id, next_vocab_id=next_vocab_id, 61 | defaults={'frequency': v}) 62 | else: 63 | relation.frequency = relation.frequency + v 64 | relation.save() 65 | cache.set(key, relation) 66 | 67 | print('tasks: {}, vocabs: {}, relations: {}'.format( 68 | task_num, len(vocab_counter), len(vocab_relation_counter))) 69 | -------------------------------------------------------------------------------- /resource_collector/crawler/voa.py: -------------------------------------------------------------------------------- 1 | import re 2 | import queue 3 | import random 4 | from django.utils import timezone 5 | from resource_collector.crawler.utils import user_agents, get_html 6 | from resource_collector.crawler.multithread_crawler import MultiThreadCrawler 7 | 8 | 9 | class VOACrawler: 10 | 11 | url_root = 'http://www.51voa.com' 12 | url_standard = url_root + "/VOA_Standard_{}_archiver.html" 13 | 14 | def __init__(self, retry=3, encoding='utf-8'): 15 | self.retry = retry 16 | self.encoding = encoding 17 | 18 | @property 19 | def headers(self): 20 | return {'User-Agent': random.choice(user_agents)} 21 | 22 | @property 23 | def standard_archiver_pages_task_queue(self): 24 | url = self.url_standard.format(1) 25 | html = get_html(url, retry=self.retry, headers=self.headers, 26 | encoding=self.encoding) 27 | page = re.search(r'页次：\d+/(\d+) 每页', 28 | html, re.S) 29 | task_queue = queue.Queue() 30 | [task_queue.put((i, MultiThreadCrawler.TASK_TYPE_ARCHIVER)) 31 | for i in range(int(page.groups()[0]))] 32 | return task_queue 33 | 34 | @property 35 | def update_todays_article_queue(self): 36 | html = get_html(self.url_root, retry=self.retry, headers=self.headers, 37 | encoding=self.encoding) 38 | urls = re.findall( 39 | r'href="(/VOA_Standard_English/[a-zA-Z0-9_-]*?.html)"', html, re.S) 40 | task_queue = queue.Queue() 41 | for url in urls: 42 | task_queue.put( 43 | (self.url_root + url, MultiThreadCrawler.TASK_TYPE_ARTICLE) 44 | ) 45 | return task_queue 46 | 47 | def get_archiver_article_urls(self, page): 48 | url = self.url_standard.format(page) 49 | html = get_html(url, retry=self.retry, headers=self.headers, 50 | encoding=self.encoding) 51 | urls = re.findall( 52 | r'href="(/VOA_Standard_English/[a-zA-Z0-9_-]*?.html)"', html, re.S) 53 | return [self.url_root + u for u in urls] 54 | 55 | def get_article_content_by_url(self, url): 56 | html = get_html(url, retry=self.retry, headers=self.headers, 57 | encoding=self.encoding) 58 | contents = re.findall(r'
(.*?)
', 59 | html, re.S) 60 | return re.sub(r'<.*?>', ' ', ''.join(contents), flags=re.S) 61 | 62 | 63 | def run_crawler(cpu=1, crawl_today=False): 64 | crawler = VOACrawler() 65 | if crawl_today: 66 | print(timezone.now()) 67 | task_queue = crawler.update_todays_article_queue 68 | else: 69 | task_queue = crawler.standard_archiver_pages_task_queue 70 | threads = [] 71 | for i in range(cpu): 72 | t = MultiThreadCrawler(crawler, task_queue) 73 | t.daemon = True 74 | t.start() 75 | threads.append(t) 76 | for t in threads: 77 | t.join() 78 | -------------------------------------------------------------------------------- /predictive/utils.py: -------------------------------------------------------------------------------- 1 | from rest_framework.pagination import PageNumberPagination 2 | from rest_framework.response import Response 3 | from predictive import cache_keys 4 | from predictive.models import Vocabulary, VocabularyRelation 5 | from collections import OrderedDict 6 | from django.core.cache import cache 7 | 8 | 9 | class PageNumberPaginationExt(PageNumberPagination): 10 | 11 | max_page_size = 20 12 | max_page = 20 13 | page_size_query_param = 'ipp' 14 | 15 | def paginate_queryset(self, queryset, request, view=None): 16 | queryset = queryset[:self.max_page_size * self.max_page] 17 | return super().paginate_queryset(queryset, request, view=None) 18 | 19 | def get_paginated_response(self, data): 20 | return Response(OrderedDict([ 21 | ('ipp', self.get_page_size(self.request)), 22 | ('page', self.page.number), 23 | ('total', self.page.paginator.count), 24 | ('objects', data) 25 | ])) 26 | 27 | 28 | MAX_SIZE_OF_QUERYSET = (PageNumberPaginationExt.max_page * 29 | PageNumberPaginationExt.max_page_size) 30 | 31 | 32 | def get_data(request): 33 | if request.method == 'GET': 34 | data = request.GET 35 | else: 36 | try: 37 | data = request.data 38 | except AttributeError: 39 | data = {} 40 | try: 41 | return data.dict() 42 | except AttributeError: 43 | return data 44 | 45 | 46 | def get_ip(request): 47 | x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') 48 | if x_forwarded_for: 49 | ip = x_forwarded_for.split(',')[0] 50 | else: 51 | ip = request.META.get('REMOTE_ADDR') 52 | return ip 53 | 54 | 55 | def get_word_process_method(string): 56 | if not string: 57 | return None 58 | 59 | if string.istitle(): 60 | _word_process_method = str.title 61 | elif string.isupper(): 62 | _word_process_method = str.upper 63 | elif string[0].isupper(): 64 | _word_process_method = str.title 65 | else: 66 | _word_process_method = None 67 | return _word_process_method 68 | 69 | 70 | def get_phrase_queryset(word): 71 | if not word: 72 | return [] 73 | key = cache_keys.key_of_phrase_queryset(word) 74 | queryset = cache.get(key) 75 | if not queryset: 76 | vocab = get_vocab_by_word(word) 77 | if vocab: 78 | model = VocabularyRelation.get_sharding_model(vocab.pk) 79 | queryset = list(model.objects.filter(vocab_id=vocab.pk)) 80 | else: 81 | queryset = [] 82 | cache.set(key, queryset[:MAX_SIZE_OF_QUERYSET]) 83 | return queryset 84 | 85 | 86 | def get_vocab_by_word(word): 87 | word = word.lower() 88 | key = cache_keys.key_of_vocabulary(word) 89 | vocab = cache.get(key) 90 | if not vocab: 91 | vocab = Vocabulary.objects.filter(word=word).first() 92 | return vocab 93 | 94 | 95 | def sort_queryset_by_lru_cache(queryset, key): 96 | ids = cache.get(key, []) 97 | sort_index = {ids[i]: i for i in range(len(ids))} 98 | cached_objs = {} 99 | objs = [] 100 | for obj in queryset: 101 | if obj.pk in sort_index: 102 | cached_objs[sort_index[obj.pk]] = obj 103 | else: 104 | objs.append(obj) 105 | sorted_queryset = [cached_objs[k] for k in 106 | sorted(cached_objs.keys(), reverse=True)] + objs 107 | return sorted_queryset 108 | -------------------------------------------------------------------------------- /predictive/migrations/0003_auto_20180113_2319.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.4 on 2018-01-13 23:19 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('predictive', '0002_delete_vocabularyrelation'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='VocabularyRelation_0', 17 | fields=[ 18 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 19 | ('vocab_id', models.BigIntegerField(db_index=True)), 20 | ('next_vocab_id', models.BigIntegerField()), 21 | ('frequency', models.IntegerField(default=0)), 22 | ('created_at', models.DateTimeField(auto_now_add=True)), 23 | ('updated_at', models.DateTimeField(auto_now=True)), 24 | ], 25 | options={ 26 | 'db_table': 'predictive_vocabularyrelation_0', 27 | }, 28 | ), 29 | migrations.CreateModel( 30 | name='VocabularyRelation_1', 31 | fields=[ 32 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 33 | ('vocab_id', models.BigIntegerField(db_index=True)), 34 | ('next_vocab_id', models.BigIntegerField()), 35 | ('frequency', models.IntegerField(default=0)), 36 | ('created_at', models.DateTimeField(auto_now_add=True)), 37 | ('updated_at', models.DateTimeField(auto_now=True)), 38 | ], 39 | options={ 40 | 'db_table': 'predictive_vocabularyrelation_1', 41 | }, 42 | ), 43 | migrations.CreateModel( 44 | name='VocabularyRelation_2', 45 | fields=[ 46 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 47 | ('vocab_id', models.BigIntegerField(db_index=True)), 48 | ('next_vocab_id', models.BigIntegerField()), 49 | ('frequency', models.IntegerField(default=0)), 50 | ('created_at', models.DateTimeField(auto_now_add=True)), 51 | ('updated_at', models.DateTimeField(auto_now=True)), 52 | ], 53 | options={ 54 | 'db_table': 'predictive_vocabularyrelation_2', 55 | }, 56 | ), 57 | migrations.CreateModel( 58 | name='VocabularyRelation_3', 59 | fields=[ 60 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 61 | ('vocab_id', models.BigIntegerField(db_index=True)), 62 | ('next_vocab_id', models.BigIntegerField()), 63 | ('frequency', models.IntegerField(default=0)), 64 | ('created_at', models.DateTimeField(auto_now_add=True)), 65 | ('updated_at', models.DateTimeField(auto_now=True)), 66 | ], 67 | options={ 68 | 'db_table': 'predictive_vocabularyrelation_3', 69 | }, 70 | ), 71 | migrations.AlterUniqueTogether( 72 | name='vocabularyrelation_3', 73 | unique_together=set([('vocab_id', 'next_vocab_id')]), 74 | ), 75 | migrations.AlterUniqueTogether( 76 | name='vocabularyrelation_2', 77 | unique_together=set([('vocab_id', 'next_vocab_id')]), 78 | ), 79 | migrations.AlterUniqueTogether( 80 | name='vocabularyrelation_1', 81 | unique_together=set([('vocab_id', 'next_vocab_id')]), 82 | ), 83 | migrations.AlterUniqueTogether( 84 | name='vocabularyrelation_0', 85 | unique_together=set([('vocab_id', 'next_vocab_id')]), 86 | ), 87 | ] 88 | -------------------------------------------------------------------------------- /predictive/views.py: -------------------------------------------------------------------------------- 1 | from django.core.cache import cache 2 | from predictive import cache_keys 3 | from predictive import utils as u 4 | from predictive import serializers as s 5 | from predictive.models import Vocabulary, VocabularyRelation 6 | 7 | from rest_framework import viewsets, status 8 | from rest_framework.response import Response 9 | 10 | 11 | class VocabularyViewSet(viewsets.ModelViewSet): 12 | 13 | serializer_class = s.VocabularySerializer 14 | 15 | def list(self, request): 16 | data = u.get_data(request) 17 | prefix = data.get('prefix', '') 18 | _word_process_method = u.get_word_process_method(prefix) 19 | key = cache_keys.key_of_vocabulary_queryset(prefix) 20 | self.queryset = cache.get(key) 21 | if not self.queryset: 22 | self.queryset = list(Vocabulary.objects.filter( 23 | word__startswith=prefix.lower())[:u.MAX_SIZE_OF_QUERYSET]) 24 | cache.set(key, self.queryset) 25 | 26 | if not data.get('user_mode') == '1': 27 | resp = super().list(request) 28 | if _word_process_method: 29 | for obj in resp.data['objects']: 30 | obj['word'] = _word_process_method(obj['word']) 31 | return resp 32 | 33 | user_vocab_key = cache_keys.key_of_user_vocabulary(u.get_ip(request)) 34 | self.queryset = u.sort_queryset_by_lru_cache(self.queryset, 35 | user_vocab_key) 36 | resp = super().list(request) 37 | if _word_process_method: 38 | for obj in resp.data['objects']: 39 | obj['word'] = _word_process_method(obj['word']) 40 | return resp 41 | 42 | def update(self, request, pk): 43 | pk = int(pk) 44 | vocab = Vocabulary.objects.filter(pk=pk).first() 45 | if not vocab: 46 | return Response(data={'detail': 'not found'}, 47 | status=status.HTTP_404_NOT_FOUND) 48 | ip = u.get_ip(request) 49 | key = cache_keys.key_of_user_vocabulary(ip) 50 | user_vocabs = cache.get(key, []) 51 | if pk in user_vocabs: 52 | user_vocabs.remove(pk) 53 | user_vocabs.append(pk) 54 | cache.set(key, user_vocabs[-100:]) 55 | vocab.frequency = vocab.frequency + 5 56 | vocab.save() 57 | return Response(self.get_serializer(vocab).data) 58 | 59 | 60 | class PhraseViewSet(viewsets.ModelViewSet): 61 | 62 | serializer_class = s.PhraseSerializer 63 | 64 | def list(self, request): 65 | data = u.get_data(request) 66 | word = data.get('word') 67 | self.queryset = u.get_phrase_queryset(word) 68 | if not data.get('user_mode') == '1': 69 | return super().list(request) 70 | 71 | user_relation_key = cache_keys.key_of_user_relation(u.get_ip(request)) 72 | self.queryset = u.sort_queryset_by_lru_cache(self.queryset, 73 | user_relation_key) 74 | return super().list(request) 75 | 76 | def update(self, request, pk): 77 | pk = int(pk) 78 | try: 79 | vocab_id = int(u.get_data(request).get('vocab_id')) 80 | except (TypeError, ValueError): 81 | return Response(data={'detail': 'invalid vocab_id'}) 82 | 83 | model = VocabularyRelation.get_sharding_model(vocab_id) 84 | relation = model.objects.filter(pk=pk).first() 85 | if not relation: 86 | return Response(data={'detail': 'not found'}, 87 | status=status.HTTP_404_NOT_FOUND) 88 | ip = u.get_ip(request) 89 | key = cache_keys.key_of_user_relation(ip) 90 | user_relations = cache.get(key, []) 91 | if pk in user_relations: 92 | user_relations.remove(pk) 93 | user_relations.append(pk) 94 | cache.set(key, user_relations[-100:]) 95 | relation.frequency = relation.frequency + 5 96 | relation.save() 97 | return Response(self.get_serializer(relation).data) 98 | -------------------------------------------------------------------------------- /resource_collector/crawler/utils.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | import requests 4 | 5 | 6 | def get_html(url, retry=3, headers=None, encoding=None): 7 | while retry: 8 | try: 9 | resp = requests.get(url, headers=headers) 10 | if encoding: 11 | resp.encoding = encoding 12 | return resp.text 13 | except (TimeoutError, ConnectionError): 14 | retry -= 1 15 | raise TimeoutError('timeout error') 16 | 17 | 18 | user_agents = [ 19 | "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 20 | "Avant Browser/1.2.789rel1 (http://www.avantbrowser.com)", 21 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5", 22 | "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9", 23 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7", 24 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14", 25 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14", 26 | "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20", 27 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27", 28 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1", 29 | "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2", 30 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7", 31 | "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre", 32 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10", 33 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)", 34 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5", 35 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)", 36 | "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", 37 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", 38 | "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0", 39 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110622 Firefox/6.0a2", 40 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", 41 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre", 42 | "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0 )", 43 | "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90)", 44 | "Mozilla/5.0 (Windows; U; Windows XP) Gecko MultiZilla/1.6.1.0a", 45 | "Mozilla/2.02E (Win95; U)", 46 | "Mozilla/3.01Gold (Win95; I)", 47 | "Mozilla/4.8 [en] (Windows NT 5.1; U)", 48 | "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.4) Gecko Netscape/7.1 (ax)", 49 | "HTC_Dream Mozilla/5.0 (Linux; U; Android 1.5; en-ca; Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 50 | "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0", 51 | "Mozilla/5.0 (Linux; U; Android 1.5; en-us; sdk Build/CUPCAKE) AppleWebkit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 52 | "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 53 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 54 | "Mozilla/5.0 (Linux; U; Android 1.5; en-us; htc_bahamas Build/CRB17) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 55 | "Mozilla/5.0 (Linux; U; Android 2.1-update1; de-de; HTC Desire 1.19.161.5 Build/ERE27) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 56 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 57 | "Mozilla/5.0 (Linux; U; Android 1.5; de-ch; HTC Hero Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 58 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 59 | "Mozilla/5.0 (Linux; U; Android 2.1; en-us; HTC Legend Build/cupcake) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 60 | "Mozilla/5.0 (Linux; U; Android 1.5; de-de; HTC Magic Build/PLAT-RC33) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1 FirePHP/0.3", 61 | "Mozilla/5.0 (Linux; U; Android 1.6; en-us; HTC_TATTOO_A3288 Build/DRC79) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 62 | "Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", 63 | "Mozilla/5.0 (Linux; U; Android 1.5; en-us; T-Mobile G1 Build/CRB43) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari 525.20.1", 64 | "Mozilla/5.0 (Linux; U; Android 1.5; en-gb; T-Mobile_G2_Touch Build/CUPCAKE) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 65 | "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 66 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Droid Build/FRG22D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 67 | "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Milestone Build/ SHOLS_U2_01.03.1) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 68 | "Mozilla/5.0 (Linux; U; Android 2.0.1; de-de; Milestone Build/SHOLS_U2_01.14.0) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 69 | "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", 70 | "Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522 (KHTML, like Gecko) Safari/419.3", 71 | "Mozilla/5.0 (Linux; U; Android 1.1; en-gb; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", 72 | "Mozilla/5.0 (Linux; U; Android 2.0; en-us; Droid Build/ESD20) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 73 | "Mozilla/5.0 (Linux; U; Android 2.1; en-us; Nexus One Build/ERD62) AppleWebKit/530.17 (KHTML, like Gecko) Version/4.0 Mobile Safari/530.17", 74 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 75 | "Mozilla/5.0 (Linux; U; Android 2.2; en-us; ADR6300 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 76 | "Mozilla/5.0 (Linux; U; Android 2.2; en-ca; GT-P1000M Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 77 | "Mozilla/5.0 (Linux; U; Android 3.0.1; fr-fr; A500 Build/HRI66) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13", 78 | "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2", 79 | "Mozilla/5.0 (Linux; U; Android 1.6; es-es; SonyEricssonX10i Build/R1FA016) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 80 | "Mozilla/5.0 (Linux; U; Android 1.6; en-us; SonyEricssonX10i Build/R1AA056) AppleWebKit/528.5 (KHTML, like Gecko) Version/3.1.2 Mobile Safari/525.20.1", 81 | ] 82 | --------------------------------------------------------------------------------