├── images ├── __init__.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── admin.py ├── apps.py ├── models.py └── views.py ├── silumz ├── __init__.py ├── wsgi.py ├── urls.py └── settings.py ├── static ├── hot.png ├── logo.png ├── zde │ ├── 1.png │ ├── 2.png │ ├── 3.png │ ├── 4.png │ ├── 5.png │ ├── hot.png │ ├── next.png │ ├── timg.gif │ ├── reward.png │ ├── loading.gif │ ├── css │ │ └── fonts │ │ │ ├── iconfont.eot │ │ │ ├── iconfont.ttf │ │ │ ├── iconfont.woff │ │ │ └── iconfont.svg │ └── js │ │ ├── html5.js │ │ └── flutter-hearts-zmt.js ├── bg-ico.png ├── favicon.ico └── v_back.jpg ├── .gitignore ├── hyimg ├── 4a526605037c6.4tteiq5psd80.png └── 2c1185b91e0c4bb399334aeab5f6a011.4lxc1r4h9hg0.gif ├── requirements.txt ├── uwsgi.ini ├── templates └── zde │ ├── 404.html │ ├── tag.html │ ├── pagination.html │ ├── base.html │ ├── index.html │ ├── category.html │ ├── sort.html │ ├── video.html │ ├── page.html │ └── mVideo.html ├── config.py ├── manage.py ├── start.sh ├── crawler ├── delete_img.py ├── Compress.py ├── down_img.py ├── AutoPost.py ├── crawler_xmt.py ├── crawle_mzt.py ├── crawler_mtl.py ├── crawler_ytu.py ├── crawler_mmjpg.py ├── crawler_mm131.py ├── crawler_amn.py └── crawler_nsg.py ├── readme.md ├── 94imm.sql └── install.sh /images/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /silumz/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/hot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/hot.png -------------------------------------------------------------------------------- /static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/logo.png -------------------------------------------------------------------------------- /static/zde/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/1.png -------------------------------------------------------------------------------- /static/zde/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/2.png -------------------------------------------------------------------------------- /static/zde/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/3.png -------------------------------------------------------------------------------- /static/zde/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/4.png -------------------------------------------------------------------------------- /static/zde/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/5.png -------------------------------------------------------------------------------- /static/bg-ico.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/bg-ico.png -------------------------------------------------------------------------------- /static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/favicon.ico -------------------------------------------------------------------------------- /static/v_back.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/v_back.jpg -------------------------------------------------------------------------------- /static/zde/hot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/hot.png -------------------------------------------------------------------------------- /static/zde/next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/next.png -------------------------------------------------------------------------------- /static/zde/timg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/timg.gif -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/* 2 | /__pycache__/* 3 | /static/images/* 4 | **/__pycache__/* 5 | test.py -------------------------------------------------------------------------------- /images/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /static/zde/reward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/reward.png -------------------------------------------------------------------------------- /static/zde/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/loading.gif -------------------------------------------------------------------------------- /static/zde/css/fonts/iconfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/css/fonts/iconfont.eot -------------------------------------------------------------------------------- /static/zde/css/fonts/iconfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/css/fonts/iconfont.ttf -------------------------------------------------------------------------------- /static/zde/css/fonts/iconfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/static/zde/css/fonts/iconfont.woff -------------------------------------------------------------------------------- /hyimg/4a526605037c6.4tteiq5psd80.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/hyimg/4a526605037c6.4tteiq5psd80.png -------------------------------------------------------------------------------- /images/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ImagesConfig(AppConfig): 5 | name = 'images' 6 | -------------------------------------------------------------------------------- /hyimg/2c1185b91e0c4bb399334aeab5f6a011.4lxc1r4h9hg0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zephyr-y/94imm/HEAD/hyimg/2c1185b91e0c4bb399334aeab5f6a011.4lxc1r4h9hg0.gif -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asn1crypto==0.24.0 2 | beautifulsoup4==4.6.0 3 | certifi==2018.11.29 4 | cffi==1.11.5 5 | chardet==3.0.4 6 | cryptography==2.4.2 7 | dj-pagination==2.4.0 8 | Django==1.10.6 9 | idna==2.6 10 | mysqlclient==1.3.14 11 | pycparser==2.19 12 | PyMySQL==0.9.2 13 | requests==2.18.4 14 | six==1.12.0 15 | urllib3==1.22 16 | uWSGI==2.0.17.1 17 | -------------------------------------------------------------------------------- /silumz/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for silumz project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "silumz.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /uwsgi.ini: -------------------------------------------------------------------------------- 1 | # uwsig使用配置文件启动 2 | #[uwsgi] 3 | # 项目目录 4 | #chdir=/root/94imm/ 5 | # 指定项目的application 6 | #file=/root/94imm/silumz/wsgi.py 7 | # 指定sock的文件路径 8 | #socket=/root/94imm/uwsgi.sock 9 | # 进程个数 10 | #workers=2 11 | #pidfile=/root/94imm/uwsgi.pid 12 | # 指定IP端口 13 | #http=127.0.0.1:8000 14 | # 指定静态文件 15 | #static-map=/static=/root/94imm/static 16 | # 启动uwsgi的用户名和用户组 17 | #uid=root 18 | #gid=root 19 | # 启用主进程 20 | #master=true 21 | # 自动移除unix Socket和pid文件当服务停止的时候 22 | #vacuum=true 23 | # 序列化接受的内容，如果可能的话 24 | #thunder-lock=true 25 | # 启用线程 26 | #enable-threads=true 27 | # 设置自中断时间 28 | #harakiri=30 29 | # 设置缓冲 30 | #post-buffering=4096 31 | # 设置日志目录 32 | #daemonize=/root/94imm/uwsgi.log 33 | # 通过该端口可查看uwsgi负载情况 34 | #stats=8001 35 | -------------------------------------------------------------------------------- /templates/zde/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 404 8 | 30 | 31 | 32 | 33 |

404，您请求的文件不存在!

34 |

返回首页

35 | 36 | 37 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | # 数据库信息，一键脚本自动添加 2 | mysql_config = { 3 | 'ENGINE': 'django.db.backends.mysql', 4 | 'NAME': '94imm', 5 | 'USER': '94imm', 6 | 'PASSWORD': '94imm', 7 | 'HOST': '127.0.0.1', 8 | 'PORT': '3306', 9 | } 10 | # 数组形式，可以添加多个域名 11 | allow_url=["www.94imm.com","94imm.com"] 12 | # 缓存超时时间，服务器性能好可缩短此时 13 | cache_time=300 14 | # 使用的模板（暂时开放一个） 15 | templates="zde" 16 | # 网站名 17 | site_name="94iMM" 18 | # 一键脚本自动添加 19 | site_url = "https://www.94imm.com" 20 | # 网站关键词 21 | key_word = "关键词1,关键词2,关键词3" 22 | # 网站说明 23 | description = "这是一个高质量的自动爬虫" 24 | # 底部联系邮箱 25 | email = "admin@94imm.com" 26 | # 网站调试模式 27 | debug = False 28 | # 友联 29 | friendly_link = [{"name":"94imm","link":"https://www.94imm.com"},{"name":"获取源码","link":"https://github.com/Turnright-git/94imm.git"}] -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "silumz.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | NAME="uwsgi" 3 | 4 | function run_start(){ 5 | uwsgi --ini uwsgi.ini 6 | } 7 | 8 | function run_stop(){ 9 | if [ ! -n "$NAME" ];then 10 | echo "no arguments" 11 | exit; 12 | fi 13 | ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'` 14 | for id in $ID 15 | do 16 | kill -9 $id 17 | done 18 | } 19 | 20 | function run_clear(){ 21 | rm -rf cache/* 22 | } 23 | 24 | case "$1" in "start"|"s"|"S") 25 | run_start 26 | echo "website run successfully" 27 | ;; 28 | "restart"|"r"|"S") 29 | run_stop 30 | run_clear 31 | run_start 32 | echo "website restart successfully" 33 | ;; 34 | "clear"|"c"|"C") 35 | run_clear 36 | echo "cache cleared" 37 | ;; 38 | "stop") 39 | run_stop 40 | echo "website closed" 41 | ;; 42 | *) 43 | echo -e "Use command to:\n-s start website\n-r restart website\n-c clear cache\n-stop stop website" 44 | ;; 45 | esac 46 | -------------------------------------------------------------------------------- /images/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | class Page(models.Model): 5 | typeid = models.IntegerField() 6 | sendtime=models.DateField() 7 | title=models.CharField(max_length=200) 8 | firstimg=models.CharField(max_length=200) 9 | tagid=models.CharField(max_length=200) 10 | hot=models.IntegerField() 11 | 12 | class Image(models.Model): 13 | pageid=models.IntegerField() 14 | imageurl=models.URLField() 15 | 16 | class Type(models.Model): 17 | type=models.CharField(max_length=200) 18 | 19 | class Tag(models.Model): 20 | tag = models.CharField(max_length=200) 21 | 22 | 23 | class Video(models.Model): 24 | url = models.CharField(max_length=500) 25 | user_id = models.CharField(max_length=15) 26 | date_time = models.CharField(max_length=30) 27 | v_name = models.CharField(max_length=255) 28 | v_path = models.CharField(max_length=50) 29 | source = models.CharField(max_length=10) -------------------------------------------------------------------------------- /silumz/urls.py: -------------------------------------------------------------------------------- 1 | """silumz URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url 17 | from django.contrib import admin 18 | from images import views 19 | 20 | urlpatterns = [ 21 | url(r'^$', views.index), 22 | url(r'^article/(?P\d+)/$', views.page, name='article'), 23 | url(r'^tag/(?P\d+)/$', views.tag, name='tag'), 24 | url(r'^type/(?P\d+)/$', views.type, name='type'), 25 | url(r'^search/', views.search), 26 | url(r'^get_video/', views.getVideo), 27 | url(r'^video/', views.pVideo), 28 | url(r'^mvideo/', views.mVideo), 29 | url(r'^tag/', views.HotTag), 30 | url(r'^sort/(?P\w+)/$', views.SortBy, name='sort'), 31 | ] 32 | -------------------------------------------------------------------------------- /images/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.6 on 2018-12-09 04:41 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Image', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('imageid', models.IntegerField()), 21 | ('pageid', models.IntegerField()), 22 | ('imageurl', models.URLField()), 23 | ], 24 | ), 25 | migrations.CreateModel( 26 | name='Page', 27 | fields=[ 28 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 29 | ('pageid', models.IntegerField()), 30 | ('typeid', models.IntegerField()), 31 | ('pageurl', models.URLField()), 32 | ('time', models.DateField()), 33 | ('title', models.CharField(max_length=200)), 34 | ('numbs', models.IntegerField()), 35 | ('firstimg', models.IntegerField()), 36 | ('otherimg', models.IntegerField()), 37 | ], 38 | ), 39 | ] 40 | -------------------------------------------------------------------------------- /crawler/delete_img.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('../') 4 | import pymysql,os 5 | from config import mysql_config 6 | 7 | dbhost = { 8 | "host": mysql_config['HOST'], 9 | "dbname": mysql_config['NAME'], 10 | "user": mysql_config['USER'], 11 | "password": mysql_config['PASSWORD'] 12 | } 13 | 14 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), dbhost.get("dbname")) 15 | cursor = db.cursor() 16 | 17 | def del_page(id): 18 | cursor.execute("Delete FROM images_page WHERE id=" + "'" + id + "'") 19 | cursor.execute("SELECT imageurl FROM images_image WHERE pageid =" + "'" + id + "'") 20 | for img in cursor.fetchall(): 21 | try: 22 | os.remove(img[0]) 23 | except FileNotFoundError: 24 | print("***************") 25 | print("图片不存在或未下载") 26 | pass 27 | except Exception as e: 28 | print("***************") 29 | print("图片删除失败，错误信息：",e) 30 | cursor.execute("Delete FROM images_image WHERE pageid=" + "'" + id + "'") 31 | print("***************") 32 | print("图片删除成功") 33 | print("***************") 34 | print("采集数据删除成功") 35 | try: 36 | ls = os.listdir('../cache') 37 | for i in ls: 38 | os.remove("../cache/"+i) 39 | os.system("sh ../restart.sh") 40 | print("***************") 41 | print("缓存更新成功") 42 | except Exception as e: 43 | print("***************") 44 | print("缓存更新失败，错误信息：",e) 45 | 46 | if __name__=="__main__": 47 | print("请输入要删除的图集ID") 48 | id=input() 49 | del_page(id) -------------------------------------------------------------------------------- /templates/zde/tag.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}标签云 - {{ siteName }}{% endblock title %} 5 | {% block keywords %}{% for t in keyword %}{{ t.tag }},{% endfor %}{% endblock keywords %} 6 | {% block description %}{{ description }}{% endblock description %} 7 | 8 | {% block focusbox %} 9 |

热门标签

10 | {% endblock focusbox %} 11 | 12 | {% block menu %} 13 | 30 | {% endblock menu %} 31 | 32 | 33 | {% block section %} 34 |
35 |
    36 | {% for t in data %} 37 |
  • {{ t.tag }}×{{t.viwe}}
    • 38 | {% endfor %} 39 |
40 | 41 |
42 | {% endblock section %} 43 | 44 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # 安装说明： 2 | ## 自动安装： 3 | ``` 4 | wget -O install.sh https://github.com/Turnright-git/94imm/blob/master/install.sh && chmod +x install.sh &&./install.sh 5 | ``` 6 | 安装过程中需要输入 7 | ``` 8 | > * allow url:www.94imm.com # 防盗链允许的使用的域名 9 | > * site_name:94imm # 网站名称，将显示在网站底部网站title和底部 10 | > * site_url:www.94imm.com # 网站url 11 | > * Create databases:94imm # 添加数据库 12 | > * Create databases password: # 数据库账号(root)如未安装mysql，此处将设置为root密码 13 | > * Password for root: # 如本机以安装mysql，此处需输入root密码 14 | ``` 15 | 参数配置 16 | > 配置文件为根目录下的config.py 17 | ```python 18 | # 数据库信息，一键脚本自动添加 19 | mysql_config = { 20 | 'ENGINE': 'django.db.backends.mysql', 21 | 'NAME': '94imm', 22 | 'USER': '94imm', 23 | 'PASSWORD': '94imm', 24 | 'HOST': '127.0.0.1', 25 | 'PORT': '3306', 26 | } 27 | # 数组形式，可以添加多个域名 28 | allow_url=["www.94imm.com","94imm.com"] 29 | # 缓存超时时间，服务器性能好可缩短此时 30 | cache_time=300 31 | # 使用的模板（暂时开放一个） 32 | templates="zde" 33 | # 网站名 34 | site_name="94iMM" 35 | # 一键脚本自动添加 36 | site_url = "https://www.94imm.com" 37 | # 网站关键词 38 | key_word = "关键词1,关键词2,关键词3" 39 | # 网站说明 40 | description = "这是一个高质量的自动爬虫" 41 | # 底部联系邮箱 42 | email = "admin@94imm.com" 43 | # 网站调试模式 44 | debug = False 45 | # 页面底部友情链接 46 | friendly_link = [{"name":"94imm","link":"https://www.94imm.com"},{"name":"获取源码","link":"https://github.com/Turnright-git/94imm.git"}] 47 | 48 | ``` 49 | 50 | 使用说明 51 | > 进入项目根目录 52 | ```shell 53 | 启动网站 54 | ./start s 55 | 关闭网站 56 | ./start stop 57 | 重启网站 58 | ./start r 59 | 清空网站缓存（使所做的修改立即生效） 60 | ./start c 61 | ``` 62 | > 项目模板目录templates，base.html中可直接添加统计代码 63 | 64 | 手动安装说明 65 | > 项目依赖python3.6 mysql5.6 66 | 67 | ``` 68 | git clone https://github.com/Turnright-git/94imm.git 69 | cd 94imm 70 | vi config.py #参照配置说明修改 71 | vi uwsgi.ini #修改uwsgi配置 72 | ``` 73 | 如需使用反向代理，在nginx.conf中添加如下server段 74 | ``` 75 | server { 76 | listen 80; 77 | server_name localhost; # 网站域名 78 | 79 | location / { 80 | proxy_pass http://127.0.0.1:8000; 81 | } 82 | ``` 83 | -------------------------------------------------------------------------------- /templates/zde/pagination.html: -------------------------------------------------------------------------------- 1 | {% if is_paginated %} 2 | {% load i18n %} 3 |
    4 | {% block previouslink %} 5 | {% if page_obj.has_previous %} 6 | {% if disable_link_for_first_page and page_obj.previous_page_number == 1 %} 7 |
  • {{ previous_link_decorator|safe }}{% trans "" %}
    • 8 | {% else %} 9 |
  • {{ previous_link_decorator|safe }}{% trans "" %}
    • 10 | {% endif %} 11 | {% else %} 12 | {% if display_disabled_previous_link %} 13 |
  • {{ previous_link_decorator|safe }}{% trans "" %}
    • 14 | {% endif %} 15 | {% endif %} 16 | {% endblock previouslink %} 17 | {% block pagelinks %} 18 | {% if display_page_links %} 19 | {% for page in pages %} 20 | {% if page %} 21 | {% ifequal page page_obj.number %} 22 |
  • {{ page }}
    • 23 | {% else %} 24 | {% if disable_link_for_first_page and page == 1 %} 25 |
  • {{ page }}
    • 26 | {% else %} 27 |
  • {{ page }}
    • 28 | {% endif %} 29 | {% endifequal %} 30 | {% else %} 31 |
  • ...
    • 32 | {% endif %} 33 | {% endfor %} 34 | {% endif %} 35 | {% endblock pagelinks %} 36 | {% block nextlink %} 37 | {% if page_obj.has_next %} 38 |
  • {% trans "" %}{{ next_link_decorator|safe }}
    • 39 | {% else %} 40 | {% if display_disabled_next_link %} 41 |
  • {% trans "" %}{{ next_link_decorator|safe }}
    • 42 | {% endif %} 43 | {% endif %} 44 | {% endblock nextlink %} 45 | 46 | {% endif %} 47 | -------------------------------------------------------------------------------- /static/zde/js/html5.js: -------------------------------------------------------------------------------- 1 | (function(a,b){function h(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function i(){var a=l.elements;return typeof a=="string"?a.split(" "):a}function j(a){var b={},c=a.createElement,f=a.createDocumentFragment,g=f();a.createElement=function(a){if(!l.shivMethods)return c(a);var f;return b[a]?f=b[a].cloneNode():e.test(a)?f=(b[a]=c(a)).cloneNode():f=c(a),f.canHaveChildren&&!d.test(a)?g.appendChild(f):f},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+i().join().replace(/\w+/g,function(a){return c(a),g.createElement(a),'c("'+a+'")'})+");return n}")(l,g)}function k(a){var b;return a.documentShived?a:(l.shivCSS&&!f&&(b=!!h(a,"article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio{display:none}canvas,video{display:inline-block;*display:inline;*zoom:1}[hidden]{display:none}audio[controls]{display:inline-block;*display:inline;*zoom:1}mark{background:#FF0;color:#000}")),g||(b=!j(a)),b&&(a.documentShived=b),a)}var c=a.html5||{},d=/^<|^(?:button|form|map|select|textarea|object|iframe|option|optgroup)$/i,e=/^<|^(?:a|b|button|code|div|fieldset|form|h1|h2|h3|h4|h5|h6|i|iframe|img|input|label|li|link|ol|option|p|param|q|script|select|span|strong|style|table|tbody|td|textarea|tfoot|th|thead|tr|ul)$/i,f,g;(function(){var c=b.createElement("a");c.innerHTML="",f="hidden"in c,f&&typeof injectElementWithStyles=="function"&&injectElementWithStyles("#modernizr{}",function(b){b.hidden=!0,f=(a.getComputedStyle?getComputedStyle(b,null):b.currentStyle).display=="none"}),g=c.childNodes.length==1||function(){try{b.createElement("a")}catch(a){return!0}var c=b.createDocumentFragment();return typeof c.cloneNode=="undefined"||typeof c.createDocumentFragment=="undefined"||typeof c.createElement=="undefined"}()})();var l={elements:c.elements||"abbr article aside audio bdi canvas data datalist details figcaption figure footer header hgroup mark meter nav output progress section summary time video",shivCSS:c.shivCSS!==!1,shivMethods:c.shivMethods!==!1,type:"default",shivDocument:k};a.html5=l,k(b)})(this,document) -------------------------------------------------------------------------------- /crawler/Compress.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | sys.path.append('../') 5 | from PIL import Image as Img 6 | import os,threading,platform 7 | 8 | class Compress(): 9 | file_list=[] 10 | rlock = threading.RLock() 11 | def __init__(self,file_dir,new_dir=None,th_num=10): 12 | self.file_dir=file_dir 13 | self.new_dir=new_dir 14 | self.th_num=th_num 15 | 16 | sysstr = platform.system() 17 | if sysstr == "Windows": 18 | p="\\" 19 | else: 20 | p="/" 21 | 22 | def get_file_name(self): 23 | for files in os.walk(self.file_dir): 24 | for name in files[2]: 25 | file=files[0]+self.p+name 26 | size=int(os.path.getsize(file) / 1024) 27 | if size>400: 28 | self.file_list.append(file) 29 | 30 | def pl_compress(self,file_path): 31 | path = file_path.split(self.p) 32 | name = path[-1] 33 | image = Img.open(file_path) 34 | image.save("/".join(path[0:-1])+self.p+name, quality=85) 35 | print("压缩完成：" + file_path) 36 | 37 | def pl_compress_new(self,file_path): 38 | path = file_path.split(self.p) 39 | name = path[-1] 40 | image = Img.open(file_path) 41 | new_name=self.new_dir+self.p+"/".join(path[-2:-1])+self.p 42 | is_ex=os.path.exists(new_name) 43 | if not is_ex: 44 | os.makedirs(new_name) 45 | image.save(new_name+name, quality=85) 46 | print("压缩完成：" + file_path) 47 | 48 | def do_work(self): 49 | while True: 50 | Compress.rlock.acquire() 51 | if len(Compress.file_list) == 0: 52 | Compress.rlock.release() 53 | break 54 | else: 55 | file_path = Compress.file_list.pop() 56 | Compress.rlock.release() 57 | try: 58 | if new_dir == None: 59 | self.pl_compress(file_path) 60 | else: 61 | self.pl_compress_new(file_path) 62 | except Exception as e: 63 | pass 64 | 65 | 66 | def run(self): 67 | for i in range(self.th_num): 68 | download_t = threading.Thread(target=self.do_work) 69 | download_t.start() 70 | 71 | 72 | if __name__ == "__main__": 73 | print("输入源图片所在路径") 74 | dir_name = input("") 75 | print("1.覆盖原图片，2.压缩到新路径") 76 | in_num=input("") 77 | new_dir=None 78 | if in_num=="2": 79 | print("输入保存路径") 80 | new_dir = input("") 81 | compress=Compress(dir_name,new_dir,10) 82 | compress.get_file_name() 83 | compress.run() -------------------------------------------------------------------------------- /templates/zde/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | 9 | 10 | 11 | {% block title %} - {% endblock title %} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 23 | {% block js_head %}{% endblock js_head %} 24 | 25 | 27 | 28 | 29 |
    30 |
    31 |

    {{ siteName }}

    32 |
    33 | {% block menu %}{% endblock menu %} 34 |
    35 | 36 | 37 | 38 | 39 |
    40 | 41 | 42 |
    43 |
    44 |
    45 | {% block focusbox %}{% endblock focusbox %} 46 |
    {% block section %}{% endblock section %}
    47 | {% block recommend %}{% endblock recommend %} 48 |
    49 |

    © 2020 {{ siteName }}

    50 |

    美图分享,每日更新.

    51 |
      友情链接： 52 | {% for links in friendly_link %} 53 |
    • {{ links.name }}
      • 54 | {% endfor %} 55 |
    56 |

    POWERED BY 94iMM | 本站数据来自网络，如侵害您的权利请联系本站删除相关信息，邮箱：{{ email }}

    57 |
    58 | 59 | {% block js %}{% endblock js %} 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /static/zde/js/flutter-hearts-zmt.js: -------------------------------------------------------------------------------- 1 | !function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):t.BubbleHearts=e()}(this,function(){"use strict";function t(t,n,r){var i=e.uniformDiscrete(89,91)/100,o=1-i,u=(e.uniformDiscrete(45,60)+e.uniformDiscrete(45,60))/100,a=function(t){return t>i?Math.max(((1-t)/o).toFixed(2),.1)*u:u},c=e.uniformDiscrete(-30,30),f=function(t){return c},h=10,s=n.width/2+e.uniformDiscrete(-h,h),d=(n.width-Math.sqrt(Math.pow(t.width,2)+Math.pow(t.height,2)))/2-h,l=e.uniformDiscrete(.8*d,d)*(e.uniformDiscrete(0,1)?1:-1),m=e.uniformDiscrete(250,400),w=function(t){return t>i?s:s+l*Math.sin(m*(i-t)*Math.PI/180)},v=function(e){return t.height/2+(n.height-t.height/2)*e},p=e.uniformDiscrete(14,18)/100,g=function(t){return t>p?1:1-((p-t)/p).toFixed(2)};return function(e){if(!(e>=0))return!0;r.save();var n=a(e),i=f(e),o=w(e),u=v(e);r.translate(o,u),r.scale(n,n),r.rotate(i*Math.PI/180),r.globalAlpha=g(e),r.drawImage(t,-t.width/2,-t.height/2,t.width,t.height),r.restore()}} 2 | var e=function(t){var e=t,n=Math.floor,r=Math.random;return t.uniform=function(t,e){return t+(e-t)*r()},t.uniformDiscrete=function(t,r){return t+n((r-t+1)*e.uniform(0,1))},t}({}),n=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")},r=function(){function t(t,e){for(var n=0;n1&&void 0!==arguments[1]?arguments[1]:e.uniformDiscrete(1200,2400),i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:t(n,this.canvas,this.context);return this._children.push({render:i,duration:r,timestamp:+new Date}),this}}]),o}();return o});var assets=['/static/zde/1.png','/static/zde/2.png','/static/zde/3.png','/static/zde/4.png','/static/zde/5.png',];assets.forEach(function(src,index){assets[index]=new Promise(function(resolve){var img=new Image;img.onload=resolve.bind(null,img);img.src=src;});});Promise.all(assets).then(function(images){var random={uniform:function(min,max){return min+(max-min)*Math.random();},uniformDiscrete:function(i,j){return i+Math.floor((j-i+1)*random.uniform(0,1));},};var stage=new BubbleHearts();var canvas=stage.canvas;canvas.width=170;canvas.height=300;canvas.style['width']='170px';canvas.style['height']='300px';document.body.appendChild(canvas);document.getElementsByClassName('journal-reward')[0].addEventListener('click',function(){stage.bubble(images[random.uniformDiscrete(0,images.length-1)]);},false);}); -------------------------------------------------------------------------------- /templates/zde/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}{{ siteName }}{% endblock title %} 5 | {% block keywords %}{{ keyWord }}{% endblock keywords %} 6 | {% block description %}{{ description }}{% endblock description %} 7 | 8 | 9 | {% block menu %} 10 | 32 | {% endblock menu %} 33 | 34 | 35 | {% block section %} 36 |
    37 |
    38 | {% load pagination_tags %} 39 | {% autopaginate data 10 %} 40 | {% for imglist in data %} 41 | 55 | {% endfor %} 56 |
    57 |
    58 | 59 | {% endblock section%} 60 | 61 | 62 | {% block js %} 63 | 73 | {% endblock js %} 74 | -------------------------------------------------------------------------------- /crawler/down_img.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('../') 4 | import threading, pymysql,os,requests 5 | from config import mysql_config 6 | 7 | 8 | url_list=[] 9 | origin_list=[] 10 | 11 | 12 | rlock = threading.RLock() 13 | dbhost = { 14 | "host": mysql_config['HOST'], 15 | "dbname": mysql_config['NAME'], 16 | "user": mysql_config['USER'], 17 | "password": mysql_config['PASSWORD'] 18 | } 19 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), dbhost.get("dbname")) 20 | cursor = db.cursor() 21 | 22 | def del_page(): 23 | cursor.execute("Delete FROM images_page WHERE firstimg='1'") 24 | 25 | def img_url(): 26 | cursor.execute("SELECT * FROM images_image") 27 | for img_url in cursor.fetchall(): 28 | isExiststag=os.path.exists(".."+img_url[2]) 29 | host_sql = "SELECT crawler FROM images_page WHERE id =" + "'" + str(img_url[1]) + "'" 30 | cursor.execute(host_sql) 31 | if not isExiststag: 32 | print("添加图片：" + img_url[-1]) 33 | url_list.append({"img_path": img_url[-2], "origin_url": img_url[-1], "host": cursor.fetchone()[0]}) 34 | elif os.path.getsize(".."+img_url[2])==0: 35 | os.remove(".."+img_url[2]) 36 | print("添加图片：" + img_url[-1]) 37 | url_list.append({"img_path": img_url[-2], "origin_url": img_url[-1], "host": cursor.fetchone()[0]}) 38 | elif "http://www.nvshenge.com" in img_url[-1]: 39 | image_path=img_url[-2].split("/")[-1][0:8] 40 | origin_url="http://nvshenge.com/uploads/image/"+image_path+"/"+img_url[-2].split("/")[-1] 41 | print("添加图片：" + origin_url) 42 | url_list.append({"img_path": img_url[-2], "origin_url": origin_url, "host": cursor.fetchone()[0]}) 43 | else: 44 | continue 45 | 46 | 47 | def down_img(): 48 | while True: 49 | rlock.acquire() 50 | if len(url_list) == 0: 51 | rlock.release() 52 | break 53 | else: 54 | img_url = url_list.pop() 55 | rlock.release() 56 | try: 57 | headers = { 58 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 59 | "Referer": img_url.get("host") 60 | } 61 | img_path=img_url.get("img_path") 62 | path = "/".join(img_path.split("/")[1:-1]) 63 | is_path=os.path.exists("../"+path) 64 | if "http://nvshenge.com" == img_url.get("origin_url")[0:19]: 65 | continue 66 | if not is_path: 67 | os.makedirs("../" + path) 68 | origin_url=img_url.get("origin_url") 69 | with open (".."+img_path,"wb") as w: 70 | w.write(requests.get(origin_url, headers=headers,verify=False,timeout=5).content) 71 | print("下载完成："+origin_url) 72 | except Exception as e: 73 | # url_list.append({"img_path":img_url.get("img_path"),"origin_url":img_url.get("origin_url"),"host":img_url.get("host")}) 74 | # print("下载失败，重新添加到列表中") 75 | pass 76 | 77 | if __name__ == "__main__": 78 | del_page() 79 | try: 80 | img_url() 81 | except Exception as e: 82 | print(e) 83 | for i in range(10): 84 | t2 = threading.Thread(target=down_img) 85 | t2.start() -------------------------------------------------------------------------------- /templates/zde/category.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}妹子图 - {{ siteName }}{% endblock title %} 5 | {% block keywords %}{{ keyWord }}{% endblock keywords %} 6 | {% block description %}{{ description }}{% endblock description %} 7 | 8 | 9 | {% block menu %} 10 | 37 | {% endblock menu %} 38 | 39 | 40 | {% block section %} 41 |
    42 |
    43 | {% load pagination_tags %} 44 | {% autopaginate data 10 %} 45 | {% for imglist in data %} 46 | 60 | {% endfor %} 61 |
    62 |
    63 | 64 | {% endblock section %} 65 | 66 | 67 | {% block js %} 68 | 78 | {% endblock js %} 79 | -------------------------------------------------------------------------------- /crawler/AutoPost.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | sys.path.append('../') 5 | import pymysql,time,os,random,shutil,platform 6 | from config import mysql_config 7 | 8 | dbhost = { 9 | "host": mysql_config['HOST'], 10 | "dbname": mysql_config['NAME'], 11 | "user": mysql_config['USER'], 12 | "password": mysql_config['PASSWORD'] 13 | } 14 | 15 | def do_post(file_dir,sleep_time="0"): 16 | db = pymysql.connect(dbhost.get("host"),dbhost.get("user"), dbhost.get("password"),dbhost.get("dbname")) 17 | cursor = db.cursor() 18 | for files in os.walk(file_dir): 19 | tagidlist = [] 20 | sysstr = platform.system() 21 | if sysstr == "Windows": 22 | title=files[0].split("\\")[-1] 23 | os_path=file_dir.split("\\")[-1] 24 | elif sysstr == "Linux": 25 | title = files[0].split("/")[-1] 26 | os_path = file_dir.split("/")[-1] 27 | if title != os_path: 28 | tags=['cosplay','萝莉','美腿','丝袜','少女'] 29 | isExists = cursor.execute("SELECT * FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 30 | if isExists != 0: 31 | print("已存在：" + title) 32 | else: 33 | for tag in tags: 34 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 35 | isExiststag = cursor.execute(sqltag) 36 | if isExiststag != 1: 37 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 38 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 39 | for id in cursor.fetchall(): 40 | tagidlist.append(id[0]) 41 | p = (title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), "1", "1") 42 | cursor.execute("INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg) VALUES (%s,%s,%s,%s,%s)", 43 | p) 44 | pageid = cursor.lastrowid 45 | rpath = "".join(random.sample('abcdefghijklmnopqrstuvwxyz', 7)) 46 | count = 1 47 | for name in files[2]: 48 | path=files[0]+"/"+name 49 | rename=str(count)+"."+name.split(".")[-1] 50 | path_isExists=os.path.exists("../static/images/"+rpath) 51 | if not path_isExists: 52 | os.makedirs("../static/images/"+rpath) 53 | try: 54 | shutil.move(path, "../static/images/"+rpath+"/"+rename) 55 | imgp = "/static/images/" + rpath+"/"+rename 56 | if count==1: 57 | cursor.execute( 58 | "UPDATE images_page SET firstimg = %s WHERE id=%s",(imgp,pageid)) 59 | cursor.execute("INSERT INTO images_image (pageid,imageurl) VALUES (%s,%s)", (pageid,imgp)) 60 | 61 | except Exception as e: 62 | print(e) 63 | break 64 | count+=1 65 | try: 66 | os.removedirs(files[0]) 67 | except: 68 | print("目录不为空，无法删除") 69 | print("发布完成：" + title) 70 | time.sleep(int(sleep_time)) 71 | 72 | # do_post("输入图片所在目录","发布间隔时间，默认0，单位秒") 73 | if __name__ == "__main__": 74 | print("图片所在目录：") 75 | path=input("") 76 | print("自动发布间隔，0为全部发布，单位秒") 77 | send_time=input("") 78 | do_post(path,send_time) 79 | 80 | -------------------------------------------------------------------------------- /templates/zde/sort.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}{% ifequal typeid "new" %}最新发布 - {{ siteName }}{{ else }}最受欢迎- {{ siteName }}{% endifequal %}{% endblock title %} 5 | {% block keywords %}{{ keyWord }}{% endblock keywords %} 6 | {% block description %}{{ description }}{% endblock description %} 7 | 8 | 9 | {% block menu %} 10 | 39 | {% endblock menu %} 40 | 41 | 42 | {% block section %} 43 |
    44 |
    45 | {% load pagination_tags %} 46 | {% autopaginate data 10 %} 47 | {% for imglist in data %} 48 | 62 | {% endfor %} 63 |
    64 |
    65 | 66 | {% endblock section %} 67 | 68 | 69 | {% block js %} 70 | 80 | {% endblock js %} 81 | -------------------------------------------------------------------------------- /94imm.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat Premium Data Transfer 3 | 4 | Source Server : localhost 5 | Source Server Type : MySQL 6 | Source Server Version : 50629 7 | Source Host : localhost:3306 8 | Source Schema : 94imm 9 | 10 | Target Server Type : MySQL 11 | Target Server Version : 50629 12 | File Encoding : 65001 13 | 14 | Date: 15/04/2020 20:13:30 15 | */ 16 | 17 | SET NAMES utf8mb4; 18 | SET FOREIGN_KEY_CHECKS = 0; 19 | 20 | -- ---------------------------- 21 | -- Table structure for django_migrations 22 | -- ---------------------------- 23 | DROP TABLE IF EXISTS `django_migrations`; 24 | CREATE TABLE `django_migrations` ( 25 | `id` int(11) NOT NULL AUTO_INCREMENT, 26 | `app` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 27 | `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 28 | `applied` datetime(0) NOT NULL, 29 | PRIMARY KEY (`id`) USING BTREE 30 | ) ENGINE = MyISAM AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; 31 | 32 | -- ---------------------------- 33 | -- Table structure for images_image 34 | -- ---------------------------- 35 | DROP TABLE IF EXISTS `images_image`; 36 | CREATE TABLE `images_image` ( 37 | `id` int(11) NOT NULL AUTO_INCREMENT, 38 | `pageid` int(11) NOT NULL, 39 | `imageurl` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 40 | `originurl` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 41 | PRIMARY KEY (`id`) USING BTREE 42 | ) ENGINE = MyISAM AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; 43 | 44 | -- ---------------------------- 45 | -- Table structure for images_page 46 | -- ---------------------------- 47 | DROP TABLE IF EXISTS `images_page`; 48 | CREATE TABLE `images_page` ( 49 | `id` int(11) NOT NULL AUTO_INCREMENT, 50 | `typeid` int(11) NOT NULL, 51 | `sendtime` date NOT NULL, 52 | `title` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 53 | `firstimg` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 54 | `tagid` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 55 | `crawler` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 56 | `hot` int(10) NULL DEFAULT 0, 57 | PRIMARY KEY (`id`) USING BTREE 58 | ) ENGINE = MyISAM AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; 59 | 60 | -- ---------------------------- 61 | -- Table structure for images_tag 62 | -- ---------------------------- 63 | DROP TABLE IF EXISTS `images_tag`; 64 | CREATE TABLE `images_tag` ( 65 | `id` int(4) NOT NULL AUTO_INCREMENT, 66 | `tag` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 67 | `uid` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 68 | `date` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 69 | PRIMARY KEY (`id`) USING BTREE 70 | ) ENGINE = MyISAM AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; 71 | 72 | -- ---------------------------- 73 | -- Table structure for images_type 74 | -- ---------------------------- 75 | DROP TABLE IF EXISTS `images_type`; 76 | CREATE TABLE `images_type` ( 77 | `id` int(11) NOT NULL AUTO_INCREMENT, 78 | `type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL, 79 | PRIMARY KEY (`id`) USING BTREE 80 | ) ENGINE = MyISAM AUTO_INCREMENT = 7 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic; 81 | 82 | -- ---------------------------- 83 | -- Table structure for images_video 84 | -- ---------------------------- 85 | DROP TABLE IF EXISTS `images_video`; 86 | CREATE TABLE `images_video` ( 87 | `id` int(11) NOT NULL AUTO_INCREMENT, 88 | `url` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 89 | `user_id` varchar(15) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 90 | `date_time` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 91 | `v_name` varchar(255) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL, 92 | `v_path` varchar(50) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL, 93 | `source` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 94 | PRIMARY KEY (`id`) USING BTREE 95 | ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = latin1 COLLATE = latin1_swedish_ci ROW_FORMAT = Compact; 96 | 97 | SET FOREIGN_KEY_CHECKS = 1; 98 | -------------------------------------------------------------------------------- /silumz/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for silumz project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.6. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import os 14 | from config import mysql_config,allow_url,cache_time,templates,debug 15 | 16 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 17 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 18 | 19 | 20 | # Quick-start development settings - unsuitable for production 21 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 22 | 23 | # SECURITY WARNING: keep the secret key used in production secret! 24 | SECRET_KEY = 'ge)(a+37gny_zn9c(+(kq+^yqw!jvblb67ck5allkpgv6(wi@^' 25 | 26 | # SECURITY WARNING: don't run with debug turned on in production! 27 | DEBUG = debug 28 | 29 | ALLOWED_HOSTS = allow_url 30 | 31 | 32 | # Application definition 33 | 34 | INSTALLED_APPS = [ 35 | 'django.contrib.admin', 36 | 'django.contrib.auth', 37 | 'django.contrib.contenttypes', 38 | 'django.contrib.sessions', 39 | 'django.contrib.messages', 40 | 'django.contrib.staticfiles', 41 | 'images', 42 | 'dj_pagination' 43 | ] 44 | 45 | MIDDLEWARE = [ 46 | 'django.middleware.cache.UpdateCacheMiddleware', 47 | 'django.middleware.security.SecurityMiddleware', 48 | 'django.middleware.gzip.GZipMiddleware', 49 | 'django.contrib.sessions.middleware.SessionMiddleware', 50 | 'django.middleware.common.CommonMiddleware', 51 | # 'django.middleware.csrf.CsrfViewMiddleware', 52 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 53 | 'django.contrib.messages.middleware.MessageMiddleware', 54 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 55 | 'dj_pagination.middleware.PaginationMiddleware', 56 | 'django.middleware.cache.FetchFromCacheMiddleware', 57 | ] 58 | 59 | CACHES = { 60 | 'default': { 61 | 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 62 | 'LOCATION': 'cache', #设置缓存文件的目录 63 | 'TIMEOUT':cache_time, 64 | 'OPTIONS':{ 65 | 'MAX_ENTRIES': 300, 66 | 'CULL_FREQUENCY': 3, 67 | } 68 | } 69 | } 70 | 71 | ROOT_URLCONF = 'silumz.urls' 72 | 73 | TEMPLATES = [ 74 | { 75 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 76 | 'DIRS': [os.path.join(BASE_DIR, 'templates'+"/"+templates)] 77 | , 78 | 'APP_DIRS': True, 79 | 'OPTIONS': { 80 | 'context_processors': [ 81 | 'django.template.context_processors.debug', 82 | 'django.template.context_processors.request', 83 | 'django.contrib.auth.context_processors.auth', 84 | 'django.contrib.messages.context_processors.messages', 85 | 'django.template.context_processors.i18n', 86 | 'django.template.context_processors.media', 87 | 'django.contrib.auth.context_processors.auth', 88 | ], 89 | }, 90 | }, 91 | ] 92 | 93 | WSGI_APPLICATION = 'silumz.wsgi.application' 94 | 95 | 96 | # Database 97 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 98 | 99 | DATABASES = { 100 | 'default': { 101 | 'ENGINE': 'django.db.backends.mysql', 102 | 'NAME': mysql_config['NAME'], 103 | 'USER': mysql_config['USER'], 104 | 'PASSWORD': mysql_config['PASSWORD'], 105 | 'HOST': mysql_config['HOST'], 106 | 'PORT': mysql_config['PORT'], 107 | } 108 | } 109 | 110 | 111 | # Password validation 112 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 113 | 114 | AUTH_PASSWORD_VALIDATORS = [ 115 | { 116 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 117 | }, 118 | { 119 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 120 | }, 121 | { 122 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 123 | }, 124 | { 125 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 126 | }, 127 | ] 128 | 129 | 130 | # Internationalization 131 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 132 | 133 | LANGUAGE_CODE = 'en-us' 134 | 135 | TIME_ZONE = 'UTC' 136 | 137 | USE_I18N = True 138 | 139 | USE_L10N = True 140 | 141 | USE_TZ = True 142 | 143 | 144 | #= Static files (CSS, JavaScript, Images) 145 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 146 | 147 | STATIC_URL = '/static/' 148 | STATICFILES_DIRS=( 149 | os.path.join(BASE_DIR,'static'), 150 | ) 151 | 152 | # dj_pagination 153 | PAGINATION_DEFAULT_WINDOW=1 154 | -------------------------------------------------------------------------------- /templates/zde/video.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}视频小姐姐 - {{ siteName }}{% endblock title %} 5 | {% block keywords %}{% for t in keyword %}{{ t.tag }},{% endfor %}{% endblock keywords %} 6 | {% block description %}{{ description }}{% endblock description %} 7 | 8 | {% block js_head %} 9 | 10 | {% endblock js_head %} 11 | {% block focusbox %} 12 |
    13 |

    小姐姐福利视频

    14 |
    15 | 美女短视频 16 | {{ source }}短视频 17 | 小姐姐福利视频 18 |
    19 |
    20 | 点击用户ID可以关注小姐姐哦~！如查询结果为空请尝试刷新查询页面 21 |
    22 |
    23 |
    24 | {% endblock focusbox %} 25 | 26 | {% block menu %} 27 | 51 | {% endblock menu %} 52 | 53 | 54 | {% block section %} 55 | 56 |
    57 | 59 | 视频来源:[{{ source }}] 61 | 用户ID:[{{ user_id }}] 63 | 65 |
    66 | 67 | {% endblock section %} 68 | 69 | {% block js %} 70 | 71 | 123 | {% endblock js %} -------------------------------------------------------------------------------- /crawler/crawler_xmt.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import sys 3 | 4 | sys.path.append('../') 5 | from bs4 import BeautifulSoup 6 | import threading, pymysql, time, requests, os, urllib3, re 7 | from config import mysql_config 8 | 9 | requests.packages.urllib3.disable_warnings() 10 | # 数据库连接信息 11 | dbhost = { 12 | "host": mysql_config['HOST'], 13 | "dbname": mysql_config['NAME'], 14 | "user": mysql_config['USER'], 15 | "password": mysql_config['PASSWORD'] 16 | } 17 | 18 | 19 | class Spider(): 20 | headers = { 21 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' 22 | 'Chrome/65.0.3325.181 Safari/537.36', 23 | 'Referer': "http://www.xgmmtk.com/" 24 | } 25 | page_url_list = [] 26 | img_url_list = [] 27 | rlock = threading.RLock() 28 | # s = requests.session() 29 | 30 | def __init__(self, img_path='imgdir', thread_number=5): 31 | self.spider_url = 'http://www.xgmmtk.com/' 32 | self.img_path = img_path 33 | self.thread_num = thread_number 34 | 35 | def get_url(self): 36 | page = requests.get("http://www.xgmmtk.com/") 37 | soup = BeautifulSoup(page.text, "html.parser") 38 | a_soup = soup.find_all("a") 39 | for a in a_soup: 40 | url = "http://www.xgmmtk.com/" + a.get("href") 41 | self.page_url_list.append(url) 42 | 43 | def get_img(self): 44 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), dbhost.get("dbname")) 45 | cursor = db.cursor() 46 | while True: 47 | self.rlock.acquire() 48 | if len(self.page_url_list) == 0: 49 | self.rlock.release() 50 | break 51 | else: 52 | page_url = self.page_url_list.pop() 53 | self.rlock.release() 54 | page = requests.get(page_url) 55 | soup=BeautifulSoup(page.text,"html.parser") 56 | title=soup.title.string.replace("�","") 57 | isExists = cursor.execute( 58 | "SELECT title FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 59 | if isExists == 0: 60 | print("添加采集：",title) 61 | if "袜" in title or "丝" in title or "腿" in title: 62 | type_id = 2 63 | tagidlist=[3679,3700,3719,3628] 64 | elif "青春" in title or "清纯" in title or "萝莉" in title: 65 | tagidlist=[3694,3627,3635] 66 | type_id = 3 67 | else: 68 | tagidlist=[3630,3623,3618,3642] 69 | type_id = 1 70 | p = ( 71 | title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), type_id, 72 | "1", 73 | page_url) 74 | cursor.execute( 75 | "INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", 76 | p) 77 | pageid = cursor.lastrowid 78 | img = soup.find_all("img") 79 | i=0 80 | page_id=page_url[page_url.find("?id=")+4:-1] 81 | img_path = self.img_path + time.strftime('%Y%m%d', time.localtime( 82 | time.time())) + "/" +page_id + "/" 83 | for imgurl in img: 84 | imgsrc = "http://www.xgmmtk.com/" + imgurl.get("src") 85 | self.img_url_list.append( 86 | {"img_url": imgsrc, "Referer": page_url, 87 | "id": page_id}) 88 | if i==0: 89 | cursor.execute( 90 | "UPDATE images_page SET firstimg = " + "'" + img_path+imgsrc.split("/")[-1] + "'" + " WHERE id=" + "'" + str( 91 | pageid) + "'") 92 | i+=1 93 | else: 94 | print("已采集") 95 | pass 96 | def down_img(self,imgsrc,Referer,id): 97 | headers = { 98 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 99 | "Referer": Referer 100 | } 101 | path = self.img_path + time.strftime('%Y%m%d', time.localtime(time.time())) + "/" 102 | page_id = id 103 | isdata = os.path.exists("../" + path + page_id) 104 | if not isdata: 105 | os.makedirs("../" + path + page_id) 106 | with open("../" + path + page_id + "/" + imgsrc.split("/")[-1], "wb") as f: 107 | print("已保存：" ,imgsrc) 108 | f.write(requests.get(imgsrc, headers=headers,verify=False).content) 109 | 110 | 111 | 112 | def run_img(self): 113 | while True: 114 | Spider.rlock.acquire() 115 | if len(self.img_url_list) == 0 : 116 | Spider.rlock.release() 117 | continue 118 | else: 119 | urls = self.img_url_list.pop() 120 | url = urls.get("img_url") 121 | Referer = urls.get("Referer") 122 | id = urls.get("id") 123 | Spider.rlock.release() 124 | try: 125 | self.down_img(url, Referer, id) 126 | except Exception as e: 127 | pass 128 | 129 | def run(self): 130 | # 启动thread_num个来下载图片 131 | for img_th in range(self.thread_num): 132 | download_t = threading.Thread(target=self.run_img) 133 | download_t.start() 134 | 135 | for img_th in range(self.thread_num): 136 | run_t = threading.Thread(target=self.get_img) 137 | run_t.start() 138 | 139 | if __name__ == "__main__": 140 | spider=Spider(img_path='/static/images/',thread_number=10) 141 | spider.get_url() 142 | spider.run() 143 | -------------------------------------------------------------------------------- /templates/zde/page.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | 4 | {% block title %}{{ title }} - {{ siteName }}{% endblock title %} 5 | {% block keywords %}{% for t in tag %}{{ t.tname }},{% endfor %}{% endblock keywords %} 6 | {% block description %}由{{ siteName }}为您整理的高清《{{ title }}》图集{% endblock description %} 7 | 8 | {% block focusbox %} 9 |
    10 |

    {{ title }}

    11 |
    12 | {% for t in tag %} 13 | {{ t.tname }} 14 | {% endfor %} 15 |
    16 |
    17 | 点击图片自动播放 18 |
    19 |
    20 |
    21 | {% endblock focusbox %} 22 | 23 | {% block menu %} 24 | 50 | {% endblock menu %} 51 | 52 | 53 | {% block section %} 54 | 55 | 56 |
    57 | {% for img in data %} 58 |

    {{ title }} 59 | 60 | {% endfor %} 61 |

    62 |
    63 | {% endblock section %} 64 | 65 | {% block recommend %} 66 |
    相关推荐
    67 |
    68 |
    69 | {% for items in similar %} 70 | 86 | {% endfor %} 87 |
    88 |
    89 | {% endblock recommend %} 90 | 91 | {% block js %} 92 | 147 | {% endblock js %} 148 | -------------------------------------------------------------------------------- /crawler/crawle_mzt.py: -------------------------------------------------------------------------------- 1 | # coding='UTF-8' 2 | import sys 3 | 4 | sys.path.append('../') 5 | from bs4 import BeautifulSoup 6 | import threading, pymysql, time, requests, os, urllib3 7 | from config import mysql_config 8 | 9 | requests.packages.urllib3.disable_warnings() 10 | 11 | 12 | class Spider(): 13 | headers = { 14 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' 15 | 'Chrome/65.0.3325.181 Safari/537.36', 16 | 'Referer': "https://www.mzitu.com" 17 | } 18 | page_url_list = [] 19 | img_url_list = [] 20 | rlock = threading.RLock() 21 | s = requests.session() 22 | s.keep_alive = False 23 | dbhost = { 24 | "host": mysql_config['HOST'], 25 | "dbname": mysql_config['NAME'], 26 | "user": mysql_config['USER'], 27 | "password": mysql_config['PASSWORD'] 28 | } 29 | 30 | def __init__(self, page_num=10, img_path='imgdir', thread_num=5, type="xinggan", type_id=1): 31 | self.spider_url = 'https://www.mzitu.com/' 32 | self.page_number = int(page_num) 33 | self.img_path = img_path 34 | self.thread_num = thread_num 35 | self.type = type 36 | self.type_id = type_id 37 | 38 | def get_url(self): 39 | for i in range(1, self.page_number + 1): 40 | if i ==1: 41 | page = self.s.get(self.spider_url +"/"+self.type ,verify=False).text 42 | page=self.s.get(self.spider_url +"/"+self.type+"/page/"+str(i),verify=False).text 43 | soup = BeautifulSoup(page, "html.parser") 44 | page_base_url = soup.find("div",class_="postlist").find_all("li") 45 | for page_url in page_base_url: 46 | url = page_url.find("a").get("href") 47 | self.page_url_list.append(url) 48 | i = i + 1 49 | 50 | def get_img_url(self): 51 | db = pymysql.connect(self.dbhost.get("host"), self.dbhost.get("user"), self.dbhost.get("password"), 52 | self.dbhost.get("dbname")) 53 | cursor = db.cursor() 54 | for img_base_url in self.page_url_list: 55 | tagidlist = [] 56 | img_soup = BeautifulSoup(self.s.get(img_base_url,verify=False).text, "html.parser") 57 | img_num = img_soup.find("div", class_="pagenavi").text.split("…")[-1][0:-5] 58 | img_url = img_soup.find("div", class_="main-image").find("img").get("src").split("/")[0:-1] 59 | img_surl = "/".join(img_url) 60 | title = img_soup.find("h2", class_="main-title").text 61 | isExists = cursor.execute("SELECT * FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 62 | tag_list = img_soup.find("div", class_="main-tags").find_all("a") 63 | if isExists == 1: 64 | print("已采集：" + title) 65 | else: 66 | for tags in tag_list: 67 | tag=tags.text 68 | print(tag) 69 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 70 | isExiststag = cursor.execute(sqltag) 71 | if isExiststag != 1: 72 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 73 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 74 | for id in cursor.fetchall(): 75 | tagidlist.append(id[0]) 76 | p = (title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), self.type_id, "1") 77 | cursor.execute("INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg) VALUES (%s,%s,%s,%s,%s)", 78 | p) 79 | print("开始采集：" + title) 80 | pageid = cursor.lastrowid 81 | for i in range(1, int(img_num)): 82 | temp_url = img_soup.find("div", class_="main-image").find("img").get("src").split("/") 83 | path = temp_url[-1][0:3] 84 | new_url = img_surl + "/" + path + str("%02d" % i) + ".jpg" 85 | img_src = temp_url[-3] + "/" + temp_url[-2] + "/" + path + str("%02d" % i) + ".jpg" 86 | imgp = pageid, self.img_path + img_src 87 | cursor.execute("INSERT INTO images_image (pageid,imageurl) VALUES (%s,%s)", imgp) 88 | if i == 1: 89 | cursor.execute( 90 | "UPDATE images_page SET firstimg = " + "'" + self.img_path + img_src + "'" + " WHERE title=" + "'" + title + "'") 91 | self.img_url_list.append(new_url) 92 | i = i + 1 93 | db.close() 94 | 95 | def down_img(self, imgsrc): 96 | path = imgsrc.split("/")[-3] + "/" + imgsrc.split("/")[-2] 97 | isdata = os.path.exists("../" + self.img_path + path) 98 | if isdata == False: 99 | os.makedirs("../" + self.img_path + path) 100 | with open("../" + self.img_path + path + "/" + imgsrc.split("/")[-1], "wb")as f: 101 | print("下载图片：" + self.img_path + path + "/" + imgsrc.split("/")[-1]) 102 | f.write(requests.get(imgsrc, headers=self.headers, verify=False).content) 103 | 104 | def down_url(self): 105 | while True: 106 | Spider.rlock.acquire() 107 | if len(Spider.img_url_list) == 0: 108 | Spider.rlock.release() 109 | break 110 | else: 111 | img_url = Spider.img_url_list.pop() 112 | Spider.rlock.release() 113 | try: 114 | self.down_img(img_url) 115 | except Exception as e: 116 | pass 117 | 118 | def run(self): 119 | # 启动thread_num个来下载图片 120 | for img_th in range(self.thread_num): 121 | download_t = threading.Thread(target=self.down_url) 122 | download_t.start() 123 | 124 | 125 | if __name__ == '__main__': 126 | for i in [{"page": 1, "type": "xinggan", "type_id": 1},]: 127 | spider = Spider(page_num=i.get("page"), img_path='/static/images/', thread_num=10, type_id=i.get("type_id"), 128 | type=i.get("type")) 129 | spider.get_url() 130 | spider.get_img_url() 131 | spider.run() 132 | -------------------------------------------------------------------------------- /crawler/crawler_mtl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import sys 3 | 4 | sys.path.append('../') 5 | from bs4 import BeautifulSoup 6 | import threading,pymysql,time,requests,os,urllib3 7 | from config import mysql_config 8 | requests.packages.urllib3.disable_warnings() 9 | requests.adapters.DEFAULT_RETRIES = 5 10 | 11 | class Spider(): 12 | headers = { 13 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' 14 | 'Chrome/65.0.3325.181 Safari/537.36', 15 | 'Referer': "https://www.meitulu.com" 16 | } 17 | page_url_list = [] 18 | img_url_list = [] 19 | rlock = threading.RLock() 20 | s=requests.session() 21 | s.keep_alive = False 22 | dbhost = { 23 | "host": mysql_config['HOST'], 24 | "dbname": mysql_config['NAME'], 25 | "user": mysql_config['USER'], 26 | "password": mysql_config['PASSWORD'] 27 | } 28 | 29 | def __init__(self,page_number=10,img_path='imgdir',thread_number=5,type='xinggan',type_id=1): 30 | self.spider_url = 'https://www.meitulu.com/t/'+type 31 | self.page_number = int(page_number) 32 | self.img_path = img_path 33 | self.thread_num = thread_number 34 | self.type_id = type_id 35 | 36 | def get_url(self): 37 | db = pymysql.connect(self.dbhost.get("host"), self.dbhost.get("user"), self.dbhost.get("password"), 38 | self.dbhost.get("dbname")) 39 | cursor = db.cursor() 40 | for i in range(1, self.page_number+1): 41 | page_base_url = BeautifulSoup(requests.get(self.spider_url + "/" + str(i) + ".html").content.decode("utf-8"), 42 | "html.parser") 43 | if i == 1: 44 | page_base_url = BeautifulSoup(requests.get(self.spider_url).content.decode("utf-8"), "html.parser") 45 | img_ul = page_base_url.find("ul", class_="img").find_all("li") 46 | for img_li in img_ul: 47 | page_url = img_li.find("p", class_="p_title").find("a").get("href") 48 | self.page_url_list.append(page_url) 49 | db.close() 50 | 51 | def get_img_url(self): 52 | db = pymysql.connect(self.dbhost.get("host"), self.dbhost.get("user"), self.dbhost.get("password"), self.dbhost.get("dbname")) 53 | cursor = db.cursor() 54 | for page_url in reversed(self.page_url_list): 55 | tagidlist = [] 56 | img_div_soup = BeautifulSoup(requests.get(page_url).content.decode("utf-8"), "html.parser") 57 | img_base_url = img_div_soup.find("img", class_="content_img").get("src").split("/") 58 | img_url = "/".join(img_base_url[0:-1]) 59 | title = img_div_soup.find("div", class_="weizhi").find("h1").text.replace(" 萝莉丝袜写真套图","") 60 | isExists =cursor.execute("SELECT * FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 61 | if isExists != 0: 62 | print ("已采集:"+title) 63 | else: 64 | tag_list = img_div_soup.find("div", class_="fenxiang_l").find_all("a") 65 | for tag in tag_list: 66 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag.text + "'" + " limit 1;" 67 | isExiststag = cursor.execute(sqltag) 68 | if isExiststag == 0: 69 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag.text) 70 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag.text + "'") 71 | for id in cursor.fetchall(): 72 | tagidlist.append(id[0]) 73 | p = ( 74 | title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), self.type_id, "1", page_url) 75 | cursor.execute( 76 | "INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", 77 | p) 78 | pageid =cursor.lastrowid 79 | ima_num_tem = img_div_soup.find("div", id="pages").text 80 | img_num = ima_num_tem[-6:-4] 81 | i = 1 82 | for i in range(1, int(img_num)): 83 | img_src = img_url + "/" + str(i) + "." + img_base_url[-1].split(".")[-1] 84 | img_loc_path = self.img_path + img_base_url[-2]+"/"+ str(i) + "." + img_base_url[-1].split(".")[-1] 85 | imgp = pageid, img_loc_path,img_src 86 | if i == 1: 87 | cursor.execute( 88 | "UPDATE images_page SET firstimg = " + "'" + img_loc_path + "'" + " WHERE title=" + "'" + title + "'") 89 | i = i + 1 90 | cursor.execute("INSERT INTO images_image (pageid,imageurl,originurl) VALUES (%s,%s,%s)", imgp) 91 | self.img_url_list.append(img_src) 92 | print("添加："+title) 93 | db.close() 94 | 95 | def down_img(self,imgsrc): 96 | path = imgsrc.split("/")[-2] 97 | isdata = os.path.exists("../" + self.img_path + path) 98 | if isdata == False: 99 | os.makedirs("../" + self.img_path + path) 100 | with open("../" + self.img_path + path + "/" + imgsrc.split("/")[-1], "wb")as f: 101 | f.write(requests.get(imgsrc, headers=self.headers, verify=False).content) 102 | 103 | def down_url(self): 104 | while True: 105 | Spider.rlock.acquire() 106 | if len(Spider.img_url_list) == 0: 107 | Spider.rlock.release() 108 | break 109 | else: 110 | img_url = Spider.img_url_list.pop() 111 | Spider.rlock.release() 112 | try: 113 | self.down_img(img_url) 114 | except Exception as e: 115 | pass 116 | 117 | 118 | def run(self): 119 | # print("开始下载") 120 | # 启动thread_num个进程来爬去具体的img url 链接 121 | for th in range(self.thread_num): 122 | add_pic_t = threading.Thread(target=self.get_img_url) 123 | add_pic_t.start() 124 | 125 | # 启动thread_num个来下载图片 126 | for img_th in range(self.thread_num): 127 | download_t = threading.Thread(target=self.down_url) 128 | download_t.start() 129 | 130 | 131 | if __name__ == '__main__': 132 | for i in [{"page": 3, "type": "1290", "type_id": 4}]: 133 | spider = Spider(page_number=i.get("page"), img_path='/static/images/', thread_number=10,type=i.get("type"),type_id=i.get("type_id")) 134 | spider.get_url() 135 | spider.get_img_url() 136 | spider.run() -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function install_mysql(){ 4 | cd ~/$tmp 5 | yum install ncurses-devel libaio-devel cmake gcc gcc-c++ make autoconf -y 6 | wget http://dev.mysql.com/get/Downloads/MySQL-5.6/mysql-5.6.21.tar.gz 7 | tar -zxvf mysql-5.6.21.tar.gz 8 | cd mysql-5.6.21 9 | 10 | cmake . 11 | make 12 | sudo make install 13 | 14 | 15 | sudo groupadd mysql 16 | sudo useradd -r -g mysql mysql 17 | 18 | cd /usr/local/mysql/ 19 | sudo chown -R root . 20 | sudo chown -R mysql data 21 | 22 | 23 | sudo yum install perl-Data-Dumper -y 24 | 25 | sudo scripts/mysql_install_db --user=mysql 26 | sudo cp support-files/my-default.cnf /etc/my.cnf 27 | 28 | sudo cp support-files/mysql.server /etc/init.d/mysql 29 | sudo chmod u+x /etc/init.d/mysql 30 | sudo chkconfig --add mysql 31 | # MySQL 环境变量 32 | cd ~ 33 | echo 'if [ -d "/usr/local/mysql/bin" ] ; then 34 | PATH=$PATH:/usr/local/mysql/bin 35 | export PATH 36 | fi' > env_mysql.sh 37 | sudo cp env_mysql.sh /etc/profile.d/env_mysql.sh 38 | touch /usr/local/vagrant.mysql.lock 39 | ln -s /usr/local/mysql/bin/mysql /usr/bin 40 | systemctl start mysql 41 | mysql -uroot -e "CREATE DATABASE $db_name;" 42 | echo "Mysql install successful" 43 | } 44 | 45 | function install_python(){ 46 | # the version of python 47 | version="3.8.0" 48 | # the installation directory of python 49 | python3_install_dir="/usr/local/python3" 50 | cd ~/$tmp 51 | file_name="Python-$version.tgz" 52 | sudo yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gcc make libffi-devel 53 | rm `pwd`"/$file_name" 54 | wget "https://www.python.org/ftp/python/$version/$file_name" 55 | mkdir $tmp 56 | tar -xf $file_name -C $tmp 57 | make_dir="$tmp/Python-$version" 58 | cd $make_dir 59 | mkdir -p $python3_install_dir 60 | ./configure --prefix=$python3_install_dir --with-ssl 61 | sudo make 62 | sudo make install 63 | ln -s /usr/local/python3/bin/python3 /usr/bin/python3 64 | cd ~/tmp 65 | wget --no-check-certificate https://pypi.python.org/packages/source/s/setuptools/setuptools-19.6.tar.gz 66 | tar -zxvf setuptools-19.6.tar.gz 67 | cd setuptools-19.6 68 | python3 setup.py build 69 | python3 setup.py install 70 | ln -s /usr/local/python3/bin/pip3 /usr/bin/pip3 71 | rm -rf ~/$tmp 72 | echo "all in well !" 73 | } 74 | 75 | # ----------------------- 安装mysql ------------------------------ 76 | # 编译MySQL时间比较长，需要等很长时间 77 | read -p "Allow Url: " allow_url 78 | read -p "Site Name: " site_name 79 | read -p "Site Url: " site_url 80 | yum install wget git -y 81 | git clone https://github.com/Turnright-git/94imm.git 82 | yum install gcc mariadb-devel -y 83 | cd "94imm" 84 | path=$(pwd) 85 | yum install yum install -y python3-devel 86 | tmp="tmp" 87 | mkdir ~/$tmp 88 | if ! [ -x "$(command -v python3)" ]; then 89 | echo "Start the Python3 installation process" 90 | install_python 91 | fi 92 | if ! [ -x "$(command -v mysql)" ]; then 93 | echo "编译MySQL时间比较长，需要等很长时间,可自安装。行输入n退出" 94 | read -p "(y , n):" isinstallmysql56 95 | case "$isinstallmysql56" in 96 | n|N|No|NO) 97 | exit 98 | ;; 99 | *) 100 | esac 101 | echo "Start the MySQL installation process" 102 | install_mysql 103 | systemctl start mysql 104 | read -p "Create databases : " db_name 105 | read -p "Create databases password: " db_pass 106 | create_db_sql="create database IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;" 107 | create_user="update mysql.user set password=password('${db_pass}') where user='root';" 108 | mysql -uroot -e "${create_db_sql}" 109 | mysql -uroot -e "${create_user}" 110 | mysql -uroot -e "${grant_user}" 111 | mysql -uroot -e "flush privileges;" 112 | else 113 | read -p "Create databases : " db_name 114 | read -p "Password for root: " db_pass 115 | create_db_sql="create database IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;" 116 | create_user="update mysql.user set password=password('${db_pass}') where user='root';" 117 | mysql -uroot -p$db_pass -e "${create_db_sql}" 118 | fi 119 | if ! [ -x "$(command -v nginx)" ]; then 120 | cd ~/$tmp 121 | wget https://nginx.org/download/nginx-1.16.0.tar.gz 122 | tar zxvf nginx-1.16.0.tar.gz 123 | cd nginx-1.16.0 124 | ./configure --user=nobody --group=nobody --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --with-http_sub_module --with-http_ssl_module 125 | make && make install 126 | cd $path 127 | cat>/lib/systemd/system/nginx.service</usr/local/nginx/conf/nginx.conf<"$path/uwsgi.ini"<"$path/config.py"< 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 视频小姐姐-手机版-94iMM 19 | 70 | 71 | 72 | 73 |
    74 | 75 |
    76 | 视频来源:[{{ source }}] 77 | 用户ID:[{{ user_id}}] 78 | ←左滑切换视频 79 | 点击播放/暂停 80 | 视频采集自网络 | 返回94imm 81 |
    82 | 83 | 84 | 85 | 178 | 182 | 183 | 194 | -------------------------------------------------------------------------------- /crawler/crawler_mmjpg.py: -------------------------------------------------------------------------------- 1 | #coding='UTF-8' 2 | 3 | import sys 4 | 5 | sys.path.append('../') 6 | from bs4 import BeautifulSoup 7 | from requests.adapters import HTTPAdapter 8 | import threading,pymysql,time,requests,os,urllib3,re,random 9 | from config import mysql_config 10 | 11 | requests.packages.urllib3.disable_warnings() 12 | requests.adapters.DEFAULT_RETRIES = 5 13 | s = requests.session() 14 | s.keep_alive = False 15 | s.mount('http://', HTTPAdapter(max_retries=3)) 16 | # 数据库连接信息 17 | dbhost = { 18 | "host": mysql_config['HOST'], 19 | "dbname": mysql_config['NAME'], 20 | "user": mysql_config['USER'], 21 | "password": mysql_config['PASSWORD'] 22 | } 23 | 24 | class Spider(): 25 | rlock = threading.RLock() 26 | page_url_list=[] 27 | img_url_list=[] 28 | proxy_dict = "" 29 | base_url="http://www.mmmjpg.com/" 30 | def __init__(self,start_page_num,end_page_num,img_path,thread_num,type): 31 | self.start_page_num=start_page_num 32 | self.end_page_num=end_page_num 33 | self.img_path=img_path 34 | self.thread_num=thread_num 35 | self.type=type 36 | 37 | def get_url(self): 38 | for i in range(self.start_page_num,self.end_page_num+1): 39 | if i==0: 40 | page=s.get(self.base_url) 41 | else: 42 | page=s.get(self.base_url+self.type+"/"+str(i)) 43 | soup=BeautifulSoup(page.text, "html.parser") 44 | url_soup=soup.find("div",class_="pic").find("ul").find_all("li") 45 | for li in url_soup: 46 | url=li.find("a").get("href") 47 | self.page_url_list.append(url) 48 | 49 | def get_img(self,url): 50 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"),dbhost.get("dbname")) 51 | cursor = db.cursor() 52 | tagidlist=[] 53 | page_id = url.split("/")[-1] 54 | page_url=self.base_url+"mm/"+page_id 55 | headers = { 56 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 57 | "Referer": page_url 58 | } 59 | info_page = s.get(self.base_url+"mm/" + page_id,headers=headers) 60 | info_page.encoding="utf-8" 61 | info_soup = BeautifulSoup(info_page.text,"html.parser") 62 | title=info_soup.find("div",class_="article").find("h1").text 63 | if "袜" in title or "丝" in title or "腿" in title: 64 | type_id = 2 65 | elif "青春" in title or "清纯" in title: 66 | type_id = 3 67 | elif "萝莉" in title: 68 | type_id = 4 69 | else: 70 | type_id = 1 71 | isExists = cursor.execute("SELECT title FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 72 | img_m_src=info_soup.find("div",class_="content").find("a").find("img").get("src").split("/")[-3] 73 | if isExists != 0: 74 | print("已采集：" + title) 75 | else: 76 | tags=info_soup.find("div",class_="tags").find_all("a") 77 | for tag_soup in tags: 78 | tag=tag_soup.text 79 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 80 | isExiststag = cursor.execute(sqltag) 81 | if isExiststag == 0: 82 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 83 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 84 | for id in cursor.fetchall(): 85 | tagidlist.append(id[0]) 86 | p = (title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), self.type_id, "1",page_url) 87 | cursor.execute("INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", p) 88 | print("开始采集："+title) 89 | pageid = cursor.lastrowid 90 | page=s.get(page_url,headers=headers) 91 | soup=BeautifulSoup(page.text,"html.parser") 92 | img_base=soup.find("div",class_="content").find("img").get("src").split("/") 93 | img_base_url="http://"+img_base[2]+"/" 94 | img_num=soup.find("div",class_="page").text.replace("全部图片下一页","").split("...")[-1] 95 | img_path = self.img_path + time.strftime('%Y%m%d', time.localtime( 96 | time.time())) + "/" + img_base[-2] +"/" 97 | for i in range(1,int(img_num)): 98 | img_loc_path=img_path+str(i)+".jpg" 99 | imgp = pageid, img_loc_path, img_base_url+img_base[-2]+"/"+str(i)+".jpg" 100 | cursor.execute("INSERT INTO images_image (pageid,imageurl,originurl) VALUES (%s,%s,%s)", imgp) 101 | if i == 1: 102 | cursor.execute( 103 | "UPDATE images_page SET firstimg = " + "'" + img_loc_path + "'" + " WHERE id=" + "'" + str( 104 | pageid) + "'") 105 | self.img_url_list.append({"img_url": img_base_url+img_base[-2]+"/"+str(i)+".jpg", "Referer": url, "id": img_base[-2]}) 106 | # print({"img_url": img_base_url+img_path+str(i)+".jpg", "Referer": img_base_url+img_base[-2], "id": img_base[-2]}) 107 | 108 | 109 | 110 | def down_img(self,imgsrc,Referer,id): 111 | headers = { 112 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 113 | "Referer": Referer 114 | } 115 | path = self.img_path + time.strftime('%Y%m%d', time.localtime(time.time())) + "/" 116 | page_id = id 117 | isdata = os.path.exists("../" + path + page_id) 118 | if not isdata: 119 | os.makedirs("../" + path + page_id) 120 | with open("../" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg", "wb") as f: 121 | print("已保存：" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg") 122 | f.write(s.get(imgsrc, headers=headers,verify=False).content) 123 | 124 | def run_page(self): 125 | while True: 126 | Spider.rlock.acquire() 127 | if len(self.page_url_list) == 0: 128 | Spider.rlock.release() 129 | break 130 | else: 131 | page_url = self.page_url_list.pop() 132 | Spider.rlock.release() 133 | try: 134 | self.get_img(page_url) 135 | except Exception as e: 136 | pass 137 | 138 | def run_img(self): 139 | while True: 140 | Spider.rlock.acquire() 141 | if len(self.img_url_list) == 0 : 142 | Spider.rlock.release() 143 | break 144 | else: 145 | urls = self.img_url_list.pop() 146 | url = urls.get("img_url") 147 | Referer = urls.get("Referer") 148 | id = urls.get("id") 149 | Spider.rlock.release() 150 | try: 151 | self.down_img(url, Referer, id) 152 | except Exception as e: 153 | pass 154 | 155 | def run_1(self): 156 | # 启动thread_num个进程来爬去具体的img url 链接 157 | url_threa_list=[] 158 | for th in range(self.thread_num): 159 | add_pic_t = threading.Thread(target=self.run_page) 160 | url_threa_list.append(add_pic_t) 161 | 162 | for t in url_threa_list: 163 | t.setDaemon(True) 164 | t.start() 165 | 166 | for t in url_threa_list: 167 | t.join() 168 | 169 | def run_2(self): 170 | # 启动thread_num个来下载图片 171 | for img_th in range(self.thread_num): 172 | download_t = threading.Thread(target=self.run_img) 173 | download_t.start() 174 | 175 | # start_page是采集开始也，end是采集结束页，type不用修改，自动分类 176 | if __name__ == "__main__": 177 | for i in [{"start_page": 1,"end_page":1, "type": "home"}]: 178 | spider=Spider(start_page_num=i.get("start_page"),end_page_num=i.get("end_page"),img_path='/static/images/',thread_num=10,type=i.get("type")) 179 | spider.get_url() 180 | spider.run_1() 181 | spider.run_2() -------------------------------------------------------------------------------- /crawler/crawler_mm131.py: -------------------------------------------------------------------------------- 1 | # coding='UTF-8' 2 | 3 | import sys 4 | 5 | sys.path.append('../') 6 | from bs4 import BeautifulSoup 7 | import threading, pymysql, time, requests, os, urllib3, re,random 8 | from config import mysql_config 9 | 10 | requests.packages.urllib3.disable_warnings() 11 | requests.adapters.DEFAULT_RETRIES = 5 12 | s = requests.session() 13 | s.keep_alive = False 14 | # 数据库连接信息 15 | dbhost = { 16 | "host": mysql_config['HOST'], 17 | "dbname": mysql_config['NAME'], 18 | "user": mysql_config['USER'], 19 | "password": mysql_config['PASSWORD'] 20 | } 21 | base_url="https://mm131.pro" 22 | 23 | class Spider(): 24 | rlock = threading.RLock() 25 | page_url_list = [] 26 | img_url_list = [] 27 | headers = { 28 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", 29 | "Referer": base_url 30 | } 31 | 32 | def __init__(self, page_num, img_path, thread_num, type_id=1, type="home",tagslist=["性感美女","诱惑美女","大胸美女","萌妹子"]): 33 | self.page_num = page_num 34 | self.img_path = img_path 35 | self.thread_num = thread_num 36 | self.type_id = type_id 37 | self.type = type 38 | self.tagslist= tagslist 39 | 40 | def get_url(self): 41 | for i in range(self.page_num): 42 | page = s.get(base_url+"/e/action/ListInfo/?classid="+str(self.type_id), headers=self.headers,verify=False) 43 | soup = BeautifulSoup(page.text, "html.parser") 44 | try: 45 | page_div = soup.find("dl", class_="list-left public-box").find_all("dd") 46 | except: 47 | print("采集错误，跳过本条") 48 | continue 49 | del page_div[-1] 50 | for dd in page_div: 51 | url = dd.find("a").get("href") 52 | self.page_url_list.append(base_url+url) 53 | 54 | def get_img(self,url): 55 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), dbhost.get("dbname")) 56 | cursor = db.cursor() 57 | tagidlist = [] 58 | page = s.get(url, headers=self.headers) 59 | page.encoding='UTF-8' 60 | soup = BeautifulSoup(page.text, "html.parser") 61 | # page_div = soup.find("div", class_="content-pic") 62 | title = soup.title.string.replace("_znns.com宅男钕神",'') 63 | isExists = cursor.execute("SELECT title FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 64 | if isExists != 0: 65 | print("isExists:" + title) 66 | else: 67 | tagslist = re.findall('', page.text) 68 | for tags in tagslist: 69 | for tag in tags.split(","): 70 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 71 | isExiststag = cursor.execute(sqltag) 72 | if isExiststag == 0: 73 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 74 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 75 | for id in cursor.fetchall(): 76 | tagidlist.append(id[0]) 77 | p = ( 78 | title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), self.type_id, "1", url) 79 | cursor.execute( 80 | "INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", p) 81 | print("down：" + title) 82 | pageid = cursor.lastrowid 83 | img_num_soup = soup.find("div", class_="content-page").find("span").text 84 | img_num = "".join(re.findall(r"\d", img_num_soup)) 85 | for i in range(1, int(img_num)): 86 | headers = self.headers.copy() 87 | headers.update({"Referer":url}) 88 | id = url.split("/")[-1].split(".")[0] 89 | if i==1: 90 | img_page_url=url 91 | else: 92 | img_page_url = "/".join(url.split("/")[0:-1]) + "/" + id + "_" + str(i) + ".html" 93 | img_page=s.get(img_page_url,headers=headers,verify=False) 94 | # page.encoding = 'utf-8' 95 | img_soup=BeautifulSoup(img_page.text,"html.parser") 96 | img_url = img_soup.find("div",class_="content-pic").find("img").get("src") 97 | img_name =img_url.split("/")[-1] 98 | id=url.split("/")[-1].split(".")[0] 99 | img_loc_path = self.img_path + time.strftime('%Y%m%d', time.localtime( 100 | time.time())) + "/" + id + "/" +img_name 101 | if i == 1: 102 | cursor.execute( 103 | "UPDATE images_page SET firstimg = " + "'" + img_loc_path + "'" + " WHERE id=" + "'" + str( 104 | pageid) + "'") 105 | imgp = pageid, img_loc_path,img_url 106 | cursor.execute("INSERT INTO images_image (pageid,imageurl,originurl) VALUES (%s,%s,%s)", imgp) 107 | i += 1 108 | data={"img_url":img_url,"Referer":url,"id":id} 109 | if data in self.img_url_list: 110 | continue 111 | else: 112 | self.img_url_list.append(data) 113 | 114 | def down_img(self,imgsrc,Referer,id): 115 | headers = { 116 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 117 | "Referer": Referer 118 | } 119 | path = self.img_path + time.strftime('%Y%m%d', time.localtime(time.time())) + "/" 120 | page_id = id 121 | isdata = os.path.exists("../" + path + page_id) 122 | if not isdata: 123 | os.makedirs("../" + path + page_id) 124 | with open("../" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg", "wb") as f: 125 | print("已保存：" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg") 126 | f.write(s.get(imgsrc, headers=headers,verify=False).content) 127 | 128 | def run_page(self): 129 | while True: 130 | Spider.rlock.acquire() 131 | if len(self.page_url_list) == 0: 132 | Spider.rlock.release() 133 | break 134 | else: 135 | try: 136 | page_url = self.page_url_list.pop() 137 | except Exception as e: 138 | print(e) 139 | pass 140 | Spider.rlock.release() 141 | try: 142 | self.get_img(page_url) 143 | except Exception as e: 144 | print(e) 145 | pass 146 | 147 | def run_img(self): 148 | while True: 149 | Spider.rlock.acquire() 150 | if len(self.img_url_list) == 0 : 151 | Spider.rlock.release() 152 | break 153 | else: 154 | urls = self.img_url_list.pop() 155 | url = urls.get("img_url") 156 | Referer = urls.get("Referer") 157 | id = urls.get("id") 158 | Spider.rlock.release() 159 | try: 160 | self.down_img(url, Referer, id) 161 | except Exception as e: 162 | print(e) 163 | pass 164 | 165 | def run_1(self): 166 | # 启动thread_num个进程来爬去具体的img url 链接 167 | url_threa_list=[] 168 | for th in range(self.thread_num): 169 | add_pic_t = threading.Thread(target=self.run_page) 170 | url_threa_list.append(add_pic_t) 171 | 172 | for t in url_threa_list: 173 | t.setDaemon(True) 174 | t.start() 175 | 176 | for t in url_threa_list: 177 | t.join() 178 | 179 | def run_2(self): 180 | # 启动thread_num个来下载图片 181 | for img_th in range(self.thread_num): 182 | download_t = threading.Thread(target=self.run_img) 183 | download_t.start() 184 | 185 | 186 | # page是采集深度，从1开始，采集第一页即采集最新发布。type是源站分类，type_id是对应本站分类的id 187 | if __name__ == "__main__": 188 | for i in [{"page": 1, "type": "xinggan", "type_id": 1},{"page":1,"type":"qingchun","type_id": 2}]: 189 | spider = Spider(page_num=i.get("page"), img_path='/static/images/', thread_num=10, type_id=i.get("type_id"), 190 | type=i.get("type"),tagslist=["性感美女","诱惑美女","大胸美女","萌妹子"]) 191 | spider.get_url() 192 | spider.run_1() 193 | spider.run_2() -------------------------------------------------------------------------------- /crawler/crawler_amn.py: -------------------------------------------------------------------------------- 1 | # coding='UTF-8' 2 | import sys 3 | sys.path.append('../') 4 | from bs4 import BeautifulSoup 5 | from requests.adapters import HTTPAdapter 6 | import threading,pymysql,time,requests,os,urllib3,re,random 7 | from config import mysql_config 8 | 9 | requests.packages.urllib3.disable_warnings() 10 | requests.adapters.DEFAULT_RETRIES = 5 11 | s = requests.session() 12 | s.keep_alive = False 13 | s.mount('http://', HTTPAdapter(max_retries=3)) 14 | # 数据库连接信息 15 | dbhost = { 16 | "host": mysql_config['HOST'], 17 | "dbname": mysql_config['NAME'], 18 | "user": mysql_config['USER'], 19 | "password": mysql_config['PASSWORD'] 20 | } 21 | 22 | base_url="https://www.2meinv.com/" 23 | tag_url="https://www.2meinv.com/tags-{}-{}.html" 24 | index_url="https://www.2meinv.com/index-1.html" 25 | img_path='/static/images/' 26 | 27 | class Spider(): 28 | page_url_list = [] 29 | img_url_list = [] 30 | rlock = threading.RLock() 31 | proxy_dict = "" 32 | def __init__(self, start_page_num, end_page_num,img_path, thread_num, type="home",type_id=0): 33 | self.start_page_num = start_page_num 34 | self.end_page_num=end_page_num 35 | self.img_path = img_path 36 | self.thread_num = thread_num 37 | self.type = type 38 | self.type_id=type_id 39 | 40 | def get_url(self): 41 | for i in range(self.start_page_num, self.end_page_num): 42 | if self.type_id==0: 43 | page = s.get(index_url.format(str(i)), verify=False).text 44 | else: 45 | page = s.get(tag_url.format(self.type,str(i)), verify=False).text 46 | # page = s.get(base_url + self.type+"-"+str(i)+".html", verify=False).text 47 | soup = BeautifulSoup(page, "html.parser") 48 | page_base_url = soup.find("ul", class_="detail-list").find_all("li") 49 | for page_url in page_base_url: 50 | url = page_url.find("a",class_="dl-pic").get("href") 51 | self.page_url_list.append(url) 52 | 53 | def get_img(self,url): 54 | tagidlist=[] 55 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), dbhost.get("dbname")) 56 | cursor = db.cursor() 57 | page = s.get(url,verify=False) 58 | soup = BeautifulSoup(page.text, "html.parser") 59 | title=soup.title.string.replace("_爱美女","") 60 | if self.type_id == 0: 61 | if "袜" in title or "丝" in title or "腿" in title: 62 | self.type_id = 2 63 | elif "青春" in title or "清纯" in title: 64 | self.type_id = 3 65 | elif "萝莉" in title: 66 | self.type_id = 4 67 | else: 68 | self.type_id = 1 69 | isExists = cursor.execute("SELECT title FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 70 | if isExists != 0: 71 | print("已采集：" , title) 72 | else: 73 | print("正在采集：", title) 74 | tags=soup.find(attrs={"name":"Keywords"})['content'].split(",") 75 | for tag in tags: 76 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 77 | isExiststag = cursor.execute(sqltag) 78 | if isExiststag == 0: 79 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 80 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 81 | for id in cursor.fetchall(): 82 | tagidlist.append(id[0]) 83 | p = (title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), self.type_id, "1",url) 84 | cursor.execute("INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", p) 85 | pageid = cursor.lastrowid 86 | img_soup=soup.find("div",class_="page-show").text 87 | img_nums=re.sub("\D", "", img_soup) 88 | if len(img_nums)==6: 89 | img_num=img_nums[-2:] 90 | elif len(img_nums)<6: 91 | img_num = img_nums[-1] 92 | elif len(img_nums)>6: 93 | img_num = img_nums[-3:] 94 | id=url.split("-")[-1].split(".")[0] 95 | for i in range(1,int(img_num)+1): 96 | img_page_url=base_url+"article-"+id+"-"+str(i)+".html" 97 | img_page=s.get(img_page_url) 98 | img_soup=BeautifulSoup(img_page.text, "html.parser") 99 | img_url=img_soup.find("div",class_="pp hh").find("img").get("src") 100 | img_name = img_url.split("/")[-1] 101 | img_loc_path = self.img_path + time.strftime('%Y%m%d', time.localtime( 102 | time.time())) + "/" + id + "/" + img_name 103 | imgp = pageid, img_loc_path,img_url 104 | cursor.execute("INSERT INTO images_image (pageid,imageurl,originurl) VALUES (%s,%s,%s)", imgp) 105 | if i==1: 106 | cursor.execute( 107 | "UPDATE images_page SET firstimg = " + "'" + img_loc_path + "'" + " WHERE id=" + "'" + str( 108 | pageid) + "'") 109 | self.img_url_list.append({"img_url":img_url,"Referer":url,"id":id}) 110 | 111 | def down_img(self,imgsrc,Referer,id): 112 | headers = { 113 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 114 | "Referer": Referer 115 | } 116 | path = img_path + time.strftime('%Y%m%d', time.localtime(time.time())) + "/" 117 | page_id = id 118 | isdata = os.path.exists("../" + path + page_id) 119 | if not isdata: 120 | os.makedirs("../" + path + page_id) 121 | with open("../" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg", "wb") as f: 122 | print("已保存：" + path + page_id + "/" + imgsrc.split("/")[-1].split(".")[0] + ".jpg") 123 | f.write(s.get(imgsrc, headers=headers,verify=False).content) 124 | 125 | def run_page(self): 126 | while True: 127 | Spider.rlock.acquire() 128 | if len(self.page_url_list) == 0: 129 | Spider.rlock.release() 130 | break 131 | else: 132 | page_url = self.page_url_list.pop() 133 | Spider.rlock.release() 134 | try: 135 | self.get_img(page_url) 136 | except Exception as e: 137 | print(e) 138 | pass 139 | 140 | def run_img(self): 141 | while True: 142 | Spider.rlock.acquire() 143 | if len(self.img_url_list) == 0 : 144 | Spider.rlock.release() 145 | break 146 | else: 147 | urls = self.img_url_list.pop() 148 | url = urls.get("img_url") 149 | Referer = urls.get("Referer") 150 | id = urls.get("id") 151 | Spider.rlock.release() 152 | try: 153 | self.down_img(url, Referer, id) 154 | except Exception as e: 155 | print(e) 156 | pass 157 | 158 | def run_1(self): 159 | # 启动thread_num个进程来爬去具体的img url 链接 160 | url_threa_list=[] 161 | for th in range(self.thread_num): 162 | add_pic_t = threading.Thread(target=self.run_page) 163 | url_threa_list.append(add_pic_t) 164 | 165 | for t in url_threa_list: 166 | t.setDaemon(True) 167 | t.start() 168 | 169 | for t in url_threa_list: 170 | t.join() 171 | 172 | def run_2(self): 173 | # 启动thread_num个来下载图片 174 | for img_th in range(self.thread_num): 175 | download_t = threading.Thread(target=self.run_img) 176 | download_t.start() 177 | 178 | # start_page是采集开始也，end是采集结束页，type不用修改，自动分类，起始页为1 179 | if __name__ == "__main__": 180 | cl_list=[{"start_page": 1,"end_page":17, "type": "Cosplay", "type_id":6}, 181 | {"start_page": 1,"end_page":17, "type": "性感", "type_id":1}, 182 | {"start_page": 1, "end_page": 17, "type": "丝袜", "type_id": 2}, 183 | {"start_page": 1, "end_page": 17, "type": "美腿", "type_id": 2}, 184 | {"start_page": 1, "end_page": 17, "type": "美胸", "type_id": 1}, 185 | {"start_page": 1, "end_page": 17, "type": "制服诱惑", "type_id": 2} 186 | ] 187 | 188 | 189 | for i in cl_list: 190 | spider = Spider(start_page_num=i.get("start_page"),end_page_num=i.get("end_page"), img_path='/static/images/', thread_num=10, 191 | type=i.get("type"),type_id=i.get("type_id")) 192 | spider.get_url() 193 | spider.run_1() 194 | spider.run_2() -------------------------------------------------------------------------------- /crawler/crawler_nsg.py: -------------------------------------------------------------------------------- 1 | # coding='UTF-8' 2 | 3 | import sys 4 | 5 | sys.path.append('../') 6 | from bs4 import BeautifulSoup 7 | from requests.adapters import HTTPAdapter 8 | import threading,pymysql,time,requests,os,urllib3,re,random 9 | from config import mysql_config 10 | 11 | requests.packages.urllib3.disable_warnings() 12 | requests.adapters.DEFAULT_RETRIES = 5 13 | s = requests.session() 14 | s.keep_alive = False 15 | s.mount('http://', HTTPAdapter(max_retries=3)) 16 | # 数据库连接信息 17 | dbhost = { 18 | "host": mysql_config['HOST'], 19 | "dbname": mysql_config['NAME'], 20 | "user": mysql_config['USER'], 21 | "password": mysql_config['PASSWORD'] 22 | } 23 | 24 | # dbhost = { 25 | # "host": "192.168.1.67", 26 | # "dbname": "silumz", 27 | # "user": "silumz", 28 | # "password": "fendou2009" 29 | # } 30 | 31 | base_url="http://www.nvshenge.com/mntp/" 32 | img_path='/static/images/' 33 | 34 | class Spider(): 35 | page_url_list = [] 36 | img_url_list = [] 37 | rlock = threading.RLock() 38 | proxy_dict = "" 39 | def __init__(self, start_page_num, end_page_num,img_path, thread_num, type="home"): 40 | self.start_page_num = start_page_num 41 | self.end_page_num=end_page_num 42 | self.img_path = img_path 43 | self.thread_num = thread_num 44 | self.type = type 45 | 46 | def get_url(self): 47 | for i in range(self.start_page_num -1, self.end_page_num -1): 48 | if i==0: 49 | page=s.get(base_url, verify=False).text 50 | else: 51 | page = s.get(base_url + "list_"+str(i)+".html", verify=False).text 52 | soup = BeautifulSoup(page, "html.parser") 53 | all_list = soup.find_all("a", class_="PicTxt") 54 | i = 0 55 | for info_soup in all_list: 56 | url=info_soup.get("href") 57 | title=info_soup.text 58 | self.page_url_list.append({"url":url,"title":title}) 59 | i += 1 60 | 61 | 62 | def get_img(self): 63 | db = pymysql.connect(dbhost.get("host"), dbhost.get("user"), dbhost.get("password"), 64 | dbhost.get("dbname")) 65 | cursor = db.cursor() 66 | while True: 67 | self.rlock.acquire() 68 | if len(self.page_url_list) == 0: 69 | self.rlock.release() 70 | break 71 | else: 72 | page_info= self.page_url_list.pop() 73 | page_url = page_info.get("url") 74 | title = page_info.get("title") 75 | if "袜" in title or "丝" in title or "腿" in title: 76 | type_id = 2 77 | elif "青春" in title or "清纯" in title: 78 | type_id = 3 79 | elif "萝莉" in title: 80 | type_id = 4 81 | else: 82 | type_id = 1 83 | self.rlock.release() 84 | try: 85 | tagidlist = [] 86 | page = s.get(page_url, verify=False).text 87 | soup = BeautifulSoup(page, "html.parser") 88 | img_num_soup = soup.find("div", class_="articleTop yh").find("h1").text 89 | img_num = int(img_num_soup[img_num_soup.find("(1/") + 3:img_num_soup.find("）")]) 90 | isExists = cursor.execute( 91 | "SELECT title FROM images_page WHERE title =" + "'" + title + "'" + " limit 1;") 92 | if isExists != 0: 93 | print("已采集：" + title) 94 | else: 95 | taglist = soup.find("div",class_="articleTag l").find_all("dd") 96 | for tag_soup in taglist: 97 | tag=tag_soup.text 98 | sqltag = "SELECT * FROM images_tag WHERE tag =" + "'" + tag + "'" + " limit 1;" 99 | isExiststag = cursor.execute(sqltag) 100 | if isExiststag == 0: 101 | cursor.execute("INSERT INTO images_tag (tag) VALUES (%s)", tag) 102 | cursor.execute("SELECT id FROM images_tag WHERE tag =" + "'" + tag + "'") 103 | for id in cursor.fetchall(): 104 | tagidlist.append(id[0]) 105 | p = ( 106 | title, str(tagidlist), time.strftime('%Y-%m-%d', time.localtime(time.time())), type_id, 107 | "1", page_url) 108 | cursor.execute( 109 | "INSERT INTO images_page (title,tagid,sendtime,typeid,firstimg,crawler) VALUES (%s,%s,%s,%s,%s,%s)", 110 | p) 111 | print("开始采集：" + title) 112 | pageid = cursor.lastrowid 113 | for i in range(0, int(img_num)): 114 | img_id = page_url.split("/")[-1].split(".")[0] 115 | if i==0: 116 | url=page_url 117 | else: 118 | url = "/".join(page_url.split("/")[0:-1])+"/"+img_id+"_"+str(i)+".html" 119 | img_page=s.get(url, verify=False).text 120 | img_soup= BeautifulSoup(img_page, "html.parser") 121 | img_src=img_soup.find("div",id="ArticlePicBox1").find("img").get("src") 122 | img_loc_path = self.img_path + time.strftime('%Y%m%d', time.localtime( 123 | time.time())) + "/"+img_id+"/"+img_src.split("/")[-1] 124 | if i == 0: 125 | cursor.execute( 126 | "UPDATE images_page SET firstimg = " + "'" + img_loc_path + "'" + " WHERE title=" + "'" + title + "'") 127 | imgp = pageid, img_loc_path, img_src 128 | cursor.execute("INSERT INTO images_image (pageid,imageurl,originurl) VALUES (%s,%s,%s)", 129 | imgp) 130 | self.img_url_list.append({"url": img_src, "path": img_loc_path, "referer": page_url}) 131 | except Exception as e: 132 | cursor.execute("Delete FROM images_page WHERE title=" + "'" + title + "'") 133 | print("采集失败(已删除)：",title) 134 | print("连接地址：", page_url) 135 | print("错误信息：", e) 136 | db.close() 137 | 138 | def down_img(self, imgsrc, imgpath, referer): 139 | headers = { 140 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", 141 | "Referer": referer 142 | } 143 | isdata = os.path.exists(".." +"/".join(imgpath.split("/")[0:-1])) 144 | if not isdata: 145 | os.makedirs(".."+"/".join(imgpath.split("/")[0:-1])) 146 | with open(".."+ imgpath, "wb")as f: 147 | f.write(requests.get(imgsrc, headers=headers, verify=False).content) 148 | print("下载图片：" + imgpath) 149 | 150 | def down_url(self): 151 | while True: 152 | Spider.rlock.acquire() 153 | if len(Spider.img_url_list) == 0: 154 | Spider.rlock.release() 155 | break 156 | else: 157 | img_url = Spider.img_url_list.pop() 158 | Spider.rlock.release() 159 | try: 160 | url = img_url.get("url") 161 | path = img_url.get("path") 162 | referer = img_url.get("referer") 163 | self.down_img(url, path, referer) 164 | except Exception as e: 165 | print(e) 166 | self.img_url_list.append( 167 | {"url": img_url.get("url"), "path": img_url.get("path"), "referer": img_url.get("referer")}) 168 | pass 169 | 170 | def run_1(self): 171 | # 启动thread_num个进程来爬去具体的img url 链接 172 | url_threa_list = [] 173 | for th in range(self.thread_num): 174 | add_pic_t = threading.Thread(target=self.get_img) 175 | url_threa_list.append(add_pic_t) 176 | 177 | for t in url_threa_list: 178 | t.setDaemon(True) 179 | t.start() 180 | 181 | for t in url_threa_list: 182 | t.join() 183 | 184 | def run_2(self): 185 | # 启动thread_num个来下载图片 186 | for img_th in range(self.thread_num): 187 | download_t = threading.Thread(target=self.down_url) 188 | download_t.start() 189 | 190 | 191 | # start_page是采集开始也，end是采集结束页，type不用修改，自动分类，起始页为1 192 | if __name__ == "__main__": 193 | for i in [{"start_page": 1,"end_page":2, "type": "index"}]: 194 | spider = Spider(start_page_num=i.get("start_page"),end_page_num=i.get("end_page"), img_path='/static/images/', thread_num=10, 195 | type=i.get("type")) 196 | spider.get_url() 197 | spider.run_1() 198 | spider.run_2() -------------------------------------------------------------------------------- /static/zde/css/fonts/iconfont.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Created by FontForge Q2wB39 20120731 at Fri Nov 18 13:25:10 2016 6 | By admin 7 | 8 | 9 | 10 | 24 | 26 | 28 | 30 | 32 | 34 | 38 | 41 | 43 | 50 | 54 | 57 | 60 | 62 | 67 | 70 | 74 | 76 | 79 | 83 | 85 | 87 | 89 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /images/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from images.models import * 3 | import random, json 4 | from django.http import HttpResponse 5 | from config import site_name, site_url, key_word, description, email,friendly_link 6 | 7 | 8 | 9 | def index(request): 10 | if request.method == "GET": 11 | imgs = [] 12 | page_list = Page.objects.all().order_by('?')[:50] 13 | typedict, typelist = type_list() 14 | for pid in page_list: 15 | id = pid.id 16 | title = pid.title 17 | firstimg = pid.firstimg 18 | sendtime = pid.sendtime 19 | hot = pid.hot 20 | type_id = pid.typeid 21 | imgs.append({"pid": id, "firstimg": firstimg, "title": title, "sendtime": sendtime, "hot": hot, 22 | "type": typedict[type_id], "type_id": type_id}) 23 | return render(request, 'index.html', 24 | {"data": imgs, "typelist": typelist, "siteName": site_name, "keyWord": key_word, 25 | "description": description, "siteUrl": site_url, "email": email}) 26 | 27 | 28 | def page(request, i_id): 29 | # try: 30 | page_arr = Page.objects.get(id=i_id) 31 | imgs = [] 32 | tags = [] 33 | typedict, typelist = type_list() 34 | page_hot = page_arr.hot 35 | page_arr.hot = page_hot + 1 36 | page_arr.save() 37 | time = page_arr.sendtime 38 | typeid = page_arr.typeid 39 | pagetype = Type.objects.get(id=typeid).type 40 | title = page_arr.title 41 | taglist = page_arr.tagid 42 | tag_arr = taglist.replace("[", "").replace("]", "").split(",") 43 | for t_id in tag_arr: 44 | tagid = t_id.strip(" ") 45 | tag = Tag.objects.get(id=tagid).tag 46 | tags.append({"tname": tag, "tid": tagid}) 47 | imglist = Image.objects.filter(pageid=i_id) 48 | for img_arr in imglist: 49 | img = img_arr.imageurl 50 | imgs.append(img) 51 | if len(tags) > 4: 52 | tags = random.sample(tags, 4) 53 | typename = typedict[typeid] 54 | return render(request, 'page.html', 55 | {"data": imgs, "tag": tags, "title": title, "type": pagetype, "typeid": str(typeid), "time": time, 56 | "similar": page_similar(typeid), "typelist": typelist, "pageid": i_id, "siteName": site_name, 57 | "keyWord": key_word, "description": description, "typeName": typename, "siteUrl": site_url, 58 | "email": email,"friendly_link":friendly_link}) 59 | 60 | 61 | def tag(request, tid): 62 | if request.method == "GET": 63 | imgs = [] 64 | page_list = Page.objects.all().order_by("-id") 65 | typedict, typelist = type_list() 66 | for pid in page_list: 67 | if tid in pid.tagid: 68 | id = pid.id 69 | title = pid.title 70 | firstimg = pid.firstimg 71 | type_id = pid.typeid 72 | sendtime = pid.sendtime 73 | hot = pid.hot 74 | imgs.append({"pid": id, "firstimg": firstimg, "title": title, "sendtime": sendtime, "hot": hot, 75 | "type": typedict[type_id], "type_id": type_id}) 76 | return render(request, 'index.html', 77 | {"data": imgs, "typelist": typelist, "siteName": site_name, "keyWord": key_word, 78 | "description": description, "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 79 | 80 | 81 | def type(request, typeid): 82 | if request.method == "GET": 83 | imgs = [] 84 | typedict, typelist = type_list() 85 | page_list = Page.objects.filter(typeid=typeid).order_by("-id") 86 | for pid in page_list: 87 | title = pid.title 88 | firstimg = pid.firstimg 89 | id = pid.id 90 | hot = pid.hot 91 | type_id = pid.typeid 92 | sendtime = pid.sendtime 93 | imgs.append({"pid": id, "firstimg": firstimg, "title": title, "sendtime": sendtime, "hot": hot, 94 | "type": typedict[type_id], "type_id": type_id}) 95 | return render(request, 'category.html', 96 | {"data": imgs, "typelist": typelist, "typeid": str(typeid), "siteName": site_name, 97 | "keyWord": key_word, "description": description, "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 98 | 99 | 100 | def page_similar(id): 101 | similarlist = [] 102 | sidlist = Page.objects.filter(typeid=id).order_by("?") 103 | type = Type.objects.get(id=id).type 104 | i = 0 105 | for s in sidlist: 106 | if i < 20: 107 | stitle = s.title 108 | pid = s.id 109 | tid = s.typeid 110 | firstimg = s.firstimg 111 | sendtime = s.sendtime 112 | hot = s.hot 113 | if pid != id: 114 | similarlist.append( 115 | {"stitle": stitle, "tid": tid, "pid": pid, "firstimg": firstimg, "sendtime": sendtime, "hot": hot, 116 | "type": type, "type_id": tid 117 | }) 118 | i += 1 119 | return similarlist 120 | 121 | 122 | def search(request): 123 | if "s" in request.GET: 124 | imgs = [] 125 | typedict, typelist = type_list() 126 | context = request.GET['s'] 127 | pagelist = Page.objects.filter(title__contains=context).order_by("-id") 128 | for pid in pagelist: 129 | title = pid.title 130 | firstimg = pid.firstimg 131 | id = pid.id 132 | hot = pid.hot 133 | type_id = pid.typeid 134 | sendtime = pid.sendtime 135 | imgs.append({"pid": id, "firstimg": firstimg, "title": title, "sendtime": sendtime, "hot": hot, 136 | "type": typedict[type_id], "type_id": type_id}) 137 | return render(request, 'index.html', 138 | {"data": imgs, "typelist": typelist, "siteName": site_name, "keyWord": key_word, 139 | "description": description, "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 140 | 141 | 142 | def HotTag(request): 143 | tag_sql = Tag.objects.all().order_by("?") 144 | tag_dict = {} 145 | tag_id_list = [] 146 | page_sql = Page.objects.all() 147 | page_dict = {} 148 | return_list = [] 149 | typedict, typelist = type_list() 150 | for alltag in tag_sql: 151 | tag_dict.update({str(alltag.id).strip(): alltag.tag}) 152 | for page in page_sql: 153 | title = page.title 154 | pid = page.id 155 | tag_id = page.tagid.replace("[", "").replace("]", "").split(",") 156 | for t in tag_id: 157 | if str(t).strip() == '': 158 | pass 159 | else: 160 | if str(t).strip() not in tag_id_list: 161 | page_dict.update({str(t).strip(): 1}) 162 | tag_id_list.append(str(t).strip()) 163 | else: 164 | view = page_dict[str(t).strip()] 165 | page_dict.update({str(t).strip(): view + 1}) 166 | 167 | page_dict_sort = sorted(page_dict.items(), key=lambda d: d[1], reverse=True) 168 | for i in page_dict_sort: 169 | if page_dict[str(i[0])] > 20: 170 | return_list.append( 171 | {"tid": i[0], "tag": tag_dict[str(i[0]).strip()], "viwe": page_dict[str(i[0].strip())]} 172 | ) 173 | return render(request, 'tag.html', 174 | {"data": return_list, "typelist": typelist, "keyword": return_list[0:10], "siteName": site_name, 175 | "keyWord": key_word, "description": description, "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 176 | 177 | 178 | def SortBy(request, method): 179 | if request.method == "GET": 180 | if method == "new": 181 | page_list = Page.objects.all().order_by("-id")[:100] 182 | else: 183 | page_list = Page.objects.all().order_by("-hot")[:100] 184 | imgs = [] 185 | type_dict, typelist = type_list() 186 | for pid in page_list: 187 | title = pid.title 188 | firstimg = pid.firstimg 189 | id = pid.id 190 | hot = pid.hot 191 | type_id = pid.typeid 192 | sendtime = pid.sendtime 193 | imgs.append({"pid": id, "firstimg": firstimg, "title": title, "sendtime": sendtime, "hot": hot, 194 | "type": type_dict[type_id], "type_id": type_id}) 195 | 196 | return render(request, 'sort.html', 197 | {"data": imgs, "typelist": typelist, "method": method, "siteName": site_name, "keyWord": key_word, 198 | "description": description, "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 199 | 200 | 201 | def getVideo(request): 202 | count = Video.objects.count() 203 | video_info = '' 204 | while True: 205 | vid = random.randint(1, count) 206 | try: 207 | video_info = Video.objects.get(id=vid) 208 | break 209 | except: 210 | continue 211 | url = video_info.url 212 | user_id = video_info.user_id 213 | source = video_info.source 214 | return HttpResponse(json.dumps({"url": url, "user_id": user_id, "source": source})) 215 | 216 | 217 | def mVideo(request): 218 | if request.method == "GET": 219 | count = Video.objects.count() 220 | video_info = '' 221 | while True: 222 | vid = random.randint(1, count) 223 | try: 224 | video_info = Video.objects.get(id=vid) 225 | break 226 | except: 227 | continue 228 | url = "https:"+video_info.url 229 | return render(request, 'mVideo.html', { 230 | "url": url, 231 | "user_id": video_info.user_id, 232 | "date_time": video_info.date_time, 233 | "v_name": video_info.v_name, 234 | "source": video_info.source, "siteName": site_name, "keyWord": key_word, "description": description, 235 | "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 236 | 237 | 238 | def pVideo(request): 239 | if request.method == "GET": 240 | typedict, typelist = type_list() 241 | count = Video.objects.count() 242 | video_info = '' 243 | while True: 244 | vid = random.randint(1, count) 245 | try: 246 | video_info = Video.objects.get(id=vid) 247 | break 248 | except: 249 | continue 250 | url="https:"+video_info.url 251 | return render(request, 'video.html', { 252 | "url": url, 253 | "user_id": video_info.user_id, 254 | "date_time": video_info.date_time, 255 | "v_name": video_info.v_name, 256 | "source": video_info.source, 257 | "typelist": typelist, "siteName": site_name, "keyWord": key_word, "description": description, 258 | "siteUrl": site_url, "email": email,"friendly_link":friendly_link}) 259 | 260 | 261 | def type_list(): 262 | typelist = [] 263 | type_list = Type.objects.all().order_by("id") 264 | type_dict = {} 265 | for type_arr in type_list: 266 | type = type_arr.type 267 | type_id = type_arr.id 268 | typelist.append({"type": type, "type_id": str(type_id)}) 269 | type_dict.update({type_id: type}) 270 | return type_dict, typelist 271 | --------------------------------------------------------------------------------