├── templates ├── feed.xml ├── error.html ├── wrong_edit.html ├── update_status.html ├── entry_list.html ├── block.html ├── login.html ├── home.html.bak ├── edit.html ├── addrss.html └── home.html ├── requirements.txt ├── screen ├── Selection_001.png ├── Selection_002.png └── Selection_003.png ├── supervisor ├── conf.d │ └── rss-from-web.conf └── supervisord.conf ├── static ├── rss-from-web.css ├── js.cookie.min.js ├── jquery_cookie.js ├── js.cookie-2.2.1.min.js └── font-awesome.min.css ├── rsslog.py ├── selectsql.py ├── LICENSE ├── genfeedxpath.py ├── config.py ├── rssfeed.py ├── genrss.py ├── createsql.py ├── timerevent.py ├── README.md ├── initiate.py ├── rssbody.py ├── rsssql.py ├── entrylink.py └── main.py /templates/feed.xml: -------------------------------------------------------------------------------- 1 | {% raw rss_content %} -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tornado 2 | psycopg2 3 | lxml 4 | asyncpg 5 | aiopg 6 | rfeed 7 | -------------------------------------------------------------------------------- /screen/Selection_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/su-sanzhou/rss-from-web/HEAD/screen/Selection_001.png -------------------------------------------------------------------------------- /screen/Selection_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/su-sanzhou/rss-from-web/HEAD/screen/Selection_002.png -------------------------------------------------------------------------------- /screen/Selection_003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/su-sanzhou/rss-from-web/HEAD/screen/Selection_003.png -------------------------------------------------------------------------------- /supervisor/conf.d/rss-from-web.conf: -------------------------------------------------------------------------------- 1 | [program:rss-from-web] 2 | command=/usr/bin/python3 /var/rss-from-web/main.py 3 | directory=/var/rss-from-web/ 4 | user=ubuntu 5 | autorestart=true 6 | redirect_stderr=true 7 | stdout_logfile=/var/rss-from-web/rss-from-web.log 8 | loglevel=info 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /static/rss-from-web.css: -------------------------------------------------------------------------------- 1 | .hideSlow{ 2 | background: #a40808; 3 | color: #fff; 4 | animation: hidetip 5s 1; 5 | animation-fill-mode: forwards; 6 | -webkit-animation: showtip 5s 1; 7 | -webkit-animation-fill-mode: forwards; 8 | } 9 | 10 | @keyframes hidetip { 11 | from { 12 | opcity: 1; 13 | } 14 | to { 15 | opacity: 0; 16 | } 17 | } 18 | @-webkit-keyframes hidetip { 19 | from { 20 | opcity: 1; 21 | } 22 | to { 23 | opacity: 0; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rsslog.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from logging.handlers import RotatingFileHandler 3 | import config 4 | 5 | class RssLog(object): 6 | def __init__(self): 7 | self.rss_log_file = config.get_other_config()["log_file"] 8 | 9 | self.rss_logger = logging.getLogger("rss") 10 | self.rss_logger.setLevel(logging.INFO) 11 | 12 | file_handler = RotatingFileHandler(self.rss_log_file, 13 | mode='w', 14 | maxBytes=1000, 15 | backupCount=3, 16 | encoding='utf-8') 17 | 18 | formatter = logging.Formatter('%(asctime)s - %(lineno)d - %(levelname)s - %(message)s') 19 | file_handler.setFormatter(formatter) 20 | 21 | self.rss_logger.addHandler(file_handler) 22 | 23 | #useage 24 | #self.rss_logger.info('info messages') -------------------------------------------------------------------------------- /selectsql.py: -------------------------------------------------------------------------------- 1 | import asyncpg 2 | 3 | class SelectSql(object): 4 | def __init__(self,database): 5 | self.dbname = database.get("dbname") 6 | self.dbuser = database.get("user") 7 | self.dbpasswd = database.get("password") 8 | self.dbhost = database.get("host") 9 | self.dbport = database.get("port") 10 | 11 | async def sql_conn(self): 12 | try: 13 | conn = await asyncpg.connect(database = self.dbname, 14 | user = self.dbuser, 15 | password = self.dbpasswd, 16 | host = self.dbhost, 17 | port = self.dbport 18 | ) 19 | except: 20 | print("I am unable to connect to the database.") 21 | await conn.close() 22 | return {"conn","wrong_conn"} 23 | else: 24 | return conn 25 | 26 | 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 su-sanzhou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /genfeedxpath.py: -------------------------------------------------------------------------------- 1 | from entrylink import EntryLink 2 | from rssbody import RssBody 3 | from rssfeed import RssFeed 4 | 5 | class GenFeedXpath(object): 6 | 7 | async def gen_feed_from_xpath(self, xpath): 8 | entry_link = EntryLink(xpath["site_url"], 9 | xpath["entry_css"], 10 | xpath["entry_link_css"], 11 | xpath["add_base_url"], 12 | xpath["rss_link_prefix"], 13 | xpath["site_title_css"], 14 | xpath["site_motto_css"]) 15 | await entry_link.start() # get entry_and_link,stored in entry_link 16 | 17 | rss_body = RssBody(entry_link.entry_and_link, 18 | xpath["entry_content_css"], 19 | xpath["author_css"], 20 | xpath["datetime_css"]) 21 | 22 | await rss_body.get_rss_body() # get rss_body,stored in rss_body 23 | 24 | rss_feed = RssFeed(rss_body.rss_body, entry_link) 25 | gened_rss = await rss_feed.start() 26 | 27 | return gened_rss 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | def get_database_config(): 4 | database = {"dbname": "rss_from_web", 5 | "user": "ubuntu", 6 | "password": "password", 7 | "host": "127.0.0.1", 8 | } 9 | return database 10 | 11 | def get_app_config(): 12 | base_dir = os.path.dirname(__file__) 13 | app_settings = { 14 | "cookie_secret": "bZJc2sWbQLKos6GkHn/VB9oXwQt8S0R0kRvJ5xxxJ89E=", 15 | #the home.html,login.html dir:./templates 16 | 'template_path': os.path.join(base_dir, "templates"), 17 | #the .css,.js dir:./static 18 | 'static_path': os.path.join(base_dir, "static"), 19 | "xsrf_cookies": True, 20 | } 21 | 22 | return app_settings 23 | 24 | def get_other_config(): 25 | other_settings = { 26 | "listen_port": 8000, 27 | # the same with nginx proxy loation 28 | "home_uri": "/rss-from-web/", 29 | #the url prefix when visit the rss 30 | "absolute_uri_prefix": "http://localhost:8000/rss-from-web/", 31 | "rss_site_uri": "http://localost:8000/", 32 | "refresh_interval": 12 * 60, #minutes 33 | "log_file": "/var/rss-from-web/rfw.log", 34 | 35 | } 36 | return other_settings 37 | -------------------------------------------------------------------------------- /static/js.cookie.min.js: -------------------------------------------------------------------------------- 1 | !function(e){var n;if("function"==typeof define&&define.amd&&(define(e),n=!0),"object"==typeof exports&&(module.exports=e(),n=!0),!n){var t=window.Cookies,o=window.Cookies=e();o.noConflict=function(){return window.Cookies=t,o}}}(function(){function f(){for(var e=0,n={};e 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Rss from web 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 50 | 51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 |

{{ error_msg }}

60 | 61 | 62 |
63 |
64 |
65 |
66 |
67 | 68 | 69 | 70 | 71 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /templates/wrong_edit.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Rss from web 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 50 | 51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 |

{{ rss[0]["rss_wrong"] }}

60 | 61 | 62 |
63 |
64 |
65 |
66 |
67 | 68 | 69 | 70 | 71 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /templates/update_status.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Rss from web 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 50 | 51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | {% if res > 0 %} 59 |

Update success!

60 | {% else %} 61 |

Something wrong happened!

62 | {% end %} 63 | 64 |
65 |
66 |
67 |
68 |
69 | 70 | 71 | 72 | 73 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /templates/entry_list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Entry List 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 50 | 51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 |

Entry List

60 | 61 |
62 | {% for item in entry_list %} 63 | 66 | 69 | {% end %} 70 |
71 |
72 |
73 |
74 |
75 | 76 | 77 | 78 | 79 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /templates/block.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Rss from web 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 33 | 51 | 52 | 53 | 54 |
55 |
56 |
57 |
58 |
59 | 60 |

Block

61 | 62 |
63 | 64 | 65 |
66 | 67 | 68 | 69 | 70 |
71 |
72 |
73 |
74 |
75 |
76 | 77 | 78 | 79 | 80 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /createsql.py: -------------------------------------------------------------------------------- 1 | #import psycopg2 2 | #import datetime 3 | from selectsql import SelectSql 4 | 5 | class CreateTable(object): 6 | """ 7 | when initiate,create all table in postgresql 8 | """ 9 | def __init__(self,database): 10 | self.database = database 11 | #self.user = user_table 12 | #self.xpath = xpath_table 13 | #self.rss = rss_table 14 | 15 | async def create_user_table(self): 16 | select_sql = SelectSql(self.database) 17 | conn = await select_sql.sql_conn() 18 | 19 | res = await conn.execute(""" 20 | CREATE TABLE rss_user( 21 | user_id serial PRIMARY KEY, 22 | user_name varchar NOT NULL DEFAULT 'guest', 23 | email varchar NOT NULL , 24 | password varchar NOT NULL,phone varchar, 25 | paid numeric,cash numeric,priviage varchar, 26 | verified bool NOT NULL DEFAULT TRUE, 27 | created_date timestamp NOT NULL , 28 | last_login timestamp NOT NULL 29 | ); 30 | """) 31 | await conn.close() 32 | 33 | return res 34 | 35 | 36 | async def create_xpath_table(self): 37 | select_sql = SelectSql(self.database) 38 | conn = await select_sql.sql_conn() 39 | 40 | res = await conn.execute(""" 41 | CREATE TABLE xpath( 42 | xpath_id serial PRIMARY KEY, 43 | user_id integer REFERENCES rss_user(user_id), 44 | site_url varchar NOT NULL, 45 | entry_css varchar NOT NULL, 46 | entry_link_css varchar NOT NULL, 47 | add_base_url bool NOT NULL, 48 | rss_link_prefix varchar NOT NULL, 49 | site_title_css varchar NOT NULL, 50 | site_motto_css varchar NOT NULL, 51 | entry_content_css varchar NOT NULL, 52 | author_css varchar NOT NULL, 53 | datetime_css varchar NOT NULL, 54 | interval_time bigint NOT NULL DEFAULT 30 55 | ); 56 | """) 57 | await conn.close() 58 | 59 | return res 60 | 61 | async def creat_rss_table(self): 62 | select_sql = SelectSql(self.database) 63 | conn = await select_sql.sql_conn() 64 | 65 | res = await conn.execute(""" 66 | CREATE TABLE rss( 67 | user_id integer REFERENCES rss_user(user_id), 68 | xpath_id integer REFERENCES xpath(xpath_id), 69 | site_title varchar NOT NULL, 70 | rss_url_name varchar NOT NULL, 71 | rss_content varchar NOT NULL, 72 | rss_last_build_time varchar NOT NULL, 73 | rss_sha256sum varchar NOT NULL 74 | ); 75 | """) 76 | await conn.close() 77 | 78 | return res 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /timerevent.py: -------------------------------------------------------------------------------- 1 | from tornado import ioloop 2 | from entrylink import EntryLink 3 | from rssbody import RssBody 4 | from rssfeed import RssFeed 5 | from rsssql import RssSql 6 | import datetime 7 | import hashlib 8 | import tornado.gen 9 | import time 10 | import asyncio 11 | 12 | class TimerEvent(object): 13 | def __init__(self,mini_interval): 14 | self.mini_interval = mini_interval #seonds 15 | self.xpath_id_interval = [] #all xpath_id and interval,inner is tuple 16 | self.xpath_id_time = {} # all xpath_id and last execute timestamp 17 | self.xpath_id_rss_sha256sum = {} # all xpath_id and rss shasum 18 | self.now_stamp = datetime.datetime.now().timestamp() 19 | 20 | async def timer_execute(self): 21 | rss_sql = RssSql() 22 | new_stamp = datetime.datetime.now().timestamp() 23 | #if self.mini_interval == -1: 24 | self.xpath_id_interval = await rss_sql.get_xpath_id_interval_all() 25 | for id_interval in self.xpath_id_interval: 26 | self.xpath_id_time[id_interval["xpath_id"]] = self.now_stamp 27 | 28 | try: 29 | for id_interval in self.xpath_id_interval: 30 | if (new_stamp - self.xpath_id_time[id_interval["xpath_id"]]) > id_interval["interval_time"]: 31 | self.now_stamp = new_stamp 32 | xpath = await rss_sql.get_xpath_one_from_xpath_id(id_interval["xpath_id"]) 33 | gened_rss = await self.gen_feed_from_xpath(xpath) 34 | 35 | 36 | sha256 = hashlib.sha256() 37 | sha256.update(gened_rss.rss_xml.encode('utf-8')) 38 | res = sha256.hexdigest() 39 | 40 | #if res != self.xpath_id_rss_sha256sum[id_interval["xpath_id"]]: 41 | await rss_sql.update_one_rss_xpath_id(gened_rss.rss_xml, 42 | res, 43 | id_interval["xpath_id"]) 44 | except: 45 | print(f"update some url have problem,maybe the url have been shutdown") 46 | 47 | #print(f"mini_interval is: {self.mini_interval}") 48 | ioloop.IOLoop.current().add_timeout(time.time() + self.mini_interval, 49 | self.timer_execute) 50 | 51 | async def gen_feed_from_xpath(self,xpath): 52 | entry_link = EntryLink(xpath["site_url"], 53 | xpath["entry_css"], 54 | xpath["entry_link_css"], 55 | xpath["add_base_url"], 56 | xpath["rss_link_prefix"], 57 | xpath["site_title_css"], 58 | xpath["site_motto_css"], 59 | xpath["base_url"]) 60 | await entry_link.start() #get entry_and_link,stored in entry_link 61 | 62 | rss_body = RssBody(entry_link.entry_and_link, 63 | xpath["entry_content_css"], 64 | xpath["author_css"], 65 | xpath["datetime_css"]) 66 | 67 | await rss_body.get_rss_body() # get rss_body,stored in rss_body 68 | 69 | rss_feed = RssFeed(rss_body.rss_body,entry_link) 70 | await rss_feed.start() 71 | 72 | return rss_feed 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Because my huginn jobs as rss source always show as not working,so i write a rss source from web,replace huginn. 2 | the grab time interval can be configured ,in the config.py,the init value is 12 hours 3 | # 0.known bugs 4 | All kinds of css can not contain only a single quote ' or only a double quote " which is not belong to the xpath format,such as "//title's tr" or "//title tr",must be "//titletr" 5 | 6 | these bugs exist because lxml can not handle these situation 7 | # 1.how it works 8 | If a web do not have a rss,this project can get a rss source from the web 9 | 10 | This project based on tornado and postgresql,after login,tornado shows you a html page which can be used to input the rss contents xpath css,after click save,the tornaodo will fetch the web contents use the "site_url",then get the articl title and article hyperlink,then tornado fetch every article from the article hyperlink,and generated a rss file. 11 | 12 | I use postgresql stored the rss file's contents and its hyperlink,you can subscribe the rss using the hyper link. 13 | 14 | Every time interval in the config.py,tornado will regenerate the rss through the above process. 15 | 16 | All process in tornado is async. 17 | # 2.prerequiste 18 | - ubuntu 18.04 19 | ``` 20 | lsb_release -a 21 | No LSB modules are available. 22 | Distributor ID: Ubuntu 23 | Description: Ubuntu 18.04.3 LTS 24 | Release: 18.04 25 | Codename: bionic 26 | ``` 27 | - ubuntu user:ubuntu 28 | - python3.6 29 | ``` 30 | python --version 31 | Python 3.6.9 32 | ``` 33 | - pip3 34 | ``` 35 | pip --version 36 | pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6) 37 | ``` 38 | # 3.install 39 | ## 3.1 install command 40 | ``` 41 | p3 install git+https://github.com/Supervisor/supervisor 42 | sudo apt-get install postgresql libpq5 postgresql-contrib postgresql-server-dev-10 43 | cd 44 | git clone https://github.com/su-sanzhou/rss-from-web.git 45 | sudo cp -rv rss-from-web /var 46 | sudo chown -R ubuntu rss-from-web 47 | sudo chgrp -R ubuntu rss-from-web 48 | cd /var/rss-from-web 49 | pip3 install -r requirements.txt 50 | 51 | ``` 52 | 53 | ## 3.2 config postgresql 54 | - create user ubuntu for postgresql 55 | - create database rss_from_web 56 | - setup the password as "password" for user ubuntu when accessing database rss_from_web 57 | # 4.run 58 | ``` 59 | cd /var/rss-from-web/ 60 | supervisord -c supervisor/supervisord.conf 61 | lsof -i | grep python 62 | ``` 63 | if you see listen 8000,congratulations,it's working 64 | visit "http://localhost:8000/rss-from-web" ,input user_name:admin and password:password 65 | you will see the home page,it looks like this: 66 | ![image](https://github.com/su-sanzhou/rss-from-web/blob/master/screen/Selection_001.png) 67 | 68 | # 5.usage 69 | click the "Add a rss" on the right up corner,then you could add something like this: 70 | ![image](https://github.com/su-sanzhou/rss-from-web/blob/master/screen/Selection_002.png) 71 | 72 | you can copy all the xpath css here: 73 | ``` 74 | https://sanzhou.live/ 75 | 76 | //section/article/header/h1/a 77 | 78 | //section/article/header/h1/a 79 | 80 | False 81 | 82 | //header/div/div[1]/div/a/span[2] 83 | 84 | //header/div/div[1]/p 85 | 86 | //article 87 | 88 | //header/div/div[1]/div/a/span[2] 89 | 90 | //article/header/div/span[1]/time 91 | ``` 92 | 93 | then click the save button,you will see the rss source,like this: 94 | ![image](https://github.com/su-sanzhou/rss-from-web/blob/master/screen/Selection_003.png) 95 | 96 | then you can use your rss reader subscribe it. 97 | # 6.todo 98 | - fix the **only one** single or double quotes in the xpath css 99 | - ~~fix the error when generate rss from the next web~~ 100 | 101 | ``` 102 | https://bookdown.org/baydap/papasdiary/ 103 | http://fishandhappiness.blogspot.com/ 104 | ``` 105 | - ~~add the tornado log(in fact,add the python log) ,so that can debug easily~~ 106 | - add the xpath error to the browser when the some error happened 107 | - add to the pypi library ,so that install easily 108 | - construct a good document page 109 | - ~~reform the index.html,so that it looks more suitable~~ 110 | - ~~add the exist feed or rss entrance,so that can add an exist feed or rss to the database~~ 111 | -------------------------------------------------------------------------------- /templates/login.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Rss from web 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 50 | 51 | 52 | 53 |
54 |
55 |
56 |
57 |
58 | 59 | {% if incorrect_times != 0 %} 60 |
61 | Wrong user or password 62 |
63 | {% end %} 64 | 65 |

Login

66 | 67 |
68 | 69 |
70 | 71 |
72 | 73 | 74 |
75 | 76 |
77 | 78 | 79 |
80 | 81 |
82 |
83 | 84 |
85 | 86 | 87 |
88 | {% raw xsrf_form_html() %} 89 |
90 | 91 |
92 |
93 |
94 |
95 |
96 |
97 | 98 | 99 | 100 | 101 | 108 | 109 | 110 | 111 | 112 | 126 | -------------------------------------------------------------------------------- /initiate.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import hashlib 3 | import config 4 | import psycopg2 5 | 6 | def syncpg_connect(database): 7 | database = config.get_database_config() 8 | try: 9 | conn = psycopg2.connect(dbname = database["dbname"], 10 | user = database["user"], 11 | password = database["password"], 12 | host = database["host"]) 13 | except: 14 | print("can not connect to database using psycopg2") 15 | print(f"the conn is {conn}") 16 | else: 17 | cur = conn.cursor() 18 | return conn,cur 19 | 20 | def syncpg_create_user_table(cur): 21 | try: 22 | res = cur.execute(""" 23 | CREATE TABLE rss_user( 24 | user_id serial PRIMARY KEY, 25 | user_name varchar NOT NULL DEFAULT 'guest', 26 | email varchar NOT NULL , 27 | password varchar NOT NULL,phone varchar, 28 | paid numeric,cash numeric,priviage varchar, 29 | verified bool NOT NULL DEFAULT TRUE, 30 | created_date timestamp NOT NULL , 31 | last_login timestamp NOT NULL 32 | ); 33 | """) 34 | except: 35 | print("can not create db rss_user using psycopg2") 36 | else: 37 | print(f"create rss_user table success") 38 | 39 | 40 | def syncpg_create_xpath_table( cur): 41 | try: 42 | res = cur.execute(""" 43 | CREATE TABLE xpath( 44 | xpath_id serial PRIMARY KEY, 45 | user_id integer REFERENCES rss_user(user_id), 46 | site_url varchar NOT NULL, 47 | entry_css varchar NOT NULL, 48 | entry_link_css varchar NOT NULL, 49 | add_base_url bool NOT NULL, 50 | rss_link_prefix varchar NOT NULL, 51 | site_title_css varchar NOT NULL, 52 | site_motto_css varchar NOT NULL, 53 | entry_content_css varchar NOT NULL, 54 | author_css varchar NOT NULL, 55 | datetime_css varchar NOT NULL, 56 | interval_time bigint NOT NULL DEFAULT 30, 57 | rss_link varchar NOT NULL, 58 | base_url varchar NOT NULL 59 | ); 60 | """) 61 | except: 62 | print("can not create db xpath using psycopg2") 63 | else: 64 | print(f"create table xpath success") 65 | 66 | def syncpg_create_rss_table( cur): 67 | try: 68 | res = cur.execute(""" 69 | CREATE TABLE rss( 70 | user_id integer REFERENCES rss_user(user_id), 71 | xpath_id integer REFERENCES xpath(xpath_id), 72 | site_title varchar NOT NULL, 73 | rss_url_name varchar NOT NULL, 74 | rss_content varchar NOT NULL, 75 | rss_last_build_time varchar NOT NULL, 76 | rss_sha256sum varchar NOT NULL 77 | ); 78 | """) 79 | except: 80 | print("can not create db rss using psycopg2") 81 | else: 82 | print("create table rss success") 83 | 84 | def syncopg_insert_admin_user(cur): 85 | 86 | passwd = "password" 87 | sha256 = hashlib.sha256() 88 | sha256.update(passwd.encode('utf-8')) 89 | passwd_sha = sha256.hexdigest() 90 | 91 | try: 92 | res = cur.execute(""" 93 | INSERT INTO rss_user (user_name,email, 94 | password,phone,paid,cash,priviage, 95 | verified,created_date,last_login) 96 | VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s); 97 | """,("admin","admin@test.com",passwd_sha, 98 | "13810000000",0.0,0.0,"administrator", 99 | False,datetime.datetime.now(), 100 | datetime.datetime.now())) 101 | except: 102 | print("there some wong when insert admin") 103 | else: 104 | print("init admin user success") 105 | 106 | 107 | 108 | 109 | if __name__ == "__main__": 110 | database = config.get_database_config() 111 | conn,cur = syncpg_connect(database) 112 | syncpg_create_user_table(cur) 113 | conn.commit() 114 | syncpg_create_xpath_table(cur) 115 | conn.commit() 116 | syncpg_create_rss_table(cur) 117 | conn.commit() 118 | syncopg_insert_admin_user(cur) 119 | conn.commit() 120 | 121 | cur.close() 122 | conn.close() 123 | 124 | 125 | -------------------------------------------------------------------------------- /rssbody.py: -------------------------------------------------------------------------------- 1 | from entrylink import EntryLink 2 | from lxml import etree 3 | from lxml import html as lhtml 4 | from rsslog import RssLog 5 | 6 | class RssBody(object): 7 | def __init__(self, entry_and_link,entry_content_css, 8 | author_css, datetime_css): 9 | self.entry_and_link = entry_and_link 10 | self.rss_body = [] 11 | self.entry_content_css = entry_content_css 12 | self.author_css = author_css 13 | self.datetime_css = datetime_css 14 | self.do_success = "do_success" 15 | self.do_not_success = "do_not_success" 16 | self.status = {"status_rss_body": self.do_success, 17 | "status_entry_content": self.do_success, 18 | "status_rss_author": self.do_success} 19 | self.all_status = self.do_success 20 | self.rss_logger = RssLog() 21 | 22 | async def get_rss_body(self): 23 | for entry in self.entry_and_link: 24 | link = self.entry_and_link[entry] 25 | entry_link = EntryLink(link,self.entry_content_css, 26 | self.author_css, False, 27 | "nothing","nothing","nothing","base_url") 28 | http_body = await entry_link.get_http_body(link) 29 | 30 | if entry_link.status["status_http_body"] != self.do_success: 31 | # this branch just give an empty entry_content 32 | temp_entry_link = {"entry": entry, "link": link, 33 | "entry_content": " ", 34 | "author": " ", 35 | "datetime": " ", 36 | "guid": f'Guid("{link}")'} 37 | self.rss_body.append(temp_entry_link) 38 | entry_link.all_status = self.do_success 39 | self.all_status = self.do_success 40 | else: 41 | http_html = etree.HTML(http_body) 42 | entry_content = await self.get_entry_content(http_html, 43 | self.entry_content_css) 44 | author = await self.get_rss_author(http_html, self.author_css) 45 | datetime = await self.get_rss_datetime(http_html, self.datetime_css) 46 | temp_entry_link = {"entry": entry, "link": link, 47 | "entry_content": entry_content, 48 | "author": author, 49 | "datetime": datetime, 50 | "guid": f'Guid("{link}")'} 51 | self.rss_body.append(temp_entry_link) 52 | 53 | async def get_entry_content(self, http_html,entry_content_css): 54 | entry_contents = http_html.xpath(entry_content_css) 55 | string_entry_content = "" 56 | if len(entry_contents) == 0: 57 | log_info = f"can not parse contents from entry_contents using {entry_content_css}" 58 | self.rss_logger.rss_logger.info(log_info) 59 | self.status["status_entry_centent"] = f"No contents for every {entry_content_css}" 60 | else: 61 | for entry_content in entry_contents: 62 | string_entry_content += etree.tostring(entry_content,encoding = "utf-8").decode(encoding="utf-8") 63 | #string_entry_content += bytes.decode(etree.tostring(entry_content)) 64 | 65 | if not string_entry_content: 66 | log_info = f"there are no contents in string_entry_content:{string_entry_content}" 67 | self.rss_logger.rss_logger.info(log_info) 68 | self.status["status_entry_centent"] = f"No contents for every {entry_contents}" 69 | 70 | return string_entry_content 71 | 72 | async def get_rss_author(self, http_html, author_css): 73 | author_all = http_html.xpath(author_css) 74 | if len(author_all) == 0: 75 | log_info = f"can not parse author from author_all using {author_css}" 76 | self.rss_logger.rss_logger.info(log_info) 77 | return "" 78 | else: 79 | return etree.tostring(author_all[0], method="text", encoding="utf-8").decode(encoding="utf-8") 80 | 81 | async def get_rss_datetime(self, http_html, datetime_css): 82 | datetime_all = http_html.xpath(datetime_css) 83 | if len(datetime_all) == 0: 84 | log_info = f"can not parse datetime from http_html using {datetime_css}" 85 | self.rss_logger.rss_logger.info(log_info) 86 | return "" 87 | else: 88 | return etree.tostring(datetime_all[0], method="text", encoding="utf-8").decode(encoding="utf-8") 89 | 90 | async def start(self): 91 | await self.get_rss_body() 92 | if self.status["status_rss_body"] != self.do_success \ 93 | or self.status["status_entry_content"] != self.do_success \ 94 | or self.status["status_rss_author"] != self.do_success: 95 | self.all_status = self.do_not_success 96 | 97 | -------------------------------------------------------------------------------- /templates/home.html.bak: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Rss from web 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 57 | 58 | 59 | 60 | 61 | 62 | 83 | 84 | 85 | 86 |
87 |
88 |
89 |
90 |
91 | 92 |

Your rss from web

93 | 94 |
95 | {% for item in rss %} 96 | {% if item["site_title"] == "rss_is_none" %} 97 | 100 | {% elif item["site_title"] == "no_site_title" %} 101 | 118 | {% else %} 119 | 137 | {% end %} 138 | {% end %} 139 |
140 |
141 |
142 |
143 |
144 | 145 | 146 | 147 | 148 | 155 | 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /templates/edit.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Rss from web 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 33 | 52 | 53 | 54 | 55 | 56 |
57 |
58 |
59 |
60 |
61 | 62 |

Input web css for rss

63 | 64 |
65 | 66 |
67 | 68 |
69 | 70 | {% for item in rss %} 71 | 72 |
73 | 74 |
75 | 76 | 77 |
78 | 79 |
80 | 81 | 82 |
83 | 84 |
85 | 86 | 87 |
88 | 89 |
90 | 91 | 92 |
93 | 94 |
95 | 96 | 97 |
98 | 99 |
100 | 101 | 102 |
103 | 104 |
105 | 106 | 107 |
108 | 109 |
110 | 111 | 112 |
113 | 114 |
115 | 116 | 117 |
118 | 119 |
120 | 121 | 122 |
123 | 124 |
125 | 126 | 127 |
128 | 129 |
130 | 131 | {% end %} 132 |
133 | 134 |
135 | 136 | 137 |
138 | {% raw xsrf_form_html() %} 139 | 140 | 141 | 142 |
143 |
144 |
145 |
146 |
147 |
148 | 149 | 150 | 151 | 152 | 159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /templates/addrss.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Rss from web 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 33 | 52 | 53 | 54 | 55 | 56 |
57 |
58 |
59 |
60 |
61 | 62 |

Input web css for rss

63 | 64 |
65 | 66 | 67 | 68 |
69 | 70 | 71 |
72 | 73 |
74 | 75 | 76 |
77 | 78 |
79 | 80 | 81 |
82 | 83 |
84 | 85 | 86 |
87 | 88 |
89 | 90 | 91 |
92 | 93 |
94 | 95 |
96 | 97 |
98 | 99 | 100 |
101 | 102 |
103 | 104 | 105 |
106 | 107 |
108 | 109 | 110 |
111 | 112 |
113 | 114 | 115 |
116 | 117 |
118 | 119 | 120 |
121 | 122 |
123 | 124 |
125 | 126 |
127 | 128 | 129 | 130 |
131 | {% raw xsrf_form_html() %} 132 | 133 | 134 | 135 |
136 |
137 |
138 |
139 |
140 |
141 | 142 | 143 | 144 | 145 | 152 | 153 | 154 | 155 | 156 | -------------------------------------------------------------------------------- /templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Rss from web 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 22 | 23 | 29 | 30 | 31 | 54 | 55 | 56 | 57 | 58 | 59 | 80 | 81 | 82 | 83 |
84 |
85 |
86 |
87 |
88 | 89 |

Your rss from web

90 | 91 |
92 | {% for item in rss %} 93 | 160 | {% end %} 161 |
162 |
163 |
164 |
165 |
166 | 167 | 168 | 169 | 170 | 177 | 178 | 179 | 180 | 181 | -------------------------------------------------------------------------------- /rsssql.py: -------------------------------------------------------------------------------- 1 | import config 2 | from selectsql import SelectSql 3 | 4 | class RssSql(object): 5 | def __init__(self): 6 | self.database = config.get_database_config() 7 | self.select_sql = SelectSql(self.database) 8 | self.do_not_success = "do_not_success" 9 | self.do_success = "do_success" 10 | self.user = {} 11 | self.xpath = {} 12 | self.xpath_id = -1 13 | 14 | #not success,return [] 15 | async def get_user_id_password(self,user_name): 16 | conn = await self.select_sql.sql_conn() 17 | res = await conn.fetchrow(""" 18 | SELECT user_id,user_name,password FROM rss_user WHERE user_name = $1 19 | """,user_name) 20 | await conn.close() 21 | return res 22 | 23 | #not success,return [] 24 | async def insert_xpath(self,user_id, 25 | site_url, 26 | entry_css, 27 | entry_link_css, 28 | add_base_url, 29 | rss_link_prefix, 30 | site_title_css, 31 | site_motto_css, 32 | entry_content_css, 33 | author_css, 34 | datetime_css, 35 | interval_time, 36 | rss_link, 37 | base_url): 38 | 39 | conn = await self.select_sql.sql_conn() 40 | res = await conn.fetchrow(""" 41 | INSERT INTO xpath (user_id,site_url, 42 | entry_css,entry_link_css,add_base_url, 43 | rss_link_prefix,site_title_css,site_motto_css, 44 | entry_content_css,author_css,datetime_css, 45 | interval_time,rss_link,base_url) 46 | VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) 47 | RETURNING xpath_id; 48 | """,user_id,site_url,entry_css,entry_link_css, 49 | add_base_url,rss_link_prefix, 50 | site_title_css,site_motto_css,entry_content_css, 51 | author_css,datetime_css,interval_time,rss_link,base_url) 52 | await conn.close() 53 | return res 54 | 55 | #not success,return [] 56 | async def get_xpath_interval_one(self,xpath_id): 57 | conn = await self.select_sql.sql_conn() 58 | res = await conn.fetchrow(""" 59 | SELECT xpath_id,interval_time FROM xpath WHERE xpath_id = $1 60 | """,xpath_id) 61 | await conn.close() 62 | 63 | return res 64 | 65 | #not success,return [] 66 | async def get_xpath_id_interval_all(self): 67 | conn = await self.select_sql.sql_conn() 68 | res = await conn.fetch(""" 69 | SELECT xpath_id,interval_time FROM xpath 70 | """) 71 | await conn.close() 72 | 73 | return res 74 | 75 | 76 | 77 | #not success,return [] 78 | async def get_xpath_from_user_id(self,user_id): 79 | conn = await self.select_sql.sql_conn() 80 | res = await conn.fetch(""" 81 | SELECT * FROM xpath WHERE user_id = $1 82 | """, user_id) 83 | await conn.close() 84 | 85 | return res 86 | 87 | 88 | #not success,return [] 89 | async def get_xpath_one_from_xpath_id(self,xpath_id): 90 | conn = await self.select_sql.sql_conn() 91 | res = await conn.fetchrow(""" 92 | SELECT * FROM xpath WHERE xpath_id = $1 93 | """, xpath_id) 94 | await conn.close() 95 | 96 | return res 97 | 98 | #not success,return [] 99 | async def get_xpath_one_from_url_name(self,url_name): 100 | conn = await self.select_sql.sql_conn() 101 | res = await conn.fetch(""" 102 | SELECT * FROM xpath WHERE rss_link = $1 103 | """, url_name) 104 | await conn.close() 105 | #print(f"the user_id is:{user_id}") 106 | #print(f"the rss is:{res}") 107 | 108 | return res 109 | 110 | #not success,return [] 111 | async def update_xpath_one_from_rss_link(self, 112 | site_url, 113 | entry_css, 114 | entry_link_css, 115 | add_base_url, 116 | site_title_css, 117 | site_motto_css, 118 | entry_content_css, 119 | author_css, 120 | datetime_css, 121 | interval_time, 122 | rss_link, 123 | base_url 124 | ): 125 | 126 | conn = await self.select_sql.sql_conn() 127 | res = await conn.fetchrow(""" 128 | UPDATE xpath SET site_url = $1, 129 | entry_css = $2,entry_link_css = $3,add_base_url = $4, 130 | site_title_css = $5,site_motto_css = $6,entry_content_css = $7, 131 | author_css = $8,datetime_css = $9,interval_time = $10, 132 | base_url = $11 133 | WHERE rss_link = $12 RETURNING xpath_id 134 | """,site_url,entry_css,entry_link_css,add_base_url, 135 | site_title_css,site_motto_css,entry_content_css, 136 | author_css,datetime_css,interval_time,base_url, 137 | rss_link) 138 | 139 | await conn.close() 140 | return res 141 | 142 | 143 | #not success,return [] 144 | async def insert_rss(self,user_id,xpath_id,site_title,rss_url_name, 145 | rss_content,rss_last_build_time,rss_sha256sum): 146 | 147 | conn = await self.select_sql.sql_conn() 148 | res = await conn.fetchrow(""" 149 | INSERT INTO rss (user_id,xpath_id,site_title,rss_url_name, 150 | rss_content,rss_last_build_time,rss_sha256sum) 151 | VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING xpath_id 152 | """, user_id, 153 | xpath_id, 154 | site_title, 155 | rss_url_name, 156 | rss_content, 157 | rss_last_build_time, 158 | rss_sha256sum) 159 | await conn.close() 160 | return res 161 | 162 | 163 | #not success,return [] 164 | async def get_one_rss_from_userid_xpathid(self,user_id,xpath_id): 165 | conn = await self.select_sql.sql_conn() 166 | res = await conn.fetchrow(""" 167 | SELECT * FROM rss WHERE user_id = $1 AND xpath_id = $2; 168 | """, user_id,xpath_id) 169 | await conn.close() 170 | 171 | return res 172 | 173 | 174 | #not success,return [] 175 | async def get_all_rss_from_userid(self,user_id): 176 | conn = await self.select_sql.sql_conn() 177 | res = await conn.fetch(""" 178 | SELECT * FROM rss WHERE user_id = $1 179 | """, user_id) 180 | await conn.close() 181 | #print(f"the user_id is:{user_id}") 182 | #print(f"the rss is:{res}") 183 | 184 | if len(res) == 0: 185 | res = [{"site_title": "rss_is_none","rss_url_name": "no_url"}] 186 | return res 187 | 188 | #not success,return [] 189 | async def get_one_rss_from_url_name(self,url_name): 190 | conn = await self.select_sql.sql_conn() 191 | res = await conn.fetch(""" 192 | SELECT * FROM rss WHERE rss_url_name = $1 193 | """, url_name) 194 | await conn.close() 195 | #print(f"the user_id is:{user_id}") 196 | #print(f"the rss is:{res}") 197 | 198 | if len(res) == 0: 199 | res = [{"rss_content": "no rss,maybe deleted","rss_url_name": "no_url"}] 200 | return res 201 | 202 | #not success,return "do_not_success" 203 | async def update_one_rss_xpath_id(self,rss_content, 204 | rss_sha256sum,xpath_id): 205 | 206 | conn = await self.select_sql.sql_conn() 207 | try: 208 | res = await conn.execute(""" 209 | UPDATE rss SET rss_content = $1, 210 | rss_sha256sum = $2 WHERE xpath_id = $3 211 | """,rss_content, 212 | rss_sha256sum,xpath_id) 213 | 214 | await conn.close() 215 | except: 216 | res = self.do_not_success 217 | return res 218 | else: 219 | return res 220 | 221 | #not success,return [] 222 | async def delete_one_rss_from_url_name(self,url_name): 223 | conn = await self.select_sql.sql_conn() 224 | res1 = await conn.fetchrow(""" 225 | DELETE FROM rss WHERE rss_url_name = $1 RETURNING * 226 | """, url_name) 227 | res2 = await conn.fetchrow(""" 228 | DELETE FROM xpath WHERE rss_link = $1 RETURNING * 229 | """,url_name) 230 | await conn.close() 231 | #print(f"the user_id is:{user_id}") 232 | #print(f"the rss is:{res}") 233 | 234 | if len(res1) != 0 and len(res2) != 0: 235 | res = self.do_success 236 | else: 237 | res = self.do_not_success 238 | return res 239 | 240 | 241 | 242 | 243 | 244 | -------------------------------------------------------------------------------- /entrylink.py: -------------------------------------------------------------------------------- 1 | from tornado import httpclient as hc 2 | from lxml import etree 3 | from lxml import objectify 4 | from urllib.parse import urlparse 5 | import string 6 | import random 7 | from rsslog import RssLog 8 | 9 | class EntryLink(object): 10 | def __init__(self,site_url,entry_css,entry_link_css, 11 | add_base_url,rss_link_prefix, 12 | site_title_css,site_motto_css,base_url): 13 | self.site_url = site_url 14 | self.entry_css = entry_css 15 | self.entry_link_css = entry_link_css 16 | self.add_base_url = add_base_url 17 | self.rss_link_prefix = rss_link_prefix 18 | self.site_title_css = site_title_css 19 | self.site_motto_css = site_motto_css 20 | self.entry_and_link = {} 21 | self.other_for_rss = {} 22 | self.base_url = base_url 23 | self.do_success = "do_success" 24 | self.do_not_success = "do_not_success" 25 | self.status = {"status_http_body": self.do_success, 26 | "status_entry": self.do_success, 27 | "status_entry_link": self.do_success, 28 | "status_entry_and_entry_link": self.do_success} 29 | self.all_status = self.do_success 30 | self.rss_logger = RssLog() 31 | 32 | 33 | async def get_http_body(self,site_url): 34 | http_user_agent = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/79.0.3945.79 Chrome/79.0.3945.79 Safari/537.36'} 35 | http_request = hc.HTTPRequest(url = site_url, 36 | method = 'GET', 37 | headers = http_user_agent, 38 | connect_timeout = 2000, 39 | request_timeout = 6000) 40 | http_client = hc.AsyncHTTPClient() 41 | try: 42 | response = await http_client.fetch(http_request) 43 | except Exception as e: 44 | #add it as a log later 45 | status_http_body = f"can not get content from site_url:{site_url},the Error is{e}" 46 | self.rss_logger.rss_logger.info(status_http_body) 47 | 48 | self.status["status_http_body"] = status_http_body 49 | else: 50 | if not response: 51 | log_info = f"there is no content in site_url:{site_url}" 52 | self.rss_logger.rss_logger.info(log_info) 53 | return response.body.decode() 54 | 55 | async def get_entry_and_link(self,body,entry_css,entry_link_css, 56 | add_base_url,base_url): 57 | #html = etree.HTML(body) 58 | #html = etree.XML(body) 59 | parser = etree.XMLParser(recover=True, ns_clean=True) 60 | xml = etree.fromstring(body.encode(), parser) 61 | tree = etree.ElementTree(xml) 62 | html = tree.getroot() 63 | for elem in html.getiterator(): 64 | if not hasattr(elem.tag, 'find'): continue 65 | i = elem.tag.find('}') 66 | if i >= 0: 67 | elem.tag = elem.tag[i + 1:] 68 | objectify.deannotate(html, cleanup_namespaces=True) 69 | 70 | entrys = await self.get_entry(html,entry_css) 71 | entry_links = await self.get_entry_link(html,entry_link_css,add_base_url,base_url) 72 | entry_and_link = {} 73 | if len(entrys) != len(entry_links): 74 | #add to log later 75 | log_info = f"the wrong title css or wrong link css" 76 | self.rss_logger.rss_logger.info(log_info) 77 | self.status["status_entry_and_entry_link"] = f"the wrong title css or wrong link css" 78 | else: 79 | for i in range(len(entrys)): 80 | entry_and_link[entrys[i]] = entry_links[i] 81 | return entry_and_link 82 | 83 | async def get_entry(self,html,entry_css): 84 | entrys = html.xpath(entry_css) 85 | if len(entrys) == 0: 86 | log_info = f"there are no contents in entrys: {entrys}" 87 | self.rss_logger.rss_logger.info(log_info) 88 | self.status["status_entry"] = f"Can not get entry from {entry_css}" 89 | 90 | entry_list = [] 91 | 92 | for entry in entrys: 93 | entry_list.append(etree.tostring(entry, method="text", encoding="utf-8").decode(encoding="utf-8")) 94 | return entry_list 95 | 96 | async def get_entry_link(self,html,entry_link_css,add_base_url,base_url): 97 | entry_links = html.xpath(entry_link_css) 98 | if len(entry_links) == 0: 99 | log_info = f"there are no contents in entry_links: {entry_links}" 100 | self.rss_logger.rss_logger.info(log_info) 101 | self.status["status_entry_link"] = f"Can not get entry link from {entry_link_css}" 102 | entry_link_list = [] 103 | 104 | for entry_link in entry_links: 105 | href = entry_link.attrib["href"] 106 | if not href: 107 | log_info = f"there are no href in {href}" 108 | self.rss_logger.rss_logger.info(log_info) 109 | self.status["status_entry_link"] = f"Can not get the href attr from the {entry_link_css}" 110 | if add_base_url: 111 | base_and_href = base_url + href 112 | undup_base_and_href = base_and_href.replace("//","/") 113 | if "http://" in base_url: 114 | add_http_or_https_url = undup_base_and_href.replace("http:/","http://") 115 | elif "https://" in base_url: 116 | add_http_or_https_url = undup_base_and_href.replace("https:/","https://") 117 | else: 118 | log_info = "something wrong hanppened when get the url" 119 | self.rss_logger.rss_logger.info(log_info) 120 | self.status["status_entry_link"] = f"Something wrong happened when add base url:{base_url}" 121 | entry_link_list.append(add_http_or_https_url) 122 | else: 123 | entry_link_list.append(href) 124 | 125 | return entry_link_list 126 | 127 | async def get_other(self,http_body,rss_link_prefix,site_url, 128 | site_title_css,site_motto_css): 129 | #html = etree.HTML(http_body) 130 | #html = etree.XML(http_body) 131 | 132 | parser = etree.XMLParser(recover=True, ns_clean=True) 133 | xml = etree.fromstring(http_body.encode(), parser) 134 | tree = etree.ElementTree(xml) 135 | html = tree.getroot() 136 | for elem in html.getiterator(): 137 | if not hasattr(elem.tag, 'find'): continue 138 | i = elem.tag.find('}') 139 | if i >= 0: 140 | elem.tag = elem.tag[i + 1:] 141 | objectify.deannotate(html, cleanup_namespaces=True) 142 | 143 | site_title = html.xpath(site_title_css) 144 | if len(site_title) == 0: 145 | if "/" not in site_title_css: 146 | self.other_for_rss["site_title"] = site_title_css 147 | else: 148 | log_info = f"there are no site_tile" 149 | self.rss_logger.rss_logger.info(log_info) 150 | self.other_for_rss["site_title"] = "no_site_title" 151 | else: 152 | res = etree.tostring(site_title[0], method="text", encoding="utf-8").decode(encoding="utf-8") 153 | self.other_for_rss["site_title"] = res 154 | 155 | site_motto = html.xpath(site_motto_css) 156 | if len(site_motto) == 0: 157 | if "/" not in site_motto_css: 158 | self.other_for_rss["site_motto"] = site_motto_css 159 | else: 160 | self.other_for_rss["site_motto"] = " " 161 | else: 162 | res = etree.tostring(site_motto[0], method="text", encoding="utf-8").decode(encoding="utf-8") 163 | self.other_for_rss["site_motto"] = res 164 | # parse_url = urlparse(site_url) 165 | # if not parse_url: 166 | # log_info = f"there are no contents in parse_url: {parse_url}" 167 | # self.rss_logger.rss_logger.info(log_info) 168 | # domain = '{uri.netloc}'.format(uri = parse_url) 169 | # domain = domain.split(".") 170 | domain = site_url.replace("/","") 171 | domain = domain.replace("http:","") 172 | domain = domain.replace("https:","") 173 | domain = domain.replace("www","") 174 | domain = domain.replace(".","") 175 | 176 | rand_str = ''.join(random.sample(string.ascii_letters + string.digits,30)) 177 | 178 | #if len(domain) >= 2: 179 | # self.other_for_rss["rss_link"] = rss_link_prefix +\ 180 | # domain[-2] + rand_str + ".rss" 181 | #else: 182 | self.other_for_rss["rss_link"] = rss_link_prefix + domain + rand_str + ".rss" 183 | 184 | 185 | async def start(self): 186 | http_body = await self.get_http_body(self.site_url) 187 | #def start(self): 188 | # http_body = self.get_http_body(self.site_url) 189 | if self.status["status_http_body"] == self.do_success: 190 | self.entry_and_link = await self.get_entry_and_link(http_body, 191 | self.entry_css, 192 | self.entry_link_css, 193 | self.add_base_url, 194 | self.base_url) 195 | await self.get_other(http_body,self.rss_link_prefix, 196 | self.site_url,self.site_title_css,self.site_motto_css) 197 | if self.status["status_entry"] != self.do_success \ 198 | or self.status["status_entry_link"] != self.do_success \ 199 | or self.status["status_entry_and_entry_link"] != self.do_success: 200 | self.all_status = self.do_not_success 201 | 202 | -------------------------------------------------------------------------------- /supervisor/supervisord.conf: -------------------------------------------------------------------------------- 1 | ; Sample supervisor config file. 2 | ; 3 | ; For more information on the config file, please see: 4 | ; http://supervisord.org/configuration.html 5 | ; 6 | ; Notes: 7 | ; - Shell expansion ("~" or "$HOME") is not supported. Environment 8 | ; variables can be expanded using this syntax: "%(ENV_HOME)s". 9 | ; - Quotes around values are not supported, except in the case of 10 | ; the environment= options as shown below. 11 | ; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". 12 | ; - Command will be truncated if it looks like a config file comment, e.g. 13 | ; "command=bash -c 'foo ; bar'" will truncate to "command=bash -c 'foo ". 14 | ; 15 | ; Warning: 16 | ; Paths throughout this example file use /tmp because it is available on most 17 | ; systems. You will likely need to change these to locations more appropriate 18 | ; for your system. Some systems periodically delete older files in /tmp. 19 | ; Notably, if the socket file defined in the [unix_http_server] section below 20 | ; is deleted, supervisorctl will be unable to connect to supervisord. 21 | 22 | [unix_http_server] 23 | file=/tmp/supervisor.sock ; the path to the socket file 24 | ;chmod=0700 ; socket file mode (default 0700) 25 | ;chown=nobody:nogroup ; socket file uid:gid owner 26 | ;username=user ; default is no username (open server) 27 | ;password=123 ; default is no password (open server) 28 | 29 | ; Security Warning: 30 | ; The inet HTTP server is not enabled by default. The inet HTTP server is 31 | ; enabled by uncommenting the [inet_http_server] section below. The inet 32 | ; HTTP server is intended for use within a trusted environment only. It 33 | ; should only be bound to localhost or only accessible from within an 34 | ; isolated, trusted network. The inet HTTP server does not support any 35 | ; form of encryption. The inet HTTP server does not use authentication 36 | ; by default (see the username= and password= options to add authentication). 37 | ; Never expose the inet HTTP server to the public internet. 38 | 39 | ;[inet_http_server] ; inet (TCP) server disabled by default 40 | ;port=127.0.0.1:9001 ; ip_address:port specifier, *:port for all iface 41 | ;username=user ; default is no username (open server) 42 | ;password=123 ; default is no password (open server) 43 | 44 | [supervisord] 45 | logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log 46 | logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB 47 | logfile_backups=10 ; # of main logfile backups; 0 means none, default 10 48 | loglevel=info ; log level; default info; others: debug,warn,trace 49 | pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid 50 | nodaemon=false ; start in foreground if true; default false 51 | silent=false ; no logs to stdout if true; default false 52 | minfds=1024 ; min. avail startup file descriptors; default 1024 53 | minprocs=200 ; min. avail process descriptors;default 200 54 | ;umask=022 ; process file creation umask; default 022 55 | ;user=supervisord ; setuid to this UNIX account at startup; recommended if root 56 | ;identifier=supervisor ; supervisord identifier, default is 'supervisor' 57 | ;directory=/tmp ; default is not to cd during start 58 | ;nocleanup=true ; don't clean up tempfiles at start; default false 59 | ;childlogdir=/tmp ; 'AUTO' child log dir, default $TEMP 60 | ;environment=KEY="value" ; key value pairs to add to environment 61 | ;strip_ansi=false ; strip ansi escape codes in logs; def. false 62 | 63 | ; The rpcinterface:supervisor section must remain in the config file for 64 | ; RPC (supervisorctl/web interface) to work. Additional interfaces may be 65 | ; added by defining them in separate [rpcinterface:x] sections. 66 | 67 | [rpcinterface:supervisor] 68 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 69 | 70 | ; The supervisorctl section configures how supervisorctl will connect to 71 | ; supervisord. configure it match the settings in either the unix_http_server 72 | ; or inet_http_server section. 73 | 74 | [supervisorctl] 75 | serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket 76 | ;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket 77 | ;username=chris ; should be same as in [*_http_server] if set 78 | ;password=123 ; should be same as in [*_http_server] if set 79 | ;prompt=mysupervisor ; cmd line prompt (default "supervisor") 80 | ;history_file=~/.sc_history ; use readline history if available 81 | 82 | ; The sample program section below shows all possible program subsection values. 83 | ; Create one or more 'real' program: sections to be able to control them under 84 | ; supervisor. 85 | 86 | 87 | 88 | 89 | 90 | 91 | ;[program:theprogramname] 92 | ;command=/bin/cat ; the program (relative uses PATH, can take args) 93 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s) 94 | ;numprocs=1 ; number of processes copies to start (def 1) 95 | ;directory=/tmp ; directory to cwd to before exec (def no cwd) 96 | ;umask=022 ; umask for process (default None) 97 | ;priority=999 ; the relative start priority (default 999) 98 | ;autostart=true ; start at supervisord start (default: true) 99 | ;startsecs=1 ; # of secs prog must stay up to be running (def. 1) 100 | ;startretries=3 ; max # of serial start failures when starting (default 3) 101 | ;autorestart=unexpected ; when to restart if exited after running (def: unexpected) 102 | ;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) 103 | ;stopsignal=QUIT ; signal used to kill process (default TERM) 104 | ;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) 105 | ;stopasgroup=false ; send stop signal to the UNIX process group (default false) 106 | ;killasgroup=false ; SIGKILL the UNIX process group (def false) 107 | ;user=chrism ; setuid to this UNIX account to run the program 108 | ;redirect_stderr=true ; redirect proc stderr to stdout (default false) 109 | ;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO 110 | ;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) 111 | ;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) 112 | ;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) 113 | ;stdout_events_enabled=false ; emit events on stdout writes (default false) 114 | ;stdout_syslog=false ; send stdout to syslog with process name (default false) 115 | ;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO 116 | ;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) 117 | ;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) 118 | ;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) 119 | ;stderr_events_enabled=false ; emit events on stderr writes (default false) 120 | ;stderr_syslog=false ; send stderr to syslog with process name (default false) 121 | ;environment=A="1",B="2" ; process environment additions (def no adds) 122 | ;serverurl=AUTO ; override serverurl computation (childutils) 123 | 124 | ; The sample eventlistener section below shows all possible eventlistener 125 | ; subsection values. Create one or more 'real' eventlistener: sections to be 126 | ; able to handle event notifications sent by supervisord. 127 | 128 | ;[eventlistener:theeventlistenername] 129 | ;command=/bin/eventlistener ; the program (relative uses PATH, can take args) 130 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s) 131 | ;numprocs=1 ; number of processes copies to start (def 1) 132 | ;events=EVENT ; event notif. types to subscribe to (req'd) 133 | ;buffer_size=10 ; event buffer queue size (default 10) 134 | ;directory=/tmp ; directory to cwd to before exec (def no cwd) 135 | ;umask=022 ; umask for process (default None) 136 | ;priority=-1 ; the relative start priority (default -1) 137 | ;autostart=true ; start at supervisord start (default: true) 138 | ;startsecs=1 ; # of secs prog must stay up to be running (def. 1) 139 | ;startretries=3 ; max # of serial start failures when starting (default 3) 140 | ;autorestart=unexpected ; autorestart if exited after running (def: unexpected) 141 | ;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) 142 | ;stopsignal=QUIT ; signal used to kill process (default TERM) 143 | ;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) 144 | ;stopasgroup=false ; send stop signal to the UNIX process group (default false) 145 | ;killasgroup=false ; SIGKILL the UNIX process group (def false) 146 | ;user=chrism ; setuid to this UNIX account to run the program 147 | ;redirect_stderr=false ; redirect_stderr=true is not allowed for eventlisteners 148 | ;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO 149 | ;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) 150 | ;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) 151 | ;stdout_events_enabled=false ; emit events on stdout writes (default false) 152 | ;stdout_syslog=false ; send stdout to syslog with process name (default false) 153 | ;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO 154 | ;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) 155 | ;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) 156 | ;stderr_events_enabled=false ; emit events on stderr writes (default false) 157 | ;stderr_syslog=false ; send stderr to syslog with process name (default false) 158 | ;environment=A="1",B="2" ; process environment additions 159 | ;serverurl=AUTO ; override serverurl computation (childutils) 160 | 161 | ; The sample group section below shows all possible group values. Create one 162 | ; or more 'real' group: sections to create "heterogeneous" process groups. 163 | 164 | ;[group:thegroupname] 165 | ;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions 166 | ;priority=999 ; the relative start priority (default 999) 167 | 168 | ; The [include] section can just contain the "files" setting. This 169 | ; setting can list multiple files (separated by whitespace or 170 | ; newlines). It can also contain wildcards. The filenames are 171 | ; interpreted as relative to this file. Included files *cannot* 172 | ; include files themselves. 173 | 174 | [include] 175 | ;files = relative/directory/*.ini 176 | files = /var/rss-from-web/supervisor/conf.d/*.conf 177 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from tornado.web import RequestHandler, HTTPError 2 | import tornado.web 3 | import tornado.gen 4 | from tornado import escape 5 | import hashlib 6 | from rsssql import RssSql 7 | import config 8 | from timerevent import TimerEvent 9 | from entrylink import EntryLink 10 | from rssbody import RssBody 11 | from rssfeed import RssFeed 12 | import datetime 13 | import tornado.ioloop 14 | import asyncio 15 | 16 | home_uri = "" 17 | absolute_uri_prefix = "" 18 | refresh_interval = 0 19 | 20 | class HomeHandler(RequestHandler): 21 | async def get(self): 22 | user_id = self.get_secure_cookie("user_id") 23 | if not user_id: 24 | self.redirect(self.reverse_url("login")) 25 | return 26 | 27 | rss_sql = RssSql() 28 | rss = await rss_sql.get_all_rss_from_userid(int(user_id)) 29 | 30 | await self.render("home.html",rss = rss, 31 | request_uri = home_uri) 32 | 33 | async def post(self): 34 | user_id = self.get_secure_cookie("user_id") 35 | if not user_id: 36 | self.redirect(self.reverse_url("login")) 37 | return 38 | 39 | rss_sql = RssSql() 40 | rss = await rss_sql.get_all_rss_from_userid(int(user_id)) 41 | 42 | await self.render("home.html",rss = rss, 43 | request_uri = home_uri) 44 | 45 | 46 | class LoginHandler(RequestHandler): 47 | async def get(self): 48 | incorrect = self.get_secure_cookie("incorrect") or 0 49 | if int(incorrect) > 10: 50 | await self.render("block.html") 51 | return 52 | 53 | user_id = self.get_secure_cookie("user_id") 54 | if not user_id: 55 | temp_uri = home_uri + "login" 56 | await self.render("login.html",request_uri = temp_uri, 57 | incorrect_times = incorrect) 58 | return 59 | 60 | 61 | async def post(self): 62 | incorrect = self.get_secure_cookie("incorrect") or 0 63 | if int(incorrect) > 10: 64 | await self.render("block.html") 65 | return 66 | 67 | user_id = self.get_secure_cookie("user_id") 68 | if not user_id: 69 | user_name = escape.xhtml_escape(self.get_body_argument("user_name")) 70 | password = escape.xhtml_escape(self.get_body_argument("password")) 71 | 72 | sha256 = hashlib.sha256() 73 | sha256.update(password.encode('utf-8')) 74 | res_password = sha256.hexdigest() 75 | 76 | rss_sql = RssSql() 77 | user_id_name_passwd = await rss_sql.get_user_id_password(user_name) 78 | 79 | if not user_id_name_passwd: 80 | incorrect = int(incorrect) + 1 81 | #0.01 means 0.01 day,14minutes later will reset? 82 | self.set_secure_cookie("incorrect", str(incorrect),0.01) 83 | 84 | temp_uri = home_uri + "login" 85 | await self.render("login.html", request_uri=temp_uri, 86 | incorrect_times=incorrect) 87 | return 88 | 89 | else: 90 | if res_password == user_id_name_passwd["password"]: 91 | # use user_id as cookie 92 | self.set_secure_cookie("user_id", 93 | str(user_id_name_passwd["user_id"])) 94 | self.redirect(self.reverse_url("home")) 95 | return 96 | else: 97 | incorrect = int(incorrect) + 1 98 | self.set_secure_cookie("incorrect", str(incorrect),0.01) 99 | 100 | temp_uri = home_uri + "login" 101 | await self.render("login.html", request_uri=temp_uri, 102 | incorrect_times=incorrect) 103 | return 104 | 105 | 106 | 107 | class LogoutHandler(RequestHandler): 108 | async def get(self): 109 | self.clear_cookie("user_id") 110 | self.redirect(self.reverse_url("login")) 111 | return 112 | 113 | 114 | class AddrssHandler(RequestHandler): 115 | async def get(self): 116 | print(f"come here?") 117 | user_id = self.get_secure_cookie("user_id") 118 | if not user_id: 119 | self.redirect(self.reverse_url("login")) 120 | return 121 | 122 | await self.render("addrss.html",home_uri = home_uri, 123 | request_uri = home_uri + "add_rss") 124 | 125 | async def post(self): 126 | user_id = self.get_secure_cookie("user_id") 127 | if not user_id: 128 | self.redirect(self.reverse_url("login")) 129 | return 130 | 131 | site_url = escape.xhtml_escape(self.get_body_argument("site_url")) 132 | entry_css = self.get_body_argument("entry_css") 133 | entry_link_css = self.get_body_argument("entry_link_css") 134 | add_base_url = escape.xhtml_escape(self.get_body_argument("add_base_url")) 135 | site_title_css = self.get_body_argument("site_title_css") 136 | site_motto_css = self.get_body_argument("site_motto_css") 137 | entry_content_css = self.get_body_argument("entry_content_css") 138 | author_css = self.get_body_argument("author_css") 139 | datetime_css = self.get_body_argument("datetime_css") 140 | base_url = self.get_body_argument("base_url") 141 | #print(f"the entry_content_css is:{self.get_body_argument('entry_content_css')}") 142 | 143 | #xpath css can not contain space " " 144 | site_url = site_url.replace(" ","") 145 | entry_css = entry_css.replace(" ","") 146 | entry_link_css = entry_link_css.replace(" ","") 147 | add_base_url = add_base_url.replace(" ","") 148 | site_title_css = site_title_css.replace(" ","") 149 | site_motto_css = site_motto_css.replace(" ","") 150 | entry_content_css = entry_content_css.replace(" ","") 151 | author_css = author_css.replace(" ","") 152 | datetime_css = datetime_css.replace(" ","") 153 | base_url = base_url.replace(" ","") 154 | 155 | 156 | if self.get_arguments('dry_run_list'): 157 | save_button = "dry_run_list" 158 | elif self.get_arguments('save'): 159 | save_button = "save" 160 | else: 161 | save_button = "dry_run_content" 162 | 163 | if str.lower(add_base_url) == "true": 164 | add_base_url = True 165 | else: 166 | add_base_url = False 167 | 168 | 169 | entry_link = EntryLink(site_url,entry_css, 170 | entry_link_css,add_base_url, 171 | absolute_uri_prefix, 172 | site_title_css, 173 | site_motto_css,base_url) 174 | try: 175 | await entry_link.start() 176 | except: 177 | pass 178 | 179 | if entry_link.all_status != entry_link.do_success: 180 | await self.render("error.html",request_uri = home_uri, 181 | error_msg = entry_link.status) 182 | return 183 | 184 | 185 | 186 | if save_button == "dry_run_list": 187 | await self.render("entry_list.html", 188 | entry_list = entry_link.entry_and_link, 189 | request_uri = home_uri) 190 | return 191 | 192 | 193 | 194 | rss_body = RssBody(entry_link.entry_and_link, 195 | entry_content_css,author_css, 196 | datetime_css) 197 | try: 198 | await rss_body.start() 199 | except: 200 | pass 201 | if rss_body.all_status != rss_body.do_success: 202 | await self.render("error.html", request_uri=home_uri, 203 | error_msg=rss_body.status) 204 | return 205 | 206 | rss_feed = RssFeed(rss_body.rss_body,entry_link) 207 | try: 208 | await rss_feed.start() 209 | except: 210 | pass 211 | 212 | rss_sql = RssSql() 213 | res_xpath = await rss_sql.insert_xpath(int(user_id),site_url, 214 | entry_css,entry_link_css, 215 | add_base_url,absolute_uri_prefix, 216 | site_title_css,site_motto_css, 217 | entry_content_css,author_css, 218 | datetime_css,refresh_interval, 219 | entry_link.other_for_rss["rss_link"], 220 | base_url) 221 | xpath_id = res_xpath["xpath_id"] 222 | 223 | sha256 = hashlib.sha256() 224 | sha256.update(rss_feed.rss_xml.encode('utf-8')) 225 | res_rss = sha256.hexdigest() 226 | 227 | 228 | last_build_time = datetime.datetime.now().strftime("%Y-%m-%d") 229 | await rss_sql.insert_rss(int(user_id),xpath_id, 230 | entry_link.other_for_rss["site_title"], 231 | entry_link.other_for_rss["rss_link"], 232 | #escape.xhtml_unescape(rss_feed.rss_xml), 233 | rss_feed.rss_xml, 234 | last_build_time, 235 | res_rss) 236 | 237 | self.redirect(self.reverse_url("home")) 238 | 239 | 240 | 241 | class RssHandler(RequestHandler): 242 | async def get(self): 243 | req_uri = self.request.uri 244 | req_uri = req_uri.split("/") 245 | rss_url_name = absolute_uri_prefix + req_uri[-1] 246 | 247 | rss_sql = RssSql() 248 | rss = await rss_sql.get_one_rss_from_url_name(rss_url_name) 249 | #self.set_header("Content-Type", "application/atom+xml") 250 | await self.render("feed.xml",rss_content = rss[0]["rss_content"]) 251 | #self.write(rss[0]["rss_content"]) 252 | 253 | async def post(self): 254 | req_uri = self.request.uri 255 | req_uri = req_uri.split("/") 256 | rss_url_name = absolute_uri_prefix + req_uri[-1] 257 | 258 | rss_sql = RssSql() 259 | rss = await rss_sql.get_one_rss_from_url_name(rss_url_name) 260 | #self.set_header("Content-Type", "application/atom+xml") 261 | await self.render("feed.xml", rss_content=rss[0]["rss_content"]) 262 | #self.write(rss[0]["rss_content"]) 263 | 264 | class PrivaterssHandler(RequestHandler): 265 | async def get(self): 266 | user_id = self.get_secure_cookie("user_id") 267 | if not user_id: 268 | self.redirect(self.reverse_url("login")) 269 | return 270 | 271 | rss_sql = RssSql() 272 | rss = await rss_sql.get_all_rss_from_userid(int(user_id)) 273 | 274 | self.set_header("Content-Type", "application/atom+xml") 275 | self.write(rss[0]["rss_content"]) 276 | 277 | async def post(self): 278 | user_id = self.get_secure_cookie("user_id") 279 | if not user_id: 280 | self.redirect(self.reverse_url("login")) 281 | return 282 | 283 | rss_sql = RssSql() 284 | rss = await rss_sql.get_all_rss_from_userid(int(user_id)) 285 | self.set_header("Content-Type", "application/atom+xml") 286 | self.write(rss[0]["rss_content"]) 287 | 288 | class DeleterssHandler(RequestHandler): 289 | async def post(self): 290 | user_id = self.get_secure_cookie("user_id") 291 | if not user_id: 292 | self.redirect(self.reverse_url("login")) 293 | return 294 | 295 | url_name = escape.xhtml_escape(self.get_body_argument("url_name")) 296 | #print(f"the url_name is{url_name}") 297 | 298 | rss_sql = RssSql() 299 | res = await rss_sql.delete_one_rss_from_url_name(url_name) 300 | self.write(res) 301 | 302 | class EditrssHandler(RequestHandler): 303 | async def post(self): 304 | user_id = self.get_secure_cookie("user_id") 305 | if not user_id: 306 | self.redirect(self.reverse_url("login")) 307 | return 308 | 309 | url_name = escape.xhtml_escape(self.get_body_argument("url_name")) 310 | 311 | rss_sql = RssSql() 312 | rss = await rss_sql.get_xpath_one_from_url_name(url_name) 313 | 314 | await self.render("edit.html", rss=rss, 315 | request_uri=home_uri) 316 | return 317 | 318 | 319 | class UpdaterssHandler(RequestHandler): 320 | async def post(self): 321 | user_id = self.get_secure_cookie("user_id") 322 | if not user_id: 323 | self.redirect(self.reverse_url("login")) 324 | return 325 | 326 | site_url = self.get_body_argument("site_url") 327 | entry_css = self.get_body_argument("entry_css") 328 | entry_link_css = self.get_body_argument("entry_link_css") 329 | add_base_url = self.get_body_argument("add_base_url") 330 | site_title_css = self.get_body_argument("site_title_css") 331 | site_motto_css = self.get_body_argument("site_motto_css") 332 | entry_content_css = self.get_body_argument("entry_content_css") 333 | author_css = self.get_body_argument("author_css") 334 | datetime_css = self.get_body_argument("datetime_css") 335 | interval_time = float(self.get_body_argument("interval_time")) * 60 * 60 336 | interval_time = int(interval_time) 337 | rss_link = escape.xhtml_escape(self.get_body_argument("rss_link")) 338 | base_url = self.get_body_argument("base_url") 339 | 340 | if str.lower(add_base_url) == "true": 341 | add_base_url = True 342 | else: 343 | add_base_url = False 344 | 345 | rss_sql = RssSql() 346 | res = await rss_sql.update_xpath_one_from_rss_link(site_url,entry_css, 347 | entry_link_css,add_base_url, 348 | site_title_css,site_motto_css,entry_content_css, 349 | author_css,datetime_css,interval_time, 350 | rss_link,base_url) 351 | await self.render("update_status.html",res = int(res["xpath_id"]),request_uri = home_uri) 352 | 353 | 354 | 355 | class Application(tornado.web.Application): 356 | def __init__(self,home_uri,settings): 357 | tornado.web.Application.__init__(self, [ 358 | tornado.web.url(f'{home_uri}?', HomeHandler, 359 | name="home"), 360 | tornado.web.url(f'{home_uri}login', LoginHandler, 361 | name="login"), 362 | tornado.web.url(f'{home_uri}logout', LogoutHandler, 363 | name="logout"), 364 | tornado.web.url(f'{home_uri}add_rss', AddrssHandler, 365 | name="add_rss"), 366 | tornado.web.url(f'{home_uri}.*\.rss', RssHandler, 367 | name="get_rss"), 368 | tornado.web.url(f'{home_uri}private_rss', PrivaterssHandler, 369 | name="private_rss"), 370 | tornado.web.url(f'{home_uri}delete_rss', DeleterssHandler, 371 | name="delete_rss"), 372 | tornado.web.url(f'{home_uri}edit_rss', EditrssHandler, 373 | name="edit_rss"), 374 | tornado.web.url(f'{home_uri}update_rss', UpdaterssHandler, 375 | name="update_rss"), 376 | ], **settings) 377 | 378 | 379 | 380 | if __name__ == "__main__": 381 | 382 | settings = config.get_app_config() 383 | other_configs = config.get_other_config() 384 | home_uri = other_configs["home_uri"] 385 | listen_port = other_configs["listen_port"] 386 | absolute_uri_prefix = other_configs["absolute_uri_prefix"] 387 | rss_site_uri = other_configs["rss_site_uri"] 388 | refresh_interval = other_configs["refresh_interval"] * 60 389 | 390 | 391 | app = Application(home_uri,settings) 392 | app.listen(listen_port) 393 | 394 | timer_event = TimerEvent(refresh_interval) 395 | 396 | io_loop = tornado.ioloop.IOLoop.current() 397 | io_loop.add_callback(timer_event.timer_execute) 398 | 399 | 400 | io_loop.start() 401 | -------------------------------------------------------------------------------- /static/font-awesome.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.7.0');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'),url('../fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} 5 | --------------------------------------------------------------------------------