├── README.md
├── subtitles.ipynb
├── crawler_postnauka.ipynb
├── add_proza_ru.py
├── add_stihi_ru.py
├── youtube_crawl_demo.ipynb
├── crawler_vecher_moskva.ipynb
└── иа_панорама.ipynb


/README.md:
--------------------------------------------------------------------------------
 1 | # crawlers
 2 | Краулеры для проекта Taiga Corpus и Taiga Parser, скачивание ресурсов из открытых источников
 3 | 
 4 | Ресурсы к скачиванию:
 5 | Новости
 6 | 
 7 |     Фонтанка (Оля)
 8 |     Ведомости (Оля)
 9 |     Известия (Оля)
10 |     Интерфакс (Таня)
11 |     Комсомольская Правда (Таня)
12 |     Лента ру (Таня)
13 |     Газета ру (Оля)
14 | 
15 | Худлит 
16 | 
17 |     Журнальный зал (Таня)
18 | 
19 | Остальное 
20 | 
21 |     Прожито ру (?)
22 |     oral history (Оля)
23 |     nplus1 (Таня)
24 |     postnauka (Таня)
25 |     Стихи ру (Таня)
26 |     Проза ру (Таня)
27 |     Арзамас (Оля)
28 | 
29 | Все скачанные данными скриптами материалы необходимо проверять на наличие лишних тегов, достоверность метатекстовой разметки и дедублицировать отдельно.
30 | 


--------------------------------------------------------------------------------
/subtitles.ipynb:
--------------------------------------------------------------------------------
 1 | {
 2 |  "cells": [
 3 |   {
 4 |    "cell_type": "code",
 5 |    "execution_count": 16,
 6 |    "metadata": {
 7 |     "collapsed": false
 8 |    },
 9 |    "outputs": [],
10 |    "source": [
11 |     "import os\n",
12 |     "import pysrt"
13 |    ]
14 |   },
15 |   {
16 |    "cell_type": "code",
17 |    "execution_count": 17,
18 |    "metadata": {
19 |     "collapsed": false
20 |    },
21 |    "outputs": [],
22 |    "source": [
23 |     "\n",
24 |     "for file in os.listdir(r\"/home/mi_air/Downloads/18 to Life\"):\n",
25 |     "    if file.endswith(\"ru.srt\"):\n",
26 |     "        \n",
27 |     "        filename = r\"/home/mi_air/Downloads/18 to Life/\" + file\n",
28 |     "        filenametxt = filename + \".txt\"\n",
29 |     "        out = open(filenametxt, \"w\", encoding=\"utf-8\")\n",
30 |     "        fl = open(filename, \"r\", encoding=\"utf-8\")\n",
31 |     "        out.write(fl.read())"
32 |    ]
33 |   },
34 |   {
35 |    "cell_type": "code",
36 |    "execution_count": 22,
37 |    "metadata": {
38 |     "collapsed": false
39 |    },
40 |    "outputs": [
41 |     {
42 |      "name": "stdout",
43 |      "output_type": "stream",
44 |      "text": [
45 |       "126271\n"
46 |      ]
47 |     }
48 |    ],
49 |    "source": [
50 |     "wordcount=0\n",
51 |     "for file in os.listdir(r\"/home/mi_air/Downloads/18 to Life\"):\n",
52 |     "    if file.endswith(\"ru.srt.txt\"):\n",
53 |     "        filename = r\"/home/mi_air/Downloads/18 to Life/\" + file\n",
54 |     "        fl = open(filename, \"r\", encoding=\"utf-8\")\n",
55 |     "        for line in fl:\n",
56 |     "            wordlist = line.split(\" \")\n",
57 |     "            if line.isdigit()==False:\n",
58 |     "                #print(line)\n",
59 |     "                wordcount += len(wordlist)\n",
60 |     "print (wordcount)"
61 |    ]
62 |   },
63 |   {
64 |    "cell_type": "code",
65 |    "execution_count": null,
66 |    "metadata": {
67 |     "collapsed": true
68 |    },
69 |    "outputs": [],
70 |    "source": []
71 |   }
72 |  ],
73 |  "metadata": {
74 |   "anaconda-cloud": {},
75 |   "kernelspec": {
76 |    "display_name": "Python [default]",
77 |    "language": "python",
78 |    "name": "python3"
79 |   },
80 |   "language_info": {
81 |    "codemirror_mode": {
82 |     "name": "ipython",
83 |     "version": 3
84 |    },
85 |    "file_extension": ".py",
86 |    "mimetype": "text/x-python",
87 |    "name": "python",
88 |    "nbconvert_exporter": "python",
89 |    "pygments_lexer": "ipython3",
90 |    "version": "3.5.2"
91 |   }
92 |  },
93 |  "nbformat": 4,
94 |  "nbformat_minor": 2
95 | }
96 | 


--------------------------------------------------------------------------------
/crawler_postnauka.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "Краулер для постнауки,\n",
  8 |     "\n",
  9 |     "только чистые тексты FAQ\n",
 10 |     "848 публикаций\n",
 11 |     "\n",
 12 |     "https://postnauka.ru/faq"
 13 |    ]
 14 |   },
 15 |   {
 16 |    "cell_type": "code",
 17 |    "execution_count": 1,
 18 |    "metadata": {
 19 |     "collapsed": true
 20 |    },
 21 |    "outputs": [],
 22 |    "source": [
 23 |     "# Импортируем необходимые библиотеки:\n",
 24 |     "import random\n",
 25 |     "import time\n",
 26 |     "import requests # http-запросы,\n",
 27 |     "import re # регулярные выражения,\n",
 28 |     "from bs4 import BeautifulSoup # удаление тегов html,\n",
 29 |     "from tqdm import tqdm # красотуля для анализа прогресса."
 30 |    ]
 31 |   },
 32 |   {
 33 |    "cell_type": "code",
 34 |    "execution_count": 2,
 35 |    "metadata": {
 36 |     "collapsed": true
 37 |    },
 38 |    "outputs": [],
 39 |    "source": [
 40 |     "#список всех ссылок\n",
 41 |     "hrefs = []\n",
 42 |     "out = open(r\"/home/mi_air/Downloads/spisok_postnauka_hrefs.txt\",\"w\", encoding=\"utf-8\")"
 43 |    ]
 44 |   },
 45 |   {
 46 |    "cell_type": "code",
 47 |    "execution_count": 3,
 48 |    "metadata": {
 49 |     "collapsed": true
 50 |    },
 51 |    "outputs": [],
 52 |    "source": [
 53 |     "start = 72417\n",
 54 |     "faq = \"https://postnauka.ru/faq/\"\n",
 55 |     "special = \"https://postnauka.ru/specials/\""
 56 |    ]
 57 |   },
 58 |   {
 59 |    "cell_type": "code",
 60 |    "execution_count": 6,
 61 |    "metadata": {
 62 |     "collapsed": true
 63 |    },
 64 |    "outputs": [],
 65 |    "source": [
 66 |     "def getHref(link):\n",
 67 |     "    bl = 0\n",
 68 |     "    r = requests.get(link)\n",
 69 |     "    for line in r.text:\n",
 70 |     "        if '<link rel=\"image_src\" href=\"http://postnauka.ru/img/' in line:\n",
 71 |     "            bl+=1\n",
 72 |     "    return(bl)"
 73 |    ]
 74 |   },
 75 |   {
 76 |    "cell_type": "code",
 77 |    "execution_count": null,
 78 |    "metadata": {
 79 |     "collapsed": false
 80 |    },
 81 |    "outputs": [],
 82 |    "source": [
 83 |     "while start >25000:\n",
 84 |     "    link = faq + str(start)\n",
 85 |     "    if getHref(link)==1:\n",
 86 |     "        hrefs.append(link)\n",
 87 |     "    link2 = special + str(start)\n",
 88 |     "    if getHref(link2)==1:\n",
 89 |     "        hrefs.append(link2)\n",
 90 |     "    start -= 1"
 91 |    ]
 92 |   },
 93 |   {
 94 |    "cell_type": "code",
 95 |    "execution_count": 8,
 96 |    "metadata": {
 97 |     "collapsed": true
 98 |    },
 99 |    "outputs": [],
100 |    "source": [
101 |     "hrefs = list(set(hrefs))"
102 |    ]
103 |   },
104 |   {
105 |    "cell_type": "code",
106 |    "execution_count": 9,
107 |    "metadata": {
108 |     "collapsed": true
109 |    },
110 |    "outputs": [],
111 |    "source": [
112 |     "for i in hrefs:\n",
113 |     "    out.write(i+\"\\n\")"
114 |    ]
115 |   },
116 |   {
117 |    "cell_type": "code",
118 |    "execution_count": null,
119 |    "metadata": {
120 |     "collapsed": true
121 |    },
122 |    "outputs": [],
123 |    "source": []
124 |   }
125 |  ],
126 |  "metadata": {
127 |   "anaconda-cloud": {},
128 |   "kernelspec": {
129 |    "display_name": "Python [default]",
130 |    "language": "python",
131 |    "name": "python3"
132 |   },
133 |   "language_info": {
134 |    "codemirror_mode": {
135 |     "name": "ipython",
136 |     "version": 3
137 |    },
138 |    "file_extension": ".py",
139 |    "mimetype": "text/x-python",
140 |    "name": "python",
141 |    "nbconvert_exporter": "python",
142 |    "pygments_lexer": "ipython3",
143 |    "version": "3.5.2"
144 |   }
145 |  },
146 |  "nbformat": 4,
147 |  "nbformat_minor": 2
148 | }
149 | 


--------------------------------------------------------------------------------
/add_proza_ru.py:
--------------------------------------------------------------------------------
  1 | 
  2 | # coding: utf-8
  3 | 
  4 | # In[3]:
  5 | 
  6 | # Импортируем необходимые библиотеки:
  7 | import random
  8 | import time
  9 | import requests # http-запросы,
 10 | import os
 11 | import re # регулярные выражения,
 12 | from bs4 import BeautifulSoup # удаление тегов html,
 13 | from tqdm import tqdm # красотуля для анализа прогресса.
 14 | import unify
 15 | 
 16 | 
 17 | # In[29]:
 18 | 
 19 | genre_dic = {"миниатюры":"Малые формы",\
 20 | "новеллы":"Малые формы",\
 21 | "рассказы":"Малые формы",\
 22 | "репортажи":"Малые формы",\
 23 | "повести":"Крупные формы",\
 24 | "романы":"Крупные формы",\
 25 | "драматургия":"Жанровые произведения",\
 26 | "детективы":"Жанровые произведения",\
 27 | "приключения":"Жанровые произведения",\
 28 | "фантастика":"Жанровые произведения",\
 29 | "фэнтези":"Жанровые произведения",\
 30 | "ужасы":"Жанровые произведения",\
 31 | "киберпанк":"Жанровые произведения",\
 32 | "эротическая проза":"Жанровые произведения",\
 33 | "юмористическая проза":"Юмор",\
 34 | "ироническая проза":"Юмор",\
 35 | "фельетоны":"Юмор",\
 36 | "анекдоты":"Юмор",\
 37 | "байки":"Юмор",\
 38 | "история и политика":"Эссе и статьи",\
 39 | "литературоведение":"Эссе и статьи",\
 40 | "естествознание":"Эссе и статьи",\
 41 | "публицистика":"Эссе и статьи",\
 42 | "философия":"Эссе и статьи",\
 43 | "религия":"Эссе и статьи",\
 44 | "мистика":"Эссе и статьи",\
 45 | "мемуары":"Эссе и статьи",\
 46 | "критические статьи":"Литературная критика",\
 47 | "литературные обзоры":"Литературная критика",\
 48 | "музыкальные и кинообзоры":"Литературная критика",\
 49 | "литература для детей":"Детские разделы",\
 50 | "рассказы о детях":"Детские разделы",\
 51 | "сказки":"Детские разделы",\
 52 | "детское творчество":"Детские разделы",\
 53 | "стихи":"Поэзия",\
 54 | "стихотворения в прозе":"Поэзия",\
 55 | "литературные переводы":"Переводы и проза на других языках",\
 56 | "проза на других языках":"Переводы и проза на других языках"}
 57 | 
 58 | 
 59 | # In[30]:
 60 | 
 61 | rubrics = {"05":"миниатюры",\
 62 | "21":"новеллы",\
 63 | "02":"рассказы",\
 64 | "30":"репортажи",\
 65 | "01":"повести",\
 66 | "04":"романы",\
 67 | "13":"драматургия",\
 68 | "07":"детективы",\
 69 | "23":"приключения",\
 70 | "06":"фантастика",\
 71 | "24":"фэнтези",\
 72 | "25":"ужасы",\
 73 | "26":"киберпанк",\
 74 | "03":"эротическая проза",\
 75 | "08":"юмористическая проза",\
 76 | "16":"ироническая проза",\
 77 | "09":"фельетоны",\
 78 | "27":"анекдоты",\
 79 | "28":"байки",\
 80 | "31":"история и политика",\
 81 | "10":"литературоведение",\
 82 | "32":"естествознание",\
 83 | "11":"публицистика",\
 84 | "33":"философия",\
 85 | "34":"религия",\
 86 | "35":"мистика",\
 87 | "18":"мемуары",\
 88 | "12":"критические статьи",\
 89 | "41":"литературные обзоры",\
 90 | "42":"музыкальные и кинообзоры",\
 91 | "17":"литература для детей",\
 92 | "51":"рассказы о детях",\
 93 | "52":"сказки",\
 94 | "50":"детское творчество",\
 95 | "39":"стихи",\
 96 | "43":"стихотворения в прозе",\
 97 | "15":"литературные переводы",\
 98 | "44":"проза на других языках"}
 99 | 
100 | 
101 | # In[31]:
102 | 
103 | #задаем хэдеры - они понадобятся еще много раз
104 | user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'
105 | headers = { 'User-Agent' : user_agent }
106 | 
107 | 
108 | # In[32]:
109 | 
110 | def ensure_dir(directory):
111 |     
112 |     if not os.path.exists(directory):
113 |         os.makedirs(directory, mode=0o777, exist_ok=False)
114 |     return directory
115 | 
116 | 
117 | # In[25]:
118 | 
119 | def make_daily_link(year, month, day, topic='all'):
120 |     return 'http://www.proza.ru/poems/list.html?day='+str(day)+'&month='+str(month)+'&year='+str(year)+'&topic='+str(topic)
121 | 
122 | 
123 | # In[33]:
124 | 
125 | def get_number_poems(link):
126 |     r = requests.get(link, headers=headers)
127 |     
128 |     num = int(re.split(' по', re.split('<p>Произведения в обратном порядке с ', r.text)[1])[0])
129 |     return num
130 | 
131 | 
132 | # In[74]:
133 | 
134 | def get_poem_links(link):
135 |     #ссылка на текст, заголовок, имя автора, ссылка на автора, дата и время
136 |     r = requests.get(link, headers=headers)
137 |     allinks = re.split('"textlink nounline', re.split('опубликовать произведение', r.text)[1])[0].split('</ul>')
138 |     textinfo = []
139 |     
140 |     for part in allinks:
141 |         
142 |         if 'Авторские анонсы' not in part:
143 |             for l in part.split('\n'):
144 |                 
145 |                 if "poemlink" in l:
146 |                     poemlink = 'http://www.proza.ru'+re.split('" ', re.split('<li><a href="', l)[1])[0]
147 |                     title = re.split('</a>', re.split('class="poemlink">', l)[1])[0]
148 |                     author = re.split('</a>', re.split('class="authorlink">', l)[1])[0]
149 |                     authorlink = 'http://www.proza.ru/avtor/'+re.split('" class="authorlink', re.split('href="/avtor/', l)[1])[0]
150 |                     datetime = re.split('</small></li>', re.split('<small>- ', l)[1])[0]
151 |                     date = datetime.split()[0]
152 |                     time = datetime.split()[1]
153 |                     textinfo.append([poemlink, title, author, authorlink,date,time])
154 |     return textinfo
155 | 
156 | 
157 | # In[76]:
158 | 
159 | def get_poem_links_by_date(daily_link):
160 |     r = requests.get(daily_link, headers=headers)
161 |     text_info = []
162 |     
163 |     lines = r.text.split('\n')
164 |     starts = ['http://www.proza.ru'+re.split('">', re.split('<a href="', l)[1])[0] for l in lines if daily_link.strip('http://www.proza.ru')+'&start=' in l]
165 |     text_info = get_poem_links(daily_link)
166 |     for i in starts:
167 |         text_info += get_poem_links(i)
168 |     return text_info
169 | 
170 | 
171 | # In[6]:
172 | 
173 | def make_poem_link(year, month, day, textid):
174 |     
175 |     if year<2008 and int(month) < 10:
176 |         try:
177 |             textlink = 'http://www.proza.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'-' + str(textid)
178 |         except:
179 |             textlink = 'http://www.proza.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'/' + str(textid)
180 |     else:
181 |         textlink = 'http://www.proza.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'/' + str(textid)
182 |     return textlink
183 | 
184 | 
185 | # In[8]:
186 | 
187 | # Функия принимает адрес статьи на nplus1.ru и возвращает текст статьи и метаинформацию по ней.
188 | def getTextStihi(textlink):
189 |     r = requests.get(textlink,headers=headers)  
190 |     text = re.split("</div>", re.split('<div class="text">', r.text)[1])[0]
191 | # "Откусываем" оставшиеся теги.
192 |     beaux_text=BeautifulSoup(text, "lxml")
193 |     n_text = beaux_text.get_text() 
194 |     n_text = re.sub('\xa0', '', n_text)
195 |     n_text = unify.unify_sym(n_text)
196 |     return(n_text)
197 | 
198 | 
199 | # In[77]:
200 | 
201 | #Теперь список авторов нам нужно превратить с список данных автора и ссылки на его тексты:
202 | def getAuthorInfo(authorlink):
203 |     r = requests.get(authorlink, headers=headers)
204 |     
205 |     try:
206 |         
207 |         author_items = re.split("</b>", re.split("Произведений: <b>", r.text)[1])[0]
208 |         author_readeres = re.split("</b>", re.split("Читателей</a>: <b>", r.text)[1])[0]
209 |         return author_items, author_readeres
210 | 
211 |     except:
212 |         return '', ''
213 | 
214 | 
215 | # In[12]:
216 | 
217 | WDIR = ensure_dir(r'/home/tsha/proza_ru/texts')
218 | 
219 | 
220 | # In[ ]:
221 | 
222 | for year in range(2005,2008)[::-1]:
223 |     metatable_texts = open(ensure_dir(r'/home/tsha/proza_ru/meta/'+str(year))+'/metatable_texts.txt', 'a', encoding='utf8')
224 |     metatable_texts.write('textid\tURL\ttitle\tauthor\tauthorlink\tdate\ttime\tpath\tauthor_readers\tauthor_texts\ttopic\tgenre\n')
225 |     #textid, poemlink, title, author, authorlink,date,time, path, author_readers,author_poems,topic,genre
226 |     for month in range(1,13)[::-1]:
227 |         if month < 10:
228 |             month = "0" + str(month)
229 |         path = ensure_dir(WDIR + "/"+str(year)+"/"+str(month))
230 |         for day in range(1, 32)[::-1]:
231 |             if day < 10:
232 |                 day = "0" + str(day)
233 |             if year==2007 and int(month)==12 and int(day)>=3 :
234 |                 pass
235 |             else:
236 |                 for topic in rubrics:
237 |                     print(year, month, day,rubrics[topic] )
238 |                     link = make_daily_link(year, month, day, topic)
239 |                     text_info = get_poem_links_by_date(link)
240 |                     
241 |                     #вот здесь по-другому
242 |                     for i in tqdm(range(len(text_info))):
243 |                         textid = str(year)+str(month)+str(day)+str(i)+str(topic)
244 |                         textlink = text_info[i][0]
245 |                         
246 |                 
247 |                         try:
248 |                             text = getTextStihi(textlink)
249 |                             textfile = open(os.path.join(path, textid+'.txt'), 'w', encoding='utf8')
250 |                             textfile.write(text)
251 |                             textfile.close()
252 |                             author_poems, author_readers  = getAuthorInfo(text_info[i][3])
253 |                             genre = genre_dic[rubrics[topic]]
254 |                             textfeats = [textid]+text_info[i] + [os.path.join(path, textid+'.txt'),author_poems, author_readers, topic, genre]
255 |                             metatable_texts.write("\t".join(textfeats)+'\n')
256 |                         except:
257 |                             continue
258 |                             print(textlink)
259 |         metatable_texts.close()
260 | 
261 | 
262 | 
263 | 


--------------------------------------------------------------------------------
/add_stihi_ru.py:
--------------------------------------------------------------------------------
  1 | 
  2 | # coding: utf-8
  3 | 
  4 | # In[3]:
  5 | 
  6 | # Импортируем необходимые библиотеки:
  7 | import random
  8 | import time
  9 | import requests # http-запросы,
 10 | import os
 11 | import re # регулярные выражения,
 12 | from bs4 import BeautifulSoup # удаление тегов html,
 13 | from tqdm import tqdm # красотуля для анализа прогресса.
 14 | import unify
 15 | 
 16 | 
 17 | # In[29]:
 18 | 
 19 | genre_dic = {'любовная лирика': 'лирика', 'гражданская лирика': 'лирика лирика', 'пейзажная лирика': 'лирика','городская лирика': 'лирика','религиозная лирика': 'лирика','философская лирика': 'лирика','мистика и эзотерика': 'лирика',            'циклы стихов': 'Крупные формы', 'поэмы': 'Крупные формы','пьесы': 'Крупные формы','запад: сонеты, канцоны, рондо':'Твердые формы','без рубрики':'Стихи без рубрики',            'восток: рубаи, хокку, танка':'Твердые формы', 'акростихи':'Твердые формы', 'верлибр':'Свободные формы и проза',            'белый и вольный стих':'Свободные формы и проза', 'cтихотворения в прозе':'Свободные формы и проза',            'прозаические миниатюры':'Свободные формы и проза','эссе и статьи':'Свободные формы и проза', 'афоризмы':'Свободные формы и проза',            'пародии':'Пародии и юмор','подражания':'Пародии и юмор','шуточные стихи':'Пародии и юмор','иронические стихи':'Пародии и юмор','сатирические стихи':'Пародии и юмор','басни':'Пародии и юмор',            'стихи для детей':'Детские разделы','детское творчество':'Детские разделы','авторская песня':'Музыкальное творчество',            'эстрадная песня':'Музыкальное творчество','русский рок':'Музыкальное творчество','либретто':'Музыкальное творчество','шансон':'Музыкальное творчество',            'переводы песен':'Музыкальное творчество','переделки песен':'Музыкальное творчество','поэтические переводы':'Переводы и стихи на других языках',            'стихи на других языках':'Переводы и стихи на других языках','переводы песен':'Переводы и стихи на других языках','без рубрики':'Стихи без рубрики'}
 20 | 
 21 | 
 22 | # In[30]:
 23 | 
 24 | rubrics = {"01":"любовная лирика",\
 25 | "14":"гражданская лирика",\
 26 | "02":"пейзажная лирика",\
 27 | "08":"городская лирика",\
 28 | "19":"религиозная лирика",\
 29 | "17":"философская лирика",\
 30 | "13":"мистика и эзотерика",\
 31 | "04":"циклы стихов",\
 32 | "46":"поэмы",\
 33 | "47":"пьесы",\
 34 | "22":"запад: сонеты, канцоны, рондо",\
 35 | "18":"восток: рубаи, хокку, танка",\
 36 | "44":"акростихи",\
 37 | "32":"верлибр",\
 38 | "15":"белый и вольный стих",\
 39 | "39":"cтихотворения в прозе",\
 40 | "05":"прозаические миниатюры",\
 41 | "40":"эссе и статьи",\
 42 | "06":"афоризмы",\
 43 | "12":"пародии",\
 44 | "21":"подражания",\
 45 | "11":"шуточные стихи",\
 46 | "16":"иронические стихи",\
 47 | "34":"сатирические стихи",\
 48 | "41":"басни",\
 49 | "10":"стихи для детей",\
 50 | "50":"детское творчество",\
 51 | "23":"авторская песня",\
 52 | "24":"эстрадная песня",\
 53 | "25":"русский рок",\
 54 | "26":"либретто",\
 55 | "37":"шансон",\
 56 | "43":"переводы песен",\
 57 | "33":"переделки песен",\
 58 | "20":"поэтические переводы",\
 59 | "36":"стихи на других языках",\
 60 | "43":"переводы песен",\
 61 | "03":"без рубрики"}
 62 | 
 63 | 
 64 | # In[31]:
 65 | 
 66 | #задаем хэдеры - они понадобятся еще много раз
 67 | user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'
 68 | headers = { 'User-Agent' : user_agent }
 69 | 
 70 | 
 71 | # In[32]:
 72 | 
 73 | def ensure_dir(directory):
 74 |     
 75 |     if not os.path.exists(directory):
 76 |         os.makedirs(directory, mode=0o777, exist_ok=True)
 77 |     return directory
 78 | 
 79 | 
 80 | # In[25]:
 81 | 
 82 | def make_daily_link(year, month, day, topic='all'):
 83 |     return 'http://www.stihi.ru/poems/list.html?day='+str(day)+'&month='+str(month)+'&year='+str(year)+'&topic='+str(topic)
 84 | 
 85 | 
 86 | # In[33]:
 87 | 
 88 | def get_number_poems(link):
 89 |     r = requests.get(link, headers=headers)
 90 |     
 91 |     num = int(re.split(' по', re.split('<p>Произведения в обратном порядке с ', r.text)[1])[0])
 92 |     return num
 93 | 
 94 | 
 95 | # In[74]:
 96 | 
 97 | def get_poem_links(link):
 98 |     #ссылка на текст, заголовок, имя автора, ссылка на автора, дата и время
 99 |     r = requests.get(link, headers=headers)
100 |     allinks = re.split('"textlink nounline', re.split('опубликовать произведение', r.text)[1])[0].split('</ul>')
101 |     textinfo = []
102 |     
103 |     for part in allinks:
104 |         
105 |         if 'Авторские анонсы' not in part:
106 |             for l in part.split('\n'):
107 |                 
108 |                 if "poemlink" in l:
109 |                     poemlink = 'http://www.stihi.ru'+re.split('" ', re.split('<li><a href="', l)[1])[0]
110 |                     title = re.split('</a>', re.split('class="poemlink">', l)[1])[0]
111 |                     author = re.split('</a>', re.split('class="authorlink">', l)[1])[0]
112 |                     authorlink = 'http://www.stihi.ru/avtor/'+re.split('" class="authorlink', re.split('href="/avtor/', l)[1])[0]
113 |                     datetime = re.split('</small></li>', re.split('<small>- ', l)[1])[0]
114 |                     date = datetime.split()[0]
115 |                     time = datetime.split()[1]
116 |                     textinfo.append([poemlink, title, author, authorlink,date,time])
117 |     return textinfo
118 | 
119 | 
120 | # In[76]:
121 | 
122 | def get_poem_links_by_date(daily_link):
123 |     r = requests.get(daily_link, headers=headers)
124 |     text_info = []
125 |     
126 |     lines = r.text.split('\n')
127 |     starts = ['http://www.stihi.ru'+re.split('">', re.split('<a href="', l)[1])[0] for l in lines if daily_link.strip('http://www.stihi.ru')+'&start=' in l]
128 |     text_info = get_poem_links(daily_link)
129 |     for i in starts:
130 |         text_info += get_poem_links(i)
131 |     return text_info
132 | 
133 | 
134 | # In[6]:
135 | 
136 | def make_poem_link(year, month, day, textid):
137 |     
138 |     if year<2008 and int(month) < 10:
139 |         try:
140 |             textlink = 'http://www.stihi.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'-' + str(textid)
141 |         except:
142 |             textlink = 'http://www.stihi.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'/' + str(textid)
143 |     else:
144 |         textlink = 'http://www.stihi.ru/'+str(year) + '/' + str(month) + '/' + str(day) +'/' + str(textid)
145 |     return textlink
146 | 
147 | 
148 | # In[8]:
149 | 
150 | # Функия принимает адрес статьи на nplus1.ru и возвращает текст статьи и метаинформацию по ней.
151 | def getTextStihi(textlink):
152 |     r = requests.get(textlink,headers=headers)  
153 |     text = re.split("</div>", re.split('<div class="text">', r.text)[1])[0]
154 | # "Откусываем" оставшиеся теги.
155 |     beaux_text=BeautifulSoup(text, "lxml")
156 |     n_text = beaux_text.get_text() 
157 |     n_text = re.sub('\xa0', '', n_text)
158 |     n_text = unify.unify_sym(n_text)
159 |     return(n_text)
160 | 
161 | 
162 | # In[77]:
163 | 
164 | #Теперь список авторов нам нужно превратить с список данных автора и ссылки на его тексты:
165 | def getAuthorInfo(authorlink):
166 |     r = requests.get(authorlink, headers=headers)
167 |     
168 |     try:
169 |         
170 |         author_items = re.split("</b>", re.split("Произведений: <b>", r.text)[1])[0]
171 |         author_readeres = re.split("</b>", re.split("Читателей</a>: <b>", r.text)[1])[0]
172 |         return author_items, author_readeres
173 | 
174 |     except:
175 |         return '', ''
176 | 
177 | 
178 | # In[12]:
179 | 
180 | WDIR = ensure_dir(r'/home/tsha/stihi_ru/texts')
181 | 
182 | 
183 | # In[ ]:
184 | 
185 | for year in range(2005,2016)[::-1]:
186 |     metatable_texts = open(ensure_dir(r'/home/tsha/stihi_ru/meta/'+str(year))+'/metatable_texts.txt', 'a', encoding='utf8')
187 |     metatable_texts.write('textid\tURL\ttitle\tauthor\tauthorlink\tdate\ttime\tpath\tauthor_readers\tauthor_poems\ttopic\tgenre\n')
188 |     #textid, poemlink, title, author, authorlink,date,time, path, author_readers,author_poems,topic,genre
189 |     for month in range(1,13)[::-1]:
190 |         if month < 10:
191 |             month = "0" + str(month)
192 |         path = ensure_dir(WDIR + "/"+str(year)+"/"+str(month))
193 |         for day in range(1, 32)[::-1]:
194 |             if day < 10:
195 |                 day = "0" + str(day)
196 |             if year==2015 and int(month)==12 :
197 |                 pass
198 |             elif year==2015 and int(month)==11 and int(day)>=11:
199 |                 pass
200 |             else:
201 |                 for topic in rubrics:
202 |                     print(year, month, day,rubrics[topic] )
203 |                     link = make_daily_link(year, month, day, topic)
204 |                     text_info = get_poem_links_by_date(link)
205 |                     
206 |                     #вот здесь по-другому
207 |                     for i in tqdm(range(len(text_info))):
208 |                         textid = str(year)+str(month)+str(day)+str(i)+str(topic)
209 |                         textlink = text_info[i][0]
210 |                         
211 |                 
212 |                         try:
213 |                             text = getTextStihi(textlink)
214 |                             textfile = open(os.path.join(path, textid+'.txt'), 'w', encoding='utf8')
215 |                             textfile.write(text)
216 |                             textfile.close()
217 |                             author_poems, author_readers  = getAuthorInfo(text_info[i][3])
218 |                             genre = genre_dic[rubrics[topic]]
219 |                             textfeats = [textid]+text_info[i] + [os.path.join(path, textid+'.txt'),author_poems, author_readers, topic, genre]
220 |                             metatable_texts.write("\t".join(textfeats)+'\n')
221 |                         except:
222 |                             continue
223 |                             print(textlink)
224 |         metatable_texts.close()
225 | 
226 | 
227 | # In[ ]:
228 | 
229 | 
230 | 
231 | 


--------------------------------------------------------------------------------
/youtube_crawl_demo.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 2,
  6 |    "metadata": {},
  7 |    "outputs": [
  8 |     {
  9 |      "name": "stderr",
 10 |      "output_type": "stream",
 11 |      "text": [
 12 |       "/home/mi_air/.local/lib/python3.5/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.24.1) or chardet (2.3.0) doesn't match a supported version!\n",
 13 |       "  RequestsDependencyWarning)\n"
 14 |      ]
 15 |     }
 16 |    ],
 17 |    "source": [
 18 |     "from __future__ import print_function\n",
 19 |     "\n",
 20 |     "import os\n",
 21 |     "import sys\n",
 22 |     "import time\n",
 23 |     "import json\n",
 24 |     "import requests\n",
 25 |     "import lxml.html\n",
 26 |     "import io\n",
 27 |     "from tqdm import tqdm\n",
 28 |     "from lxml.cssselect import CSSSelector\n",
 29 |     "\n",
 30 |     "#used to make the Browser Working\n",
 31 |     "from selenium import webdriver\n",
 32 |     "#Send keycodes to Elements\n",
 33 |     "from selenium.webdriver.common.keys import Keys\n",
 34 |     "#scrape the url's and comments\n",
 35 |     "from bs4 import BeautifulSoup\n",
 36 |     "\n",
 37 |     "import re\n",
 38 |     "import datetime\n",
 39 |     "import time\n",
 40 |     "import codecs\n",
 41 |     "\n",
 42 |     "\n"
 43 |    ]
 44 |   },
 45 |   {
 46 |    "cell_type": "code",
 47 |    "execution_count": 5,
 48 |    "metadata": {
 49 |     "collapsed": true
 50 |    },
 51 |    "outputs": [],
 52 |    "source": [
 53 |     "\n",
 54 |     "\n",
 55 |     "YOUTUBE_COMMENTS_URL = 'https://www.youtube.com/all_comments?v={youtube_id}'\n",
 56 |     "YOUTUBE_COMMENTS_AJAX_URL = 'https://www.youtube.com/comment_ajax'\n",
 57 |     "\n",
 58 |     "USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'\n"
 59 |    ]
 60 |   },
 61 |   {
 62 |    "cell_type": "code",
 63 |    "execution_count": 3,
 64 |    "metadata": {},
 65 |    "outputs": [
 66 |     {
 67 |      "data": {
 68 |       "text/plain": [
 69 |        "'2019-04-10'"
 70 |       ]
 71 |      },
 72 |      "execution_count": 3,
 73 |      "metadata": {},
 74 |      "output_type": "execute_result"
 75 |     }
 76 |    ],
 77 |    "source": [
 78 |     "str(datetime.date.today())"
 79 |    ]
 80 |   },
 81 |   {
 82 |    "cell_type": "code",
 83 |    "execution_count": 4,
 84 |    "metadata": {},
 85 |    "outputs": [
 86 |     {
 87 |      "name": "stdout",
 88 |      "output_type": "stream",
 89 |      "text": [
 90 |       "{'/watch?v=ew-8i7UUpLY', '/watch?v=oOxDLuTXyCo', '/watch?v=32Rj7dv2IRE', '/watch?v=RtSS0SJE8oE', '/watch?v=N0CGM956z18', '/watch?v=fhCLQNYowcE', '/watch?v=AbXahmBSLTk', '/watch?v=ocekebVtZvw', '/watch?v=JM-GwDh73Wc', '/watch?v=BLHrjzTEr0c', '/watch?v=H6Kl8kheGBg', '/watch?v=cnn-z4U_S50', '/watch?v=F1B9Fk_SgI0', '/watch?v=qr4AaWAkf34', '/watch?v=knBhDpMXsQo', '/watch?v=DEfgiRorfbM', '/watch?v=DVH0CzurtIE', '/watch?v=p1JPKLa-Ofc', '/watch?v=nhcDl3S5sXQ', '/watch?v=TcMBFSGVi1c', '/watch?v=b52lho8lD6Q', '/watch?v=kPg3M4C9N9w', '/watch?v=XlmaJ-yU46U', '/watch?v=U_90XNCBatY', '/watch?v=6pdfriFuFt8', '/watch?v=vw2SaHkGfss', '/watch?v=yzZIS1TtXjw', '/watch?v=q97nD5dOS5M', '/watch?v=31OnT5iSLA0', '/watch?v=3y-O-4IL-PU', '/watch?v=RzfO1FbUCo8', '/watch?v=IRKwwk7CXBQ', '/watch?v=ZD9OkKE0TfA', '/watch?v=35adpxPiNlU', '/watch?v=4yXU8K-9SIw', '/watch?v=nDq6TstdEi8', '/watch?v=3XNDaISqFX8', '/watch?v=3p1fHBNILhM', '/watch?v=W1j28DRcFBQ', '/watch?v=qywZ6lUcNo8', '/watch?v=_XFzT9GMmw8', '/watch?v=hE2Ira-Cwxo', '/watch?v=A8N4_cjLXH8', '/watch?v=kvvLXVDYl6I', '/watch?v=vEUlnLOQG8k', '/watch?v=ssbNmaOmVMk', '/watch?v=XW_KhFq4LQo', '/watch?v=CaBq3SvO0a4', '/watch?v=6Z6zfRWTotY', '/watch?v=3t195yz9xCc', '/watch?v=YbiKtZSqmB4', '/watch?v=wzjWIxXBs_s', '/watch?v=4D8ezH0iXh8', '/watch?v=cyzqxRHLPpk', '/watch?v=CiL-yTNa6QY', '/watch?v=eKmRkS1os7k', '/watch?v=ufI6DCB6X2U', '/watch?v=c18WvLeJn-I', '/watch?v=f30Jq8BQPQo', '/watch?v=7gvqArR7nlA', '/watch?v=el00pNoRB34', '/watch?v=Ba44js56nF4', '/watch?v=gporsZ8WnsM', '/watch?v=P2qOZDuiYlM', '/watch?v=x865r5EqKDo', '/watch?v=PHgc8Q6qTjc', '/watch?v=TwFvvcHf7Dw', '/watch?v=_nf8GV0AvtI', '/watch?v=AKzFFJXMDyE', '/watch?v=iP0MrLN4xso', '/watch?v=o39KwSswsgw', '/watch?v=iloh1SUe42g', '/watch?v=2apVwq-pX9E', '/watch?v=u4x9YyRnFDE', '/watch?v=hsGOT_0L16U', '/watch?v=kO9bzwqCNgo', '/watch?v=IRUihzQvBMo', '/watch?v=zUyH3XhpLTo', '/watch?v=DFia7FhVmuM', '/watch?v=buCD-_1UPn4', '/watch?v=mFlrc16xjik', '/watch?v=qcGNoZ3r9t8', '/watch?v=XmAsgB4EMR8', '/watch?v=KCSNFZKbhZE', '/watch?v=nMfPqeZjc2c', '/watch?v=eEd2K1FxNQY', '/watch?v=3NycM9lYdRI', '/watch?v=gmU9PBDS-0k', '/watch?v=3fEdoqHCaM8', '/watch?v=jCC8fPQOaxU', '/watch?v=VUArb3AIpm4', '/watch?v=z6buCeA4ZSc', '/watch?v=K3Qzzggn--s', '/watch?v=jwxI0OX3GsA', '/watch?v=xcg_e-FY_Vs', '/watch?v=nvRjW2oYBiU', '/watch?v=b2AcxL88DoI', '/watch?v=ZeTWW47yhC4', '/watch?v=sxt4YCIsn2I', '/watch?v=S1gp0m4B5p8', '/watch?v=laoUmXqscdk', '/watch?v=hC8CH0Z3L54', '/watch?v=eQHo2zo58no', '/watch?v=lZIq7A9zKFs', '/watch?v=C5Gm8UvxKlU', '/watch?v=2KBFD0aoZy8', '/watch?v=JKeG1iJNxGs', '/watch?v=OdV6SkGZb3g', '/watch?v=g9bzrGBzSC4', '/watch?v=b5W9t62t10I', '/watch?v=66Ki5_-E0n4', '/watch?v=yMRoNNKWuqQ', '/watch?v=4cx9apL7HhY', '/watch?v=9DzSGPad_z4', '/watch?v=4H9jTQKmR3Q', '/watch?v=4zI6guqVqiI', '/watch?v=sCD9zjf_YRU', '/watch?v=jfjfzKf85Ac', '/watch?v=Yxnsxg4rs0E', '/watch?v=lFcSrYw-ARY', '/watch?v=1ZYbU82GVz4', '/watch?v=1nnRC6jDOCI', '/watch?v=7Jj83FOlBF8', '/watch?v=L5cLq1mIC70', '/watch?v=4NcoqtHH2IE', '/watch?v=jJys1BM8x8k', '/watch?v=tF0uHeLy1v0', '/watch?v=gXKPjSkCSMM', '/watch?v=9uIk_91GQYI', '/watch?v=njHvGxZgTPk', '/watch?v=l8kLiUZDbQ4', '/watch?v=t433PEQGErc', '/watch?v=GRTS9yZJREk', '/watch?v=emKhAptPqg4', '/watch?v=Z-0FXUgVsVs', '/watch?v=CX17qmYO0o0', '/watch?v=WzfRhSU9_qA', '/watch?v=kHkKihbfsXQ', '/watch?v=GMFewiplIbw', '/watch?v=xpVfcZ0ZcFM', '/watch?v=x4o5g_PGkiA', '/watch?v=qvzW_CJTlmM', '/watch?v=VfNvJs7-RM4', '/watch?v=LS_-ZMcGnow', '/watch?v=r34Isj_erU4', '/watch?v=zXtsGAkyeIo', '/watch?v=tKMmMHyLBCE', '/watch?v=0fUMyQlzujU', '/watch?v=4IrkawvzGE8', '/watch?v=k4YRWT_Aldo', '/watch?v=-UOMvxh4MYU', '/watch?v=NwSIgDKvMHk', '/watch?v=au2n7VVGv_c', '/watch?v=HmH4W8JOifg', '/watch?v=IV6IuCTg6MU', '/watch?v=8xQrvclhJrU', '/watch?v=RbMqcFvtMN8', '/watch?v=5nxD4PY39xw', '/watch?v=jC7eeYwKrg0', '/watch?v=txQ6t4yPIM0', '/watch?v=WPni755-Krg', '/watch?v=NymS69shfkc', '/watch?v=_YuMfMLC8FA', '/watch?v=jHcbLgNQ4Co', '/watch?v=wi0q0y7U75c', '/watch?v=JeTzND6XrB0', '/watch?v=r8EF3X8EI2o', '/watch?v=s2Gw6r6HooA', '/watch?v=9QbltzIUV6w', '/watch?v=B8yo1HPW2O4', '/watch?v=JnfP9qKAbk8', '/watch?v=o7W7OvETO40', '/watch?v=vjjS92Q0lYs', '/watch?v=mzs7lmETE90', '/watch?v=qO2Y6BHYhHw', '/watch?v=HO2AJneTjAM', '/watch?v=GFEcOvs6YWk', '/watch?v=rTKodwXQi78', '/watch?v=57p6K-5ZSNc', '/watch?v=-twm7ldMOtI', '/watch?v=nvm6RzVLjWo', '/watch?v=J3UXp9jIr-U', '/watch?v=Cfd6PknS0Fw', '/watch?v=waU75jdUnYw', '/watch?v=ERUugjLmwuY', '/watch?v=MikD7plCDQg', '/watch?v=zxeTC0wKPXs', '/watch?v=OjWsugnahJ0', '/watch?v=zS-Og_RfdNc', '/watch?v=eDuRoPIOBjE', '/watch?v=23e9_o5rxsA', '/watch?v=STZZso9GUhA', '/watch?v=nEDKVNoE2ws', '/watch?v=m8UQ4O7UiDs', '/watch?v=EIeUJcP3T0Q', '/watch?v=7ysFgElQtjI', '/watch?v=jsHX1cFL41w', '/watch?v=qOXXvttM-e8', '/watch?v=9z1nTwP2n0w', '/watch?v=LH4Y1ZUUx2g', '/watch?v=0HpYEZ86Wuc', '/watch?v=2sap-GTtCiU', '/watch?v=8Ap7aJsfaXQ', '/watch?v=IPPYI64aHno', '/watch?v=ygyz3Mqjh0k', '/watch?v=-QKP4iVCaiY', '/watch?v=-bmA1D00B4o', '/watch?v=01ouUdAEFdU', '/watch?v=wWdXfX4Vpm8'}\n"
 91 |      ]
 92 |     }
 93 |    ],
 94 |    "source": [
 95 |     "\n",
 96 |     "# The List where the links to the videos are stored\n",
 97 |     "links = set()\n",
 98 |     "\n",
 99 |     "comments = list()\n",
100 |     "\n",
101 |     "homePage = 'https:www.youtube.com'\n",
102 |     "linksSize = 10\n",
103 |     "driver = webdriver.Firefox()\n",
104 |     "\n",
105 |     "output = open(\"/media/mi_air/0F0B7DDE62EEA81E/youtube/\"+str(datetime.date.today())+\".txt\",\"w\")\n",
106 |     "\n",
107 |     "def loadFullPage(Timeout):\n",
108 |     "    reachedbottom = None\n",
109 |     "    while not reachedbottom:\n",
110 |     "        #scroll one pane down\n",
111 |     "        driver.execute_script(\"window.scrollTo(0,Math.max(document.documentElement.scrollHeight,document.body.scrollHeight,document.documentElement.clientHeight));\");\n",
112 |     "        time.sleep(Timeout)\n",
113 |     "        #check if the bottom is reached\n",
114 |     "        a = driver.execute_script(\"return document.documentElement.scrollTop;\")\n",
115 |     "        b = driver.execute_script(\"return document.documentElement.scrollHeight - document.documentElement.clientHeight;\")\n",
116 |     "        relativeHeight = a / b\n",
117 |     "        if(relativeHeight==1):\n",
118 |     "            reachedbottom = True\n",
119 |     "def getComments(link):\n",
120 |     "    driver.get(url='https:youtube.com'+link)\n",
121 |     "    loadFullPage(1)\n",
122 |     "\n",
123 |     "\n",
124 |     "def main():\n",
125 |     "    driver.get(url=homePage)\n",
126 |     "    enoughLinks = None\n",
127 |     "\n",
128 |     "    while not enoughLinks:\n",
129 |     "        loadFullPage(1)\n",
130 |     "\n",
131 |     "        soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
132 |     "\n",
133 |     "        for link in soup.find_all(\"a\",class_=\"yt-simple-endpoint style-scope ytd-grid-video-renderer\", href=True):\n",
134 |     "            if link not in links:\n",
135 |     "                links.add(link['href'])\n",
136 |     "\n",
137 |     "        if len(links) < linksSize:\n",
138 |     "            driver.refresh()\n",
139 |     "        else:\n",
140 |     "            #for i in range(len(links)-1000):\n",
141 |     "                #links.pop()\n",
142 |     "            enoughLinks = True\n",
143 |     "\n",
144 |     "    #links.sort()\n",
145 |     "    for link in links:\n",
146 |     "        output.write(link)\n",
147 |     "        output.write(\"\\n\")\n",
148 |     "    output.close()\n",
149 |     "    print(links)\n",
150 |     "\n",
151 |     "\n",
152 |     "if __name__ == '__main__':\n",
153 |     "    main()"
154 |    ]
155 |   },
156 |   {
157 |    "cell_type": "code",
158 |    "execution_count": null,
159 |    "metadata": {
160 |     "collapsed": true
161 |    },
162 |    "outputs": [],
163 |    "source": [
164 |     "def find_value(html, key, num_chars=2):\n",
165 |     "    pos_begin = html.find(key) + len(key) + num_chars\n",
166 |     "    pos_end = html.find('\"', pos_begin)\n",
167 |     "    return html[pos_begin: pos_end]\n",
168 |     "\n",
169 |     "\n",
170 |     "def extract_comments(html):\n",
171 |     "    tree = lxml.html.fromstring(html)\n",
172 |     "    item_sel = CSSSelector('.comment-item')\n",
173 |     "    text_sel = CSSSelector('.comment-text-content')\n",
174 |     "    time_sel = CSSSelector('.time')\n",
175 |     "    author_sel = CSSSelector('.user-name')\n",
176 |     "\n",
177 |     "    for item in item_sel(tree):\n",
178 |     "        yield {'cid': item.get('data-cid'),\n",
179 |     "               'text': text_sel(item)[0].text_content(),\n",
180 |     "               'time': time_sel(item)[0].text_content().strip(),\n",
181 |     "               'author': author_sel(item)[0].text_content()}\n",
182 |     "\n",
183 |     "\n",
184 |     "def extract_reply_cids(html):\n",
185 |     "    tree = lxml.html.fromstring(html)\n",
186 |     "    sel = CSSSelector('.comment-replies-header > .load-comments')\n",
187 |     "    return [i.get('data-cid') for i in sel(tree)]\n",
188 |     "\n",
189 |     "\n",
190 |     "def ajax_request(session, url, params, data, retries=10, sleep=20):\n",
191 |     "    for _ in range(retries):\n",
192 |     "        response = session.post(url, params=params, data=data)\n",
193 |     "        if response.status_code == 200:\n",
194 |     "            response_dict = json.loads(response.text)\n",
195 |     "            return response_dict.get('page_token', None), response_dict['html_content']\n",
196 |     "        else:\n",
197 |     "            time.sleep(sleep)\n",
198 |     "\n",
199 |     "\n",
200 |     "def download_comments(youtube_id, sleep=1):\n",
201 |     "    session = requests.Session()\n",
202 |     "    session.headers['User-Agent'] = USER_AGENT\n",
203 |     "\n",
204 |     "    # Get Youtube page with initial comments\n",
205 |     "    response = session.get(YOUTUBE_COMMENTS_URL.format(youtube_id=youtube_id))\n",
206 |     "    html = response.text\n",
207 |     "    reply_cids = extract_reply_cids(html)\n",
208 |     "\n",
209 |     "    ret_cids = []\n",
210 |     "    for comment in extract_comments(html):\n",
211 |     "        ret_cids.append(comment['cid'])\n",
212 |     "        yield comment\n",
213 |     "\n",
214 |     "    page_token = find_value(html, 'data-token')\n",
215 |     "    session_token = find_value(html, 'XSRF_TOKEN', 4)\n",
216 |     "\n",
217 |     "    first_iteration = True\n",
218 |     "\n",
219 |     "    # Get remaining comments (the same as pressing the 'Show more' button)\n",
220 |     "    while page_token:\n",
221 |     "        data = {'video_id': youtube_id,\n",
222 |     "                'session_token': session_token}\n",
223 |     "\n",
224 |     "        params = {'action_load_comments': 1,\n",
225 |     "                  'order_by_time': True,\n",
226 |     "                  'filter': youtube_id}\n",
227 |     "\n",
228 |     "        if first_iteration:\n",
229 |     "            params['order_menu'] = True\n",
230 |     "        else:\n",
231 |     "            data['page_token'] = page_token\n",
232 |     "\n",
233 |     "        response = ajax_request(session, YOUTUBE_COMMENTS_AJAX_URL, params, data)\n",
234 |     "        if not response:\n",
235 |     "            break\n",
236 |     "\n",
237 |     "        page_token, html = response\n",
238 |     "\n",
239 |     "        reply_cids += extract_reply_cids(html)\n",
240 |     "        for comment in extract_comments(html):\n",
241 |     "            if comment['cid'] not in ret_cids:\n",
242 |     "                ret_cids.append(comment['cid'])\n",
243 |     "                yield comment\n",
244 |     "\n",
245 |     "        first_iteration = False\n",
246 |     "        time.sleep(sleep)\n",
247 |     "\n",
248 |     "    # Get replies (the same as pressing the 'View all X replies' link)\n",
249 |     "    for cid in reply_cids:\n",
250 |     "        data = {'comment_id': cid,\n",
251 |     "                'video_id': youtube_id,\n",
252 |     "                'can_reply': 1,\n",
253 |     "                'session_token': session_token}\n",
254 |     "\n",
255 |     "        params = {'action_load_replies': 1,\n",
256 |     "                  'order_by_time': True,\n",
257 |     "                  'filter': youtube_id,\n",
258 |     "                  'tab': 'inbox'}\n",
259 |     "\n",
260 |     "        response = ajax_request(session, YOUTUBE_COMMENTS_AJAX_URL, params, data)\n",
261 |     "        if not response:\n",
262 |     "            break\n",
263 |     "\n",
264 |     "        _, html = response\n",
265 |     "\n",
266 |     "        for comment in extract_comments(html):\n",
267 |     "            if comment['cid'] not in ret_cids:\n",
268 |     "                ret_cids.append(comment['cid'])\n",
269 |     "                yield comment\n",
270 |     "        time.sleep(sleep)\n",
271 |     "\n",
272 |     "\n",
273 |     "def main(youtube_id, output, limit=100):\n",
274 |     "\n",
275 |     "    try:\n",
276 |     "\n",
277 |     "        if not youtube_id or not output:\n",
278 |     "            parser.print_usage()\n",
279 |     "            raise ValueError('you need to specify a Youtube ID and an output filename')\n",
280 |     "\n",
281 |     "        print('Downloading Youtube comments for video:', youtube_id)\n",
282 |     "        count = 0\n",
283 |     "        with io.open(output, 'w', encoding='utf8') as fp:\n",
284 |     "            for comment in download_comments(youtube_id):\n",
285 |     "                sys.stdout.write(json.dumps(comment, ensure_ascii=False))\n",
286 |     "                count += 1\n",
287 |     "                sys.stdout.write('Downloaded %d comment(s)\\r' % count)\n",
288 |     "                sys.stdout.flush()\n",
289 |     "                if limit and count >= limit:\n",
290 |     "                    break\n",
291 |     "        print('\\nDone!')\n",
292 |     "\n",
293 |     "\n",
294 |     "    except Exception as e:\n",
295 |     "        print('Error:', str(e))\n",
296 |     "        sys.exit(1)\n",
297 |     "\n"
298 |    ]
299 |   },
300 |   {
301 |    "cell_type": "code",
302 |    "execution_count": null,
303 |    "metadata": {},
304 |    "outputs": [
305 |     {
306 |      "name": "stderr",
307 |      "output_type": "stream",
308 |      "text": [
309 |       "\n",
310 |       "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A"
311 |      ]
312 |     },
313 |     {
314 |      "name": "stdout",
315 |      "output_type": "stream",
316 |      "text": [
317 |       "Downloading Youtube comments for video: ew-8i7UUpLY\n",
318 |       "{\"author\": \"B Solo\", \"cid\": \"UgyaulXNutDMC8e8br54AaABAg\", \"time\": \"1 hour ago\", \"text\": \"Draymond Green looks like he just escaped slavery!\"}Downloaded 20 comment(s) appearances than a guy like Bradley Beal speaks to why people think Draymond is overrated. He would not make the all star team on any other team in the league.\"}Downloaded 19 comment(s)es only one ball\\\" crap. hes better then 90 % of yalls PFs, even as a \\\"role player\\\". fuck off\"}Downloaded 16 comment(s)\r"
319 |      ]
320 |     },
321 |     {
322 |      "name": "stderr",
323 |      "output_type": "stream",
324 |      "text": [
325 |       "\n"
326 |      ]
327 |     },
328 |     {
329 |      "name": "stdout",
330 |      "output_type": "stream",
331 |      "text": [
332 |       "{\"author\": \"Infinit 0\", \"cid\": \"Ugz1AnGIkvGzByb2GJR4AaABAg\", \"time\": \"5 hours ago\", \"text\": \"Draymond needs to accept that he is hostile first off, and work on his craft. He could be a helluva rebounder, like Rodman with his attitude, but he thinks to highly of himself and he isn't putting up double digit scoring numbers. He should reevaluate and change for the better before his time runs out.\"}Downloaded 100 comment(s)n and on. Jalen makes a good point that he rose out of nothing and was a nobody coming into the league. But it's also true that he's become overrated playing on the Warriors.\"}Downloaded 92 comment(s)nd PG13 made a contested shot, but hey, it's still counted as assist. If he can average 5 blocks and 5 steal, then even he got no triple double he will be most underrated. But no, the players got this right, he can't even shoot FT. Oh btw, he's the 2nd in TO per game.\"}Downloaded 47 comment(s)\r"
333 |      ]
334 |     },
335 |     {
336 |      "name": "stderr",
337 |      "output_type": "stream",
338 |      "text": [
339 |       "\r",
340 |       " 10%|█         | 1/10 [00:07<01:11,  7.99s/it]"
341 |      ]
342 |     },
343 |     {
344 |      "name": "stdout",
345 |      "output_type": "stream",
346 |      "text": [
347 |       "\n",
348 |       "Done!\n",
349 |       "Downloading Youtube comments for video: oOxDLuTXyCo\n",
350 |       "{\"author\": \"SERBIA SRB\", \"cid\": \"UgwA19RnoZDvmDCdXpt4AaABAg\", \"time\": \"2 days ago\", \"text\": \"VIVA BARÇA 💪🏻💪🏻💪🏻\"}Downloaded 100 comment(s)day.\"}Downloaded 99 comment(s)er dribbling, better short and medium pass, strrt faster\\nRonaldo plays better with his head, he has a stronger blow, he runs faster for medium and long\"}Downloaded 97 comment(s)fensive game\\nAlso, Argentina needs 2-3 tall and powerful players in order not to yield to high and powerful teams like Germany and France.\"}Downloaded 96 comment(s) a stumbling block for anyone.\\\" While I did understood the meaning of it, I was not able to grasp the full width of those words. You see, what this young man was telling me was that he wouldn't want to play a song that would provoke myself to defile my own conscience with old sinful memories, because if he did, then he would become a stumbling block for me. Very truthful indeed. How much more then, is your conscience defiled when you see nudity, death, violence, and sinful acts or when we hear cursing, lies, rumors, gossip, and hatred? It's true, each and everyone of us is going to give an account for how we lived in this life, and we will be judged according to our works. Are you a stumbling block for others, or do guard your eyes and ears and mouth? \\\"But I say unto you, That every idle word that men shall speak, they shall give account thereof in the day of judgment\\\" (Matthew 12:36)\"}Downloaded 88 comment(s) en nuestro lugar. Jesús se convirtió en nuestro sustituto. \\n\\nEn la Biblia, un carcelero preguntó a sus prisioneros Pablo y Silas: “Señores, ¿qué tengo que hacer para ser salvo? —Cree en el Señor Jesús; así tú y tu familia serán salvos —le contestaron. \\n\\nLa Biblia es clara, cree en Jesús como aquel que cargó tus pecados, murió en tu lugar, fue enterrado y luego resucitado por Dios. Es la sangre de Cristo y la resurrección que nos aseguran la vida eterna cuando lo llamamos nuestro Señor y Salvador. “Porque todo el que invoque el nombre del Señor será salvo” (Romanos 10:13). “Todo el que” incluye a todos y cada uno de nosotros. \\n\\nPor lo tanto, si tú entiendes que eres un pecador y crees que Jesucristo vino como el único Redentor del pecado, entonces entiendes el plan de salvación. La pregunta es: ¿Estás listo para implementar el plan, recibiendo el regalo de Dios, Jesucristo? Si es así, cree en Jesucristo, arrepiéntete de tus pecados y entrégale el resto de tu vida a él como tu Señor:\\n\\n“Padre, sé que he quebrantado tus leyes y que mis pecados me han separado de ti. Estoy sinceramente arrepentido y ahora quiero apartarme de mi pasado pecaminoso y dirigirme hacia ti. Por favor, perdóname y ayúdame a no pecar de nuevo. Creo que tu hijo Jesucristo murió por mis pecados, resucitó de la muerte, está vivo y escucha mi oración. Invito a Jesús a que se convierta en el Señor de mi vida, a que gobierne y reine en mi corazón de este día en adelante. Por favor, envía tu Espíritu Santo para que me ayude a obedecerte y a hacer tu voluntad por el resto de mi vida. En el nombre de Jesús oro, amén.”“Arrepiéntanse y bautícese cada uno de ustedes en el nombre de Jesucristo para perdón de sus pecados --les contestó Pedro--, y recibirán el don del Espíritu Santo” (Hechos 2:38). \\n\\nSi decidiste convertirte en cristiano el día de hoy, bienvenido a la familia de Dios. Ahora, como una forma de crecer más cerca de él, la Biblia nos dice que sigamos adelante con nuestro compromiso\\n\\nBautízate como lo ordenó Cristo.\\n\\nDile a otra persona de tu nueva fe en Cristo.\\n\\nPasa tiempo con Dios cada día. No tiene que ser un largo periodo de tiempo. Simplemente desarrolla el hábito diario de orar y leer su Palabra. Pídele a Dios que incremente tu fe y tu comprensión de la Biblia.\\n\\nBusca estar en comunión con otros cristianos. Sé parte de un grupo de amigos Cristianos que responda tus inquietudes y te apoye.\\n\\nEncuentra una iglesia local en la que puedas adorar a Dios.\"}Downloaded 71 comment(s)\r"
351 |      ]
352 |     },
353 |     {
354 |      "name": "stderr",
355 |      "output_type": "stream",
356 |      "text": [
357 |       "\r",
358 |       " 20%|██        | 2/10 [00:16<01:05,  8.13s/it]"
359 |      ]
360 |     },
361 |     {
362 |      "name": "stdout",
363 |      "output_type": "stream",
364 |      "text": [
365 |       "\n",
366 |       "Done!\n",
367 |       "Downloading Youtube comments for video: 32Rj7dv2IRE\n",
368 |       "{\"author\": \"MSKANE302\", \"cid\": \"UgzAAxsi0mkbhqQ_3ld4AaABAg\", \"time\": \"1 week ago\", \"text\": \"19:14\\nMightyDuck:Ahhhhh\\nMom:Stop \\nMightyDuck:Believe I can fly\\nMom:Boy shut yo face.Shut yo face\\nMightyDuck:(trynna \\\"shut his face\\\") Um,I can't\\n😂🤣😂🤣😂🤣😂🤣😂🤣😂🤣😂🤣😂🤣😂🤣\"}Downloaded 100 comment(s)sing🏃 you all over .\"}Downloaded 95 comment(s)\r"
369 |      ]
370 |     },
371 |     {
372 |      "name": "stderr",
373 |      "output_type": "stream",
374 |      "text": [
375 |       "\r",
376 |       " 30%|███       | 3/10 [00:24<00:57,  8.15s/it]"
377 |      ]
378 |     },
379 |     {
380 |      "name": "stdout",
381 |      "output_type": "stream",
382 |      "text": [
383 |       "\n",
384 |       "Done!\n",
385 |       "Downloading Youtube comments for video: RtSS0SJE8oE\n",
386 |       "{\"author\": \"Lonewolf3D2\", \"cid\": \"UgyCblUz1kLybjkK3rd4AaABAg\", \"time\": \"7 hours ago\", \"text\": \"2516?\"}Downloaded 20 comment(s)on my cave painting deluxe\"}Downloaded 19 comment(s)All about space\"}Downloaded 14 comment(s)omment(s)\r"
387 |      ]
388 |     }
389 |    ],
390 |    "source": [
391 |     "wdir = r'/media/mi_air/0F0B7DDE62EEA81E/youtube/comments'\n",
392 |     "\n",
393 |     "uids = open(r'/media/mi_air/0F0B7DDE62EEA81E/youtube/'+str(datetime.date.today())+'.txt', 'r', encoding='utf8').readlines()\n",
394 |     "uids = [i.strip('/watch?v=') for i in uids]\n",
395 |     "for i in tqdm(uids[:10]):\n",
396 |     "    uid = i.strip('\\n')\n",
397 |     "    output = os.path.join(wdir, uid+'_'+str(datetime.date.today())+'.txt')\n",
398 |     "    main(uid, output)"
399 |    ]
400 |   },
401 |   {
402 |    "cell_type": "code",
403 |    "execution_count": null,
404 |    "metadata": {
405 |     "collapsed": true
406 |    },
407 |    "outputs": [],
408 |    "source": []
409 |   }
410 |  ],
411 |  "metadata": {
412 |   "anaconda-cloud": {},
413 |   "kernelspec": {
414 |    "display_name": "Python [default]",
415 |    "language": "python",
416 |    "name": "python3"
417 |   },
418 |   "language_info": {
419 |    "codemirror_mode": {
420 |     "name": "ipython",
421 |     "version": 3
422 |    },
423 |    "file_extension": ".py",
424 |    "mimetype": "text/x-python",
425 |    "name": "python",
426 |    "nbconvert_exporter": "python",
427 |    "pygments_lexer": "ipython3",
428 |    "version": "3.5.2"
429 |   }
430 |  },
431 |  "nbformat": 4,
432 |  "nbformat_minor": 2
433 | }
434 | 


--------------------------------------------------------------------------------
/crawler_vecher_moskva.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "Краулер для \"Вечерней Москвы\""
  8 |    ]
  9 |   },
 10 |   {
 11 |    "cell_type": "code",
 12 |    "execution_count": 1,
 13 |    "metadata": {
 14 |     "collapsed": true
 15 |    },
 16 |    "outputs": [],
 17 |    "source": [
 18 |     "# Импортируем необходимые библиотеки:\n",
 19 |     "import requests # http-запросы,\n",
 20 |     "import re # регулярные выражения,\n",
 21 |     "from bs4 import BeautifulSoup # удаление тегов html,\n",
 22 |     "from tqdm import tqdm # красотуля для анализа прогресса.\n",
 23 |     "import time\n",
 24 |     "import random"
 25 |    ]
 26 |   },
 27 |   {
 28 |    "cell_type": "code",
 29 |    "execution_count": 2,
 30 |    "metadata": {
 31 |     "collapsed": true
 32 |    },
 33 |    "outputs": [],
 34 |    "source": [
 35 |     "user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'\n",
 36 |     "headers = { 'User-Agent' : user_agent }\n"
 37 |    ]
 38 |   },
 39 |   {
 40 |    "cell_type": "code",
 41 |    "execution_count": 7,
 42 |    "metadata": {
 43 |     "collapsed": true
 44 |    },
 45 |    "outputs": [],
 46 |    "source": [
 47 |     "\n",
 48 |     "hreffile=open(r\"/home/mi_air/Downloads/VM/href_list_dedup.txt\", \"w\", encoding=\"utf-8\")\n"
 49 |    ]
 50 |   },
 51 |   {
 52 |    "cell_type": "code",
 53 |    "execution_count": 12,
 54 |    "metadata": {
 55 |     "collapsed": false
 56 |    },
 57 |    "outputs": [
 58 |     {
 59 |      "ename": "ConnectionError",
 60 |      "evalue": "HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f105c3320>: Failed to establish a new connection: [Errno 111] Connection refused',))",
 61 |      "output_type": "error",
 62 |      "traceback": [
 63 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
 64 |       "\u001b[0;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
 65 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36m_new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    137\u001b[0m             conn = connection.create_connection(\n\u001b[0;32m--> 138\u001b[0;31m                 (self.host, self.port), self.timeout, **extra_kw)\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 66 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/connection.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0merr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 67 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/connection.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_address\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 68 |       "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused",
 69 |       "\nDuring handling of the above exception, another exception occurred:\n",
 70 |       "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
 71 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[1;32m    593\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 594\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 72 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSocketTimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBaseSSLError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 73 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m    834\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sock'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# AppEngine might not have  `.sock`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 835\u001b[0;31m             \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 74 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m         \u001b[0;31m# Add certificate verification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_new_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 75 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36m_new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    146\u001b[0m             raise NewConnectionError(\n\u001b[0;32m--> 147\u001b[0;31m                 self, \"Failed to establish a new connection: %s\" % e)\n\u001b[0m\u001b[1;32m    148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 76 |       "\u001b[0;31mNewConnectionError\u001b[0m: <requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f105c3320>: Failed to establish a new connection: [Errno 111] Connection refused",
 77 |       "\nDuring handling of the above exception, another exception occurred:\n",
 78 |       "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
 79 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    422\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    424\u001b[0m                 )\n",
 80 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[1;32m    642\u001b[0m             retries = retries.increment(method, url, error=e, _pool=self,\n\u001b[0;32m--> 643\u001b[0;31m                                         _stacktrace=sys.exc_info()[2])\n\u001b[0m\u001b[1;32m    644\u001b[0m             \u001b[0mretries\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 81 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/retry.py\u001b[0m in \u001b[0;36mincrement\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    362\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_retry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_exhausted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mMaxRetryError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_pool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merror\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mResponseError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 82 |       "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f105c3320>: Failed to establish a new connection: [Errno 111] Connection refused',))",
 83 |       "\nDuring handling of the above exception, another exception occurred:\n",
 84 |       "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
 85 |       "\u001b[0;32m<ipython-input-12-f791895548ed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mlink\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"https://vm.ru/news/2017/02/03/\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetHrefs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
 86 |       "\u001b[0;32m<ipython-input-11-be3b9ea8cc34>\u001b[0m in \u001b[0;36mgetHrefs\u001b[0;34m(link)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m# Получаем текст страницы, которая содержит ссылки на все статьи этого дня (в примере - 03.02.2017).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mendpage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'</div>'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'<div class=\"total-pages\" >'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#листаем все новости дня\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mendpage\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 87 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 88 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
 89 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    486\u001b[0m         }\n\u001b[1;32m    487\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 90 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 609\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    611\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 91 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    485\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mProxyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mClosedPoolError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
 92 |       "\u001b[0;31mConnectionError\u001b[0m: HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f105c3320>: Failed to establish a new connection: [Errno 111] Connection refused',))"
 93 |      ]
 94 |     }
 95 |    ],
 96 |    "source": [
 97 |     "link = \"https://vm.ru/news/2017/02/03/\"\n",
 98 |     "print(getHrefs(link))"
 99 |    ]
100 |   },
101 |   {
102 |    "cell_type": "code",
103 |    "execution_count": 11,
104 |    "metadata": {
105 |     "collapsed": true
106 |    },
107 |    "outputs": [],
108 |    "source": [
109 |     "def getHrefs(link):\n",
110 |     "\n",
111 |     "    # Получаем текст страницы, которая содержит ссылки на все статьи этого дня (в примере - 03.02.2017).\n",
112 |     "    r = requests.get(link,headers=headers)\n",
113 |     "    endpage = int(re.split('</div>', re.split('<div class=\"total-pages\" >', r.text)[1])[0]) #листаем все новости дня\n",
114 |     "    for i in range(2,endpage+1):\n",
115 |     "        newlink = link+'?page=' + str(i)\n",
116 |     "    # Каждая ссылка на статью оформлена с помощью тега <article>\n",
117 |     "    refs=re.split('<a href=\"',re.split('</ul>', re.split('<ul class=\"articles-list brdtop-light mrgbtm20\">', r.text)[1])[0])\n",
118 |     "    for i in refs:\n",
119 |     "        if i.startswith(\"/news/\"):\n",
120 |     "            ilink = re.split('\">',i)[0]\n",
121 |     "            print(ilink)\n",
122 |     "            hreffile.write(\"https://vm.ru/\"+ilink+ \"\\n\")\n",
123 |     "            time.sleep(random.uniform(1,2))\n",
124 |     "       \n"
125 |    ]
126 |   },
127 |   {
128 |    "cell_type": "code",
129 |    "execution_count": 16,
130 |    "metadata": {
131 |     "collapsed": false
132 |    },
133 |    "outputs": [
134 |     {
135 |      "ename": "ConnectionError",
136 |      "evalue": "HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f1055a630>: Failed to establish a new connection: [Errno 111] Connection refused',))",
137 |      "output_type": "error",
138 |      "traceback": [
139 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
140 |       "\u001b[0;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
141 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36m_new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    137\u001b[0m             conn = connection.create_connection(\n\u001b[0;32m--> 138\u001b[0;31m                 (self.host, self.port), self.timeout, **extra_kw)\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
142 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/connection.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0merr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
143 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/connection.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_address\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msa\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
144 |       "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused",
145 |       "\nDuring handling of the above exception, another exception occurred:\n",
146 |       "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
147 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[1;32m    593\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 594\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
148 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    349\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSocketTimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBaseSSLError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
149 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m    834\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'sock'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# AppEngine might not have  `.sock`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 835\u001b[0;31m             \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
150 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m         \u001b[0;31m# Add certificate verification\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m         \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_new_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
151 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connection.py\u001b[0m in \u001b[0;36m_new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    146\u001b[0m             raise NewConnectionError(\n\u001b[0;32m--> 147\u001b[0;31m                 self, \"Failed to establish a new connection: %s\" % e)\n\u001b[0m\u001b[1;32m    148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
152 |       "\u001b[0;31mNewConnectionError\u001b[0m: <requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f1055a630>: Failed to establish a new connection: [Errno 111] Connection refused",
153 |       "\nDuring handling of the above exception, another exception occurred:\n",
154 |       "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
155 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    422\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    424\u001b[0m                 )\n",
156 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, **response_kw)\u001b[0m\n\u001b[1;32m    642\u001b[0m             retries = retries.increment(method, url, error=e, _pool=self,\n\u001b[0;32m--> 643\u001b[0;31m                                         _stacktrace=sys.exc_info()[2])\n\u001b[0m\u001b[1;32m    644\u001b[0m             \u001b[0mretries\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
157 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/packages/urllib3/util/retry.py\u001b[0m in \u001b[0;36mincrement\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    362\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_retry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_exhausted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mMaxRetryError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_pool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merror\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mResponseError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcause\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
158 |       "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f1055a630>: Failed to establish a new connection: [Errno 111] Connection refused',))",
159 |       "\nDuring handling of the above exception, another exception occurred:\n",
160 |       "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
161 |       "\u001b[0;32m<ipython-input-16-18a2fd98a189>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madapters\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEFAULT_RETRIES\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"https://vm.ru/news/2017/02/03/\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mendpage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'</div>'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'<div class=\"total-pages\" >'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#листаем все новости дня\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mendpage\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mnewlink\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlink\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'?page='\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
162 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
163 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
164 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    486\u001b[0m         }\n\u001b[1;32m    487\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
165 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 609\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    611\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
166 |       "\u001b[0;32m/home/mi_air/ioSavoy5/lib/python3.5/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    485\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mProxyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mClosedPoolError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
167 |       "\u001b[0;31mConnectionError\u001b[0m: HTTPSConnectionPool(host='vm.ru', port=443): Max retries exceeded with url: /news/2017/02/03/ (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f1055a630>: Failed to establish a new connection: [Errno 111] Connection refused',))"
168 |      ]
169 |     }
170 |    ],
171 |    "source": [
172 |     "requests.adapters.DEFAULT_RETRIES = 5\n",
173 |     "r = requests.get(\"https://vm.ru/news/2017/02/03/\",headers=headers)\n",
174 |     "endpage = int(re.split('</div>', re.split('<div class=\"total-pages\" >', r.text)[1])[0]) #листаем все новости дня\n",
175 |     "for i in range(2,endpage+1):\n",
176 |     "    newlink = link+'?page=' + str(i)\n",
177 |     "\n",
178 |     "refs=re.split('<a href=\"',re.split('</ul>', re.split('<ul class=\"articles-list brdtop-light mrgbtm20\">', r.text)[1])[0])\n",
179 |     "for i in refs:\n",
180 |     "    if i.startswith(\"/news/\"):\n",
181 |     "        ilink = re.split('\">',i)[0]\n",
182 |     "        print(ilink)\n",
183 |     "        hreffile.write(\"https://vm.ru/\"+ilink+ \"\\n\")\n",
184 |     "            "
185 |    ]
186 |   },
187 |   {
188 |    "cell_type": "code",
189 |    "execution_count": null,
190 |    "metadata": {
191 |     "collapsed": true
192 |    },
193 |    "outputs": [],
194 |    "source": [
195 |     "\n",
196 |     "for year in range(2010,2018):\n",
197 |     "    for month in range (1,13):\n",
198 |     "        if month <10:\n",
199 |     "            month = \"0\" + str(month)\n",
200 |     "        for day in range(1,32):\n",
201 |     "            if day<10:\n",
202 |     "                day = \"0\" + str(day)\n",
203 |     "            newlink = \"https://vm.ru/news/\" + str(year) +\"/\"+str(month)+\"/\"+str(day)+\"/\"\n",
204 |     "            print(newlink)\n",
205 |     "            try:\n",
206 |     "                getHrefs(newlink)\n",
207 |     "                \n",
208 |     "            except:\n",
209 |     "                pass\n"
210 |    ]
211 |   }
212 |  ],
213 |  "metadata": {
214 |   "anaconda-cloud": {},
215 |   "kernelspec": {
216 |    "display_name": "Python [default]",
217 |    "language": "python",
218 |    "name": "python3"
219 |   },
220 |   "language_info": {
221 |    "codemirror_mode": {
222 |     "name": "ipython",
223 |     "version": 3
224 |    },
225 |    "file_extension": ".py",
226 |    "mimetype": "text/x-python",
227 |    "name": "python",
228 |    "nbconvert_exporter": "python",
229 |    "pygments_lexer": "ipython3",
230 |    "version": "3.5.2"
231 |   }
232 |  },
233 |  "nbformat": 4,
234 |  "nbformat_minor": 2
235 | }
236 | 


--------------------------------------------------------------------------------
/иа_панорама.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "code",
   5 |    "execution_count": null,
   6 |    "metadata": {
   7 |     "collapsed": true
   8 |    },
   9 |    "outputs": [],
  10 |    "source": [
  11 |     "#https://panorama.pub/category/news/page/83\n",
  12 |     "#https://panorama.pub/category/stati/page/3"
  13 |    ]
  14 |   },
  15 |   {
  16 |    "cell_type": "code",
  17 |    "execution_count": 1,
  18 |    "metadata": {
  19 |     "collapsed": true
  20 |    },
  21 |    "outputs": [],
  22 |    "source": [
  23 |     "# Импортируем необходимые библиотеки:\n",
  24 |     "import requests # http-запросы,\n",
  25 |     "import re # регулярные выражения,\n",
  26 |     "from bs4 import BeautifulSoup # удаление тегов html,\n",
  27 |     "import time\n",
  28 |     "import random\n",
  29 |     "import os\n",
  30 |     "import tqdm"
  31 |    ]
  32 |   },
  33 |   {
  34 |    "cell_type": "code",
  35 |    "execution_count": 2,
  36 |    "metadata": {
  37 |     "collapsed": true
  38 |    },
  39 |    "outputs": [],
  40 |    "source": [
  41 |     "user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/600.3.18 (KHTML, like Gecko) Version/8.0.3 Safari/600.3.18'\n",
  42 |     "headers = { 'User-Agent' : user_agent }"
  43 |    ]
  44 |   },
  45 |   {
  46 |    "cell_type": "code",
  47 |    "execution_count": 3,
  48 |    "metadata": {
  49 |     "collapsed": true
  50 |    },
  51 |    "outputs": [],
  52 |    "source": [
  53 |     "startlink = r'https://panorama.pub/category/news/page/'\n",
  54 |     "hreflist = []"
  55 |    ]
  56 |   },
  57 |   {
  58 |    "cell_type": "code",
  59 |    "execution_count": 19,
  60 |    "metadata": {
  61 |     "collapsed": true
  62 |    },
  63 |    "outputs": [],
  64 |    "source": [
  65 |     "def GetLinks(startlink, rang=84):\n",
  66 |     "    lst = []\n",
  67 |     "    for letter in range(1,rang):\n",
  68 |     "        link = startlink +str(letter)+'/'\n",
  69 |     "        lst.append(link)\n",
  70 |     "    return lst"
  71 |    ]
  72 |   },
  73 |   {
  74 |    "cell_type": "code",
  75 |    "execution_count": 9,
  76 |    "metadata": {
  77 |     "collapsed": true
  78 |    },
  79 |    "outputs": [],
  80 |    "source": [
  81 |     "links = GetLinks(startlink)"
  82 |    ]
  83 |   },
  84 |   {
  85 |    "cell_type": "code",
  86 |    "execution_count": 15,
  87 |    "metadata": {},
  88 |    "outputs": [],
  89 |    "source": [
  90 |     "def GetTextLinks(link):\n",
  91 |     "    lst = []\n",
  92 |     "    r = requests.get(link,headers=headers)\n",
  93 |     "    links = re.split(\"<h2 class=\", r.text)\n",
  94 |     "    for l in links[1:-1]:\n",
  95 |     "        newlink = re.split('\" rel=\"bookmark\"',re.split('\"entry-title\"><a href=\"', l)[1])[0]\n",
  96 |     "        lst.append(newlink)\n",
  97 |     "    return lst"
  98 |    ]
  99 |   },
 100 |   {
 101 |    "cell_type": "code",
 102 |    "execution_count": 16,
 103 |    "metadata": {},
 104 |    "outputs": [],
 105 |    "source": [
 106 |     "listoftexts = []\n",
 107 |     "for l in links:\n",
 108 |     "    listoftexts+=GetTextLinks(l)"
 109 |    ]
 110 |   },
 111 |   {
 112 |    "cell_type": "code",
 113 |    "execution_count": 18,
 114 |    "metadata": {},
 115 |    "outputs": [
 116 |     {
 117 |      "data": {
 118 |       "text/plain": [
 119 |        "826"
 120 |       ]
 121 |      },
 122 |      "execution_count": 18,
 123 |      "metadata": {},
 124 |      "output_type": "execute_result"
 125 |     }
 126 |    ],
 127 |    "source": [
 128 |     "len(listoftexts)"
 129 |    ]
 130 |   },
 131 |   {
 132 |    "cell_type": "code",
 133 |    "execution_count": 20,
 134 |    "metadata": {
 135 |     "collapsed": true
 136 |    },
 137 |    "outputs": [],
 138 |    "source": [
 139 |     "stst_links =  GetLinks('https://panorama.pub/category/stati/page/',4)"
 140 |    ]
 141 |   },
 142 |   {
 143 |    "cell_type": "code",
 144 |    "execution_count": 21,
 145 |    "metadata": {
 146 |     "collapsed": true
 147 |    },
 148 |    "outputs": [],
 149 |    "source": [
 150 |     "listofstats = []\n",
 151 |     "for l in stst_links:\n",
 152 |     "    listofstats+=GetTextLinks(l)"
 153 |    ]
 154 |   },
 155 |   {
 156 |    "cell_type": "code",
 157 |    "execution_count": 23,
 158 |    "metadata": {
 159 |     "collapsed": true
 160 |    },
 161 |    "outputs": [],
 162 |    "source": [
 163 |     "dicofl = {}\n",
 164 |     "for l in listoftexts:\n",
 165 |     "    dicofl[l] = 'news'\n",
 166 |     "for l in listofstats:\n",
 167 |     "    dicofl[l] = 'article'"
 168 |    ]
 169 |   },
 170 |   {
 171 |    "cell_type": "code",
 172 |    "execution_count": 24,
 173 |    "metadata": {
 174 |     "collapsed": true
 175 |    },
 176 |    "outputs": [],
 177 |    "source": [
 178 |     "alllinks = listoftexts+listofstats"
 179 |    ]
 180 |   },
 181 |   {
 182 |    "cell_type": "code",
 183 |    "execution_count": 33,
 184 |    "metadata": {
 185 |     "collapsed": true
 186 |    },
 187 |    "outputs": [],
 188 |    "source": [
 189 |     "def GetText(link):\n",
 190 |     "    try:\n",
 191 |     "        lst = []\n",
 192 |     "        r = requests.get(link,headers=headers)\n",
 193 |     "        text = re.split('<div class=\"at-below-post addthis_tool',re.split('</div><p>', r.text)[1])[0]\n",
 194 |     "        text = re.sub('&nbsp;', ' ', text)\n",
 195 |     "        text = re.sub('&mdash;', ' -- ', text)\n",
 196 |     "        text = re.sub('&shy;', '', text)\n",
 197 |     "        text = re.sub('&laquo;', '\"', text)\n",
 198 |     "        text = re.sub('&raquo;', '\"', text)\n",
 199 |     "        beaux_text=BeautifulSoup(text, \"lxml\")\n",
 200 |     "        text=beaux_text.get_text() \n",
 201 |     "        text = re.sub('\\xa0', ' ', text)\n",
 202 |     "        text = re.sub('\\u2009', ' ', text)\n",
 203 |     "        title = re.split('\" />',re.split('<meta property=\"og:title\" content=\"', r.text)[1])[0] #\n",
 204 |     "        date = re.split('\" />',re.split('<meta itemprop=\"datePublished\" content=\"', r.text)[1])[0] #\n",
 205 |     "        author = re.split('\" />',re.split('<meta itemprop=\"author\" content=\"', r.text)[1])[0] #\n",
 206 |     "        return text, title, date, author\n",
 207 |     "    except:\n",
 208 |     "        print(link)\n",
 209 |     "        return [],[],[],[]"
 210 |    ]
 211 |   },
 212 |   {
 213 |    "cell_type": "code",
 214 |    "execution_count": 34,
 215 |    "metadata": {},
 216 |    "outputs": [
 217 |     {
 218 |      "data": {
 219 |       "text/plain": [
 220 |        "('Расхождение во взглядах стало поводом для массовой драки в самом сердце Праги. В столице Чехии отношения между собой выясняли пацифисты и вегетарианцы.\\nПо сообщениям очевидцев, во время санкционированного шествия против насилия, представитель вегетарианцев, проводивших свою акцию в этом же месте, назвал пацифиста «лицемерным трупоедом, не уважающим право животных на жизнь» и кинул в оппонента пустой пластиковой бутылкой. После этого завязалась потасовка, к которой подключились участники обеих акций.\\nВ результате потасовки вегетарианцы захватили нескольких пацифистов в плен, а их лидер, польский эмигрант Кутас Буракевич заявил, что будет кормить пленных фалафелем* и не отпустит их до тех пор, пока все участники шествия не согласятся с требованиями вегетарианцев об участии в совместном митинге о прекращении насилия над животными.\\nПацифисты отрицательно отнеслись к предложению противоположной стороны и предприняли попытку отбить заложников при помощи палок, поддерживающих плакаты с символикой.\\nДальнейшую эскалацию насилия на антинасильственном митинге предотвратила полиция. Правоохранители освободили пленников, а Буракевич с сообщниками были арестованы. Шестеро участников шествия были госпитализированы с переломами и другими травмами.\\n\\n * Фалафель — блюдо, представляющее собой жареные во фритюре шарики из измельчённых бобовых, иногда с добавлением фасоли, приправленные пряностями.\\n\\n \\n',\n",
 221 |        " 'Пражские вегетарианцы подрались с пацифистами на марше против насилия',\n",
 222 |        " '2018-06-26',\n",
 223 |        " 'Григорий Касаткин')"
 224 |       ]
 225 |      },
 226 |      "execution_count": 34,
 227 |      "metadata": {},
 228 |      "output_type": "execute_result"
 229 |     }
 230 |    ],
 231 |    "source": [
 232 |     "GetText(alllinks[147])"
 233 |    ]
 234 |   },
 235 |   {
 236 |    "cell_type": "code",
 237 |    "execution_count": 35,
 238 |    "metadata": {},
 239 |    "outputs": [
 240 |     {
 241 |      "name": "stderr",
 242 |      "output_type": "stream",
 243 |      "text": [
 244 |       "\n",
 245 |       "  0%|                                                  | 0/847 [00:00<?, ?it/s]\n",
 246 |       "  0%|                                          | 1/847 [00:00<13:11,  1.07it/s]\n",
 247 |       "  0%|                                          | 2/847 [00:01<13:16,  1.06it/s]\n",
 248 |       "  0%|▏                                         | 3/847 [00:02<13:17,  1.06it/s]\n",
 249 |       "  0%|▏                                         | 4/847 [00:03<13:05,  1.07it/s]\n",
 250 |       " 17%|██████▉                                 | 146/847 [02:11<10:31,  1.11it/s]"
 251 |      ]
 252 |     },
 253 |     {
 254 |      "name": "stdout",
 255 |      "output_type": "stream",
 256 |      "text": [
 257 |       "https://panorama.pub/4541-erdogan-obyavil-ob-osmanskoj-imperii.html\n"
 258 |      ]
 259 |     },
 260 |     {
 261 |      "name": "stderr",
 262 |      "output_type": "stream",
 263 |      "text": [
 264 |       " 20%|████████                                | 170/847 [02:33<09:59,  1.13it/s]"
 265 |      ]
 266 |     },
 267 |     {
 268 |      "name": "stdout",
 269 |      "output_type": "stream",
 270 |      "text": [
 271 |       "https://panorama.pub/4382-duhovoj-orkestr-minoborony-rf-oshtrafovali-v-latvii-za-ispolnenie-gimna-rossii.html\n"
 272 |      ]
 273 |     },
 274 |     {
 275 |      "name": "stderr",
 276 |      "output_type": "stream",
 277 |      "text": [
 278 |       " 41%|████████████████▎                       | 346/847 [05:10<07:27,  1.12it/s]"
 279 |      ]
 280 |     },
 281 |     {
 282 |      "name": "stdout",
 283 |      "output_type": "stream",
 284 |      "text": [
 285 |       "https://panorama.pub/3199-timiryazevskaya-akademiya-rulit.html\n"
 286 |      ]
 287 |     },
 288 |     {
 289 |      "name": "stderr",
 290 |      "output_type": "stream",
 291 |      "text": [
 292 |       " 46%|██████████████████▍                     | 391/847 [05:51<06:52,  1.11it/s]"
 293 |      ]
 294 |     },
 295 |     {
 296 |      "name": "stdout",
 297 |      "output_type": "stream",
 298 |      "text": [
 299 |       "https://panorama.pub/2903-vserossijskaya-aktsiya-spasyom-deripasku-nabiraet-populyarnost.html\n"
 300 |      ]
 301 |     },
 302 |     {
 303 |      "name": "stderr",
 304 |      "output_type": "stream",
 305 |      "text": [
 306 |       " 78%|███████████████████████████████         | 657/847 [09:54<02:48,  1.13it/s]"
 307 |      ]
 308 |     },
 309 |     {
 310 |      "name": "stdout",
 311 |      "output_type": "stream",
 312 |      "text": [
 313 |       "https://panorama.pub/1423-patriarh-kirill-sportsmeny-smogut-uchastvovat-v-olimpiade-duhovno.html\n"
 314 |      ]
 315 |     },
 316 |     {
 317 |      "name": "stderr",
 318 |      "output_type": "stream",
 319 |      "text": [
 320 |       " 86%|██████████████████████████████████▌     | 731/847 [11:01<01:46,  1.09it/s]"
 321 |      ]
 322 |     },
 323 |     {
 324 |      "name": "stdout",
 325 |      "output_type": "stream",
 326 |      "text": [
 327 |       "https://panorama.pub/862-vo-frantsii-vyshel-sbornik-eroticheskih-rasskazov-makrona.html\n"
 328 |      ]
 329 |     },
 330 |     {
 331 |      "name": "stderr",
 332 |      "output_type": "stream",
 333 |      "text": [
 334 |       " 87%|██████████████████████████████████▌     | 733/847 [11:03<01:45,  1.08it/s]"
 335 |      ]
 336 |     },
 337 |     {
 338 |      "name": "stdout",
 339 |      "output_type": "stream",
 340 |      "text": [
 341 |       "https://panorama.pub/838-makron-predlozhil-vnesti-lyagushachi-lapki-v-spisok-obektov-naslediya-yunesko.html\n"
 342 |      ]
 343 |     },
 344 |     {
 345 |      "name": "stderr",
 346 |      "output_type": "stream",
 347 |      "text": [
 348 |       " 88%|███████████████████████████████████▎    | 749/847 [11:17<01:27,  1.12it/s]"
 349 |      ]
 350 |     },
 351 |     {
 352 |      "name": "stdout",
 353 |      "output_type": "stream",
 354 |      "text": [
 355 |       "https://panorama.pub/707-v-ssha-naznachili-spetsprokurora-po-vmeshatelstvu-rossii-v-klimat.html\n"
 356 |      ]
 357 |     },
 358 |     {
 359 |      "name": "stderr",
 360 |      "output_type": "stream",
 361 |      "text": [
 362 |       " 97%|██████████████████████████████████████▊ | 823/847 [12:27<00:21,  1.12it/s]"
 363 |      ]
 364 |     },
 365 |     {
 366 |      "name": "stdout",
 367 |      "output_type": "stream",
 368 |      "text": [
 369 |       "https://panorama.pub/67-estoniya-zapustit-kriptovalyutu-cherez-20-let.html\n"
 370 |      ]
 371 |     },
 372 |     {
 373 |      "name": "stderr",
 374 |      "output_type": "stream",
 375 |      "text": [
 376 |       "100%|████████████████████████████████████████| 847/847 [12:50<00:00,  1.09it/s]\n"
 377 |      ]
 378 |     }
 379 |    ],
 380 |    "source": [
 381 |     "dic = {}\n",
 382 |     "for l in tqdm.tqdm(alllinks):\n",
 383 |     "    text_info = {}\n",
 384 |     "    text_info['text'], text_info['title'], text_info['date'], text_info['author'] = GetText(l)\n",
 385 |     "    text_info['link']=l\n",
 386 |     "    text_info['type'] = dicofl[l]\n",
 387 |     "    dic[l]=text_info"
 388 |    ]
 389 |   },
 390 |   {
 391 |    "cell_type": "code",
 392 |    "execution_count": 39,
 393 |    "metadata": {
 394 |     "collapsed": true
 395 |    },
 396 |    "outputs": [],
 397 |    "source": [
 398 |     "def writetext(wdir, tid, text):\n",
 399 |     "    try:\n",
 400 |     "        f = open(os.path.join(wdir,str(tid)+'.txt'),'w', encoding='utf8')\n",
 401 |     "        f.write(text)\n",
 402 |     "        f.close\n",
 403 |     "        return os.path.join(wdir,str(tid)+'.txt')\n",
 404 |     "    except:\n",
 405 |     "        print(text)"
 406 |    ]
 407 |   },
 408 |   {
 409 |    "cell_type": "code",
 410 |    "execution_count": 41,
 411 |    "metadata": {},
 412 |    "outputs": [
 413 |     {
 414 |      "name": "stderr",
 415 |      "output_type": "stream",
 416 |      "text": [
 417 |       "\r",
 418 |       "  0%|                                                  | 0/843 [00:00<?, ?it/s]"
 419 |      ]
 420 |     },
 421 |     {
 422 |      "name": "stdout",
 423 |      "output_type": "stream",
 424 |      "text": [
 425 |       "[]\n",
 426 |       "[]\n"
 427 |      ]
 428 |     },
 429 |     {
 430 |      "name": "stderr",
 431 |      "output_type": "stream",
 432 |      "text": [
 433 |       "\r",
 434 |       " 29%|███████████▏                          | 247/843 [00:00<00:00, 2445.29it/s]"
 435 |      ]
 436 |     },
 437 |     {
 438 |      "name": "stdout",
 439 |      "output_type": "stream",
 440 |      "text": [
 441 |       "[]\n",
 442 |       "[]\n"
 443 |      ]
 444 |     },
 445 |     {
 446 |      "name": "stderr",
 447 |      "output_type": "stream",
 448 |      "text": [
 449 |       "\r",
 450 |       " 65%|████████████████████████▋             | 549/843 [00:00<00:00, 2593.28it/s]"
 451 |      ]
 452 |     },
 453 |     {
 454 |      "name": "stdout",
 455 |      "output_type": "stream",
 456 |      "text": [
 457 |       "[]\n",
 458 |       "[]\n",
 459 |       "[]\n",
 460 |       "[]\n",
 461 |       "[]\n"
 462 |      ]
 463 |     },
 464 |     {
 465 |      "name": "stderr",
 466 |      "output_type": "stream",
 467 |      "text": [
 468 |       "100%|██████████████████████████████████████| 843/843 [00:00<00:00, 2763.66it/s]\n"
 469 |      ]
 470 |     }
 471 |    ],
 472 |    "source": [
 473 |     "metaname = r\"D:\\Panorama\\metatable.csv\"\n",
 474 |     "foldername = r\"D:\\Panorama\\texts\"\n",
 475 |     "metatable = open(metaname, \"w\", encoding=\"utf8\")\n",
 476 |     "order = ['link', 'title', 'date', 'author', 'type']\n",
 477 |     "line = 'text_id'+ '\\t' + '\\t'.join(order)+'\\t' +'filename'+\"\\n\"\n",
 478 |     "metatable.write(line)\n",
 479 |     "idcount = 0\n",
 480 |     "for l in tqdm.tqdm(dic):\n",
 481 |     "    try:\n",
 482 |     "        filename = writetext(foldername, idcount, dic[l]['text'])\n",
 483 |     "        metatable.write(str(idcount)+'\\t'+'\\t'.join([dic[l][k] for k in order])+'\\t'+filename+'\\n')                                    \n",
 484 |     "        idcount +=1\n",
 485 |     "    except:\n",
 486 |     "        pass\n",
 487 |     "\n",
 488 |     "metatable.close()"
 489 |    ]
 490 |   },
 491 |   {
 492 |    "cell_type": "code",
 493 |    "execution_count": 44,
 494 |    "metadata": {},
 495 |    "outputs": [
 496 |     {
 497 |      "name": "stderr",
 498 |      "output_type": "stream",
 499 |      "text": [
 500 |       "\r",
 501 |       "0it [00:00, ?it/s]"
 502 |      ]
 503 |     },
 504 |     {
 505 |      "name": "stdout",
 506 |      "output_type": "stream",
 507 |      "text": [
 508 |       "D:\\Panorama\\texts\\0.txt\n",
 509 |       "D:\\Panorama\\tagged_texts\\0.txt\n",
 510 |       "D:\\Panorama\\texts\\1.txt\n",
 511 |       "D:\\Panorama\\tagged_texts\\1.txt\n",
 512 |       "D:\\Panorama\\texts\\10.txt\n",
 513 |       "D:\\Panorama\\tagged_texts\\10.txt\n",
 514 |       "D:\\Panorama\\texts\\100.txt\n",
 515 |       "D:\\Panorama\\tagged_texts\\100.txt\n",
 516 |       "D:\\Panorama\\texts\\101.txt\n",
 517 |       "D:\\Panorama\\tagged_texts\\101.txt\n",
 518 |       "D:\\Panorama\\texts\\102.txt\n",
 519 |       "D:\\Panorama\\tagged_texts\\102.txt\n",
 520 |       "D:\\Panorama\\texts\\103.txt\n",
 521 |       "D:\\Panorama\\tagged_texts\\103.txt\n",
 522 |       "D:\\Panorama\\texts\\104.txt\n",
 523 |       "D:\\Panorama\\tagged_texts\\104.txt\n",
 524 |       "D:\\Panorama\\texts\\105.txt\n",
 525 |       "D:\\Panorama\\tagged_texts\\105.txt\n",
 526 |       "D:\\Panorama\\texts\\106.txt\n",
 527 |       "D:\\Panorama\\tagged_texts\\106.txt\n",
 528 |       "D:\\Panorama\\texts\\107.txt\n",
 529 |       "D:\\Panorama\\tagged_texts\\107.txt\n",
 530 |       "D:\\Panorama\\texts\\108.txt\n",
 531 |       "D:\\Panorama\\tagged_texts\\108.txt\n",
 532 |       "D:\\Panorama\\texts\\109.txt\n",
 533 |       "D:\\Panorama\\tagged_texts\\109.txt\n",
 534 |       "D:\\Panorama\\texts\\11.txt\n",
 535 |       "D:\\Panorama\\tagged_texts\\11.txt\n",
 536 |       "D:\\Panorama\\texts\\110.txt\n",
 537 |       "D:\\Panorama\\tagged_texts\\110.txt\n",
 538 |       "D:\\Panorama\\texts\\111.txt\n",
 539 |       "D:\\Panorama\\tagged_texts\\111.txt\n",
 540 |       "D:\\Panorama\\texts\\112.txt\n",
 541 |       "D:\\Panorama\\tagged_texts\\112.txt\n",
 542 |       "D:\\Panorama\\texts\\113.txt\n",
 543 |       "D:\\Panorama\\tagged_texts\\113.txt\n",
 544 |       "D:\\Panorama\\texts\\114.txt\n",
 545 |       "D:\\Panorama\\tagged_texts\\114.txt\n",
 546 |       "D:\\Panorama\\texts\\115.txt\n",
 547 |       "D:\\Panorama\\tagged_texts\\115.txt\n",
 548 |       "D:\\Panorama\\texts\\116.txt\n",
 549 |       "D:\\Panorama\\tagged_texts\\116.txt\n",
 550 |       "D:\\Panorama\\texts\\117.txt\n",
 551 |       "D:\\Panorama\\tagged_texts\\117.txt\n",
 552 |       "D:\\Panorama\\texts\\118.txt\n",
 553 |       "D:\\Panorama\\tagged_texts\\118.txt\n",
 554 |       "D:\\Panorama\\texts\\119.txt\n",
 555 |       "D:\\Panorama\\tagged_texts\\119.txt\n",
 556 |       "D:\\Panorama\\texts\\12.txt\n",
 557 |       "D:\\Panorama\\tagged_texts\\12.txt\n",
 558 |       "D:\\Panorama\\texts\\120.txt\n",
 559 |       "D:\\Panorama\\tagged_texts\\120.txt\n",
 560 |       "D:\\Panorama\\texts\\121.txt\n",
 561 |       "D:\\Panorama\\tagged_texts\\121.txt\n",
 562 |       "D:\\Panorama\\texts\\122.txt\n",
 563 |       "D:\\Panorama\\tagged_texts\\122.txt\n",
 564 |       "D:\\Panorama\\texts\\123.txt\n",
 565 |       "D:\\Panorama\\tagged_texts\\123.txt\n",
 566 |       "D:\\Panorama\\texts\\124.txt\n",
 567 |       "D:\\Panorama\\tagged_texts\\124.txt\n",
 568 |       "D:\\Panorama\\texts\\125.txt\n",
 569 |       "D:\\Panorama\\tagged_texts\\125.txt\n",
 570 |       "D:\\Panorama\\texts\\126.txt\n",
 571 |       "D:\\Panorama\\tagged_texts\\126.txt\n",
 572 |       "D:\\Panorama\\texts\\127.txt\n",
 573 |       "D:\\Panorama\\tagged_texts\\127.txt\n",
 574 |       "D:\\Panorama\\texts\\128.txt\n",
 575 |       "D:\\Panorama\\tagged_texts\\128.txt\n",
 576 |       "D:\\Panorama\\texts\\129.txt\n",
 577 |       "D:\\Panorama\\tagged_texts\\129.txt\n",
 578 |       "D:\\Panorama\\texts\\13.txt\n",
 579 |       "D:\\Panorama\\tagged_texts\\13.txt\n",
 580 |       "D:\\Panorama\\texts\\130.txt\n",
 581 |       "D:\\Panorama\\tagged_texts\\130.txt\n",
 582 |       "D:\\Panorama\\texts\\131.txt\n",
 583 |       "D:\\Panorama\\tagged_texts\\131.txt\n",
 584 |       "D:\\Panorama\\texts\\132.txt\n",
 585 |       "D:\\Panorama\\tagged_texts\\132.txt\n",
 586 |       "D:\\Panorama\\texts\\133.txt\n",
 587 |       "D:\\Panorama\\tagged_texts\\133.txt\n",
 588 |       "D:\\Panorama\\texts\\134.txt\n",
 589 |       "D:\\Panorama\\tagged_texts\\134.txt\n",
 590 |       "D:\\Panorama\\texts\\135.txt\n",
 591 |       "D:\\Panorama\\tagged_texts\\135.txt\n",
 592 |       "D:\\Panorama\\texts\\136.txt\n",
 593 |       "D:\\Panorama\\tagged_texts\\136.txt\n",
 594 |       "D:\\Panorama\\texts\\137.txt\n",
 595 |       "D:\\Panorama\\tagged_texts\\137.txt\n",
 596 |       "D:\\Panorama\\texts\\138.txt\n",
 597 |       "D:\\Panorama\\tagged_texts\\138.txt\n",
 598 |       "D:\\Panorama\\texts\\139.txt\n",
 599 |       "D:\\Panorama\\tagged_texts\\139.txt\n",
 600 |       "D:\\Panorama\\texts\\14.txt\n",
 601 |       "D:\\Panorama\\tagged_texts\\14.txt\n",
 602 |       "D:\\Panorama\\texts\\140.txt\n",
 603 |       "D:\\Panorama\\tagged_texts\\140.txt\n",
 604 |       "D:\\Panorama\\texts\\141.txt\n",
 605 |       "D:\\Panorama\\tagged_texts\\141.txt\n",
 606 |       "D:\\Panorama\\texts\\142.txt\n",
 607 |       "D:\\Panorama\\tagged_texts\\142.txt\n",
 608 |       "D:\\Panorama\\texts\\143.txt\n",
 609 |       "D:\\Panorama\\tagged_texts\\143.txt\n",
 610 |       "D:\\Panorama\\texts\\144.txt\n",
 611 |       "D:\\Panorama\\tagged_texts\\144.txt\n",
 612 |       "D:\\Panorama\\texts\\145.txt\n",
 613 |       "D:\\Panorama\\tagged_texts\\145.txt\n",
 614 |       "D:\\Panorama\\texts\\146.txt\n",
 615 |       "D:\\Panorama\\tagged_texts\\146.txt\n",
 616 |       "D:\\Panorama\\texts\\147.txt\n",
 617 |       "D:\\Panorama\\tagged_texts\\147.txt\n",
 618 |       "D:\\Panorama\\texts\\148.txt\n",
 619 |       "D:\\Panorama\\tagged_texts\\148.txt\n",
 620 |       "D:\\Panorama\\texts\\149.txt\n",
 621 |       "D:\\Panorama\\tagged_texts\\149.txt\n",
 622 |       "D:\\Panorama\\texts\\15.txt\n",
 623 |       "D:\\Panorama\\tagged_texts\\15.txt\n",
 624 |       "D:\\Panorama\\texts\\150.txt\n",
 625 |       "D:\\Panorama\\tagged_texts\\150.txt\n",
 626 |       "D:\\Panorama\\texts\\151.txt\n",
 627 |       "D:\\Panorama\\tagged_texts\\151.txt\n",
 628 |       "D:\\Panorama\\texts\\152.txt\n",
 629 |       "D:\\Panorama\\tagged_texts\\152.txt\n",
 630 |       "D:\\Panorama\\texts\\153.txt\n",
 631 |       "D:\\Panorama\\tagged_texts\\153.txt\n",
 632 |       "D:\\Panorama\\texts\\154.txt\n",
 633 |       "D:\\Panorama\\tagged_texts\\154.txt\n",
 634 |       "D:\\Panorama\\texts\\155.txt\n",
 635 |       "D:\\Panorama\\tagged_texts\\155.txt\n",
 636 |       "D:\\Panorama\\texts\\156.txt\n",
 637 |       "D:\\Panorama\\tagged_texts\\156.txt\n",
 638 |       "D:\\Panorama\\texts\\157.txt\n",
 639 |       "D:\\Panorama\\tagged_texts\\157.txt\n",
 640 |       "D:\\Panorama\\texts\\158.txt\n",
 641 |       "D:\\Panorama\\tagged_texts\\158.txt\n",
 642 |       "D:\\Panorama\\texts\\159.txt\n",
 643 |       "D:\\Panorama\\tagged_texts\\159.txt\n",
 644 |       "D:\\Panorama\\texts\\16.txt\n",
 645 |       "D:\\Panorama\\tagged_texts\\16.txt\n",
 646 |       "D:\\Panorama\\texts\\160.txt\n",
 647 |       "D:\\Panorama\\tagged_texts\\160.txt\n",
 648 |       "D:\\Panorama\\texts\\161.txt\n",
 649 |       "D:\\Panorama\\tagged_texts\\161.txt\n",
 650 |       "D:\\Panorama\\texts\\162.txt\n",
 651 |       "D:\\Panorama\\tagged_texts\\162.txt\n",
 652 |       "D:\\Panorama\\texts\\163.txt\n",
 653 |       "D:\\Panorama\\tagged_texts\\163.txt\n",
 654 |       "D:\\Panorama\\texts\\164.txt\n",
 655 |       "D:\\Panorama\\tagged_texts\\164.txt\n",
 656 |       "D:\\Panorama\\texts\\165.txt\n",
 657 |       "D:\\Panorama\\tagged_texts\\165.txt\n",
 658 |       "D:\\Panorama\\texts\\166.txt\n",
 659 |       "D:\\Panorama\\tagged_texts\\166.txt\n",
 660 |       "D:\\Panorama\\texts\\167.txt\n",
 661 |       "D:\\Panorama\\tagged_texts\\167.txt\n",
 662 |       "D:\\Panorama\\texts\\168.txt\n",
 663 |       "D:\\Panorama\\tagged_texts\\168.txt\n",
 664 |       "D:\\Panorama\\texts\\169.txt\n",
 665 |       "D:\\Panorama\\tagged_texts\\169.txt\n",
 666 |       "D:\\Panorama\\texts\\17.txt\n",
 667 |       "D:\\Panorama\\tagged_texts\\17.txt\n",
 668 |       "D:\\Panorama\\texts\\170.txt\n",
 669 |       "D:\\Panorama\\tagged_texts\\170.txt\n",
 670 |       "D:\\Panorama\\texts\\171.txt\n",
 671 |       "D:\\Panorama\\tagged_texts\\171.txt\n",
 672 |       "D:\\Panorama\\texts\\172.txt\n",
 673 |       "D:\\Panorama\\tagged_texts\\172.txt\n",
 674 |       "D:\\Panorama\\texts\\173.txt\n",
 675 |       "D:\\Panorama\\tagged_texts\\173.txt\n",
 676 |       "D:\\Panorama\\texts\\174.txt\n",
 677 |       "D:\\Panorama\\tagged_texts\\174.txt\n",
 678 |       "D:\\Panorama\\texts\\175.txt\n",
 679 |       "D:\\Panorama\\tagged_texts\\175.txt\n",
 680 |       "D:\\Panorama\\texts\\176.txt\n",
 681 |       "D:\\Panorama\\tagged_texts\\176.txt\n",
 682 |       "D:\\Panorama\\texts\\177.txt\n",
 683 |       "D:\\Panorama\\tagged_texts\\177.txt\n",
 684 |       "D:\\Panorama\\texts\\178.txt\n",
 685 |       "D:\\Panorama\\tagged_texts\\178.txt\n",
 686 |       "D:\\Panorama\\texts\\179.txt\n",
 687 |       "D:\\Panorama\\tagged_texts\\179.txt\n",
 688 |       "D:\\Panorama\\texts\\18.txt\n",
 689 |       "D:\\Panorama\\tagged_texts\\18.txt\n",
 690 |       "D:\\Panorama\\texts\\180.txt\n",
 691 |       "D:\\Panorama\\tagged_texts\\180.txt\n",
 692 |       "D:\\Panorama\\texts\\181.txt\n",
 693 |       "D:\\Panorama\\tagged_texts\\181.txt\n",
 694 |       "D:\\Panorama\\texts\\182.txt\n",
 695 |       "D:\\Panorama\\tagged_texts\\182.txt\n",
 696 |       "D:\\Panorama\\texts\\183.txt\n",
 697 |       "D:\\Panorama\\tagged_texts\\183.txt\n",
 698 |       "D:\\Panorama\\texts\\184.txt\n",
 699 |       "D:\\Panorama\\tagged_texts\\184.txt\n",
 700 |       "D:\\Panorama\\texts\\185.txt\n",
 701 |       "D:\\Panorama\\tagged_texts\\185.txt\n",
 702 |       "D:\\Panorama\\texts\\186.txt\n",
 703 |       "D:\\Panorama\\tagged_texts\\186.txt\n",
 704 |       "D:\\Panorama\\texts\\187.txt\n",
 705 |       "D:\\Panorama\\tagged_texts\\187.txt\n",
 706 |       "D:\\Panorama\\texts\\188.txt\n",
 707 |       "D:\\Panorama\\tagged_texts\\188.txt\n",
 708 |       "D:\\Panorama\\texts\\189.txt\n",
 709 |       "D:\\Panorama\\tagged_texts\\189.txt\n",
 710 |       "D:\\Panorama\\texts\\19.txt\n",
 711 |       "D:\\Panorama\\tagged_texts\\19.txt\n",
 712 |       "D:\\Panorama\\texts\\190.txt\n",
 713 |       "D:\\Panorama\\tagged_texts\\190.txt\n",
 714 |       "D:\\Panorama\\texts\\191.txt\n",
 715 |       "D:\\Panorama\\tagged_texts\\191.txt\n",
 716 |       "D:\\Panorama\\texts\\192.txt\n",
 717 |       "D:\\Panorama\\tagged_texts\\192.txt\n",
 718 |       "D:\\Panorama\\texts\\193.txt\n",
 719 |       "D:\\Panorama\\tagged_texts\\193.txt\n",
 720 |       "D:\\Panorama\\texts\\194.txt\n",
 721 |       "D:\\Panorama\\tagged_texts\\194.txt\n",
 722 |       "D:\\Panorama\\texts\\195.txt\n",
 723 |       "D:\\Panorama\\tagged_texts\\195.txt\n",
 724 |       "D:\\Panorama\\texts\\196.txt\n",
 725 |       "D:\\Panorama\\tagged_texts\\196.txt\n",
 726 |       "D:\\Panorama\\texts\\197.txt\n",
 727 |       "D:\\Panorama\\tagged_texts\\197.txt\n",
 728 |       "D:\\Panorama\\texts\\198.txt\n",
 729 |       "D:\\Panorama\\tagged_texts\\198.txt\n",
 730 |       "D:\\Panorama\\texts\\199.txt\n",
 731 |       "D:\\Panorama\\tagged_texts\\199.txt\n",
 732 |       "D:\\Panorama\\texts\\2.txt\n",
 733 |       "D:\\Panorama\\tagged_texts\\2.txt\n",
 734 |       "D:\\Panorama\\texts\\20.txt\n",
 735 |       "D:\\Panorama\\tagged_texts\\20.txt\n",
 736 |       "D:\\Panorama\\texts\\200.txt\n",
 737 |       "D:\\Panorama\\tagged_texts\\200.txt\n",
 738 |       "D:\\Panorama\\texts\\201.txt\n",
 739 |       "D:\\Panorama\\tagged_texts\\201.txt\n",
 740 |       "D:\\Panorama\\texts\\202.txt\n",
 741 |       "D:\\Panorama\\tagged_texts\\202.txt\n",
 742 |       "D:\\Panorama\\texts\\203.txt\n",
 743 |       "D:\\Panorama\\tagged_texts\\203.txt\n",
 744 |       "D:\\Panorama\\texts\\204.txt\n",
 745 |       "D:\\Panorama\\tagged_texts\\204.txt\n",
 746 |       "D:\\Panorama\\texts\\205.txt\n",
 747 |       "D:\\Panorama\\tagged_texts\\205.txt\n",
 748 |       "D:\\Panorama\\texts\\206.txt\n",
 749 |       "D:\\Panorama\\tagged_texts\\206.txt\n",
 750 |       "D:\\Panorama\\texts\\207.txt\n",
 751 |       "D:\\Panorama\\tagged_texts\\207.txt\n",
 752 |       "D:\\Panorama\\texts\\208.txt\n",
 753 |       "D:\\Panorama\\tagged_texts\\208.txt\n",
 754 |       "D:\\Panorama\\texts\\209.txt\n",
 755 |       "D:\\Panorama\\tagged_texts\\209.txt\n",
 756 |       "D:\\Panorama\\texts\\21.txt\n",
 757 |       "D:\\Panorama\\tagged_texts\\21.txt\n",
 758 |       "D:\\Panorama\\texts\\210.txt\n",
 759 |       "D:\\Panorama\\tagged_texts\\210.txt\n",
 760 |       "D:\\Panorama\\texts\\211.txt\n",
 761 |       "D:\\Panorama\\tagged_texts\\211.txt\n",
 762 |       "D:\\Panorama\\texts\\212.txt\n",
 763 |       "D:\\Panorama\\tagged_texts\\212.txt\n",
 764 |       "D:\\Panorama\\texts\\213.txt\n",
 765 |       "D:\\Panorama\\tagged_texts\\213.txt\n",
 766 |       "D:\\Panorama\\texts\\214.txt\n",
 767 |       "D:\\Panorama\\tagged_texts\\214.txt\n",
 768 |       "D:\\Panorama\\texts\\215.txt\n",
 769 |       "D:\\Panorama\\tagged_texts\\215.txt\n",
 770 |       "D:\\Panorama\\texts\\216.txt\n",
 771 |       "D:\\Panorama\\tagged_texts\\216.txt\n",
 772 |       "D:\\Panorama\\texts\\217.txt\n",
 773 |       "D:\\Panorama\\tagged_texts\\217.txt\n",
 774 |       "D:\\Panorama\\texts\\218.txt\n",
 775 |       "D:\\Panorama\\tagged_texts\\218.txt\n",
 776 |       "D:\\Panorama\\texts\\219.txt\n",
 777 |       "D:\\Panorama\\tagged_texts\\219.txt\n",
 778 |       "D:\\Panorama\\texts\\22.txt\n",
 779 |       "D:\\Panorama\\tagged_texts\\22.txt\n",
 780 |       "D:\\Panorama\\texts\\220.txt\n",
 781 |       "D:\\Panorama\\tagged_texts\\220.txt\n",
 782 |       "D:\\Panorama\\texts\\221.txt\n",
 783 |       "D:\\Panorama\\tagged_texts\\221.txt\n",
 784 |       "D:\\Panorama\\texts\\222.txt\n",
 785 |       "D:\\Panorama\\tagged_texts\\222.txt\n",
 786 |       "D:\\Panorama\\texts\\223.txt\n"
 787 |      ]
 788 |     },
 789 |     {
 790 |      "name": "stdout",
 791 |      "output_type": "stream",
 792 |      "text": [
 793 |       "D:\\Panorama\\tagged_texts\\223.txt\n",
 794 |       "D:\\Panorama\\texts\\224.txt\n",
 795 |       "D:\\Panorama\\tagged_texts\\224.txt\n",
 796 |       "D:\\Panorama\\texts\\225.txt\n",
 797 |       "D:\\Panorama\\tagged_texts\\225.txt\n",
 798 |       "D:\\Panorama\\texts\\226.txt\n",
 799 |       "D:\\Panorama\\tagged_texts\\226.txt\n",
 800 |       "D:\\Panorama\\texts\\227.txt\n",
 801 |       "D:\\Panorama\\tagged_texts\\227.txt\n",
 802 |       "D:\\Panorama\\texts\\228.txt\n",
 803 |       "D:\\Panorama\\tagged_texts\\228.txt\n",
 804 |       "D:\\Panorama\\texts\\229.txt\n",
 805 |       "D:\\Panorama\\tagged_texts\\229.txt\n",
 806 |       "D:\\Panorama\\texts\\23.txt\n",
 807 |       "D:\\Panorama\\tagged_texts\\23.txt\n",
 808 |       "D:\\Panorama\\texts\\230.txt\n",
 809 |       "D:\\Panorama\\tagged_texts\\230.txt\n",
 810 |       "D:\\Panorama\\texts\\231.txt\n",
 811 |       "D:\\Panorama\\tagged_texts\\231.txt\n",
 812 |       "D:\\Panorama\\texts\\232.txt\n",
 813 |       "D:\\Panorama\\tagged_texts\\232.txt\n",
 814 |       "D:\\Panorama\\texts\\233.txt\n",
 815 |       "D:\\Panorama\\tagged_texts\\233.txt\n",
 816 |       "D:\\Panorama\\texts\\234.txt\n",
 817 |       "D:\\Panorama\\tagged_texts\\234.txt\n",
 818 |       "D:\\Panorama\\texts\\235.txt\n",
 819 |       "D:\\Panorama\\tagged_texts\\235.txt\n",
 820 |       "D:\\Panorama\\texts\\236.txt\n",
 821 |       "D:\\Panorama\\tagged_texts\\236.txt\n",
 822 |       "D:\\Panorama\\texts\\237.txt\n",
 823 |       "D:\\Panorama\\tagged_texts\\237.txt\n",
 824 |       "D:\\Panorama\\texts\\238.txt\n",
 825 |       "D:\\Panorama\\tagged_texts\\238.txt\n",
 826 |       "D:\\Panorama\\texts\\239.txt\n",
 827 |       "D:\\Panorama\\tagged_texts\\239.txt\n",
 828 |       "D:\\Panorama\\texts\\24.txt\n",
 829 |       "D:\\Panorama\\tagged_texts\\24.txt\n",
 830 |       "D:\\Panorama\\texts\\240.txt\n",
 831 |       "D:\\Panorama\\tagged_texts\\240.txt\n",
 832 |       "D:\\Panorama\\texts\\241.txt\n",
 833 |       "D:\\Panorama\\tagged_texts\\241.txt\n",
 834 |       "D:\\Panorama\\texts\\242.txt\n",
 835 |       "D:\\Panorama\\tagged_texts\\242.txt\n",
 836 |       "D:\\Panorama\\texts\\243.txt\n",
 837 |       "D:\\Panorama\\tagged_texts\\243.txt\n",
 838 |       "D:\\Panorama\\texts\\244.txt\n",
 839 |       "D:\\Panorama\\tagged_texts\\244.txt\n",
 840 |       "D:\\Panorama\\texts\\245.txt\n",
 841 |       "D:\\Panorama\\tagged_texts\\245.txt\n",
 842 |       "D:\\Panorama\\texts\\246.txt\n",
 843 |       "D:\\Panorama\\tagged_texts\\246.txt\n",
 844 |       "D:\\Panorama\\texts\\247.txt\n",
 845 |       "D:\\Panorama\\tagged_texts\\247.txt\n",
 846 |       "D:\\Panorama\\texts\\248.txt\n",
 847 |       "D:\\Panorama\\tagged_texts\\248.txt\n",
 848 |       "D:\\Panorama\\texts\\249.txt\n",
 849 |       "D:\\Panorama\\tagged_texts\\249.txt\n",
 850 |       "D:\\Panorama\\texts\\25.txt\n",
 851 |       "D:\\Panorama\\tagged_texts\\25.txt\n",
 852 |       "D:\\Panorama\\texts\\250.txt\n",
 853 |       "D:\\Panorama\\tagged_texts\\250.txt\n",
 854 |       "D:\\Panorama\\texts\\251.txt\n",
 855 |       "D:\\Panorama\\tagged_texts\\251.txt\n",
 856 |       "D:\\Panorama\\texts\\252.txt\n",
 857 |       "D:\\Panorama\\tagged_texts\\252.txt\n",
 858 |       "D:\\Panorama\\texts\\253.txt\n",
 859 |       "D:\\Panorama\\tagged_texts\\253.txt\n",
 860 |       "D:\\Panorama\\texts\\254.txt\n",
 861 |       "D:\\Panorama\\tagged_texts\\254.txt\n",
 862 |       "D:\\Panorama\\texts\\255.txt\n",
 863 |       "D:\\Panorama\\tagged_texts\\255.txt\n",
 864 |       "D:\\Panorama\\texts\\256.txt\n",
 865 |       "D:\\Panorama\\tagged_texts\\256.txt\n",
 866 |       "D:\\Panorama\\texts\\257.txt\n",
 867 |       "D:\\Panorama\\tagged_texts\\257.txt\n",
 868 |       "D:\\Panorama\\texts\\258.txt\n",
 869 |       "D:\\Panorama\\tagged_texts\\258.txt\n",
 870 |       "D:\\Panorama\\texts\\259.txt\n",
 871 |       "D:\\Panorama\\tagged_texts\\259.txt\n",
 872 |       "D:\\Panorama\\texts\\26.txt\n",
 873 |       "D:\\Panorama\\tagged_texts\\26.txt\n",
 874 |       "D:\\Panorama\\texts\\260.txt\n",
 875 |       "D:\\Panorama\\tagged_texts\\260.txt\n",
 876 |       "D:\\Panorama\\texts\\261.txt\n",
 877 |       "D:\\Panorama\\tagged_texts\\261.txt\n",
 878 |       "D:\\Panorama\\texts\\262.txt\n",
 879 |       "D:\\Panorama\\tagged_texts\\262.txt\n",
 880 |       "D:\\Panorama\\texts\\263.txt\n",
 881 |       "D:\\Panorama\\tagged_texts\\263.txt\n",
 882 |       "D:\\Panorama\\texts\\264.txt\n",
 883 |       "D:\\Panorama\\tagged_texts\\264.txt\n",
 884 |       "D:\\Panorama\\texts\\265.txt\n",
 885 |       "D:\\Panorama\\tagged_texts\\265.txt\n",
 886 |       "D:\\Panorama\\texts\\266.txt\n",
 887 |       "D:\\Panorama\\tagged_texts\\266.txt\n",
 888 |       "D:\\Panorama\\texts\\267.txt\n",
 889 |       "D:\\Panorama\\tagged_texts\\267.txt\n",
 890 |       "D:\\Panorama\\texts\\268.txt\n",
 891 |       "D:\\Panorama\\tagged_texts\\268.txt\n",
 892 |       "D:\\Panorama\\texts\\269.txt\n",
 893 |       "D:\\Panorama\\tagged_texts\\269.txt\n",
 894 |       "D:\\Panorama\\texts\\27.txt\n",
 895 |       "D:\\Panorama\\tagged_texts\\27.txt\n",
 896 |       "D:\\Panorama\\texts\\270.txt\n",
 897 |       "D:\\Panorama\\tagged_texts\\270.txt\n",
 898 |       "D:\\Panorama\\texts\\271.txt\n",
 899 |       "D:\\Panorama\\tagged_texts\\271.txt\n",
 900 |       "D:\\Panorama\\texts\\272.txt\n",
 901 |       "D:\\Panorama\\tagged_texts\\272.txt\n",
 902 |       "D:\\Panorama\\texts\\273.txt\n",
 903 |       "D:\\Panorama\\tagged_texts\\273.txt\n",
 904 |       "D:\\Panorama\\texts\\274.txt\n",
 905 |       "D:\\Panorama\\tagged_texts\\274.txt\n",
 906 |       "D:\\Panorama\\texts\\275.txt\n",
 907 |       "D:\\Panorama\\tagged_texts\\275.txt\n",
 908 |       "D:\\Panorama\\texts\\276.txt\n",
 909 |       "D:\\Panorama\\tagged_texts\\276.txt\n",
 910 |       "D:\\Panorama\\texts\\277.txt\n",
 911 |       "D:\\Panorama\\tagged_texts\\277.txt\n",
 912 |       "D:\\Panorama\\texts\\278.txt\n",
 913 |       "D:\\Panorama\\tagged_texts\\278.txt\n",
 914 |       "D:\\Panorama\\texts\\279.txt\n",
 915 |       "D:\\Panorama\\tagged_texts\\279.txt\n",
 916 |       "D:\\Panorama\\texts\\28.txt\n",
 917 |       "D:\\Panorama\\tagged_texts\\28.txt\n",
 918 |       "D:\\Panorama\\texts\\280.txt\n",
 919 |       "D:\\Panorama\\tagged_texts\\280.txt\n",
 920 |       "D:\\Panorama\\texts\\281.txt\n",
 921 |       "D:\\Panorama\\tagged_texts\\281.txt\n",
 922 |       "D:\\Panorama\\texts\\282.txt\n",
 923 |       "D:\\Panorama\\tagged_texts\\282.txt\n",
 924 |       "D:\\Panorama\\texts\\283.txt\n",
 925 |       "D:\\Panorama\\tagged_texts\\283.txt\n",
 926 |       "D:\\Panorama\\texts\\284.txt\n",
 927 |       "D:\\Panorama\\tagged_texts\\284.txt\n",
 928 |       "D:\\Panorama\\texts\\285.txt\n",
 929 |       "D:\\Panorama\\tagged_texts\\285.txt\n",
 930 |       "D:\\Panorama\\texts\\286.txt\n",
 931 |       "D:\\Panorama\\tagged_texts\\286.txt\n",
 932 |       "D:\\Panorama\\texts\\287.txt\n",
 933 |       "D:\\Panorama\\tagged_texts\\287.txt\n",
 934 |       "D:\\Panorama\\texts\\288.txt\n",
 935 |       "D:\\Panorama\\tagged_texts\\288.txt\n",
 936 |       "D:\\Panorama\\texts\\289.txt\n",
 937 |       "D:\\Panorama\\tagged_texts\\289.txt\n",
 938 |       "D:\\Panorama\\texts\\29.txt\n",
 939 |       "D:\\Panorama\\tagged_texts\\29.txt\n",
 940 |       "D:\\Panorama\\texts\\290.txt\n",
 941 |       "D:\\Panorama\\tagged_texts\\290.txt\n",
 942 |       "D:\\Panorama\\texts\\291.txt\n",
 943 |       "D:\\Panorama\\tagged_texts\\291.txt\n",
 944 |       "D:\\Panorama\\texts\\292.txt\n",
 945 |       "D:\\Panorama\\tagged_texts\\292.txt\n",
 946 |       "D:\\Panorama\\texts\\293.txt\n",
 947 |       "D:\\Panorama\\tagged_texts\\293.txt\n",
 948 |       "D:\\Panorama\\texts\\294.txt\n",
 949 |       "D:\\Panorama\\tagged_texts\\294.txt\n",
 950 |       "D:\\Panorama\\texts\\295.txt\n",
 951 |       "D:\\Panorama\\tagged_texts\\295.txt\n",
 952 |       "D:\\Panorama\\texts\\296.txt\n",
 953 |       "D:\\Panorama\\tagged_texts\\296.txt\n",
 954 |       "D:\\Panorama\\texts\\297.txt\n",
 955 |       "D:\\Panorama\\tagged_texts\\297.txt\n",
 956 |       "D:\\Panorama\\texts\\298.txt\n",
 957 |       "D:\\Panorama\\tagged_texts\\298.txt\n",
 958 |       "D:\\Panorama\\texts\\299.txt\n",
 959 |       "D:\\Panorama\\tagged_texts\\299.txt\n",
 960 |       "D:\\Panorama\\texts\\3.txt\n",
 961 |       "D:\\Panorama\\tagged_texts\\3.txt\n",
 962 |       "D:\\Panorama\\texts\\30.txt\n",
 963 |       "D:\\Panorama\\tagged_texts\\30.txt\n",
 964 |       "D:\\Panorama\\texts\\300.txt\n",
 965 |       "D:\\Panorama\\tagged_texts\\300.txt\n",
 966 |       "D:\\Panorama\\texts\\301.txt\n",
 967 |       "D:\\Panorama\\tagged_texts\\301.txt\n",
 968 |       "D:\\Panorama\\texts\\302.txt\n",
 969 |       "D:\\Panorama\\tagged_texts\\302.txt\n",
 970 |       "D:\\Panorama\\texts\\303.txt\n",
 971 |       "D:\\Panorama\\tagged_texts\\303.txt\n",
 972 |       "D:\\Panorama\\texts\\304.txt\n",
 973 |       "D:\\Panorama\\tagged_texts\\304.txt\n",
 974 |       "D:\\Panorama\\texts\\305.txt\n",
 975 |       "D:\\Panorama\\tagged_texts\\305.txt\n",
 976 |       "D:\\Panorama\\texts\\306.txt\n",
 977 |       "D:\\Panorama\\tagged_texts\\306.txt\n",
 978 |       "D:\\Panorama\\texts\\307.txt\n",
 979 |       "D:\\Panorama\\tagged_texts\\307.txt\n",
 980 |       "D:\\Panorama\\texts\\308.txt\n",
 981 |       "D:\\Panorama\\tagged_texts\\308.txt\n",
 982 |       "D:\\Panorama\\texts\\309.txt\n",
 983 |       "D:\\Panorama\\tagged_texts\\309.txt\n",
 984 |       "D:\\Panorama\\texts\\31.txt\n",
 985 |       "D:\\Panorama\\tagged_texts\\31.txt\n",
 986 |       "D:\\Panorama\\texts\\310.txt\n",
 987 |       "D:\\Panorama\\tagged_texts\\310.txt\n",
 988 |       "D:\\Panorama\\texts\\311.txt\n",
 989 |       "D:\\Panorama\\tagged_texts\\311.txt\n",
 990 |       "D:\\Panorama\\texts\\312.txt\n",
 991 |       "D:\\Panorama\\tagged_texts\\312.txt\n",
 992 |       "D:\\Panorama\\texts\\313.txt\n",
 993 |       "D:\\Panorama\\tagged_texts\\313.txt\n",
 994 |       "D:\\Panorama\\texts\\314.txt\n",
 995 |       "D:\\Panorama\\tagged_texts\\314.txt\n",
 996 |       "D:\\Panorama\\texts\\315.txt\n",
 997 |       "D:\\Panorama\\tagged_texts\\315.txt\n",
 998 |       "D:\\Panorama\\texts\\316.txt\n",
 999 |       "D:\\Panorama\\tagged_texts\\316.txt\n",
1000 |       "D:\\Panorama\\texts\\317.txt\n",
1001 |       "D:\\Panorama\\tagged_texts\\317.txt\n",
1002 |       "D:\\Panorama\\texts\\318.txt\n",
1003 |       "D:\\Panorama\\tagged_texts\\318.txt\n",
1004 |       "D:\\Panorama\\texts\\319.txt\n",
1005 |       "D:\\Panorama\\tagged_texts\\319.txt\n",
1006 |       "D:\\Panorama\\texts\\32.txt\n",
1007 |       "D:\\Panorama\\tagged_texts\\32.txt\n",
1008 |       "D:\\Panorama\\texts\\320.txt\n",
1009 |       "D:\\Panorama\\tagged_texts\\320.txt\n",
1010 |       "D:\\Panorama\\texts\\321.txt\n",
1011 |       "D:\\Panorama\\tagged_texts\\321.txt\n",
1012 |       "D:\\Panorama\\texts\\322.txt\n",
1013 |       "D:\\Panorama\\tagged_texts\\322.txt\n",
1014 |       "D:\\Panorama\\texts\\323.txt\n",
1015 |       "D:\\Panorama\\tagged_texts\\323.txt\n",
1016 |       "D:\\Panorama\\texts\\324.txt\n",
1017 |       "D:\\Panorama\\tagged_texts\\324.txt\n",
1018 |       "D:\\Panorama\\texts\\325.txt\n",
1019 |       "D:\\Panorama\\tagged_texts\\325.txt\n",
1020 |       "D:\\Panorama\\texts\\326.txt\n",
1021 |       "D:\\Panorama\\tagged_texts\\326.txt\n",
1022 |       "D:\\Panorama\\texts\\327.txt\n",
1023 |       "D:\\Panorama\\tagged_texts\\327.txt\n",
1024 |       "D:\\Panorama\\texts\\328.txt\n",
1025 |       "D:\\Panorama\\tagged_texts\\328.txt\n",
1026 |       "D:\\Panorama\\texts\\329.txt\n",
1027 |       "D:\\Panorama\\tagged_texts\\329.txt\n",
1028 |       "D:\\Panorama\\texts\\33.txt\n",
1029 |       "D:\\Panorama\\tagged_texts\\33.txt\n",
1030 |       "D:\\Panorama\\texts\\330.txt\n",
1031 |       "D:\\Panorama\\tagged_texts\\330.txt\n",
1032 |       "D:\\Panorama\\texts\\331.txt\n",
1033 |       "D:\\Panorama\\tagged_texts\\331.txt\n",
1034 |       "D:\\Panorama\\texts\\332.txt\n",
1035 |       "D:\\Panorama\\tagged_texts\\332.txt\n",
1036 |       "D:\\Panorama\\texts\\333.txt\n",
1037 |       "D:\\Panorama\\tagged_texts\\333.txt\n",
1038 |       "D:\\Panorama\\texts\\334.txt\n",
1039 |       "D:\\Panorama\\tagged_texts\\334.txt\n",
1040 |       "D:\\Panorama\\texts\\335.txt\n",
1041 |       "D:\\Panorama\\tagged_texts\\335.txt\n",
1042 |       "D:\\Panorama\\texts\\336.txt\n",
1043 |       "D:\\Panorama\\tagged_texts\\336.txt\n",
1044 |       "D:\\Panorama\\texts\\337.txt\n",
1045 |       "D:\\Panorama\\tagged_texts\\337.txt\n",
1046 |       "D:\\Panorama\\texts\\338.txt\n",
1047 |       "D:\\Panorama\\tagged_texts\\338.txt\n",
1048 |       "D:\\Panorama\\texts\\339.txt\n",
1049 |       "D:\\Panorama\\tagged_texts\\339.txt\n",
1050 |       "D:\\Panorama\\texts\\34.txt\n",
1051 |       "D:\\Panorama\\tagged_texts\\34.txt\n",
1052 |       "D:\\Panorama\\texts\\340.txt\n",
1053 |       "D:\\Panorama\\tagged_texts\\340.txt\n",
1054 |       "D:\\Panorama\\texts\\341.txt\n",
1055 |       "D:\\Panorama\\tagged_texts\\341.txt\n",
1056 |       "D:\\Panorama\\texts\\342.txt\n",
1057 |       "D:\\Panorama\\tagged_texts\\342.txt\n",
1058 |       "D:\\Panorama\\texts\\343.txt\n",
1059 |       "D:\\Panorama\\tagged_texts\\343.txt\n",
1060 |       "D:\\Panorama\\texts\\344.txt\n",
1061 |       "D:\\Panorama\\tagged_texts\\344.txt\n",
1062 |       "D:\\Panorama\\texts\\345.txt\n",
1063 |       "D:\\Panorama\\tagged_texts\\345.txt\n",
1064 |       "D:\\Panorama\\texts\\346.txt\n",
1065 |       "D:\\Panorama\\tagged_texts\\346.txt\n",
1066 |       "D:\\Panorama\\texts\\347.txt\n",
1067 |       "D:\\Panorama\\tagged_texts\\347.txt\n",
1068 |       "D:\\Panorama\\texts\\348.txt\n",
1069 |       "D:\\Panorama\\tagged_texts\\348.txt\n",
1070 |       "D:\\Panorama\\texts\\349.txt\n"
1071 |      ]
1072 |     },
1073 |     {
1074 |      "name": "stdout",
1075 |      "output_type": "stream",
1076 |      "text": [
1077 |       "D:\\Panorama\\tagged_texts\\349.txt\n",
1078 |       "D:\\Panorama\\texts\\35.txt\n",
1079 |       "D:\\Panorama\\tagged_texts\\35.txt\n",
1080 |       "D:\\Panorama\\texts\\350.txt\n",
1081 |       "D:\\Panorama\\tagged_texts\\350.txt\n",
1082 |       "D:\\Panorama\\texts\\351.txt\n",
1083 |       "D:\\Panorama\\tagged_texts\\351.txt\n",
1084 |       "D:\\Panorama\\texts\\352.txt\n",
1085 |       "D:\\Panorama\\tagged_texts\\352.txt\n",
1086 |       "D:\\Panorama\\texts\\353.txt\n",
1087 |       "D:\\Panorama\\tagged_texts\\353.txt\n",
1088 |       "D:\\Panorama\\texts\\354.txt\n",
1089 |       "D:\\Panorama\\tagged_texts\\354.txt\n",
1090 |       "D:\\Panorama\\texts\\355.txt\n",
1091 |       "D:\\Panorama\\tagged_texts\\355.txt\n",
1092 |       "D:\\Panorama\\texts\\356.txt\n",
1093 |       "D:\\Panorama\\tagged_texts\\356.txt\n",
1094 |       "D:\\Panorama\\texts\\357.txt\n",
1095 |       "D:\\Panorama\\tagged_texts\\357.txt\n",
1096 |       "D:\\Panorama\\texts\\358.txt\n",
1097 |       "D:\\Panorama\\tagged_texts\\358.txt\n",
1098 |       "D:\\Panorama\\texts\\359.txt\n",
1099 |       "D:\\Panorama\\tagged_texts\\359.txt\n",
1100 |       "D:\\Panorama\\texts\\36.txt\n",
1101 |       "D:\\Panorama\\tagged_texts\\36.txt\n",
1102 |       "D:\\Panorama\\texts\\360.txt\n",
1103 |       "D:\\Panorama\\tagged_texts\\360.txt\n",
1104 |       "D:\\Panorama\\texts\\361.txt\n",
1105 |       "D:\\Panorama\\tagged_texts\\361.txt\n",
1106 |       "D:\\Panorama\\texts\\362.txt\n",
1107 |       "D:\\Panorama\\tagged_texts\\362.txt\n",
1108 |       "D:\\Panorama\\texts\\363.txt\n",
1109 |       "D:\\Panorama\\tagged_texts\\363.txt\n",
1110 |       "D:\\Panorama\\texts\\364.txt\n",
1111 |       "D:\\Panorama\\tagged_texts\\364.txt\n",
1112 |       "D:\\Panorama\\texts\\365.txt\n",
1113 |       "D:\\Panorama\\tagged_texts\\365.txt\n",
1114 |       "D:\\Panorama\\texts\\366.txt\n",
1115 |       "D:\\Panorama\\tagged_texts\\366.txt\n",
1116 |       "D:\\Panorama\\texts\\367.txt\n",
1117 |       "D:\\Panorama\\tagged_texts\\367.txt\n",
1118 |       "D:\\Panorama\\texts\\368.txt\n",
1119 |       "D:\\Panorama\\tagged_texts\\368.txt\n",
1120 |       "D:\\Panorama\\texts\\369.txt\n",
1121 |       "D:\\Panorama\\tagged_texts\\369.txt\n",
1122 |       "D:\\Panorama\\texts\\37.txt\n",
1123 |       "D:\\Panorama\\tagged_texts\\37.txt\n",
1124 |       "D:\\Panorama\\texts\\370.txt\n",
1125 |       "D:\\Panorama\\tagged_texts\\370.txt\n",
1126 |       "D:\\Panorama\\texts\\371.txt\n",
1127 |       "D:\\Panorama\\tagged_texts\\371.txt\n",
1128 |       "D:\\Panorama\\texts\\372.txt\n",
1129 |       "D:\\Panorama\\tagged_texts\\372.txt\n",
1130 |       "D:\\Panorama\\texts\\373.txt\n",
1131 |       "D:\\Panorama\\tagged_texts\\373.txt\n",
1132 |       "D:\\Panorama\\texts\\374.txt\n",
1133 |       "D:\\Panorama\\tagged_texts\\374.txt\n",
1134 |       "D:\\Panorama\\texts\\375.txt\n",
1135 |       "D:\\Panorama\\tagged_texts\\375.txt\n",
1136 |       "D:\\Panorama\\texts\\376.txt\n",
1137 |       "D:\\Panorama\\tagged_texts\\376.txt\n",
1138 |       "D:\\Panorama\\texts\\377.txt\n",
1139 |       "D:\\Panorama\\tagged_texts\\377.txt\n",
1140 |       "D:\\Panorama\\texts\\378.txt\n",
1141 |       "D:\\Panorama\\tagged_texts\\378.txt\n",
1142 |       "D:\\Panorama\\texts\\379.txt\n",
1143 |       "D:\\Panorama\\tagged_texts\\379.txt\n",
1144 |       "D:\\Panorama\\texts\\38.txt\n",
1145 |       "D:\\Panorama\\tagged_texts\\38.txt\n",
1146 |       "D:\\Panorama\\texts\\380.txt\n",
1147 |       "D:\\Panorama\\tagged_texts\\380.txt\n",
1148 |       "D:\\Panorama\\texts\\381.txt\n",
1149 |       "D:\\Panorama\\tagged_texts\\381.txt\n",
1150 |       "D:\\Panorama\\texts\\382.txt\n",
1151 |       "D:\\Panorama\\tagged_texts\\382.txt\n",
1152 |       "D:\\Panorama\\texts\\383.txt\n",
1153 |       "D:\\Panorama\\tagged_texts\\383.txt\n",
1154 |       "D:\\Panorama\\texts\\384.txt\n",
1155 |       "D:\\Panorama\\tagged_texts\\384.txt\n",
1156 |       "D:\\Panorama\\texts\\385.txt\n",
1157 |       "D:\\Panorama\\tagged_texts\\385.txt\n",
1158 |       "D:\\Panorama\\texts\\386.txt\n",
1159 |       "D:\\Panorama\\tagged_texts\\386.txt\n",
1160 |       "D:\\Panorama\\texts\\387.txt\n",
1161 |       "D:\\Panorama\\tagged_texts\\387.txt\n",
1162 |       "D:\\Panorama\\texts\\388.txt\n",
1163 |       "D:\\Panorama\\tagged_texts\\388.txt\n",
1164 |       "D:\\Panorama\\texts\\389.txt\n",
1165 |       "D:\\Panorama\\tagged_texts\\389.txt\n",
1166 |       "D:\\Panorama\\texts\\39.txt\n",
1167 |       "D:\\Panorama\\tagged_texts\\39.txt\n",
1168 |       "D:\\Panorama\\texts\\390.txt\n",
1169 |       "D:\\Panorama\\tagged_texts\\390.txt\n",
1170 |       "D:\\Panorama\\texts\\391.txt\n",
1171 |       "D:\\Panorama\\tagged_texts\\391.txt\n",
1172 |       "D:\\Panorama\\texts\\392.txt\n",
1173 |       "D:\\Panorama\\tagged_texts\\392.txt\n",
1174 |       "D:\\Panorama\\texts\\393.txt\n",
1175 |       "D:\\Panorama\\tagged_texts\\393.txt\n",
1176 |       "D:\\Panorama\\texts\\394.txt\n",
1177 |       "D:\\Panorama\\tagged_texts\\394.txt\n",
1178 |       "D:\\Panorama\\texts\\395.txt\n",
1179 |       "D:\\Panorama\\tagged_texts\\395.txt\n",
1180 |       "D:\\Panorama\\texts\\396.txt\n",
1181 |       "D:\\Panorama\\tagged_texts\\396.txt\n",
1182 |       "D:\\Panorama\\texts\\397.txt\n",
1183 |       "D:\\Panorama\\tagged_texts\\397.txt\n",
1184 |       "D:\\Panorama\\texts\\398.txt\n",
1185 |       "D:\\Panorama\\tagged_texts\\398.txt\n",
1186 |       "D:\\Panorama\\texts\\399.txt\n",
1187 |       "D:\\Panorama\\tagged_texts\\399.txt\n",
1188 |       "D:\\Panorama\\texts\\4.txt\n",
1189 |       "D:\\Panorama\\tagged_texts\\4.txt\n",
1190 |       "D:\\Panorama\\texts\\40.txt\n",
1191 |       "D:\\Panorama\\tagged_texts\\40.txt\n",
1192 |       "D:\\Panorama\\texts\\400.txt\n",
1193 |       "D:\\Panorama\\tagged_texts\\400.txt\n",
1194 |       "D:\\Panorama\\texts\\401.txt\n",
1195 |       "D:\\Panorama\\tagged_texts\\401.txt\n",
1196 |       "D:\\Panorama\\texts\\402.txt\n",
1197 |       "D:\\Panorama\\tagged_texts\\402.txt\n",
1198 |       "D:\\Panorama\\texts\\403.txt\n",
1199 |       "D:\\Panorama\\tagged_texts\\403.txt\n",
1200 |       "D:\\Panorama\\texts\\404.txt\n",
1201 |       "D:\\Panorama\\tagged_texts\\404.txt\n",
1202 |       "D:\\Panorama\\texts\\405.txt\n",
1203 |       "D:\\Panorama\\tagged_texts\\405.txt\n",
1204 |       "D:\\Panorama\\texts\\406.txt\n",
1205 |       "D:\\Panorama\\tagged_texts\\406.txt\n",
1206 |       "D:\\Panorama\\texts\\407.txt\n",
1207 |       "D:\\Panorama\\tagged_texts\\407.txt\n",
1208 |       "D:\\Panorama\\texts\\408.txt\n",
1209 |       "D:\\Panorama\\tagged_texts\\408.txt\n",
1210 |       "D:\\Panorama\\texts\\409.txt\n",
1211 |       "D:\\Panorama\\tagged_texts\\409.txt\n",
1212 |       "D:\\Panorama\\texts\\41.txt\n",
1213 |       "D:\\Panorama\\tagged_texts\\41.txt\n",
1214 |       "D:\\Panorama\\texts\\410.txt\n",
1215 |       "D:\\Panorama\\tagged_texts\\410.txt\n",
1216 |       "D:\\Panorama\\texts\\411.txt\n",
1217 |       "D:\\Panorama\\tagged_texts\\411.txt\n",
1218 |       "D:\\Panorama\\texts\\412.txt\n",
1219 |       "D:\\Panorama\\tagged_texts\\412.txt\n",
1220 |       "D:\\Panorama\\texts\\413.txt\n",
1221 |       "D:\\Panorama\\tagged_texts\\413.txt\n",
1222 |       "D:\\Panorama\\texts\\414.txt\n",
1223 |       "D:\\Panorama\\tagged_texts\\414.txt\n",
1224 |       "D:\\Panorama\\texts\\415.txt\n",
1225 |       "D:\\Panorama\\tagged_texts\\415.txt\n",
1226 |       "D:\\Panorama\\texts\\416.txt\n",
1227 |       "D:\\Panorama\\tagged_texts\\416.txt\n",
1228 |       "D:\\Panorama\\texts\\417.txt\n",
1229 |       "D:\\Panorama\\tagged_texts\\417.txt\n",
1230 |       "D:\\Panorama\\texts\\418.txt\n",
1231 |       "D:\\Panorama\\tagged_texts\\418.txt\n",
1232 |       "D:\\Panorama\\texts\\419.txt\n",
1233 |       "D:\\Panorama\\tagged_texts\\419.txt\n",
1234 |       "D:\\Panorama\\texts\\42.txt\n",
1235 |       "D:\\Panorama\\tagged_texts\\42.txt\n",
1236 |       "D:\\Panorama\\texts\\420.txt\n",
1237 |       "D:\\Panorama\\tagged_texts\\420.txt\n",
1238 |       "D:\\Panorama\\texts\\421.txt\n",
1239 |       "D:\\Panorama\\tagged_texts\\421.txt\n",
1240 |       "D:\\Panorama\\texts\\422.txt\n",
1241 |       "D:\\Panorama\\tagged_texts\\422.txt\n",
1242 |       "D:\\Panorama\\texts\\423.txt\n",
1243 |       "D:\\Panorama\\tagged_texts\\423.txt\n",
1244 |       "D:\\Panorama\\texts\\424.txt\n",
1245 |       "D:\\Panorama\\tagged_texts\\424.txt\n",
1246 |       "D:\\Panorama\\texts\\425.txt\n",
1247 |       "D:\\Panorama\\tagged_texts\\425.txt\n",
1248 |       "D:\\Panorama\\texts\\426.txt\n",
1249 |       "D:\\Panorama\\tagged_texts\\426.txt\n",
1250 |       "D:\\Panorama\\texts\\427.txt\n",
1251 |       "D:\\Panorama\\tagged_texts\\427.txt\n",
1252 |       "D:\\Panorama\\texts\\428.txt\n",
1253 |       "D:\\Panorama\\tagged_texts\\428.txt\n",
1254 |       "D:\\Panorama\\texts\\429.txt\n",
1255 |       "D:\\Panorama\\tagged_texts\\429.txt\n",
1256 |       "D:\\Panorama\\texts\\43.txt\n",
1257 |       "D:\\Panorama\\tagged_texts\\43.txt\n",
1258 |       "D:\\Panorama\\texts\\430.txt\n",
1259 |       "D:\\Panorama\\tagged_texts\\430.txt\n",
1260 |       "D:\\Panorama\\texts\\431.txt\n",
1261 |       "D:\\Panorama\\tagged_texts\\431.txt\n",
1262 |       "D:\\Panorama\\texts\\432.txt\n",
1263 |       "D:\\Panorama\\tagged_texts\\432.txt\n",
1264 |       "D:\\Panorama\\texts\\433.txt\n",
1265 |       "D:\\Panorama\\tagged_texts\\433.txt\n",
1266 |       "D:\\Panorama\\texts\\434.txt\n",
1267 |       "D:\\Panorama\\tagged_texts\\434.txt\n",
1268 |       "D:\\Panorama\\texts\\435.txt\n",
1269 |       "D:\\Panorama\\tagged_texts\\435.txt\n",
1270 |       "D:\\Panorama\\texts\\436.txt\n",
1271 |       "D:\\Panorama\\tagged_texts\\436.txt\n",
1272 |       "D:\\Panorama\\texts\\437.txt\n",
1273 |       "D:\\Panorama\\tagged_texts\\437.txt\n",
1274 |       "D:\\Panorama\\texts\\438.txt\n",
1275 |       "D:\\Panorama\\tagged_texts\\438.txt\n",
1276 |       "D:\\Panorama\\texts\\439.txt\n",
1277 |       "D:\\Panorama\\tagged_texts\\439.txt\n",
1278 |       "D:\\Panorama\\texts\\44.txt\n",
1279 |       "D:\\Panorama\\tagged_texts\\44.txt\n",
1280 |       "D:\\Panorama\\texts\\440.txt\n",
1281 |       "D:\\Panorama\\tagged_texts\\440.txt\n",
1282 |       "D:\\Panorama\\texts\\441.txt\n",
1283 |       "D:\\Panorama\\tagged_texts\\441.txt\n",
1284 |       "D:\\Panorama\\texts\\442.txt\n",
1285 |       "D:\\Panorama\\tagged_texts\\442.txt\n",
1286 |       "D:\\Panorama\\texts\\443.txt\n",
1287 |       "D:\\Panorama\\tagged_texts\\443.txt\n",
1288 |       "D:\\Panorama\\texts\\444.txt\n",
1289 |       "D:\\Panorama\\tagged_texts\\444.txt\n",
1290 |       "D:\\Panorama\\texts\\445.txt\n",
1291 |       "D:\\Panorama\\tagged_texts\\445.txt\n",
1292 |       "D:\\Panorama\\texts\\446.txt\n",
1293 |       "D:\\Panorama\\tagged_texts\\446.txt\n",
1294 |       "D:\\Panorama\\texts\\447.txt\n",
1295 |       "D:\\Panorama\\tagged_texts\\447.txt\n",
1296 |       "D:\\Panorama\\texts\\448.txt\n",
1297 |       "D:\\Panorama\\tagged_texts\\448.txt\n",
1298 |       "D:\\Panorama\\texts\\449.txt\n",
1299 |       "D:\\Panorama\\tagged_texts\\449.txt\n",
1300 |       "D:\\Panorama\\texts\\45.txt\n",
1301 |       "D:\\Panorama\\tagged_texts\\45.txt\n",
1302 |       "D:\\Panorama\\texts\\450.txt\n",
1303 |       "D:\\Panorama\\tagged_texts\\450.txt\n",
1304 |       "D:\\Panorama\\texts\\451.txt\n",
1305 |       "D:\\Panorama\\tagged_texts\\451.txt\n",
1306 |       "D:\\Panorama\\texts\\452.txt\n",
1307 |       "D:\\Panorama\\tagged_texts\\452.txt\n",
1308 |       "D:\\Panorama\\texts\\453.txt\n",
1309 |       "D:\\Panorama\\tagged_texts\\453.txt\n",
1310 |       "D:\\Panorama\\texts\\454.txt\n",
1311 |       "D:\\Panorama\\tagged_texts\\454.txt\n",
1312 |       "D:\\Panorama\\texts\\455.txt\n",
1313 |       "D:\\Panorama\\tagged_texts\\455.txt\n",
1314 |       "D:\\Panorama\\texts\\456.txt\n",
1315 |       "D:\\Panorama\\tagged_texts\\456.txt\n",
1316 |       "D:\\Panorama\\texts\\457.txt\n",
1317 |       "D:\\Panorama\\tagged_texts\\457.txt\n",
1318 |       "D:\\Panorama\\texts\\458.txt\n",
1319 |       "D:\\Panorama\\tagged_texts\\458.txt\n",
1320 |       "D:\\Panorama\\texts\\459.txt\n",
1321 |       "D:\\Panorama\\tagged_texts\\459.txt\n",
1322 |       "D:\\Panorama\\texts\\46.txt\n",
1323 |       "D:\\Panorama\\tagged_texts\\46.txt\n",
1324 |       "D:\\Panorama\\texts\\460.txt\n",
1325 |       "D:\\Panorama\\tagged_texts\\460.txt\n",
1326 |       "D:\\Panorama\\texts\\461.txt\n",
1327 |       "D:\\Panorama\\tagged_texts\\461.txt\n",
1328 |       "D:\\Panorama\\texts\\462.txt\n",
1329 |       "D:\\Panorama\\tagged_texts\\462.txt\n",
1330 |       "D:\\Panorama\\texts\\463.txt\n",
1331 |       "D:\\Panorama\\tagged_texts\\463.txt\n",
1332 |       "D:\\Panorama\\texts\\464.txt\n",
1333 |       "D:\\Panorama\\tagged_texts\\464.txt\n",
1334 |       "D:\\Panorama\\texts\\465.txt\n",
1335 |       "D:\\Panorama\\tagged_texts\\465.txt\n",
1336 |       "D:\\Panorama\\texts\\466.txt\n",
1337 |       "D:\\Panorama\\tagged_texts\\466.txt\n",
1338 |       "D:\\Panorama\\texts\\467.txt\n",
1339 |       "D:\\Panorama\\tagged_texts\\467.txt\n",
1340 |       "D:\\Panorama\\texts\\468.txt\n",
1341 |       "D:\\Panorama\\tagged_texts\\468.txt\n",
1342 |       "D:\\Panorama\\texts\\469.txt\n",
1343 |       "D:\\Panorama\\tagged_texts\\469.txt\n",
1344 |       "D:\\Panorama\\texts\\47.txt\n",
1345 |       "D:\\Panorama\\tagged_texts\\47.txt\n",
1346 |       "D:\\Panorama\\texts\\470.txt\n",
1347 |       "D:\\Panorama\\tagged_texts\\470.txt\n",
1348 |       "D:\\Panorama\\texts\\471.txt\n",
1349 |       "D:\\Panorama\\tagged_texts\\471.txt\n",
1350 |       "D:\\Panorama\\texts\\472.txt\n",
1351 |       "D:\\Panorama\\tagged_texts\\472.txt\n",
1352 |       "D:\\Panorama\\texts\\473.txt\n",
1353 |       "D:\\Panorama\\tagged_texts\\473.txt\n",
1354 |       "D:\\Panorama\\texts\\474.txt\n"
1355 |      ]
1356 |     },
1357 |     {
1358 |      "name": "stdout",
1359 |      "output_type": "stream",
1360 |      "text": [
1361 |       "D:\\Panorama\\tagged_texts\\474.txt\n",
1362 |       "D:\\Panorama\\texts\\475.txt\n",
1363 |       "D:\\Panorama\\tagged_texts\\475.txt\n",
1364 |       "D:\\Panorama\\texts\\476.txt\n",
1365 |       "D:\\Panorama\\tagged_texts\\476.txt\n",
1366 |       "D:\\Panorama\\texts\\477.txt\n",
1367 |       "D:\\Panorama\\tagged_texts\\477.txt\n",
1368 |       "D:\\Panorama\\texts\\478.txt\n",
1369 |       "D:\\Panorama\\tagged_texts\\478.txt\n",
1370 |       "D:\\Panorama\\texts\\479.txt\n",
1371 |       "D:\\Panorama\\tagged_texts\\479.txt\n",
1372 |       "D:\\Panorama\\texts\\48.txt\n",
1373 |       "D:\\Panorama\\tagged_texts\\48.txt\n",
1374 |       "D:\\Panorama\\texts\\480.txt\n",
1375 |       "D:\\Panorama\\tagged_texts\\480.txt\n",
1376 |       "D:\\Panorama\\texts\\481.txt\n",
1377 |       "D:\\Panorama\\tagged_texts\\481.txt\n",
1378 |       "D:\\Panorama\\texts\\482.txt\n",
1379 |       "D:\\Panorama\\tagged_texts\\482.txt\n",
1380 |       "D:\\Panorama\\texts\\483.txt\n",
1381 |       "D:\\Panorama\\tagged_texts\\483.txt\n",
1382 |       "D:\\Panorama\\texts\\484.txt\n",
1383 |       "D:\\Panorama\\tagged_texts\\484.txt\n",
1384 |       "D:\\Panorama\\texts\\485.txt\n",
1385 |       "D:\\Panorama\\tagged_texts\\485.txt\n",
1386 |       "D:\\Panorama\\texts\\486.txt\n",
1387 |       "D:\\Panorama\\tagged_texts\\486.txt\n",
1388 |       "D:\\Panorama\\texts\\487.txt\n",
1389 |       "D:\\Panorama\\tagged_texts\\487.txt\n",
1390 |       "D:\\Panorama\\texts\\488.txt\n",
1391 |       "D:\\Panorama\\tagged_texts\\488.txt\n",
1392 |       "D:\\Panorama\\texts\\489.txt\n",
1393 |       "D:\\Panorama\\tagged_texts\\489.txt\n",
1394 |       "D:\\Panorama\\texts\\49.txt\n",
1395 |       "D:\\Panorama\\tagged_texts\\49.txt\n",
1396 |       "D:\\Panorama\\texts\\490.txt\n",
1397 |       "D:\\Panorama\\tagged_texts\\490.txt\n",
1398 |       "D:\\Panorama\\texts\\491.txt\n",
1399 |       "D:\\Panorama\\tagged_texts\\491.txt\n",
1400 |       "D:\\Panorama\\texts\\492.txt\n",
1401 |       "D:\\Panorama\\tagged_texts\\492.txt\n",
1402 |       "D:\\Panorama\\texts\\493.txt\n",
1403 |       "D:\\Panorama\\tagged_texts\\493.txt\n",
1404 |       "D:\\Panorama\\texts\\494.txt\n",
1405 |       "D:\\Panorama\\tagged_texts\\494.txt\n",
1406 |       "D:\\Panorama\\texts\\495.txt\n",
1407 |       "D:\\Panorama\\tagged_texts\\495.txt\n",
1408 |       "D:\\Panorama\\texts\\496.txt\n",
1409 |       "D:\\Panorama\\tagged_texts\\496.txt\n",
1410 |       "D:\\Panorama\\texts\\497.txt\n",
1411 |       "D:\\Panorama\\tagged_texts\\497.txt\n",
1412 |       "D:\\Panorama\\texts\\498.txt\n",
1413 |       "D:\\Panorama\\tagged_texts\\498.txt\n",
1414 |       "D:\\Panorama\\texts\\499.txt\n",
1415 |       "D:\\Panorama\\tagged_texts\\499.txt\n",
1416 |       "D:\\Panorama\\texts\\5.txt\n",
1417 |       "D:\\Panorama\\tagged_texts\\5.txt\n",
1418 |       "D:\\Panorama\\texts\\50.txt\n",
1419 |       "D:\\Panorama\\tagged_texts\\50.txt\n",
1420 |       "D:\\Panorama\\texts\\500.txt\n",
1421 |       "D:\\Panorama\\tagged_texts\\500.txt\n",
1422 |       "D:\\Panorama\\texts\\501.txt\n",
1423 |       "D:\\Panorama\\tagged_texts\\501.txt\n",
1424 |       "D:\\Panorama\\texts\\502.txt\n",
1425 |       "D:\\Panorama\\tagged_texts\\502.txt\n",
1426 |       "D:\\Panorama\\texts\\503.txt\n",
1427 |       "D:\\Panorama\\tagged_texts\\503.txt\n",
1428 |       "D:\\Panorama\\texts\\504.txt\n",
1429 |       "D:\\Panorama\\tagged_texts\\504.txt\n",
1430 |       "D:\\Panorama\\texts\\505.txt\n",
1431 |       "D:\\Panorama\\tagged_texts\\505.txt\n",
1432 |       "D:\\Panorama\\texts\\506.txt\n",
1433 |       "D:\\Panorama\\tagged_texts\\506.txt\n",
1434 |       "D:\\Panorama\\texts\\507.txt\n",
1435 |       "D:\\Panorama\\tagged_texts\\507.txt\n",
1436 |       "D:\\Panorama\\texts\\508.txt\n",
1437 |       "D:\\Panorama\\tagged_texts\\508.txt\n",
1438 |       "D:\\Panorama\\texts\\509.txt\n",
1439 |       "D:\\Panorama\\tagged_texts\\509.txt\n",
1440 |       "D:\\Panorama\\texts\\51.txt\n",
1441 |       "D:\\Panorama\\tagged_texts\\51.txt\n",
1442 |       "D:\\Panorama\\texts\\510.txt\n",
1443 |       "D:\\Panorama\\tagged_texts\\510.txt\n",
1444 |       "D:\\Panorama\\texts\\511.txt\n",
1445 |       "D:\\Panorama\\tagged_texts\\511.txt\n",
1446 |       "D:\\Panorama\\texts\\512.txt\n",
1447 |       "D:\\Panorama\\tagged_texts\\512.txt\n",
1448 |       "D:\\Panorama\\texts\\513.txt\n",
1449 |       "D:\\Panorama\\tagged_texts\\513.txt\n",
1450 |       "D:\\Panorama\\texts\\514.txt\n",
1451 |       "D:\\Panorama\\tagged_texts\\514.txt\n",
1452 |       "D:\\Panorama\\texts\\515.txt\n",
1453 |       "D:\\Panorama\\tagged_texts\\515.txt\n",
1454 |       "D:\\Panorama\\texts\\516.txt\n",
1455 |       "D:\\Panorama\\tagged_texts\\516.txt\n",
1456 |       "D:\\Panorama\\texts\\517.txt\n",
1457 |       "D:\\Panorama\\tagged_texts\\517.txt\n",
1458 |       "D:\\Panorama\\texts\\518.txt\n",
1459 |       "D:\\Panorama\\tagged_texts\\518.txt\n",
1460 |       "D:\\Panorama\\texts\\519.txt\n",
1461 |       "D:\\Panorama\\tagged_texts\\519.txt\n",
1462 |       "D:\\Panorama\\texts\\52.txt\n",
1463 |       "D:\\Panorama\\tagged_texts\\52.txt\n",
1464 |       "D:\\Panorama\\texts\\520.txt\n",
1465 |       "D:\\Panorama\\tagged_texts\\520.txt\n",
1466 |       "D:\\Panorama\\texts\\521.txt\n",
1467 |       "D:\\Panorama\\tagged_texts\\521.txt\n",
1468 |       "D:\\Panorama\\texts\\522.txt\n",
1469 |       "D:\\Panorama\\tagged_texts\\522.txt\n",
1470 |       "D:\\Panorama\\texts\\523.txt\n",
1471 |       "D:\\Panorama\\tagged_texts\\523.txt\n",
1472 |       "D:\\Panorama\\texts\\524.txt\n",
1473 |       "D:\\Panorama\\tagged_texts\\524.txt\n",
1474 |       "D:\\Panorama\\texts\\525.txt\n",
1475 |       "D:\\Panorama\\tagged_texts\\525.txt\n",
1476 |       "D:\\Panorama\\texts\\526.txt\n",
1477 |       "D:\\Panorama\\tagged_texts\\526.txt\n",
1478 |       "D:\\Panorama\\texts\\527.txt\n",
1479 |       "D:\\Panorama\\tagged_texts\\527.txt\n",
1480 |       "D:\\Panorama\\texts\\528.txt\n",
1481 |       "D:\\Panorama\\tagged_texts\\528.txt\n",
1482 |       "D:\\Panorama\\texts\\529.txt\n",
1483 |       "D:\\Panorama\\tagged_texts\\529.txt\n",
1484 |       "D:\\Panorama\\texts\\53.txt\n",
1485 |       "D:\\Panorama\\tagged_texts\\53.txt\n",
1486 |       "D:\\Panorama\\texts\\530.txt\n",
1487 |       "D:\\Panorama\\tagged_texts\\530.txt\n",
1488 |       "D:\\Panorama\\texts\\531.txt\n",
1489 |       "D:\\Panorama\\tagged_texts\\531.txt\n",
1490 |       "D:\\Panorama\\texts\\532.txt\n",
1491 |       "D:\\Panorama\\tagged_texts\\532.txt\n",
1492 |       "D:\\Panorama\\texts\\533.txt\n",
1493 |       "D:\\Panorama\\tagged_texts\\533.txt\n",
1494 |       "D:\\Panorama\\texts\\534.txt\n",
1495 |       "D:\\Panorama\\tagged_texts\\534.txt\n",
1496 |       "D:\\Panorama\\texts\\535.txt\n",
1497 |       "D:\\Panorama\\tagged_texts\\535.txt\n",
1498 |       "D:\\Panorama\\texts\\536.txt\n",
1499 |       "D:\\Panorama\\tagged_texts\\536.txt\n",
1500 |       "D:\\Panorama\\texts\\537.txt\n",
1501 |       "D:\\Panorama\\tagged_texts\\537.txt\n",
1502 |       "D:\\Panorama\\texts\\538.txt\n",
1503 |       "D:\\Panorama\\tagged_texts\\538.txt\n",
1504 |       "D:\\Panorama\\texts\\539.txt\n",
1505 |       "D:\\Panorama\\tagged_texts\\539.txt\n",
1506 |       "D:\\Panorama\\texts\\54.txt\n",
1507 |       "D:\\Panorama\\tagged_texts\\54.txt\n",
1508 |       "D:\\Panorama\\texts\\540.txt\n",
1509 |       "D:\\Panorama\\tagged_texts\\540.txt\n",
1510 |       "D:\\Panorama\\texts\\541.txt\n",
1511 |       "D:\\Panorama\\tagged_texts\\541.txt\n",
1512 |       "D:\\Panorama\\texts\\542.txt\n",
1513 |       "D:\\Panorama\\tagged_texts\\542.txt\n",
1514 |       "D:\\Panorama\\texts\\543.txt\n",
1515 |       "D:\\Panorama\\tagged_texts\\543.txt\n",
1516 |       "D:\\Panorama\\texts\\544.txt\n",
1517 |       "D:\\Panorama\\tagged_texts\\544.txt\n",
1518 |       "D:\\Panorama\\texts\\545.txt\n",
1519 |       "D:\\Panorama\\tagged_texts\\545.txt\n",
1520 |       "D:\\Panorama\\texts\\546.txt\n",
1521 |       "D:\\Panorama\\tagged_texts\\546.txt\n",
1522 |       "D:\\Panorama\\texts\\547.txt\n",
1523 |       "D:\\Panorama\\tagged_texts\\547.txt\n",
1524 |       "D:\\Panorama\\texts\\548.txt\n",
1525 |       "D:\\Panorama\\tagged_texts\\548.txt\n",
1526 |       "D:\\Panorama\\texts\\549.txt\n",
1527 |       "D:\\Panorama\\tagged_texts\\549.txt\n",
1528 |       "D:\\Panorama\\texts\\55.txt\n",
1529 |       "D:\\Panorama\\tagged_texts\\55.txt\n",
1530 |       "D:\\Panorama\\texts\\550.txt\n",
1531 |       "D:\\Panorama\\tagged_texts\\550.txt\n",
1532 |       "D:\\Panorama\\texts\\551.txt\n",
1533 |       "D:\\Panorama\\tagged_texts\\551.txt\n",
1534 |       "D:\\Panorama\\texts\\552.txt\n",
1535 |       "D:\\Panorama\\tagged_texts\\552.txt\n",
1536 |       "D:\\Panorama\\texts\\553.txt\n",
1537 |       "D:\\Panorama\\tagged_texts\\553.txt\n",
1538 |       "D:\\Panorama\\texts\\554.txt\n",
1539 |       "D:\\Panorama\\tagged_texts\\554.txt\n",
1540 |       "D:\\Panorama\\texts\\555.txt\n",
1541 |       "D:\\Panorama\\tagged_texts\\555.txt\n",
1542 |       "D:\\Panorama\\texts\\556.txt\n",
1543 |       "D:\\Panorama\\tagged_texts\\556.txt\n",
1544 |       "D:\\Panorama\\texts\\557.txt\n",
1545 |       "D:\\Panorama\\tagged_texts\\557.txt\n",
1546 |       "D:\\Panorama\\texts\\558.txt\n",
1547 |       "D:\\Panorama\\tagged_texts\\558.txt\n",
1548 |       "D:\\Panorama\\texts\\559.txt\n",
1549 |       "D:\\Panorama\\tagged_texts\\559.txt\n",
1550 |       "D:\\Panorama\\texts\\56.txt\n",
1551 |       "D:\\Panorama\\tagged_texts\\56.txt\n",
1552 |       "D:\\Panorama\\texts\\560.txt\n",
1553 |       "D:\\Panorama\\tagged_texts\\560.txt\n",
1554 |       "D:\\Panorama\\texts\\561.txt\n",
1555 |       "D:\\Panorama\\tagged_texts\\561.txt\n",
1556 |       "D:\\Panorama\\texts\\562.txt\n",
1557 |       "D:\\Panorama\\tagged_texts\\562.txt\n",
1558 |       "D:\\Panorama\\texts\\563.txt\n",
1559 |       "D:\\Panorama\\tagged_texts\\563.txt\n",
1560 |       "D:\\Panorama\\texts\\564.txt\n",
1561 |       "D:\\Panorama\\tagged_texts\\564.txt\n",
1562 |       "D:\\Panorama\\texts\\565.txt\n",
1563 |       "D:\\Panorama\\tagged_texts\\565.txt\n",
1564 |       "D:\\Panorama\\texts\\566.txt\n",
1565 |       "D:\\Panorama\\tagged_texts\\566.txt\n",
1566 |       "D:\\Panorama\\texts\\567.txt\n",
1567 |       "D:\\Panorama\\tagged_texts\\567.txt\n",
1568 |       "D:\\Panorama\\texts\\568.txt\n",
1569 |       "D:\\Panorama\\tagged_texts\\568.txt\n",
1570 |       "D:\\Panorama\\texts\\569.txt\n",
1571 |       "D:\\Panorama\\tagged_texts\\569.txt\n",
1572 |       "D:\\Panorama\\texts\\57.txt\n",
1573 |       "D:\\Panorama\\tagged_texts\\57.txt\n",
1574 |       "D:\\Panorama\\texts\\570.txt\n",
1575 |       "D:\\Panorama\\tagged_texts\\570.txt\n",
1576 |       "D:\\Panorama\\texts\\571.txt\n",
1577 |       "D:\\Panorama\\tagged_texts\\571.txt\n",
1578 |       "D:\\Panorama\\texts\\572.txt\n",
1579 |       "D:\\Panorama\\tagged_texts\\572.txt\n",
1580 |       "D:\\Panorama\\texts\\573.txt\n",
1581 |       "D:\\Panorama\\tagged_texts\\573.txt\n",
1582 |       "D:\\Panorama\\texts\\574.txt\n",
1583 |       "D:\\Panorama\\tagged_texts\\574.txt\n",
1584 |       "D:\\Panorama\\texts\\575.txt\n",
1585 |       "D:\\Panorama\\tagged_texts\\575.txt\n",
1586 |       "D:\\Panorama\\texts\\576.txt\n",
1587 |       "D:\\Panorama\\tagged_texts\\576.txt\n",
1588 |       "D:\\Panorama\\texts\\577.txt\n",
1589 |       "D:\\Panorama\\tagged_texts\\577.txt\n",
1590 |       "D:\\Panorama\\texts\\578.txt\n",
1591 |       "D:\\Panorama\\tagged_texts\\578.txt\n",
1592 |       "D:\\Panorama\\texts\\579.txt\n",
1593 |       "D:\\Panorama\\tagged_texts\\579.txt\n",
1594 |       "D:\\Panorama\\texts\\58.txt\n",
1595 |       "D:\\Panorama\\tagged_texts\\58.txt\n",
1596 |       "D:\\Panorama\\texts\\580.txt\n",
1597 |       "D:\\Panorama\\tagged_texts\\580.txt\n",
1598 |       "D:\\Panorama\\texts\\581.txt\n",
1599 |       "D:\\Panorama\\tagged_texts\\581.txt\n",
1600 |       "D:\\Panorama\\texts\\582.txt\n",
1601 |       "D:\\Panorama\\tagged_texts\\582.txt\n",
1602 |       "D:\\Panorama\\texts\\583.txt\n",
1603 |       "D:\\Panorama\\tagged_texts\\583.txt\n",
1604 |       "D:\\Panorama\\texts\\584.txt\n",
1605 |       "D:\\Panorama\\tagged_texts\\584.txt\n",
1606 |       "D:\\Panorama\\texts\\585.txt\n",
1607 |       "D:\\Panorama\\tagged_texts\\585.txt\n",
1608 |       "D:\\Panorama\\texts\\586.txt\n",
1609 |       "D:\\Panorama\\tagged_texts\\586.txt\n",
1610 |       "D:\\Panorama\\texts\\587.txt\n",
1611 |       "D:\\Panorama\\tagged_texts\\587.txt\n",
1612 |       "D:\\Panorama\\texts\\588.txt\n",
1613 |       "D:\\Panorama\\tagged_texts\\588.txt\n",
1614 |       "D:\\Panorama\\texts\\589.txt\n",
1615 |       "D:\\Panorama\\tagged_texts\\589.txt\n",
1616 |       "D:\\Panorama\\texts\\59.txt\n",
1617 |       "D:\\Panorama\\tagged_texts\\59.txt\n",
1618 |       "D:\\Panorama\\texts\\590.txt\n",
1619 |       "D:\\Panorama\\tagged_texts\\590.txt\n",
1620 |       "D:\\Panorama\\texts\\591.txt\n",
1621 |       "D:\\Panorama\\tagged_texts\\591.txt\n",
1622 |       "D:\\Panorama\\texts\\592.txt\n",
1623 |       "D:\\Panorama\\tagged_texts\\592.txt\n",
1624 |       "D:\\Panorama\\texts\\593.txt\n",
1625 |       "D:\\Panorama\\tagged_texts\\593.txt\n",
1626 |       "D:\\Panorama\\texts\\594.txt\n",
1627 |       "D:\\Panorama\\tagged_texts\\594.txt\n",
1628 |       "D:\\Panorama\\texts\\595.txt\n",
1629 |       "D:\\Panorama\\tagged_texts\\595.txt\n",
1630 |       "D:\\Panorama\\texts\\596.txt\n",
1631 |       "D:\\Panorama\\tagged_texts\\596.txt\n",
1632 |       "D:\\Panorama\\texts\\597.txt\n",
1633 |       "D:\\Panorama\\tagged_texts\\597.txt\n",
1634 |       "D:\\Panorama\\texts\\598.txt\n",
1635 |       "D:\\Panorama\\tagged_texts\\598.txt\n",
1636 |       "D:\\Panorama\\texts\\599.txt\n",
1637 |       "D:\\Panorama\\tagged_texts\\599.txt\n",
1638 |       "D:\\Panorama\\texts\\6.txt\n"
1639 |      ]
1640 |     },
1641 |     {
1642 |      "name": "stdout",
1643 |      "output_type": "stream",
1644 |      "text": [
1645 |       "D:\\Panorama\\tagged_texts\\6.txt\n",
1646 |       "D:\\Panorama\\texts\\60.txt\n",
1647 |       "D:\\Panorama\\tagged_texts\\60.txt\n",
1648 |       "D:\\Panorama\\texts\\600.txt\n",
1649 |       "D:\\Panorama\\tagged_texts\\600.txt\n",
1650 |       "D:\\Panorama\\texts\\601.txt\n",
1651 |       "D:\\Panorama\\tagged_texts\\601.txt\n",
1652 |       "D:\\Panorama\\texts\\602.txt\n",
1653 |       "D:\\Panorama\\tagged_texts\\602.txt\n",
1654 |       "D:\\Panorama\\texts\\603.txt\n",
1655 |       "D:\\Panorama\\tagged_texts\\603.txt\n",
1656 |       "D:\\Panorama\\texts\\604.txt\n",
1657 |       "D:\\Panorama\\tagged_texts\\604.txt\n",
1658 |       "D:\\Panorama\\texts\\605.txt\n",
1659 |       "D:\\Panorama\\tagged_texts\\605.txt\n",
1660 |       "D:\\Panorama\\texts\\606.txt\n",
1661 |       "D:\\Panorama\\tagged_texts\\606.txt\n",
1662 |       "D:\\Panorama\\texts\\607.txt\n",
1663 |       "D:\\Panorama\\tagged_texts\\607.txt\n",
1664 |       "D:\\Panorama\\texts\\608.txt\n",
1665 |       "D:\\Panorama\\tagged_texts\\608.txt\n",
1666 |       "D:\\Panorama\\texts\\609.txt\n",
1667 |       "D:\\Panorama\\tagged_texts\\609.txt\n",
1668 |       "D:\\Panorama\\texts\\61.txt\n",
1669 |       "D:\\Panorama\\tagged_texts\\61.txt\n",
1670 |       "D:\\Panorama\\texts\\610.txt\n",
1671 |       "D:\\Panorama\\tagged_texts\\610.txt\n",
1672 |       "D:\\Panorama\\texts\\611.txt\n",
1673 |       "D:\\Panorama\\tagged_texts\\611.txt\n",
1674 |       "D:\\Panorama\\texts\\612.txt\n",
1675 |       "D:\\Panorama\\tagged_texts\\612.txt\n",
1676 |       "D:\\Panorama\\texts\\613.txt\n",
1677 |       "D:\\Panorama\\tagged_texts\\613.txt\n",
1678 |       "D:\\Panorama\\texts\\614.txt\n",
1679 |       "D:\\Panorama\\tagged_texts\\614.txt\n",
1680 |       "D:\\Panorama\\texts\\615.txt\n",
1681 |       "D:\\Panorama\\tagged_texts\\615.txt\n",
1682 |       "D:\\Panorama\\texts\\616.txt\n",
1683 |       "D:\\Panorama\\tagged_texts\\616.txt\n",
1684 |       "D:\\Panorama\\texts\\617.txt\n",
1685 |       "D:\\Panorama\\tagged_texts\\617.txt\n",
1686 |       "D:\\Panorama\\texts\\618.txt\n",
1687 |       "D:\\Panorama\\tagged_texts\\618.txt\n",
1688 |       "D:\\Panorama\\texts\\619.txt\n",
1689 |       "D:\\Panorama\\tagged_texts\\619.txt\n",
1690 |       "D:\\Panorama\\texts\\62.txt\n",
1691 |       "D:\\Panorama\\tagged_texts\\62.txt\n",
1692 |       "D:\\Panorama\\texts\\620.txt\n",
1693 |       "D:\\Panorama\\tagged_texts\\620.txt\n",
1694 |       "D:\\Panorama\\texts\\621.txt\n",
1695 |       "D:\\Panorama\\tagged_texts\\621.txt\n",
1696 |       "D:\\Panorama\\texts\\622.txt\n",
1697 |       "D:\\Panorama\\tagged_texts\\622.txt\n",
1698 |       "D:\\Panorama\\texts\\623.txt\n",
1699 |       "D:\\Panorama\\tagged_texts\\623.txt\n",
1700 |       "D:\\Panorama\\texts\\624.txt\n",
1701 |       "D:\\Panorama\\tagged_texts\\624.txt\n",
1702 |       "D:\\Panorama\\texts\\625.txt\n",
1703 |       "D:\\Panorama\\tagged_texts\\625.txt\n",
1704 |       "D:\\Panorama\\texts\\626.txt\n",
1705 |       "D:\\Panorama\\tagged_texts\\626.txt\n",
1706 |       "D:\\Panorama\\texts\\627.txt\n",
1707 |       "D:\\Panorama\\tagged_texts\\627.txt\n",
1708 |       "D:\\Panorama\\texts\\628.txt\n",
1709 |       "D:\\Panorama\\tagged_texts\\628.txt\n",
1710 |       "D:\\Panorama\\texts\\629.txt\n",
1711 |       "D:\\Panorama\\tagged_texts\\629.txt\n",
1712 |       "D:\\Panorama\\texts\\63.txt\n",
1713 |       "D:\\Panorama\\tagged_texts\\63.txt\n",
1714 |       "D:\\Panorama\\texts\\630.txt\n",
1715 |       "D:\\Panorama\\tagged_texts\\630.txt\n",
1716 |       "D:\\Panorama\\texts\\631.txt\n",
1717 |       "D:\\Panorama\\tagged_texts\\631.txt\n",
1718 |       "D:\\Panorama\\texts\\632.txt\n",
1719 |       "D:\\Panorama\\tagged_texts\\632.txt\n",
1720 |       "D:\\Panorama\\texts\\633.txt\n",
1721 |       "D:\\Panorama\\tagged_texts\\633.txt\n",
1722 |       "D:\\Panorama\\texts\\634.txt\n",
1723 |       "D:\\Panorama\\tagged_texts\\634.txt\n",
1724 |       "D:\\Panorama\\texts\\635.txt\n",
1725 |       "D:\\Panorama\\tagged_texts\\635.txt\n",
1726 |       "D:\\Panorama\\texts\\636.txt\n",
1727 |       "D:\\Panorama\\tagged_texts\\636.txt\n",
1728 |       "D:\\Panorama\\texts\\637.txt\n",
1729 |       "D:\\Panorama\\tagged_texts\\637.txt\n",
1730 |       "D:\\Panorama\\texts\\638.txt\n",
1731 |       "D:\\Panorama\\tagged_texts\\638.txt\n",
1732 |       "D:\\Panorama\\texts\\639.txt\n",
1733 |       "D:\\Panorama\\tagged_texts\\639.txt\n",
1734 |       "D:\\Panorama\\texts\\64.txt\n",
1735 |       "D:\\Panorama\\tagged_texts\\64.txt\n",
1736 |       "D:\\Panorama\\texts\\640.txt\n",
1737 |       "D:\\Panorama\\tagged_texts\\640.txt\n",
1738 |       "D:\\Panorama\\texts\\641.txt\n",
1739 |       "D:\\Panorama\\tagged_texts\\641.txt\n",
1740 |       "D:\\Panorama\\texts\\642.txt\n",
1741 |       "D:\\Panorama\\tagged_texts\\642.txt\n",
1742 |       "D:\\Panorama\\texts\\643.txt\n",
1743 |       "D:\\Panorama\\tagged_texts\\643.txt\n",
1744 |       "D:\\Panorama\\texts\\644.txt\n",
1745 |       "D:\\Panorama\\tagged_texts\\644.txt\n",
1746 |       "D:\\Panorama\\texts\\645.txt\n",
1747 |       "D:\\Panorama\\tagged_texts\\645.txt\n",
1748 |       "D:\\Panorama\\texts\\646.txt\n",
1749 |       "D:\\Panorama\\tagged_texts\\646.txt\n",
1750 |       "D:\\Panorama\\texts\\647.txt\n",
1751 |       "D:\\Panorama\\tagged_texts\\647.txt\n",
1752 |       "D:\\Panorama\\texts\\648.txt\n",
1753 |       "D:\\Panorama\\tagged_texts\\648.txt\n",
1754 |       "D:\\Panorama\\texts\\649.txt\n",
1755 |       "D:\\Panorama\\tagged_texts\\649.txt\n",
1756 |       "D:\\Panorama\\texts\\65.txt\n",
1757 |       "D:\\Panorama\\tagged_texts\\65.txt\n",
1758 |       "D:\\Panorama\\texts\\650.txt\n",
1759 |       "D:\\Panorama\\tagged_texts\\650.txt\n",
1760 |       "D:\\Panorama\\texts\\651.txt\n",
1761 |       "D:\\Panorama\\tagged_texts\\651.txt\n",
1762 |       "D:\\Panorama\\texts\\652.txt\n",
1763 |       "D:\\Panorama\\tagged_texts\\652.txt\n",
1764 |       "D:\\Panorama\\texts\\653.txt\n",
1765 |       "D:\\Panorama\\tagged_texts\\653.txt\n",
1766 |       "D:\\Panorama\\texts\\654.txt\n",
1767 |       "D:\\Panorama\\tagged_texts\\654.txt\n",
1768 |       "D:\\Panorama\\texts\\655.txt\n",
1769 |       "D:\\Panorama\\tagged_texts\\655.txt\n",
1770 |       "D:\\Panorama\\texts\\656.txt\n",
1771 |       "D:\\Panorama\\tagged_texts\\656.txt\n",
1772 |       "D:\\Panorama\\texts\\657.txt\n",
1773 |       "D:\\Panorama\\tagged_texts\\657.txt\n",
1774 |       "D:\\Panorama\\texts\\658.txt\n",
1775 |       "D:\\Panorama\\tagged_texts\\658.txt\n",
1776 |       "D:\\Panorama\\texts\\659.txt\n",
1777 |       "D:\\Panorama\\tagged_texts\\659.txt\n",
1778 |       "D:\\Panorama\\texts\\66.txt\n",
1779 |       "D:\\Panorama\\tagged_texts\\66.txt\n",
1780 |       "D:\\Panorama\\texts\\660.txt\n",
1781 |       "D:\\Panorama\\tagged_texts\\660.txt\n",
1782 |       "D:\\Panorama\\texts\\661.txt\n",
1783 |       "D:\\Panorama\\tagged_texts\\661.txt\n",
1784 |       "D:\\Panorama\\texts\\662.txt\n",
1785 |       "D:\\Panorama\\tagged_texts\\662.txt\n",
1786 |       "D:\\Panorama\\texts\\663.txt\n",
1787 |       "D:\\Panorama\\tagged_texts\\663.txt\n",
1788 |       "D:\\Panorama\\texts\\664.txt\n",
1789 |       "D:\\Panorama\\tagged_texts\\664.txt\n",
1790 |       "D:\\Panorama\\texts\\665.txt\n",
1791 |       "D:\\Panorama\\tagged_texts\\665.txt\n",
1792 |       "D:\\Panorama\\texts\\666.txt\n",
1793 |       "D:\\Panorama\\tagged_texts\\666.txt\n",
1794 |       "D:\\Panorama\\texts\\667.txt\n",
1795 |       "D:\\Panorama\\tagged_texts\\667.txt\n",
1796 |       "D:\\Panorama\\texts\\668.txt\n",
1797 |       "D:\\Panorama\\tagged_texts\\668.txt\n",
1798 |       "D:\\Panorama\\texts\\669.txt\n",
1799 |       "D:\\Panorama\\tagged_texts\\669.txt\n",
1800 |       "D:\\Panorama\\texts\\67.txt\n",
1801 |       "D:\\Panorama\\tagged_texts\\67.txt\n",
1802 |       "D:\\Panorama\\texts\\670.txt\n",
1803 |       "D:\\Panorama\\tagged_texts\\670.txt\n",
1804 |       "D:\\Panorama\\texts\\671.txt\n",
1805 |       "D:\\Panorama\\tagged_texts\\671.txt\n",
1806 |       "D:\\Panorama\\texts\\672.txt\n",
1807 |       "D:\\Panorama\\tagged_texts\\672.txt\n",
1808 |       "D:\\Panorama\\texts\\673.txt\n",
1809 |       "D:\\Panorama\\tagged_texts\\673.txt\n",
1810 |       "D:\\Panorama\\texts\\674.txt\n",
1811 |       "D:\\Panorama\\tagged_texts\\674.txt\n",
1812 |       "D:\\Panorama\\texts\\675.txt\n",
1813 |       "D:\\Panorama\\tagged_texts\\675.txt\n",
1814 |       "D:\\Panorama\\texts\\676.txt\n",
1815 |       "D:\\Panorama\\tagged_texts\\676.txt\n",
1816 |       "D:\\Panorama\\texts\\677.txt\n",
1817 |       "D:\\Panorama\\tagged_texts\\677.txt\n",
1818 |       "D:\\Panorama\\texts\\678.txt\n",
1819 |       "D:\\Panorama\\tagged_texts\\678.txt\n",
1820 |       "D:\\Panorama\\texts\\679.txt\n",
1821 |       "D:\\Panorama\\tagged_texts\\679.txt\n",
1822 |       "D:\\Panorama\\texts\\68.txt\n",
1823 |       "D:\\Panorama\\tagged_texts\\68.txt\n",
1824 |       "D:\\Panorama\\texts\\680.txt\n",
1825 |       "D:\\Panorama\\tagged_texts\\680.txt\n",
1826 |       "D:\\Panorama\\texts\\681.txt\n",
1827 |       "D:\\Panorama\\tagged_texts\\681.txt\n",
1828 |       "D:\\Panorama\\texts\\682.txt\n",
1829 |       "D:\\Panorama\\tagged_texts\\682.txt\n",
1830 |       "D:\\Panorama\\texts\\683.txt\n",
1831 |       "D:\\Panorama\\tagged_texts\\683.txt\n",
1832 |       "D:\\Panorama\\texts\\684.txt\n",
1833 |       "D:\\Panorama\\tagged_texts\\684.txt\n",
1834 |       "D:\\Panorama\\texts\\685.txt\n",
1835 |       "D:\\Panorama\\tagged_texts\\685.txt\n",
1836 |       "D:\\Panorama\\texts\\686.txt\n",
1837 |       "D:\\Panorama\\tagged_texts\\686.txt\n",
1838 |       "D:\\Panorama\\texts\\687.txt\n",
1839 |       "D:\\Panorama\\tagged_texts\\687.txt\n",
1840 |       "D:\\Panorama\\texts\\688.txt\n",
1841 |       "D:\\Panorama\\tagged_texts\\688.txt\n",
1842 |       "D:\\Panorama\\texts\\689.txt\n",
1843 |       "D:\\Panorama\\tagged_texts\\689.txt\n",
1844 |       "D:\\Panorama\\texts\\69.txt\n",
1845 |       "D:\\Panorama\\tagged_texts\\69.txt\n",
1846 |       "D:\\Panorama\\texts\\690.txt\n",
1847 |       "D:\\Panorama\\tagged_texts\\690.txt\n",
1848 |       "D:\\Panorama\\texts\\691.txt\n",
1849 |       "D:\\Panorama\\tagged_texts\\691.txt\n",
1850 |       "D:\\Panorama\\texts\\692.txt\n",
1851 |       "D:\\Panorama\\tagged_texts\\692.txt\n",
1852 |       "D:\\Panorama\\texts\\693.txt\n",
1853 |       "D:\\Panorama\\tagged_texts\\693.txt\n",
1854 |       "D:\\Panorama\\texts\\694.txt\n",
1855 |       "D:\\Panorama\\tagged_texts\\694.txt\n",
1856 |       "D:\\Panorama\\texts\\695.txt\n",
1857 |       "D:\\Panorama\\tagged_texts\\695.txt\n",
1858 |       "D:\\Panorama\\texts\\696.txt\n",
1859 |       "D:\\Panorama\\tagged_texts\\696.txt\n",
1860 |       "D:\\Panorama\\texts\\697.txt\n",
1861 |       "D:\\Panorama\\tagged_texts\\697.txt\n",
1862 |       "D:\\Panorama\\texts\\698.txt\n",
1863 |       "D:\\Panorama\\tagged_texts\\698.txt\n",
1864 |       "D:\\Panorama\\texts\\699.txt\n",
1865 |       "D:\\Panorama\\tagged_texts\\699.txt\n",
1866 |       "D:\\Panorama\\texts\\7.txt\n",
1867 |       "D:\\Panorama\\tagged_texts\\7.txt\n",
1868 |       "D:\\Panorama\\texts\\70.txt\n",
1869 |       "D:\\Panorama\\tagged_texts\\70.txt\n",
1870 |       "D:\\Panorama\\texts\\700.txt\n",
1871 |       "D:\\Panorama\\tagged_texts\\700.txt\n",
1872 |       "D:\\Panorama\\texts\\701.txt\n",
1873 |       "D:\\Panorama\\tagged_texts\\701.txt\n",
1874 |       "D:\\Panorama\\texts\\702.txt\n",
1875 |       "D:\\Panorama\\tagged_texts\\702.txt\n",
1876 |       "D:\\Panorama\\texts\\703.txt\n",
1877 |       "D:\\Panorama\\tagged_texts\\703.txt\n",
1878 |       "D:\\Panorama\\texts\\704.txt\n",
1879 |       "D:\\Panorama\\tagged_texts\\704.txt\n",
1880 |       "D:\\Panorama\\texts\\705.txt\n",
1881 |       "D:\\Panorama\\tagged_texts\\705.txt\n",
1882 |       "D:\\Panorama\\texts\\706.txt\n",
1883 |       "D:\\Panorama\\tagged_texts\\706.txt\n",
1884 |       "D:\\Panorama\\texts\\707.txt\n",
1885 |       "D:\\Panorama\\tagged_texts\\707.txt\n",
1886 |       "D:\\Panorama\\texts\\708.txt\n",
1887 |       "D:\\Panorama\\tagged_texts\\708.txt\n",
1888 |       "D:\\Panorama\\texts\\709.txt\n",
1889 |       "D:\\Panorama\\tagged_texts\\709.txt\n",
1890 |       "D:\\Panorama\\texts\\71.txt\n",
1891 |       "D:\\Panorama\\tagged_texts\\71.txt\n",
1892 |       "D:\\Panorama\\texts\\710.txt\n",
1893 |       "D:\\Panorama\\tagged_texts\\710.txt\n",
1894 |       "D:\\Panorama\\texts\\711.txt\n",
1895 |       "D:\\Panorama\\tagged_texts\\711.txt\n",
1896 |       "D:\\Panorama\\texts\\712.txt\n",
1897 |       "D:\\Panorama\\tagged_texts\\712.txt\n",
1898 |       "D:\\Panorama\\texts\\713.txt\n",
1899 |       "D:\\Panorama\\tagged_texts\\713.txt\n",
1900 |       "D:\\Panorama\\texts\\714.txt\n",
1901 |       "D:\\Panorama\\tagged_texts\\714.txt\n",
1902 |       "D:\\Panorama\\texts\\715.txt\n",
1903 |       "D:\\Panorama\\tagged_texts\\715.txt\n",
1904 |       "D:\\Panorama\\texts\\716.txt\n",
1905 |       "D:\\Panorama\\tagged_texts\\716.txt\n",
1906 |       "D:\\Panorama\\texts\\717.txt\n",
1907 |       "D:\\Panorama\\tagged_texts\\717.txt\n",
1908 |       "D:\\Panorama\\texts\\718.txt\n",
1909 |       "D:\\Panorama\\tagged_texts\\718.txt\n",
1910 |       "D:\\Panorama\\texts\\719.txt\n",
1911 |       "D:\\Panorama\\tagged_texts\\719.txt\n",
1912 |       "D:\\Panorama\\texts\\72.txt\n",
1913 |       "D:\\Panorama\\tagged_texts\\72.txt\n",
1914 |       "D:\\Panorama\\texts\\720.txt\n",
1915 |       "D:\\Panorama\\tagged_texts\\720.txt\n",
1916 |       "D:\\Panorama\\texts\\721.txt\n",
1917 |       "D:\\Panorama\\tagged_texts\\721.txt\n",
1918 |       "D:\\Panorama\\texts\\722.txt\n",
1919 |       "D:\\Panorama\\tagged_texts\\722.txt\n",
1920 |       "D:\\Panorama\\texts\\723.txt\n",
1921 |       "D:\\Panorama\\tagged_texts\\723.txt\n",
1922 |       "D:\\Panorama\\texts\\724.txt\n"
1923 |      ]
1924 |     },
1925 |     {
1926 |      "name": "stdout",
1927 |      "output_type": "stream",
1928 |      "text": [
1929 |       "D:\\Panorama\\tagged_texts\\724.txt\n",
1930 |       "D:\\Panorama\\texts\\725.txt\n",
1931 |       "D:\\Panorama\\tagged_texts\\725.txt\n",
1932 |       "D:\\Panorama\\texts\\726.txt\n",
1933 |       "D:\\Panorama\\tagged_texts\\726.txt\n",
1934 |       "D:\\Panorama\\texts\\727.txt\n",
1935 |       "D:\\Panorama\\tagged_texts\\727.txt\n",
1936 |       "D:\\Panorama\\texts\\728.txt\n",
1937 |       "D:\\Panorama\\tagged_texts\\728.txt\n",
1938 |       "D:\\Panorama\\texts\\729.txt\n",
1939 |       "D:\\Panorama\\tagged_texts\\729.txt\n",
1940 |       "D:\\Panorama\\texts\\73.txt\n",
1941 |       "D:\\Panorama\\tagged_texts\\73.txt\n",
1942 |       "D:\\Panorama\\texts\\730.txt\n",
1943 |       "D:\\Panorama\\tagged_texts\\730.txt\n",
1944 |       "D:\\Panorama\\texts\\731.txt\n",
1945 |       "D:\\Panorama\\tagged_texts\\731.txt\n",
1946 |       "D:\\Panorama\\texts\\732.txt\n",
1947 |       "D:\\Panorama\\tagged_texts\\732.txt\n",
1948 |       "D:\\Panorama\\texts\\733.txt\n",
1949 |       "D:\\Panorama\\tagged_texts\\733.txt\n",
1950 |       "D:\\Panorama\\texts\\734.txt\n",
1951 |       "D:\\Panorama\\tagged_texts\\734.txt\n",
1952 |       "D:\\Panorama\\texts\\735.txt\n",
1953 |       "D:\\Panorama\\tagged_texts\\735.txt\n",
1954 |       "D:\\Panorama\\texts\\736.txt\n",
1955 |       "D:\\Panorama\\tagged_texts\\736.txt\n",
1956 |       "D:\\Panorama\\texts\\737.txt\n",
1957 |       "D:\\Panorama\\tagged_texts\\737.txt\n",
1958 |       "D:\\Panorama\\texts\\738.txt\n",
1959 |       "D:\\Panorama\\tagged_texts\\738.txt\n",
1960 |       "D:\\Panorama\\texts\\739.txt\n",
1961 |       "D:\\Panorama\\tagged_texts\\739.txt\n",
1962 |       "D:\\Panorama\\texts\\74.txt\n",
1963 |       "D:\\Panorama\\tagged_texts\\74.txt\n",
1964 |       "D:\\Panorama\\texts\\740.txt\n",
1965 |       "D:\\Panorama\\tagged_texts\\740.txt\n",
1966 |       "D:\\Panorama\\texts\\741.txt\n",
1967 |       "D:\\Panorama\\tagged_texts\\741.txt\n",
1968 |       "D:\\Panorama\\texts\\742.txt\n",
1969 |       "D:\\Panorama\\tagged_texts\\742.txt\n",
1970 |       "D:\\Panorama\\texts\\743.txt\n",
1971 |       "D:\\Panorama\\tagged_texts\\743.txt\n",
1972 |       "D:\\Panorama\\texts\\744.txt\n",
1973 |       "D:\\Panorama\\tagged_texts\\744.txt\n",
1974 |       "D:\\Panorama\\texts\\745.txt\n",
1975 |       "D:\\Panorama\\tagged_texts\\745.txt\n",
1976 |       "D:\\Panorama\\texts\\746.txt\n",
1977 |       "D:\\Panorama\\tagged_texts\\746.txt\n",
1978 |       "D:\\Panorama\\texts\\747.txt\n",
1979 |       "D:\\Panorama\\tagged_texts\\747.txt\n",
1980 |       "D:\\Panorama\\texts\\748.txt\n",
1981 |       "D:\\Panorama\\tagged_texts\\748.txt\n",
1982 |       "D:\\Panorama\\texts\\749.txt\n",
1983 |       "D:\\Panorama\\tagged_texts\\749.txt\n",
1984 |       "D:\\Panorama\\texts\\75.txt\n",
1985 |       "D:\\Panorama\\tagged_texts\\75.txt\n",
1986 |       "D:\\Panorama\\texts\\750.txt\n",
1987 |       "D:\\Panorama\\tagged_texts\\750.txt\n",
1988 |       "D:\\Panorama\\texts\\751.txt\n",
1989 |       "D:\\Panorama\\tagged_texts\\751.txt\n",
1990 |       "D:\\Panorama\\texts\\752.txt\n",
1991 |       "D:\\Panorama\\tagged_texts\\752.txt\n",
1992 |       "D:\\Panorama\\texts\\753.txt\n",
1993 |       "D:\\Panorama\\tagged_texts\\753.txt\n",
1994 |       "D:\\Panorama\\texts\\754.txt\n",
1995 |       "D:\\Panorama\\tagged_texts\\754.txt\n",
1996 |       "D:\\Panorama\\texts\\755.txt\n",
1997 |       "D:\\Panorama\\tagged_texts\\755.txt\n",
1998 |       "D:\\Panorama\\texts\\756.txt\n",
1999 |       "D:\\Panorama\\tagged_texts\\756.txt\n",
2000 |       "D:\\Panorama\\texts\\757.txt\n",
2001 |       "D:\\Panorama\\tagged_texts\\757.txt\n",
2002 |       "D:\\Panorama\\texts\\758.txt\n",
2003 |       "D:\\Panorama\\tagged_texts\\758.txt\n",
2004 |       "D:\\Panorama\\texts\\759.txt\n",
2005 |       "D:\\Panorama\\tagged_texts\\759.txt\n",
2006 |       "D:\\Panorama\\texts\\76.txt\n",
2007 |       "D:\\Panorama\\tagged_texts\\76.txt\n",
2008 |       "D:\\Panorama\\texts\\760.txt\n",
2009 |       "D:\\Panorama\\tagged_texts\\760.txt\n",
2010 |       "D:\\Panorama\\texts\\761.txt\n",
2011 |       "D:\\Panorama\\tagged_texts\\761.txt\n",
2012 |       "D:\\Panorama\\texts\\762.txt\n",
2013 |       "D:\\Panorama\\tagged_texts\\762.txt\n",
2014 |       "D:\\Panorama\\texts\\763.txt\n",
2015 |       "D:\\Panorama\\tagged_texts\\763.txt\n",
2016 |       "D:\\Panorama\\texts\\764.txt\n",
2017 |       "D:\\Panorama\\tagged_texts\\764.txt\n",
2018 |       "D:\\Panorama\\texts\\765.txt\n",
2019 |       "D:\\Panorama\\tagged_texts\\765.txt\n",
2020 |       "D:\\Panorama\\texts\\766.txt\n",
2021 |       "D:\\Panorama\\tagged_texts\\766.txt\n",
2022 |       "D:\\Panorama\\texts\\767.txt\n",
2023 |       "D:\\Panorama\\tagged_texts\\767.txt\n",
2024 |       "D:\\Panorama\\texts\\768.txt\n",
2025 |       "D:\\Panorama\\tagged_texts\\768.txt\n",
2026 |       "D:\\Panorama\\texts\\769.txt\n",
2027 |       "D:\\Panorama\\tagged_texts\\769.txt\n",
2028 |       "D:\\Panorama\\texts\\77.txt\n",
2029 |       "D:\\Panorama\\tagged_texts\\77.txt\n",
2030 |       "D:\\Panorama\\texts\\770.txt\n",
2031 |       "D:\\Panorama\\tagged_texts\\770.txt\n",
2032 |       "D:\\Panorama\\texts\\771.txt\n",
2033 |       "D:\\Panorama\\tagged_texts\\771.txt\n",
2034 |       "D:\\Panorama\\texts\\772.txt\n",
2035 |       "D:\\Panorama\\tagged_texts\\772.txt\n",
2036 |       "D:\\Panorama\\texts\\773.txt\n",
2037 |       "D:\\Panorama\\tagged_texts\\773.txt\n",
2038 |       "D:\\Panorama\\texts\\774.txt\n",
2039 |       "D:\\Panorama\\tagged_texts\\774.txt\n",
2040 |       "D:\\Panorama\\texts\\775.txt\n",
2041 |       "D:\\Panorama\\tagged_texts\\775.txt\n",
2042 |       "D:\\Panorama\\texts\\776.txt\n",
2043 |       "D:\\Panorama\\tagged_texts\\776.txt\n",
2044 |       "D:\\Panorama\\texts\\777.txt\n",
2045 |       "D:\\Panorama\\tagged_texts\\777.txt\n",
2046 |       "D:\\Panorama\\texts\\778.txt\n",
2047 |       "D:\\Panorama\\tagged_texts\\778.txt\n",
2048 |       "D:\\Panorama\\texts\\779.txt\n",
2049 |       "D:\\Panorama\\tagged_texts\\779.txt\n",
2050 |       "D:\\Panorama\\texts\\78.txt\n",
2051 |       "D:\\Panorama\\tagged_texts\\78.txt\n",
2052 |       "D:\\Panorama\\texts\\780.txt\n",
2053 |       "D:\\Panorama\\tagged_texts\\780.txt\n",
2054 |       "D:\\Panorama\\texts\\781.txt\n",
2055 |       "D:\\Panorama\\tagged_texts\\781.txt\n",
2056 |       "D:\\Panorama\\texts\\782.txt\n",
2057 |       "D:\\Panorama\\tagged_texts\\782.txt\n",
2058 |       "D:\\Panorama\\texts\\783.txt\n",
2059 |       "D:\\Panorama\\tagged_texts\\783.txt\n",
2060 |       "D:\\Panorama\\texts\\784.txt\n",
2061 |       "D:\\Panorama\\tagged_texts\\784.txt\n",
2062 |       "D:\\Panorama\\texts\\785.txt\n",
2063 |       "D:\\Panorama\\tagged_texts\\785.txt\n",
2064 |       "D:\\Panorama\\texts\\786.txt\n",
2065 |       "D:\\Panorama\\tagged_texts\\786.txt\n",
2066 |       "D:\\Panorama\\texts\\787.txt\n",
2067 |       "D:\\Panorama\\tagged_texts\\787.txt\n",
2068 |       "D:\\Panorama\\texts\\788.txt\n",
2069 |       "D:\\Panorama\\tagged_texts\\788.txt\n",
2070 |       "D:\\Panorama\\texts\\789.txt\n",
2071 |       "D:\\Panorama\\tagged_texts\\789.txt\n",
2072 |       "D:\\Panorama\\texts\\79.txt\n",
2073 |       "D:\\Panorama\\tagged_texts\\79.txt\n",
2074 |       "D:\\Panorama\\texts\\790.txt\n",
2075 |       "D:\\Panorama\\tagged_texts\\790.txt\n",
2076 |       "D:\\Panorama\\texts\\791.txt\n",
2077 |       "D:\\Panorama\\tagged_texts\\791.txt\n",
2078 |       "D:\\Panorama\\texts\\792.txt\n",
2079 |       "D:\\Panorama\\tagged_texts\\792.txt\n",
2080 |       "D:\\Panorama\\texts\\793.txt\n",
2081 |       "D:\\Panorama\\tagged_texts\\793.txt\n",
2082 |       "D:\\Panorama\\texts\\794.txt\n",
2083 |       "D:\\Panorama\\tagged_texts\\794.txt\n",
2084 |       "D:\\Panorama\\texts\\795.txt\n",
2085 |       "D:\\Panorama\\tagged_texts\\795.txt\n",
2086 |       "D:\\Panorama\\texts\\796.txt\n",
2087 |       "D:\\Panorama\\tagged_texts\\796.txt\n",
2088 |       "D:\\Panorama\\texts\\797.txt\n",
2089 |       "D:\\Panorama\\tagged_texts\\797.txt\n",
2090 |       "D:\\Panorama\\texts\\798.txt\n",
2091 |       "D:\\Panorama\\tagged_texts\\798.txt\n",
2092 |       "D:\\Panorama\\texts\\799.txt\n",
2093 |       "D:\\Panorama\\tagged_texts\\799.txt\n",
2094 |       "D:\\Panorama\\texts\\8.txt\n",
2095 |       "D:\\Panorama\\tagged_texts\\8.txt\n",
2096 |       "D:\\Panorama\\texts\\80.txt\n",
2097 |       "D:\\Panorama\\tagged_texts\\80.txt\n",
2098 |       "D:\\Panorama\\texts\\800.txt\n",
2099 |       "D:\\Panorama\\tagged_texts\\800.txt\n",
2100 |       "D:\\Panorama\\texts\\801.txt\n",
2101 |       "D:\\Panorama\\tagged_texts\\801.txt\n",
2102 |       "D:\\Panorama\\texts\\802.txt\n",
2103 |       "D:\\Panorama\\tagged_texts\\802.txt\n",
2104 |       "D:\\Panorama\\texts\\803.txt\n",
2105 |       "D:\\Panorama\\tagged_texts\\803.txt\n",
2106 |       "D:\\Panorama\\texts\\804.txt\n",
2107 |       "D:\\Panorama\\tagged_texts\\804.txt\n",
2108 |       "D:\\Panorama\\texts\\805.txt\n",
2109 |       "D:\\Panorama\\tagged_texts\\805.txt\n",
2110 |       "D:\\Panorama\\texts\\806.txt\n",
2111 |       "D:\\Panorama\\tagged_texts\\806.txt\n",
2112 |       "D:\\Panorama\\texts\\807.txt\n",
2113 |       "D:\\Panorama\\tagged_texts\\807.txt\n",
2114 |       "D:\\Panorama\\texts\\808.txt\n",
2115 |       "D:\\Panorama\\tagged_texts\\808.txt\n",
2116 |       "D:\\Panorama\\texts\\809.txt\n",
2117 |       "D:\\Panorama\\tagged_texts\\809.txt\n",
2118 |       "D:\\Panorama\\texts\\81.txt\n",
2119 |       "D:\\Panorama\\tagged_texts\\81.txt\n",
2120 |       "D:\\Panorama\\texts\\810.txt\n",
2121 |       "D:\\Panorama\\tagged_texts\\810.txt\n",
2122 |       "D:\\Panorama\\texts\\811.txt\n",
2123 |       "D:\\Panorama\\tagged_texts\\811.txt\n",
2124 |       "D:\\Panorama\\texts\\812.txt\n",
2125 |       "D:\\Panorama\\tagged_texts\\812.txt\n",
2126 |       "D:\\Panorama\\texts\\813.txt\n",
2127 |       "D:\\Panorama\\tagged_texts\\813.txt\n",
2128 |       "D:\\Panorama\\texts\\814.txt\n",
2129 |       "D:\\Panorama\\tagged_texts\\814.txt\n",
2130 |       "D:\\Panorama\\texts\\815.txt\n",
2131 |       "D:\\Panorama\\tagged_texts\\815.txt\n",
2132 |       "D:\\Panorama\\texts\\816.txt\n",
2133 |       "D:\\Panorama\\tagged_texts\\816.txt\n",
2134 |       "D:\\Panorama\\texts\\817.txt\n",
2135 |       "D:\\Panorama\\tagged_texts\\817.txt\n",
2136 |       "D:\\Panorama\\texts\\818.txt\n",
2137 |       "D:\\Panorama\\tagged_texts\\818.txt\n",
2138 |       "D:\\Panorama\\texts\\819.txt\n",
2139 |       "D:\\Panorama\\tagged_texts\\819.txt\n",
2140 |       "D:\\Panorama\\texts\\82.txt\n",
2141 |       "D:\\Panorama\\tagged_texts\\82.txt\n",
2142 |       "D:\\Panorama\\texts\\820.txt\n",
2143 |       "D:\\Panorama\\tagged_texts\\820.txt\n",
2144 |       "D:\\Panorama\\texts\\821.txt\n",
2145 |       "D:\\Panorama\\tagged_texts\\821.txt\n",
2146 |       "D:\\Panorama\\texts\\822.txt\n",
2147 |       "D:\\Panorama\\tagged_texts\\822.txt\n",
2148 |       "D:\\Panorama\\texts\\823.txt\n",
2149 |       "D:\\Panorama\\tagged_texts\\823.txt\n",
2150 |       "D:\\Panorama\\texts\\824.txt\n",
2151 |       "D:\\Panorama\\tagged_texts\\824.txt\n",
2152 |       "D:\\Panorama\\texts\\825.txt\n",
2153 |       "D:\\Panorama\\tagged_texts\\825.txt\n",
2154 |       "D:\\Panorama\\texts\\826.txt\n",
2155 |       "D:\\Panorama\\tagged_texts\\826.txt\n",
2156 |       "D:\\Panorama\\texts\\827.txt\n",
2157 |       "D:\\Panorama\\tagged_texts\\827.txt\n",
2158 |       "D:\\Panorama\\texts\\828.txt\n",
2159 |       "D:\\Panorama\\tagged_texts\\828.txt\n",
2160 |       "D:\\Panorama\\texts\\829.txt\n",
2161 |       "D:\\Panorama\\tagged_texts\\829.txt\n",
2162 |       "D:\\Panorama\\texts\\83.txt\n",
2163 |       "D:\\Panorama\\tagged_texts\\83.txt\n",
2164 |       "D:\\Panorama\\texts\\830.txt\n",
2165 |       "D:\\Panorama\\tagged_texts\\830.txt\n",
2166 |       "D:\\Panorama\\texts\\831.txt\n",
2167 |       "D:\\Panorama\\tagged_texts\\831.txt\n",
2168 |       "D:\\Panorama\\texts\\832.txt\n",
2169 |       "D:\\Panorama\\tagged_texts\\832.txt\n",
2170 |       "D:\\Panorama\\texts\\833.txt\n",
2171 |       "D:\\Panorama\\tagged_texts\\833.txt\n",
2172 |       "D:\\Panorama\\texts\\84.txt\n",
2173 |       "D:\\Panorama\\tagged_texts\\84.txt\n",
2174 |       "D:\\Panorama\\texts\\85.txt\n",
2175 |       "D:\\Panorama\\tagged_texts\\85.txt\n",
2176 |       "D:\\Panorama\\texts\\86.txt\n",
2177 |       "D:\\Panorama\\tagged_texts\\86.txt\n",
2178 |       "D:\\Panorama\\texts\\87.txt\n",
2179 |       "D:\\Panorama\\tagged_texts\\87.txt\n",
2180 |       "D:\\Panorama\\texts\\88.txt\n",
2181 |       "D:\\Panorama\\tagged_texts\\88.txt\n",
2182 |       "D:\\Panorama\\texts\\89.txt\n",
2183 |       "D:\\Panorama\\tagged_texts\\89.txt\n",
2184 |       "D:\\Panorama\\texts\\9.txt\n",
2185 |       "D:\\Panorama\\tagged_texts\\9.txt\n",
2186 |       "D:\\Panorama\\texts\\90.txt\n",
2187 |       "D:\\Panorama\\tagged_texts\\90.txt\n",
2188 |       "D:\\Panorama\\texts\\91.txt\n",
2189 |       "D:\\Panorama\\tagged_texts\\91.txt\n",
2190 |       "D:\\Panorama\\texts\\92.txt\n",
2191 |       "D:\\Panorama\\tagged_texts\\92.txt\n",
2192 |       "D:\\Panorama\\texts\\93.txt\n",
2193 |       "D:\\Panorama\\tagged_texts\\93.txt\n",
2194 |       "D:\\Panorama\\texts\\94.txt\n",
2195 |       "D:\\Panorama\\tagged_texts\\94.txt\n",
2196 |       "D:\\Panorama\\texts\\95.txt\n",
2197 |       "D:\\Panorama\\tagged_texts\\95.txt\n",
2198 |       "D:\\Panorama\\texts\\96.txt\n",
2199 |       "D:\\Panorama\\tagged_texts\\96.txt\n",
2200 |       "D:\\Panorama\\texts\\97.txt\n",
2201 |       "D:\\Panorama\\tagged_texts\\97.txt\n",
2202 |       "D:\\Panorama\\texts\\98.txt\n",
2203 |       "D:\\Panorama\\tagged_texts\\98.txt\n",
2204 |       "D:\\Panorama\\texts\\99.txt\n",
2205 |       "D:\\Panorama\\tagged_texts\\99.txt\n"
2206 |      ]
2207 |     },
2208 |     {
2209 |      "name": "stderr",
2210 |      "output_type": "stream",
2211 |      "text": [
2212 |       "1it [01:33, 93.66s/it]\n"
2213 |      ]
2214 |     }
2215 |    ],
2216 |    "source": [
2217 |     "# coding: utf-8\n",
2218 |     "\n",
2219 |     "from numpy import median\n",
2220 |     "\n",
2221 |     "from nltk import word_tokenize, sent_tokenize\n",
2222 |     "import os\n",
2223 |     "import re\n",
2224 |     "import unify\n",
2225 |     "from tqdm import tqdm\n",
2226 |     "import ufal.udpipe\n",
2227 |     "global model\n",
2228 |     "\n",
2229 |     "def ensure_dir(directory):\n",
2230 |     "    \n",
2231 |     "    if not os.path.exists(directory):\n",
2232 |     "        os.makedirs(directory)\n",
2233 |     "    return directory\n",
2234 |     "\n",
2235 |     "def tag(text, model):\n",
2236 |     "    tokenizer.setText(text)\n",
2237 |     "    tokenizer.nextSentence(sentence, error)\n",
2238 |     "    model.tag(sentence, model.DEFAULT)\n",
2239 |     "    model.parse(sentence, model.DEFAULT)\n",
2240 |     "    return conlluOutput.writeSentence(sentence)\n",
2241 |     "\n",
2242 |     "model = ufal.udpipe.Model.load(r'D:\\udpipe\\models\\syntagrus-default.udpipe')\n",
2243 |     "\n",
2244 |     "tokenizer = model.newTokenizer(model.DEFAULT)\n",
2245 |     "conlluOutput = ufal.udpipe.OutputFormat.newOutputFormat(\"conllu\")\n",
2246 |     "sentence = ufal.udpipe.Sentence()\n",
2247 |     "error = ufal.udpipe.ProcessingError()\n",
2248 |     "\n",
2249 |     "\n",
2250 |     "WDIR = r'D:\\Panorama' \n",
2251 |     "wallpath = os.path.join(WDIR, 'texts')\n",
2252 |     "taggedpath = ensure_dir(os.path.join(WDIR, 'tagged_texts'))\n",
2253 |     "\n",
2254 |     "\n",
2255 |     "for path, subdirs, files in tqdm(os.walk(wallpath)):\n",
2256 |     "    for name in files:\n",
2257 |     "        file = os.path.join(path, name)\n",
2258 |     "        \n",
2259 |     "        if r\".txt\" in file:\n",
2260 |     "            print(file)\n",
2261 |     "            f = open( file, 'r', encoding='utf8').read()\n",
2262 |     "            newpath = ensure_dir(re.sub('texts', 'tagged_texts', path))\n",
2263 |     "            print(os.path.join(newpath, name))\n",
2264 |     "            out = open(os.path.join(newpath, name), 'w', encoding='utf8')\n",
2265 |     "            text = unify.unify_sym(f)\n",
2266 |     "            slist = sent_tokenize(text)\n",
2267 |     "            for s in slist:\n",
2268 |     "                s = tag(s, model)\n",
2269 |     "                out.write(s)\n",
2270 |     "            out.close()\n",
2271 |     "\n"
2272 |    ]
2273 |   },
2274 |   {
2275 |    "cell_type": "code",
2276 |    "execution_count": null,
2277 |    "metadata": {
2278 |     "collapsed": true
2279 |    },
2280 |    "outputs": [],
2281 |    "source": []
2282 |   }
2283 |  ],
2284 |  "metadata": {
2285 |   "kernelspec": {
2286 |    "display_name": "Python 3",
2287 |    "language": "python",
2288 |    "name": "python3"
2289 |   },
2290 |   "language_info": {
2291 |    "codemirror_mode": {
2292 |     "name": "ipython",
2293 |     "version": 3
2294 |    },
2295 |    "file_extension": ".py",
2296 |    "mimetype": "text/x-python",
2297 |    "name": "python",
2298 |    "nbconvert_exporter": "python",
2299 |    "pygments_lexer": "ipython3",
2300 |    "version": "3.6.1"
2301 |   }
2302 |  },
2303 |  "nbformat": 4,
2304 |  "nbformat_minor": 2
2305 | }
2306 | 


--------------------------------------------------------------------------------