├── .gitignore ├── README.md ├── celery_main ├── __init__.py ├── celery.py ├── task_receiver.py └── task_submitter.py ├── docker-compose.yml ├── dockerfile ├── handles.txt ├── proxy.py ├── requirements.txt └── user_agents.py /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.pyc 2 | .idea 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Scraping tweets quickly using celery, RabbitMQ and Docker cluster with Rotating proxy 2 | ------------------------- 3 | Please go through the below article for better understanding: 4 | 5 | https://www.pythoncircle.com/post/518/scraping-10000-tweets-in-60-seconds-using-celery-rabbitmq-and-docker-cluster-with-rotating-proxy/ 6 | 7 | ------------------------- 8 | 9 | ##### Version 10 | 11 | - Docker-compose : `docker-compose version 1.8.0, build unknown` 12 | - Docker : `Docker version 17.12.0-ce, build c97c6d6` 13 | ----------------------- 14 | 15 | 16 | ##### How to run 17 | 18 | - Run command `sudo docker-compose up` 19 | 20 | - Above command will start 1 container for each worker and rabbit 21 | 22 | - Now go inside one worker container and run `python -m celery_main.task_submitter` 23 | 24 | - this will start pushing tasks in rabitmq and workers 25 | --------------------- 26 | 27 | 28 | ##### Speedup the process 29 | 30 | - To run 5 workers and 1 rabbitmq: 31 | 32 | `sudo docker-compose up --scale worker=5` 33 | 34 | - Do not increase concurrency to too much in dockerfile as machine might not be able to handle it 35 | 36 | `ENTRYPOINT celery -A test_celery worker --concurrency=10 --loglevel=info` 37 | 38 | ------------------------------------------- 39 | 40 | ##### using host postgres from docker container 41 | - for allowing connection from anywhere: 42 | 43 | in /etc/postgres/ postgres.conf file , update listen_addresses set it to '*' to listed from all IPs 44 | 45 | - For authorizing any user from docker IPs, in pg_hba.conf file add line: 46 | 47 | `host all all 172.19.0.0/16 trust` 48 | 49 | 172.19.0.0/16 is the range of docker container. 50 | 51 | - restart the postgres: `sudo /etc/init.d/postgres restart` 52 | 53 | - Host for DB config would be your local machine IP. Change in config file. 54 | 55 | 56 | -------------------------------------------------------------------------------- /celery_main/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /celery_main/celery.py: -------------------------------------------------------------------------------- 1 | from celery import Celery 2 | 3 | app = Celery( 4 | 'celery_main', 5 | broker='amqp://myuser:mypassword@rabbit:5672', 6 | backend='rpc://', 7 | include=['celery_main.task_receiver'] 8 | ) 9 | -------------------------------------------------------------------------------- /celery_main/task_receiver.py: -------------------------------------------------------------------------------- 1 | from celery_main.celery import app 2 | import proxy 3 | from bs4 import BeautifulSoup 4 | 5 | 6 | @app.task(bind=True,default_retry_delay=10) 7 | def do_work(self, handle): 8 | print('handle received ' + handle) 9 | url = "https://twitter.com/" + handle 10 | session = proxy.get_session() 11 | response = session.get(url, timeout=5) 12 | print("-- STATUS " + str(response.status_code) + " -- " + url) 13 | if response.status_code == 200: 14 | parse_tweets(response, handle) 15 | 16 | 17 | def parse_tweets(response, handle): 18 | soup = BeautifulSoup(response.text, 'lxml') 19 | tweets_list = list() 20 | tweets = soup.find_all("li", {"data-item-type": "tweet"}) 21 | for tweet in tweets: 22 | tweets_list.append(get_tweet_text(tweet)) 23 | 24 | print(str(len(tweets_list)) + " tweets found.") 25 | # save to DB or to files 26 | 27 | 28 | def get_tweet_text(tweet): 29 | try: 30 | tweet_text_box = tweet.find("p", {"class": "TweetTextSize TweetTextSize--normal js-tweet-text tweet-text"}) 31 | images_in_tweet_tag = tweet_text_box.find_all("a", {"class": "twitter-timeline-link u-hidden"}) 32 | tweet_text = tweet_text_box.text 33 | for image_in_tweet_tag in images_in_tweet_tag: 34 | tweet_text = tweet_text.replace(image_in_tweet_tag.text, '') 35 | return tweet_text 36 | except Exception as e: 37 | return None 38 | -------------------------------------------------------------------------------- /celery_main/task_submitter.py: -------------------------------------------------------------------------------- 1 | from .task_receiver import do_work 2 | 3 | 4 | def submit_handles(handles): 5 | for handle in handles: 6 | handle = handle.strip() 7 | handle = handle.strip(".!,") 8 | do_work.delay(handle) 9 | print("submitted " + handle) 10 | 11 | 12 | if __name__ == '__main__': 13 | handles = list() 14 | with open("handles.txt", "r") as f: 15 | handles = f.readlines() 16 | 17 | submit_handles(handles) 18 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | worker: 4 | build: 5 | context: . 6 | dockerfile: dockerfile 7 | volumes: 8 | - type: bind 9 | source: . 10 | target: /app 11 | links: 12 | - rabbit 13 | depends_on: 14 | - rabbit 15 | rabbit: 16 | hostname: rabbit 17 | image: rabbitmq:latest 18 | environment: 19 | - RABBITMQ_DEFAULT_USER=myuser 20 | - RABBITMQ_DEFAULT_PASS=mypassword 21 | ports: 22 | - "5672:5672" 23 | rproxy: 24 | hostname: rproxy 25 | image: anuragrana/rotating-proxy 26 | environment: 27 | - tors=25 28 | ports: 29 | - "5566:5566" 30 | - "4444:4444" 31 | -------------------------------------------------------------------------------- /dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | ADD requirements.txt /app/requirements.txt 3 | ADD ./celery_main/ /app/ 4 | WORKDIR /app/ 5 | RUN pip install -r requirements.txt 6 | ENTRYPOINT celery -A celery_main worker --concurrency=5 --loglevel=info 7 | 8 | -------------------------------------------------------------------------------- /handles.txt: -------------------------------------------------------------------------------- 1 | AmexOffers 2 | notiven 3 | acmc_clock_euro 4 | Favstar_Bot 5 | urbandictionary 6 | narendramodi 7 | PMOIndia 8 | AnthonyBrownMD4 9 | AustinScottGA08 10 | BennieGThompson 11 | BettyMcCollum04 12 | BillPascrell 13 | BobbyScott 14 | boblatta 15 | BradSherman 16 | brettguthrie 17 | Call_Me_Dutch 18 | cathymcmorris 19 | chelliepingree 20 | Clyburn 21 | ConawayTX11 22 | congbillposey 23 | CongBoyle 24 | CongCulberson 25 | CongMikeSimpson 26 | CongPalazzo 27 | CongressmanGT 28 | CongressmanHice 29 | CongressmanRaja 30 | CongressmanRuiz 31 | DanaRohrabacher 32 | DarrellIssa 33 | daveloebsack 34 | davereichert 35 | DavidRouzer 36 | DesJarlaisTN04 37 | DonaldNorcross 38 | DorisMatsui 39 | DrNealDunnFL2 40 | DrPhilRoe 41 | EleanorNorton 42 | farenthold 43 | FrankPallone 44 | GerryConnolly 45 | GKButterfield 46 | GOPLeader 47 | gracenapolitano 48 | GreggHarper 49 | GregoryMeeks 50 | HerreraBeutler 51 | HurdOnTheHill 52 | JacksonLeeTX18 53 | jahimes 54 | janschakowsky 55 | jasoninthehouse 56 | JeffFortenberry 57 | Jim_Jordan 58 | JimLangevin 59 | JimPressOffice 60 | JoaquinCastrotx 61 | JudgeCarter 62 | JudgeTedPoe 63 | JuliaBrownley26 64 | justinamash 65 | keithellison 66 | KeithRothfus 67 | KenCalvert 68 | KYComer 69 | LamarSmithTX21 70 | louiseslaughter 71 | MacTXPress 72 | MarioDB 73 | MarkAmodeiNV2 74 | MarshaBlackburn 75 | MaxineWaters 76 | michaelcburgess 77 | MikeKellyPA 78 | NancyPelosi 79 | nikiinthehouse 80 | NitaLowey 81 | NormaJTorres 82 | NydiaVelazquez 83 | PatrickMcHenry 84 | PatTiberi 85 | PeterRoskam 86 | PeterWelch 87 | PeteSessions 88 | Raul_Labrador 89 | RepAbraham 90 | RepAdams 91 | RepAdamSchiff 92 | RepAdamSmith 93 | RepAdrianSmith 94 | RepAlexMooney 95 | RepAlGreen 96 | RepAlLawsonJr 97 | RepAmata 98 | RepAndreCarson 99 | RepAndyBarr 100 | RepAndyBiggsAZ 101 | RepAndyHarrisMD 102 | RepAnnaEshoo 103 | RepAnnieKuster 104 | RepAnnWagner 105 | RepArrington 106 | RepBarbaraLee 107 | RepBarragan 108 | RepBeatty 109 | RepBecerra 110 | RepBera 111 | RepBetoORourke 112 | RepBillFlores 113 | RepBillFoster 114 | RepBillJohnson 115 | RepBillShuster 116 | RepBlainePress 117 | repblumenauer 118 | RepBobbyRush 119 | RepBobGibbs 120 | RepBonamici 121 | RepBonnie 122 | RepBost 123 | RepBradWenstrup 124 | RepBrady 125 | RepBrianBabin 126 | RepBrianFitz 127 | RepBrianHiggins 128 | RepBrianMast 129 | RepBRochester 130 | RepBuddyCarter 131 | RepByrne 132 | RepCarbajal 133 | RepCardenas 134 | RepCartwright 135 | RepCharlieCrist 136 | RepCharlieDent 137 | RepCheri 138 | RepChrisCollins 139 | RepChrisSmith 140 | RepChrisStewart 141 | RepChuck 142 | RepCicilline 143 | repcleaver 144 | RepCohen 145 | RepComstock 146 | RepCuellar 147 | RepCummings 148 | RepCurbelo 149 | RepDanDonovan 150 | RepDanKildee 151 | RepDannyDavis 152 | RepDarrenSoto 153 | RepDaveBrat 154 | RepDaveJoyce 155 | repdavetrott 156 | RepDavid 157 | RepDavidEPrice 158 | repdavidkustoff 159 | repdavidscott 160 | RepDavidValadao 161 | RepDavidYoung 162 | RepDebDingell 163 | RepDelBene 164 | RepDennisRoss 165 | RepDennyHeck 166 | RepDerekKilmer 167 | RepDeSantis 168 | RepDeSaulnier 169 | Rep_DevinNunes 170 | RepDianaDeGette 171 | RepDianeBlack 172 | repdinatitus 173 | RepDLamborn 174 | RepDonaldPayne 175 | RepDonBacon 176 | RepDonBeyer 177 | repdonyoung 178 | RepDougCollins 179 | RepDrewFerguson 180 | RepDwightEvans 181 | RepDWStweets 182 | RepEBJ 183 | RepEdRoyce 184 | RepEliotEngel 185 | RepErikPaulsen 186 | RepEspaillat 187 | RepEsty 188 | RepEvanJenkins 189 | RepFilemonVela 190 | RepFrankLucas 191 | RepFredUpton 192 | RepFrenchHill 193 | RepGallagher 194 | RepGaramendi 195 | RepGarretGraves 196 | RepGeneGreen 197 | RepGonzalez 198 | RepGoodlatte 199 | RepGosar 200 | RepGraceMeng 201 | repgregwalden 202 | RepGrothman 203 | RepGusBilirakis 204 | RepGutierrez 205 | RepGwenMoore 206 | RepHalRogers 207 | RepHanabusa 208 | RepHankJohnson 209 | RepHartzler 210 | RepHastingsFL 211 | RepHensarling 212 | RepHolding 213 | RepHuffman 214 | RepHuizenga 215 | RepHultgren 216 | Rep_Hunter 217 | RepJackBergman 218 | RepJackyRosen 219 | RepJaredPolis 220 | RepJasonLewis 221 | RepJasonSmith 222 | RepJayapal 223 | RepJBridenstine 224 | RepJeffDenham 225 | RepJeffDuncan 226 | RepJeffries 227 | RepJenniffer 228 | RepJerryNadler 229 | RepJimBanks 230 | repjimcooper 231 | RepJimCosta 232 | RepJimmyPanetta 233 | RepJimRenacci 234 | RepJoeBarton 235 | RepJoeCourtney 236 | repjoecrowley 237 | RepJoeKennedy 238 | RepJoeWilson 239 | RepJohnConyers 240 | RepJohnDelaney 241 | RepJohnDuncanJr 242 | RepJohnFaso 243 | RepJohnKatko 244 | RepJohnLarson 245 | repjohnlewis 246 | RepJohnYarmuth 247 | RepJoseSerrano 248 | RepJoshG 249 | RepJuanVargas 250 | RepJudyChu 251 | RepKarenBass 252 | RepKathleenRice 253 | RepKayGranger 254 | RepKClark 255 | RepKenBuck 256 | RepKenMarchant 257 | RepKevinBrady 258 | RepKevinCramer 259 | RepKevinYoder 260 | RepKihuen 261 | RepKinzinger 262 | RepKristiNoem 263 | RepLaHood 264 | RepLaMalfa 265 | RepLanceNJ7 266 | RepLarryBucshon 267 | RepLawrence 268 | RepLeeZeldin 269 | RepLindaSanchez 270 | RepLipinski 271 | RepLizCheney 272 | RepLloydDoggett 273 | RepLoBiondo 274 | RepLoisFrankel 275 | RepLouBarletta 276 | RepLoudermilk 277 | replouiegohmert 278 | RepLowenthal 279 | RepLujanGrisham 280 | RepLukeMesser 281 | RepLynnJenkins 282 | RepMaloney 283 | RepMarciaFudge 284 | RepMarcyKaptur 285 | RepMarkMeadows 286 | repmarkpocan 287 | RepMarkTakano 288 | RepMarkWalker 289 | RepMarthaRoby 290 | RepMattGaetz 291 | RepMcCaul 292 | RepMcClintock 293 | RepMcEachin 294 | RepMcGovern 295 | RepMcKinley 296 | RepMcNerney 297 | RepMcSally 298 | RepMeehan 299 | RepMGriffith 300 | RepMiaLove 301 | RepMikeBishop 302 | RepMikeCapuano 303 | RepMikeCoffman 304 | RepMikeQuigley 305 | RepMikeRogersAL 306 | RepMikeTurner 307 | RepMimiWalters 308 | RepMoBrooks 309 | RepMoolenaar 310 | RepMullin 311 | RepNewhouse 312 | RepOHalleran 313 | RepPaulCook 314 | RepPaulMitchell 315 | RepPaulTonko 316 | RepPerlmutter 317 | RepPeteAguilar 318 | RepPeteKing 319 | RepPeteOlson 320 | RepPeterDeFazio 321 | reppittenger 322 | RepPoliquin 323 | RepRaskin 324 | RepRatcliffe 325 | RepRaulGrijalva 326 | RepRichardNeal 327 | RepRichHudson 328 | RepRichmond 329 | RepRickAllen 330 | RepRickCrawford 331 | RepRickLarsen 332 | RepRobBishop 333 | RepRobinKelly 334 | RepRobWoodall 335 | RepRodBlum 336 | RepRoKhanna 337 | RepRonEstes 338 | RepRonKind 339 | RepRooney 340 | RepRoybalAllard 341 | RepRubenGallego 342 | RepRussell 343 | RepRWilliams 344 | RepRyanCostello 345 | RepSamGraves 346 | repsandylevin 347 | RepSanfordSC 348 | RepSarbanes 349 | RepSchneider 350 | RepSchrader 351 | RepScottPerry 352 | RepScottPeters 353 | RepSeanDuffy 354 | RepSeanMaloney 355 | RepSheaPorter 356 | RepShimkus 357 | RepSinema 358 | RepSires 359 | RepSmucker 360 | RepSpeier 361 | RepStefanik 362 | RepStephenLynch 363 | RepStephMurphy 364 | RepSteveChabot 365 | RepSteveStivers 366 | rep_stevewomack 367 | RepSusanDavis 368 | RepSwalwell 369 | RepTedBudd 370 | RepTedDeutch 371 | RepTedLieu 372 | RepTedYoho 373 | RepTenney 374 | RepTerriSewell 375 | RepThomasMassie 376 | RepThompson 377 | RepTimMurphy 378 | RepTimRyan 379 | RepTimWalz 380 | RepTipton 381 | RepTomEmmer 382 | Rep_Tom_Garrett 383 | RepTomGraves 384 | RepTomMacArthur 385 | RepTomMarino 386 | RepTomReed 387 | RepTomRice 388 | RepTomSuozzi 389 | RepTrentFranks 390 | RepTrentKelly 391 | RepTrey 392 | RepValDemings 393 | RepVeasey 394 | RepVisclosky 395 | RepWalberg 396 | RepWalorski 397 | RepWalterJones 398 | RepWebster 399 | RepWesterman 400 | RepWilson 401 | RepYvetteClarke 402 | RepZoeLofgren 403 | Robert_Aderholt 404 | RobWittman 405 | RodneyDavis 406 | RogerMarshallMD 407 | rosadelauro 408 | RosLehtinen 409 | SamsPressShop 410 | SanfordBishop 411 | Scotttaylorva 412 | SpeakerRyan 413 | StaceyPlaskett 414 | SteveKingIA 415 | SteveKnight25 416 | SteveScalise 417 | SusanWBrooks 418 | teammoulton 419 | TGowdySC 420 | ToddRokita 421 | TomColeOK04 422 | TomRooney 423 | TulsiPress 424 | TXRandy14 425 | USRepGaryPalmer 426 | USRepKCastor 427 | USRepKeating 428 | USRepLong 429 | USRepMikeDoyle 430 | USRepRickNolan 431 | USRepRodney 432 | VernBuchanan 433 | virginiafoxx 434 | WarrenDavidson 435 | WhipHoyer -------------------------------------------------------------------------------- /proxy.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import user_agents 3 | import random 4 | 5 | 6 | def get_session(): 7 | session = requests.session() 8 | session.proxies = {'http': 'rproxy:5566', 9 | 'https': 'rproxy:5566'} 10 | session.headers = get_headers() 11 | return session 12 | 13 | 14 | def get_headers(): 15 | headers = { 16 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 17 | "accept-language": "en-GB,en-US;q=0.9,en;q=0.8", 18 | "User-Agent": random.choice(user_agents.useragents) 19 | } 20 | 21 | return headers 22 | 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | amqp==2.2.2 2 | billiard==3.5.0.3 3 | celery==4.0.2 4 | certifi==2018.1.18 5 | chardet==3.0.4 6 | idna==2.6 7 | kombu==4.1.0 8 | pytz==2018.3 9 | requests==2.18.4 10 | urllib3==1.22 11 | vine==1.1.4 12 | beautifulsoup4==4.6.0 13 | bs4==0.0.1 14 | lxml==4.1.1 15 | urllib3==1.22 16 | -------------------------------------------------------------------------------- /user_agents.py: -------------------------------------------------------------------------------- 1 | useragents = [ 2 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", 3 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", 4 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", 5 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", 6 | "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", 7 | "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", 8 | "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", 9 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", 10 | "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", 11 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", 12 | "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", 13 | "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", 14 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", 15 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", 16 | "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", 17 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", 18 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", 19 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", 20 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", 21 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", 22 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", 23 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", 24 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", 25 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", 26 | "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", 27 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", 28 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", 29 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", 30 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", 31 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", 32 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", 33 | "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", 34 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", 35 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", 36 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", 37 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", 38 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", 39 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 40 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 41 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 42 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 43 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 44 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", 45 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", 46 | "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", 47 | "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", 48 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", 49 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", 50 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", 51 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14" 52 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", 53 | "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", 54 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", 55 | "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", 56 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", 57 | "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0", 58 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0", 59 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0", 60 | "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0", 61 | "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0", 62 | "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3", 63 | "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0", 64 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", 65 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", 66 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0", 67 | "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", 68 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0", 69 | "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0", 70 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0", 71 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0", 72 | "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0", 73 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0", 74 | "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0", 75 | "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0", 76 | "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", 77 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", 78 | "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0", 79 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0", 80 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0", 81 | "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0", 82 | "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0", 83 | "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0", 84 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0", 85 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0", 86 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0", 87 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0", 88 | "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0", 89 | "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0", 90 | "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0", 91 | "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0", 92 | "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0", 93 | "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0", 94 | "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0", 95 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0", 96 | "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0", 97 | "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", 98 | "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0", 99 | "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1", 100 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0", 101 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6" 102 | ] --------------------------------------------------------------------------------