├── README.md ├── alexa_top.txt ├── assets ├── ClientConf └── roots ├── bloom ├── LICENSE └── bloom.go ├── defense.pptx ├── meek_adapter.go ├── metis.pdf ├── notes.md ├── proxy.go ├── rappor_implementation.go ├── server ├── LICENSE_trie ├── alexa_random_100.txt ├── rappor_analysis.py ├── server.go └── server.py └── testAccuracy.go /README.md: -------------------------------------------------------------------------------- 1 | # Metis 2 | 3 | Thesis statement: If we build a system that can defeat censorship by choosing intelligently between existing 4 | circumvention tools to route traffic, it will reduce the bandwidth required to avoid censorship, improve the latency 5 | experienced by users of these tools, and provide a means of collecting data on censorship across the globe. 6 | 7 | ##To install: 8 | 9 | ``` 10 | go build -o client proxy.go 11 | go build -o svr server/server.go 12 | ``` 13 | 14 | Start the server before starting the client, or the client will throw an error. 15 | ``` 16 | ./svr 17 | ./client 18 | ``` 19 | 20 | ##Set up your browser to use Metis as a proxy: 21 | **Chrome:** *Settings -> Advanced -> Open proxy settings (under System).* 22 | On Windows, click the box labeled "LAN settings." 23 | Check the "Use a proxy..." box, and set your proxy address to 127.0.0.1 and the port to 8080. 24 | 25 | **Firefox:** *Preferences -> Advanced.* Click "Settings" across from Connection. Select "Manual proxy configuration" 26 | and set the HTTP Proxy box to 127.0.0.1 and the port to 8080. 27 | -------------------------------------------------------------------------------- /alexa_top.txt: -------------------------------------------------------------------------------- 1 | google.com 2 | youtube.com 3 | facebook.com 4 | baidu.com 5 | wikipedia.org 6 | yahoo.com 7 | reddit.com 8 | google.co.in 9 | qq.com 10 | taobao.com 11 | tmall.com 12 | amazon.com 13 | twitter.com 14 | live.com 15 | google.co.jp 16 | instagram.com 17 | vk.com 18 | sohu.com 19 | jd.com 20 | sina.com.cn 21 | weibo.com 22 | gmw.cn 23 | 360.cn 24 | google.de 25 | google.co.uk 26 | google.com.br 27 | linkedin.com 28 | google.fr 29 | login.tmall.com 30 | google.ru 31 | yandex.ru 32 | netflix.com 33 | google.it 34 | google.com.hk 35 | google.es 36 | t.co 37 | yahoo.co.jp 38 | office.com 39 | twitch.tv 40 | google.ca 41 | pornhub.com 42 | google.com.mx 43 | microsoft.com 44 | xvideos.com 45 | alipay.com 46 | ebay.com 47 | microsoftonline.com 48 | bing.com 49 | ok.ru 50 | naver.com 51 | pages.tmall.com 52 | aliexpress.com 53 | wordpress.com 54 | imgur.com 55 | msn.com 56 | mail.ru 57 | imdb.com 58 | csdn.net 59 | hao123.com 60 | wikia.com 61 | tumblr.com 62 | github.com 63 | stackoverflow.com 64 | whatsapp.com 65 | google.com.tr 66 | google.com.au 67 | bongacams.com 68 | blogspot.com 69 | paypal.com 70 | amazon.co.jp 71 | google.com.tw 72 | google.pl 73 | apple.com 74 | google.co.id 75 | livejasmin.com 76 | xhamster.com 77 | deloton.com 78 | diply.com 79 | pinterest.com 80 | googleusercontent.com 81 | dropbox.com 82 | adobe.com 83 | popads.net 84 | google.com.ar 85 | tribunnews.com 86 | savefrom.net 87 | amazon.de 88 | soso.com 89 | so.com 90 | coccoc.com 91 | google.co.th 92 | amazon.co.uk 93 | google.com.eg 94 | espn.com 95 | mozilla.org 96 | cnn.com 97 | google.com.sa 98 | bbc.co.uk 99 | xnxx.com 100 | google.nl 101 | youth.cn 102 | tianya.cn 103 | detail.tmall.com 104 | amazonaws.com 105 | amazon.in 106 | booking.com 107 | detik.com 108 | google.com.ua 109 | bbc.com 110 | pixnet.net 111 | craigslist.org 112 | nytimes.com 113 | onlinesbi.com 114 | txxx.com 115 | salesforce.com 116 | coinmarketcap.com 117 | thestartmagazine.com 118 | google.com.pk 119 | google.co.ve 120 | soundcloud.com 121 | google.co.za 122 | chaturbate.com 123 | xinhuanet.com 124 | nicovideo.jp 125 | thepiratebay.org 126 | ebay.de 127 | stackexchange.com 128 | quora.com 129 | nih.gov 130 | vimeo.com 131 | rakuten.co.jp 132 | queuecosm.bid 133 | aparat.com 134 | ask.com 135 | spotify.com 136 | zhihu.com 137 | google.com.ph 138 | google.gr 139 | softonic.com 140 | theguardian.com 141 | chase.com 142 | openload.co 143 | force.com 144 | fbcdn.net 145 | google.se 146 | exdynsrv.com 147 | daum.net 148 | avito.ru 149 | iwanttodeliver.com 150 | fc2.com 151 | dailymotion.com 152 | ebay.co.uk 153 | google.az 154 | google.com.sg 155 | indeed.com 156 | google.be 157 | discordapp.com 158 | google.cn 159 | hitcpm.com 160 | google.com.vn 161 | blogger.com 162 | alibaba.com 163 | mediafire.com 164 | globo.com 165 | doubleclick.net 166 | google.com.co 167 | tokopedia.com 168 | slideshare.net 169 | ettoday.net 170 | intuit.com 171 | roblox.com 172 | washingtonpost.com 173 | steamcommunity.com 174 | douyu.com 175 | dailymail.co.uk 176 | cnet.com 177 | w3schools.com 178 | redd.it 179 | k618.cn 180 | mama.cn 181 | mercadolivre.com.br 182 | vice.com 183 | google.at 184 | deviantart.com 185 | google.com.ng 186 | rumble.com 187 | cloudfront.net 188 | exosrv.com 189 | babytree.com 190 | 4chan.org 191 | china.com.cn 192 | messenger.com 193 | buzzfeed.com 194 | google.co.kr 195 | flipkart.com 196 | etsy.com 197 | slack.com 198 | 1688.com 199 | trello.com 200 | wikihow.com 201 | google.ro 202 | blastingnews.com 203 | uol.com.br 204 | nbcsports.com 205 | godaddy.com 206 | google.no 207 | gfycat.com 208 | amazon.it 209 | google.cz 210 | google.com.pe 211 | wetransfer.com 212 | sogou.com 213 | twimg.com 214 | providr.com 215 | google.ch 216 | amazon.fr 217 | kompas.com 218 | breitbart.com 219 | google.ae 220 | google.pt 221 | onlinevideoconverter.com 222 | dailysnark.com 223 | eastday.com 224 | bankofamerica.com 225 | sciencedirect.com 226 | walmart.com 227 | metropcs.mobi 228 | bukalapak.com 229 | scribd.com 230 | redtube.com 231 | google.cl 232 | researchgate.net 233 | nfl.com 234 | china.com 235 | caijing.com.cn 236 | wellsfargo.com 237 | hclips.com 238 | mega.nz 239 | yelp.com 240 | newstrend.news 241 | bp.blogspot.com 242 | upornia.com 243 | telegram.org 244 | forbes.com 245 | people.com.cn 246 | nownews.com 247 | blackboard.com 248 | exoclick.com 249 | bilibili.com 250 | huanqiu.com 251 | rambler.ru 252 | google.dz 253 | abs-cbn.com 254 | google.ie 255 | thesaurus.com 256 | indiatimes.com 257 | steampowered.com 258 | hulu.com 259 | ladbible.com 260 | weather.com 261 | medium.com 262 | foxnews.com 263 | prostomers.com 264 | businessinsider.com 265 | ikea.com 266 | google.dk 267 | ebay-kleinanzeigen.de 268 | zillow.com 269 | files.wordpress.com 270 | bet9ja.com 271 | youporn.com 272 | wikimedia.org 273 | amazon.es 274 | livedoor.jp 275 | kakaku.com 276 | pinimg.com 277 | shutterstock.com 278 | digikala.com 279 | genius.com 280 | binance.com 281 | livejournal.com 282 | theepochtimes.com 283 | archive.org 284 | google.hu 285 | gamepedia.com 286 | gearbest.com 287 | instructure.com 288 | google.fi 289 | aliyun.com 290 | sberbank.ru 291 | speedtest.net 292 | dkn.tv 293 | leboncoin.fr 294 | allegro.pl 295 | wordreference.com 296 | freepik.com 297 | yts.am 298 | patria.org.ve 299 | 9gag.com 300 | weebly.com 301 | google.co.il 302 | bitauto.com 303 | usatoday.com 304 | canva.com 305 | tripadvisor.com 306 | huffingtonpost.com 307 | gmx.net 308 | mailchimp.com 309 | amazon.cn 310 | youm7.com 311 | doublepimpssl.com 312 | ci123.com 313 | theverge.com 314 | setn.com 315 | web.de 316 | nextoptim.com 317 | cmpkynhhmkni.com 318 | youdao.com 319 | ameblo.jp 320 | google.sk 321 | ltn.com.tw 322 | aol.com 323 | google.co.ao 324 | amazon.ca 325 | rednet.cn 326 | goodreads.com 327 | skype.com 328 | kinopoisk.ru 329 | reverso.net 330 | irctc.co.in 331 | office365.com 332 | cnbc.com 333 | momoshop.com.tw 334 | outbrain.com 335 | coinbase.com 336 | cnblogs.com 337 | hp.com 338 | google.kz 339 | orange.fr 340 | airbnb.com 341 | list.tmall.com 342 | myway.com 343 | digitaldsp.com 344 | atlassian.net 345 | spotscenered.info 346 | kissanime.ru 347 | zendesk.com 348 | ntd.tv 349 | lifedaily.com 350 | xfinity.com 351 | duckduckgo.com 352 | cbssports.com 353 | feedly.com 354 | sourceforge.net 355 | zippyshare.com 356 | varzesh3.com 357 | udemy.com 358 | leagueoflegends.com 359 | oracle.com 360 | bestbuy.com 361 | rutracker.org 362 | rt.com 363 | box.com 364 | sabah.com.tr 365 | liputan6.com 366 | icloud.com 367 | chinadaily.com.cn 368 | accuweather.com 369 | hdfcbank.com 370 | codeonclick.com 371 | haber7.com 372 | chegg.com 373 | ouo.io 374 | streamable.com 375 | olx.ua 376 | asos.com 377 | flickr.com 378 | perfecttoolmedia.com 379 | 1337x.to 380 | gamefaqs.com 381 | banvenez.com 382 | ups.com 383 | mmoframes.com 384 | usps.com 385 | hola.com 386 | bloomberg.com 387 | behance.net 388 | glassdoor.com 389 | mercadolibre.com.ar 390 | livedoor.com 391 | capitalone.com 392 | ign.com 393 | springer.com 394 | okta.com 395 | americanexpress.com 396 | google.com.kw 397 | higheurest.com 398 | banggood.com 399 | giphy.com 400 | reuters.com 401 | userapi.com 402 | jsmentry.com 403 | wiley.com 404 | flash-player.stream 405 | onet.pl 406 | ibm.com 407 | iqiyi.com 408 | popcash.net 409 | wp.pl 410 | uidai.gov.in 411 | line.me 412 | gismeteo.ru 413 | tradingview.com 414 | uzone.id 415 | onoticioso.com 416 | 17ok.com 417 | rottentomatoes.com 418 | crunchyroll.com 419 | hubspot.com 420 | manoramaonline.com 421 | billdesk.com 422 | mi.com 423 | nextlnk1.com 424 | iqoption.com 425 | uptodown.com 426 | elpais.com 427 | dictionary.com 428 | zoho.com 429 | h8vzwpv.com 430 | wix.com 431 | siteadvisor.com 432 | fedex.com 433 | t-online.de 434 | target.com 435 | taleo.net 436 | pixabay.com 437 | onedio.com 438 | ndtv.com 439 | rarbg.to 440 | wordpress.org 441 | battle.net 442 | t.me 443 | rediff.com 444 | patreon.com 445 | wish.com 446 | free.fr 447 | samsung.com 448 | dell.com 449 | go.com 450 | yadi.sk 451 | express.co.uk 452 | jrj.com.cn 453 | thebalance.com 454 | cambridge.org 455 | slickdeals.net 456 | doublepimp.com 457 | fomofriend.com 458 | 51sole.com 459 | investing.com 460 | asana.com 461 | wowhead.com 462 | ptt.cc 463 | google.bg 464 | bleacherreport.com 465 | zoom.us 466 | bldaily.com 467 | irs.gov 468 | evernote.com 469 | seasonvar.ru 470 | seznam.cz 471 | repubblica.it 472 | bet365.com 473 | taboola.com 474 | subscene.com 475 | fiverr.com 476 | gizmodo.com 477 | taringa.net 478 | naukri.com 479 | xx1.me 480 | ebay.com.au 481 | wsj.com 482 | mercadolibre.com.mx 483 | olx.pl 484 | ouedkniss.com 485 | informationvine.com 486 | google.lk 487 | ebay.it 488 | mit.edu 489 | shaparak.ir 490 | webmd.com 491 | pandora.com 492 | upwork.com 493 | sbnation.com 494 | icicibank.com 495 | velocecdn.com 496 | namnak.com 497 | eskimi.com 498 | myanimelist.net 499 | wiktionary.org 500 | khanacademy.org 501 | playstation.com 502 | libero.it 503 | sahibinden.com 504 | okdiario.com 505 | ria.ru 506 | appledaily.com 507 | givemesport.com 508 | kaskus.co.id 509 | suning.com 510 | elsevier.com 511 | yao.tmall.com 512 | media.tumblr.com 513 | goo.ne.jp 514 | beeg.com 515 | pikabu.ru 516 | rarbg.is 517 | fidelity.com 518 | spiegel.de 519 | homedepot.com 520 | adp.com 521 | telegraph.co.uk 522 | ultimate-guitar.com 523 | 123movieshub.to 524 | wixsite.com 525 | alicdn.com 526 | marca.com 527 | heavy.com 528 | google.co.nz 529 | albawabhnews.com 530 | google.by 531 | nypost.com 532 | shopify.com 533 | blog.jp 534 | ytimg.com 535 | nike.com 536 | time.com 537 | quizlet.com 538 | kapanlagi.com 539 | sarkariresult.com 540 | npr.org 541 | themeforest.net 542 | prezi.com 543 | yournewtab.com 544 | kooora.com 545 | sabq.org 546 | webex.com 547 | japanpost.jp 548 | investopedia.com 549 | squarespace.com 550 | weblio.jp 551 | naver.jp 552 | citi.com 553 | patch.com 554 | debate.com.mx 555 | moneycontrol.com 556 | google.rs 557 | discogs.com 558 | caliente.mx 559 | xe.com 560 | deadspin.com 561 | ensonhaber.com 562 | surveymonkey.com 563 | artstation.com 564 | bles.com 565 | nur.kz 566 | voc.com.cn 567 | oload.stream 568 | ieee.org 569 | espncricinfo.com 570 | google.com.ly 571 | google.com.do 572 | inquirer.net 573 | gosuslugi.ru 574 | drom.ru 575 | google.com.mm 576 | beytoote.com 577 | variety.com 578 | list-manage.com 579 | blogspot.mx 580 | emol.com 581 | xda-developers.com 582 | sindonews.com 583 | addthis.com 584 | friv.com 585 | stockstar.com 586 | coursera.org 587 | yourporn.sexy 588 | pchome.com.tw 589 | hurriyet.com.tr 590 | chip.de 591 | bandcamp.com 592 | vporn.com 593 | biobiochile.cl 594 | segmentfault.com 595 | jianshu.com 596 | lapatilla.com 597 | google.hr 598 | asus.com 599 | flvto.biz 600 | independent.co.uk 601 | pantip.com 602 | dmm.co.jp 603 | thefreedictionary.com 604 | prothomalo.com 605 | bestadbid.com 606 | idntimes.com 607 | stanford.edu 608 | okezone.com 609 | nocookie.net 610 | vidio.com 611 | reundcwkqvctq.com 612 | lenta.ru 613 | convert2mp3.net 614 | qingdaonews.com 615 | cisco.com 616 | hdzog.com 617 | bild.de 618 | dmm.com 619 | avgle.com 620 | tutorialspoint.com 621 | urbandictionary.com 622 | paytm.com 623 | ck101.com 624 | buzzadnetwork.com 625 | coursehero.com 626 | cricbuzz.com 627 | marketwatch.com 628 | primevideo.com 629 | pearsoncmg.com 630 | 5ch.net 631 | cqnews.net 632 | gmarket.co.kr 633 | mymedianetnow.com 634 | olx.com.br 635 | elmundo.es 636 | baike.com 637 | bmovies.to 638 | reallifecam.com 639 | sex.com 640 | cdstm.cn 641 | v3rjvtt.com 642 | cnzz.com 643 | chatwork.com 644 | fbsbx.com 645 | milliyet.com.tr 646 | www.gov.uk 647 | yenisafak.com 648 | chinaz.com 649 | lazada.co.id 650 | cbsnews.com 651 | cdiscount.com 652 | eventbrite.com 653 | autodesk.com 654 | souq.com 655 | wattpad.com 656 | dianping.com 657 | addroplet.com 658 | kickstarter.com 659 | hm.com 660 | linkshrink.net 661 | grammarly.com 662 | engadget.com 663 | viralvideos.pro 664 | groupon.com 665 | wp.com 666 | academia.edu 667 | ajkzd9h.com 668 | hotels.com 669 | newegg.com 670 | gyazo.com 671 | python.org 672 | nextlnk2.com 673 | ca.gov 674 | elbalad.news 675 | jiameng.com 676 | filehippo.com 677 | kizlarsoruyor.com 678 | lie2anyone.com 679 | as.com 680 | drive2.ru 681 | wtoip.com 682 | merdeka.com 683 | europa.eu 684 | zztdcl.ru 685 | telewebion.com 686 | bittrex.com 687 | divar.ir 688 | getpocket.com 689 | si.com 690 | smallpdf.com 691 | torrentz2.eu 692 | expedia.com 693 | hatena.ne.jp 694 | mercadolibre.com.ve 695 | disqus.com 696 | bancodevenezuela.com 697 | freejobalert.com 698 | drudgereport.com 699 | google.tn 700 | goo.gl 701 | ebc.net.tw 702 | wunderground.com 703 | timeanddate.com 704 | epochtimes.com 705 | newtab-tv.com 706 | google.com.gt 707 | google.tm 708 | mathrubhumi.com 709 | kinogo.cc 710 | 4shared.com 711 | att.com 712 | ali213.net 713 | gogoanime.io 714 | merriam-webster.com 715 | mathworks.com 716 | hespress.com 717 | infourok.ru 718 | harvard.edu 719 | mysearch.com 720 | namasha.com 721 | el-nacional.com 722 | interia.pl 723 | hh.ru 724 | buyma.com 725 | chouftv.ma 726 | eksisozluk.com 727 | premierleague.com 728 | lifehacker.com 729 | nature.com 730 | ecollege.com 731 | gsmarena.com 732 | goal.com 733 | mawdoo3.com 734 | nikkei.com 735 | labanquepostale.fr 736 | easypdfcombine.com 737 | lenovo.com 738 | uploaded.net 739 | pinterest.co.uk 740 | chron.com 741 | tabelog.com 742 | people.com 743 | caixa.gov.br 744 | yaplakal.com 745 | lifewire.com 746 | 3c.tmall.com 747 | banesconline.com 748 | corriere.it 749 | intel.com 750 | google.com.ec 751 | lemonde.fr 752 | uniqlo.tmall.com 753 | epicgames.com 754 | ebay.fr 755 | 104.com.tw 756 | 163.com 757 | grid.id 758 | thesun.co.uk 759 | infusionsoft.com 760 | hotstar.com 761 | lefigaro.fr 762 | realtor.com 763 | thepennyhoarder.com 764 | hatenablog.com 765 | vnexpress.net 766 | akva-komptlt.ru 767 | awarded.party 768 | brilio.net 769 | rapidgator.net 770 | subito.it 771 | kijiji.ca 772 | animeflv.net 773 | techradar.com 774 | google.com.my 775 | adblitz.withyoutube.com 776 | impress.co.jp 777 | torrent9.bz 778 | gdax.com 779 | mobile01.com 780 | google.lt 781 | coindesk.com 782 | tistory.com 783 | comcast.net 784 | livescore.com 785 | yandex.ua 786 | tamilrockers.tw 787 | healthline.com 788 | myfitnesspal.com 789 | zhanqi.tv 790 | rbc.ru 791 | discover.com 792 | ssl-images-amazon.com 793 | hcg82f2b.com 794 | myfriendlyappz.com 795 | anybunny.mobi 796 | nordstrom.com 797 | meetup.com 798 | meteofrance.com 799 | ruten.com.tw 800 | eanswers.com 801 | theatlantic.com 802 | mellowads.com 803 | bitbucket.org 804 | runoob.com 805 | tube8.com 806 | uptobox.com 807 | adhoc2.net 808 | cinecalidad.to 809 | ninisite.com 810 | issuu.com 811 | 1and1.com 812 | fromdoctopdf.com 813 | offergold.online 814 | java.com 815 | bitly.com 816 | kayak.com 817 | readms.net 818 | gstatic.com 819 | ecosia.org 820 | ccm.net 821 | duolingo.com 822 | nhk.or.jp 823 | southwest.com 824 | doodle.com 825 | service-now.com 826 | flirt4free.com 827 | metropoles.com 828 | superappbox.com 829 | 4pda.ru 830 | verizonwireless.com 831 | pole-emploi.fr 832 | kotaku.com 833 | hootsuite.com 834 | visualstudio.com 835 | rutube.ru 836 | kissasian.ch 837 | trustpilot.com 838 | google.iq 839 | zapmeta.ws 840 | deadline.com 841 | usnews.com 842 | weibo.cn 843 | sportbible.com 844 | nianhuo.tmall.com 845 | umblr.com 846 | indiamart.com 847 | superuser.com 848 | mirror.co.uk 849 | jimdo.com 850 | ask.fm 851 | barraceful.com 852 | nba.com 853 | macys.com 854 | namu.wiki 855 | zzrednet.cn 856 | zara.com 857 | blogfa.com 858 | techcrunch.com 859 | shopee.tw 860 | arxiv.org 861 | norton.com 862 | deviantart.net 863 | pexels.com 864 | udn.com 865 | digitalprivacyalert.org 866 | mobile.de 867 | znanija.com 868 | nvzhuang.tmall.com 869 | psu.edu 870 | spankbang.com 871 | google.si 872 | 11st.co.kr 873 | study.com 874 | doorblog.jp 875 | cnnindonesia.com 876 | cam4.com 877 | bankmellat.ir 878 | google.com.af 879 | qualtrics.com 880 | ukr.net 881 | nptel.ac.in 882 | sifyitest.com 883 | fb.ru 884 | vesti.ru 885 | searchencrypt.com 886 | epwk.com 887 | fextralife.com 888 | zougla.gr 889 | news.com.au 890 | gotporn.com 891 | allocine.fr 892 | tandfonline.com 893 | yandex.kz 894 | slate.com 895 | bhphotovideo.com 896 | youjizz.com 897 | commentcamarche.net 898 | axisbank.co.in 899 | auction.co.kr 900 | myworkday.com 901 | tebyan.net 902 | motherless.com 903 | jw.org 904 | creditkarma.com 905 | mobafire.com 906 | padlet.com 907 | secureserver.net 908 | utorrent.com 909 | discuss.com.hk 910 | agoda.com 911 | ivi.ru 912 | sinoptik.ua 913 | dmv.org 914 | urdupoint.com 915 | videodownloadconverter.com 916 | dafont.com 917 | tvbs.com.tw 918 | lazada.co.th 919 | blpmovies.com 920 | correios.com.br 921 | gazetaexpress.com 922 | gamespot.com 923 | badoo.com 924 | y8.com 925 | codepen.io 926 | thehill.com 927 | aastocks.com 928 | liveinternet.ru 929 | purdue.edu 930 | caf.fr 931 | state.gov 932 | popmyads.com 933 | focus.de 934 | myfreecams.com 935 | qiita.com 936 | comicbook.com 937 | costco.com 938 | android.com 939 | asahi.com 940 | tomsguide.com 941 | zing.vn 942 | 17track.net 943 | nasa.gov 944 | sputniknews.com 945 | kp.ru 946 | justdial.com 947 | lazada.com.my 948 | dwatchseries.to 949 | fatosdesconhecidos.com.br 950 | cnbeta.com 951 | livestrong.com 952 | uber.com 953 | ebay.in 954 | infobae.com 955 | socialblade.com 956 | cdninstagram.com 957 | mileroticos.com 958 | 58.com 959 | okcupid.com 960 | mercari.com 961 | instructables.com 962 | avast.com 963 | digialm.com 964 | egy.best 965 | politico.com 966 | fanfiction.net 967 | internetdownloadmanager.com 968 | bodybuilding.com 969 | n11.com 970 | mydrivers.com 971 | conservativetribune.com 972 | historyinorbit.com 973 | subject.tmall.com 974 | hawaaworld.com 975 | cbc.ca 976 | ted.com 977 | huawei.com 978 | els-cdn.com 979 | berkeley.edu 980 | unsplash.com 981 | usaa.com 982 | abcnews.go.com 983 | americanas.com.br 984 | wittyfeed.com 985 | steemit.com 986 | edx.org 987 | lolesports.com 988 | biblegateway.com 989 | kinokrad.co 990 | fanserials.org 991 | hitomi.la 992 | jeuxvideo.com 993 | itmedia.co.jp 994 | geeksforgeeks.org 995 | speed-open2.com 996 | td.com 997 | pinterest.es 998 | jqw.com 999 | cimaclub.com 1000 | voirfilms.ws 1001 | -------------------------------------------------------------------------------- /assets/ClientConf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refraction-networking/Metis/a56f9c49726972bfb6e48cff28a90940ead42714/assets/ClientConf -------------------------------------------------------------------------------- /bloom/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Will Fitzgerald. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /bloom/bloom.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //TODO: What does Metis's license file have to look like if I include this? 4 | 5 | import( 6 | "github.com/willf/bloom" 7 | ) 8 | 9 | var filter *bloom.BloomFilter 10 | 11 | func initFilter(){ 12 | //TODO: figure out how to size the filter correctly. Error rate? 13 | n := uint(1000) 14 | filter = bloom.New(20*n, 5) 15 | } 16 | 17 | func addStr(str string){ 18 | filter.Add([]byte(str)) 19 | } 20 | 21 | func testStr(str string) bool { 22 | return filter.Test([]byte(str)) 23 | } 24 | -------------------------------------------------------------------------------- /defense.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refraction-networking/Metis/a56f9c49726972bfb6e48cff28a90940ead42714/defense.pptx -------------------------------------------------------------------------------- /meek_adapter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os/exec" 5 | "os" 6 | "log" 7 | "fmt" 8 | ) 9 | 10 | func logKill(p *os.Process) error { 11 | log.Printf("killing PID %d", p.Pid) 12 | err := p.Kill() 13 | if err != nil { 14 | log.Print(err) 15 | } 16 | return err 17 | } 18 | 19 | func runMeekClient(cmdName string, args []string) (cmd *exec.Cmd, err error) { 20 | //TODO: where to put meek's command line client? 21 | //Ellipsis allows you to pass a slice as a variadic parameter 22 | cmd = exec.Command(cmdName, args...) 23 | cmd.Stdout = os.Stdout 24 | cmd.Stderr = os.Stderr 25 | log.Printf("running meek-client command %q", cmd.Args) 26 | err = cmd.Start() 27 | if err != nil { 28 | return 29 | } 30 | log.Printf("meek-client started with pid %d", cmd.Process.Pid) 31 | err = cmd.Wait() 32 | if err != nil { 33 | fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err) 34 | os.Exit(1) 35 | } 36 | return cmd, nil 37 | } 38 | 39 | func configureEnv() (error) { 40 | err := os.Setenv("TOR_PT_MANAGED_TRANSPORT_VER", "1") 41 | if err != nil { 42 | return err 43 | } 44 | err = os.Setenv("TOR_PT_CLIENT_TRANSPORTS", "meek") 45 | return nil 46 | } 47 | 48 | func main() { 49 | //TODO: put all configuration flags for PTs in a config file. 50 | 51 | configureEnv() 52 | 53 | //meek-client --url=https://meek-reflect.appspot.com/ --front=www.google.com 54 | cmd := "C:\\Users\\Audrey\\go\\src\\github.com\\arlolra\\meek\\meek-client\\meek-client.exe" 55 | args := []string{"--url=https://meek-reflect.appspot.com/", "--front=www.google.com", "--log=meek-client.log"} 56 | meekClientCmd, err := runMeekClient(cmd, args) 57 | if err != nil { 58 | log.Print(err) 59 | return 60 | } 61 | //TODO: Figure out what kind of message the client expects, and send it, because it isn't just a browser connection. 62 | defer logKill(meekClientCmd.Process) 63 | } 64 | -------------------------------------------------------------------------------- /metis.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refraction-networking/Metis/a56f9c49726972bfb6e48cff28a90940ead42714/metis.pdf -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | The process should look something like: 2 | 1) Receive a connection and bytes from a local browser (e.g. "GET" or "CONNECT" stuff) 3 | 2) Pass these bytes to some HTTP proxy library parser, that parses them and returns some HTTP request object 4 | 3) Host/destination is extracted from the request object, and we determine if the request needs a proxy or not 5 | 4) If it needs a proxy, pass the bytes to tapdance/whatever proxy system we're using. If it doesn't need a proxy, 6 | you pass the bytes to a local library that does the GET or CONNECT for you (aka, goproxy). 7 | 8 | Metis goes here: browser -> Metis -> Tapdance client or local HTTP proxy 9 | 10 | # TODO 11 | 12 | This link might be useful: https://github.com/elazarl/goproxy 13 | 14 | Notes on the Tapdance station: 15 | the station runs in an ISP 16 | you shouldn't have to worry too much about what it's doing 17 | it terminates (is the other endpoint) of the HTTP proxy though 18 | so normally, we have browser -> tapdance client 19 | and then tapdance client -> tapdance station -> squid 20 | and what the browser really sees, is that it's just talking to squid 21 | (squid is an HTTP proxy) 22 | Metis goes in between the browser and tapdance client, and decides, for each request, whether to use the tapdance client, or just fetch the request directly 23 | if it's directly though, Metis COULD fetch it "itself" (implementing a local HTTP proxy, essentially), but likely there exists a go library that will do that for you like https://github.com/elazarl/goproxy 24 | 25 | browser starts a connection to tapdance client (which starts a connection to tapdance station, (which starts a connection to squid)) 26 | then browser sends up that path the request 27 | and receives back down the response 28 | yeah, squid doesn't do any decoy routing (refraction networking) 29 | the only things that do that are the tapdance client and tapdance station 30 | you can think of it like, we provide transport of data between browser and squid 31 | the browser doesn't know it's talking to tapdance, or what any of this stuff is 32 | all it cares is: it connects to *something* that talks HTTP proxy 33 | we encode and decode and transport that something, and ultimately it ends up at a squid instance 34 | that squid instance doesn't know what connected to it (or anything about tapdance or decoy routing/refraction networking) 35 | it just knows it gets a connection, and an HTTP proxy request 36 | and then it fulfills that request, and sends a response 37 | we take that response, encapsulate it back into the tapdance protocol, get it back down to the client, and then the client sends it back to the browser 38 | 39 | but basically, the only things you'll see a browser produce is a `GET http://site.com/ HTTP/1.1` for HTTP requests, and a `CONNECT site.com:443 HTTP/1.1` for TLS 40 | https://en.wikipedia.org/wiki/Proxy_server#Implementations_of_proxies 41 | 42 | 43 | # Notes 10/2: 44 | 45 | 1) If I get a GET request, close clientConn when? While clientConn is open (while it doesn't throw an error), 46 | response = http.defaultTransport(request). 47 | Forward response to client. 48 | 2) If I get a CONNECT request, it might be followed by an SSL handshake. Assuming the http parsing logic is right after 49 | accept(), stop parsing incoming msgs as HTTP right after you get a CONNECT and send the 200 OK. Switch to byte copying 50 | from then on, copy bytes from clientConn to remoteConn which you create using net.Dial. 51 | 3) Close CONNECT clientConn when? 52 | 4) accept() should return a socket sock. 53 | 5) TODO: replace goproxy with sergey's DualStream function from forward_proxy. 54 | 6) Basically, the code I had at first is what should happen for GET requests. The code I have now should happen for CONNECTs. 55 | Except that I should replace goproxy with DualStream. 56 | 57 | tdConn, err := tapdance.Dial("tcp", "censoredsite.com:80") 58 | 59 | # Notes 11/7 60 | If a client goes to server.com/GET/getBlocked, server responds with the blocked list. RESTful API. There are libraries 61 | for this. Look at Coinbase's API for examples. Basically, each URL returns a requested piece of info. server.com/POST/addBlocked 62 | should 63 | 64 | # Notes 11/8 65 | 66 | Iran's censorship: a Lantern contributor says they determine a site to be blocked if: 67 | 1) remote address resolves to 10.10.34.34 68 | 2) response is 403 with an iframe to 10.10.34.34 69 | 3) it times out 70 | 4) EPIPE or ECONNRESET 71 | 72 | Detecting DNS poisoning works as follows: 73 | 1) Do the DNS resolution and get a lie 74 | 2) Connect to it over TCP (because you don't know it's a lie yet) 75 | 3) it either doesn't respond (timeout), responds with a RST, or tries to inject a page. 76 | If it's TLS, it won't be able to inject a page, and its certificate won't match. 77 | 78 | ##Notes 1/22 79 | 80 | When Metis is run in China, and Firefox connects to it from the US, and is asked for www.google.com, AND google isn't on 81 | the blocked list, then the connection hangs indefinitely. So whatever response Metis gets when it tries to reach Google 82 | isn't being handled as evidence of a censored connection. Actually, Chrome exhibits the same behavior. This is a 83 | critical bug, and evidence of a lack of knowledge of how to test code rigorously - something I should keep in mind for 84 | future work. Solution for this one is probably to implement my own timeouts? 85 | 86 | ##Notes 2/3 87 | This website http://english.cri.cn/4406/2010/08/09/1981s587568.htm demonstrates an instance of "Tapdance responded with 88 | 503 Service Unavailable" being displayed on the website in place of the (probably) ad meant to be there. When loaded 89 | through not-Tapdance, this item displays "comment.cri.cn’s server IP address could not be found." 90 | 91 | http://libraries.colorado.edu/record=b3535240~S3 also causes problems. 92 | 93 | ##Notes 2/18 94 | 95 | Symptoms of censorship observed so far: 96 | 1) can't curl the page and can't ping the page: traffic to that domain is being dropped. 97 | 2) Can ping, but can't curl the page: connection reset by peer, reset received, etc. Firewall sent a reset? 98 | 3) Could not resolve host: DNS poisoning OR timeout from DNS server 99 | 100 | Broad categories of censorship: 101 | 1) News 102 | 2) Social media 103 | 3) Porn 104 | 105 | OONIprobe has done this - detecting censorship 106 | How do we determine what evaluation metric tells us if a website is blocked? 107 | What kind of confidence to decide site is blocked? If curl ever gets through, out of 100 runs, site is not blocked. 108 | So how many failures to connect in a row do we need? Find how often a site connects, use that to say "odds of this many failure 109 | were 5%" or whatever. Take the minimum over all sites? 110 | 111 | TODO: 112 | 1) Find out why Metis is only 70% accurate. Timeouts? Try through Metis again and figure out how often the blocked things are blocked. 113 | 2.5) Find out how many things fail through Metis, fail through testing script, always fail - use that to create evaluation metric 114 | 2) Redesign curl script for greater certainty that things I think are blocked, actually are. Use OONIprobe. 115 | Test how often I get each error - Fermi approximation. 116 | 117 | For next Tues: ground truth and Metis's accuracy as compared to that ground truth 118 | Were the false positives from Metis actually just failed? There was a bug where they were getting logged in detour and failed. 119 | 120 | -------------------------------------------------------------------------------- /proxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "net/http" 7 | "io" 8 | "bufio" 9 | "net/url" 10 | "github.com/sergeyfrolov/gotapdance/tapdance" 11 | "fmt" 12 | "strconv" 13 | "sync" 14 | "encoding/json" 15 | "bytes" 16 | "net/http/httputil" 17 | "os" 18 | //"math/rand" 19 | "golang.org/x/net/proxy" 20 | "errors" 21 | "runtime" 22 | "strings" 23 | "time" 24 | "math/rand" 25 | ) 26 | 27 | type Endpoint struct { 28 | listener net.Listener 29 | mutex sync.RWMutex 30 | } 31 | 32 | type Website struct { 33 | Domain string `json:"domain,omitempty"` 34 | } 35 | 36 | type SvrMsg struct { 37 | Cohort int `json:"cohort,omitempty"` 38 | Reports []int `json:"reports,omitempty"` 39 | } 40 | 41 | var client = &http.Client{ 42 | Transport: &http.Transport{ 43 | Dial: (&net.Dialer{ 44 | //Limits the time spent establishing a TCP connection (if a new one is needed) 45 | //TODO: tweak this value. How? Time how long usual connections take. Valid to take avg over all domains? 46 | Timeout: 5 * time.Second, 47 | KeepAlive: 30 * time.Second, 48 | }).Dial, 49 | //limits the time spent performing the TLS handshake. 50 | TLSHandshakeTimeout: 5 * time.Second, 51 | //Limits time spent reading response headers. TODO: Possibly unnecessary? 52 | ResponseHeaderTimeout: 10 * time.Second, 53 | ExpectContinueTimeout: 1 * time.Second, 54 | }, 55 | } 56 | 57 | var transport string 58 | var hmacSecret string 59 | var cohort int 60 | 61 | /* 62 | Domains Metis is reasonably certain are censored are stored here. 63 | */ 64 | var blockedDomains []string 65 | 66 | /* 67 | Domains that Metis has trouble accessing for reasons that might not be censorship are stored here. 68 | */ 69 | var tempBlockedDomains []string 70 | 71 | func contains(slice []string, s string) bool { 72 | for _, e := range slice { 73 | if strings.Contains(s, e) { return true} 74 | } 75 | return false 76 | } 77 | 78 | func isBlocked(url *url.URL) (bool) { 79 | /*r := rand.Intn(100) 80 | if r < 50 { 81 | return false 82 | } else { 83 | return true 84 | }*/ 85 | return true 86 | //return contains(blockedDomains, url.Hostname()) || contains(tempBlockedDomains, url.Hostname()) 87 | } 88 | 89 | func remove(s []string, e string) []string { 90 | for i, ele := range s { 91 | if ele==e && i+1 < len(s){ 92 | s = append(s[:i], s[i+1:]...) 93 | } else if ele == e { 94 | s = s[:i] 95 | } 96 | } 97 | return s 98 | } 99 | 100 | func getBlockedList() (error){ 101 | req, err := http.NewRequest("GET", "HTTP://localhost:5000/blocked", nil) 102 | if err != nil { 103 | log.Println(err) 104 | return err 105 | } 106 | resp, err := client.Do(req) 107 | if err != nil { 108 | log.Println(err) 109 | return err 110 | } 111 | defer resp.Body.Close() 112 | //body, _ := ioutil.ReadAll(resp.Body) 113 | //fmt.Println("Body is: ", body) 114 | dec := json.NewDecoder(resp.Body) 115 | 116 | 117 | // read open bracket 118 | t, err := dec.Token() 119 | if err != nil { 120 | log.Println("In getBlockedList, ", err) 121 | return err 122 | } 123 | fmt.Printf("First character type: %T, character: %v\n", t, t) 124 | 125 | 126 | // while the array contains values 127 | for dec.More() { 128 | var site Website 129 | // decode an array value (Message) 130 | err := dec.Decode(&site) 131 | if err != nil { 132 | log.Println("In getBlockedList, ", err) 133 | return err 134 | } 135 | fmt.Printf("Domain: %v\n", site.Domain) 136 | if !contains(blockedDomains,site.Domain) { 137 | blockedDomains = append(blockedDomains, site.Domain) 138 | } 139 | } 140 | 141 | // read closing bracket 142 | t, err = dec.Token() 143 | if err != nil { 144 | log.Println(err) 145 | return err 146 | } 147 | fmt.Printf("%T: %v\n", t, t) 148 | return nil 149 | } 150 | 151 | func generateSecret(){ 152 | rand.Seed(time.Now().UnixNano()) 153 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 154 | b := make([]rune, 10) 155 | for i := range b { 156 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 157 | } 158 | hmacSecret = string(b) 159 | } 160 | 161 | // If the buffer gets too slow, see pipe tutorial here: 162 | // https://medium.com/stupid-gopher-tricks/streaming-data-in-go-without-buffering-3285ddd2a1e5 163 | func updateMasterList() error { 164 | var buf bytes.Buffer 165 | var s SecureIrrRand 166 | var p Params 167 | p.init() 168 | s.init(p) 169 | var e Encoder 170 | 171 | e.init(p, cohort, hmacSecret, &s) 172 | var rappor []int 173 | for i := 0; i < len(blockedDomains); i++ { 174 | //fmt.Println("Blocked domains[i]: ", blockedDomains[i]) 175 | //fmt.Println(e.Encode([]byte(blockedDomains[i]))) 176 | rappor = append(rappor, e.Encode([]byte(blockedDomains[i]))) 177 | } 178 | svrMsg := SvrMsg{cohort, rappor} 179 | err := json.NewEncoder(&buf).Encode(svrMsg) 180 | if err != nil { 181 | return err 182 | } 183 | resp, err := http.Post("HTTP://localhost:5000/blocked", "application/json", &buf) 184 | if err != nil { 185 | return errors.New("Master list update failed with error "+err.Error()) 186 | } 187 | if resp.StatusCode != 200 { 188 | return errors.New("Master list update failed with response "+string(resp.StatusCode)) 189 | } 190 | fmt.Println("Post request completed successfully.") 191 | return nil 192 | } 193 | 194 | func parseRequest(conn net.Conn)(*http.Request, error){ 195 | connReader := bufio.NewReader(conn) 196 | req, err := http.ReadRequest(connReader) 197 | if err != nil {return nil, err} 198 | return req, nil 199 | } 200 | 201 | func detectedTampering(id int, req *http.Request, resp *http.Response, err error) (bool, error) { 202 | //TODO: do resets get caught correctly here? 203 | //TODO: how to catch TLS certificate errors? 204 | netErr, ok := err.(net.Error) 205 | if ok { 206 | //Timeout, RST? 207 | log.Println(id, "Website timed out with network error ", netErr) 208 | blockedDomains = append(blockedDomains, req.URL.Hostname()) 209 | return true, nil 210 | } 211 | _, ok = err.(*net.OpError) 212 | if ok { 213 | //Finds ECONNRESET and EPIPE? 214 | log.Println(id, "Website threw net.OpError ", err) 215 | blockedDomains = append(blockedDomains, req.URL.Hostname()) 216 | return true, nil 217 | } 218 | if err != nil { 219 | log.Println(id, "Website threw unknown error ", err) 220 | //Don't add to blockedDomains because error wasn't due to censorship? 221 | return false, err 222 | } else { 223 | //HTTP poisoning: Iran only, code taken from https://github.com/getlantern/detour/blob/master/detect.go 224 | byteResp, dmpErr := httputil.DumpResponse(resp, true) 225 | if dmpErr != nil { 226 | err = errors.New("response couldn't be dumped to byte slice") 227 | return false, err 228 | } 229 | http403 := []byte("HTTP/1.1 403 Forbidden") 230 | iranIFrame := []byte(`