├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── aduana ├── __init__.py ├── bin │ ├── aduana-server-cert.py │ └── aduana-server.py └── frontera.py ├── doc ├── Doxyfile ├── Makefile ├── _static │ ├── CrawledPage-PageInfo.svg │ ├── aduana-intro-arch.svg │ ├── domain-temp-beginning.svg │ ├── domain-temp-evenly.svg │ ├── domain-temp-evolve.svg │ ├── domain-temp-moved.svg │ ├── domain-temp-strange.svg │ └── gen-domain-temp-fig.py ├── conf.py ├── index.rst ├── intro.rst ├── library.rst ├── make.bat ├── python.rst └── requirements.txt ├── examples ├── keywords │ ├── example │ │ ├── __init__.py │ │ ├── frontera │ │ │ ├── __init__.py │ │ │ ├── common_settings.py │ │ │ ├── settings.py │ │ │ └── web_settings.py │ │ ├── settings.py │ │ └── spiders │ │ │ ├── __init__.py │ │ │ └── example.py │ ├── keywords.txt │ ├── requirements.txt │ ├── scorer.py │ ├── scrapy.cfg │ ├── seeds.txt │ └── server-config.py └── locations │ ├── common_words.txt │ ├── countryInfo.txt │ ├── geonames.py │ ├── locations │ ├── __init__.py │ ├── frontera │ │ ├── __init__.py │ │ ├── common_settings.py │ │ ├── settings.py │ │ └── web_settings.py │ ├── items.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ └── locations.py │ ├── requirements.txt │ ├── scrapy.cfg │ ├── seeds.txt │ └── tokenizer.py ├── lib ├── CMakeLists.txt ├── lib │ ├── lmdb │ │ ├── COPYRIGHT │ │ ├── LICENSE │ │ ├── lmdb.h │ │ ├── mdb.c │ │ ├── midl.c │ │ └── midl.h │ ├── mman.c │ ├── mman.h │ ├── smaz.c │ ├── smaz.h │ ├── xxhash.c │ └── xxhash.h ├── src │ ├── bf_scheduler.c │ ├── bf_scheduler.h │ ├── bf_scheduler_reload.c │ ├── domain_temp.c │ ├── domain_temp.h │ ├── freq_algo.c │ ├── freq_algo.h │ ├── freq_scheduler.c │ ├── freq_scheduler.h │ ├── freq_scheduler_dump.c │ ├── hits.c │ ├── hits.h │ ├── hits_scorer.c │ ├── hits_scorer.h │ ├── link_stream.h │ ├── mmap_array.c │ ├── mmap_array.h │ ├── page_db.c │ ├── page_db.h │ ├── page_db_dump.c │ ├── page_db_find.c │ ├── page_db_links.c │ ├── page_db_path.c │ ├── page_rank.c │ ├── page_rank.h │ ├── page_rank_scorer.c │ ├── page_rank_scorer.h │ ├── scheduler.c │ ├── scheduler.h │ ├── scorer.h │ ├── txn_manager.c │ ├── txn_manager.h │ ├── util.c │ └── util.h └── test │ ├── CuTest.c │ ├── CuTest.h │ ├── test.c │ ├── test.h │ ├── test_bf_scheduler.c │ ├── test_domain_temp.c │ ├── test_freq_scheduler.c │ ├── test_hits.c │ ├── test_page_rank.c │ ├── test_pagedb.c │ └── test_util.c ├── requirements.txt ├── setup.py ├── test ├── links.txt.gz ├── nodes.txt.gz ├── requirements.txt ├── test_crawl.py └── test_freq_scheduler.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/README.md -------------------------------------------------------------------------------- /aduana/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/aduana/__init__.py -------------------------------------------------------------------------------- /aduana/bin/aduana-server-cert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/aduana/bin/aduana-server-cert.py -------------------------------------------------------------------------------- /aduana/bin/aduana-server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/aduana/bin/aduana-server.py -------------------------------------------------------------------------------- /aduana/frontera.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/aduana/frontera.py -------------------------------------------------------------------------------- /doc/Doxyfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/Doxyfile -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/Makefile -------------------------------------------------------------------------------- /doc/_static/CrawledPage-PageInfo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/CrawledPage-PageInfo.svg -------------------------------------------------------------------------------- /doc/_static/aduana-intro-arch.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/aduana-intro-arch.svg -------------------------------------------------------------------------------- /doc/_static/domain-temp-beginning.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/domain-temp-beginning.svg -------------------------------------------------------------------------------- /doc/_static/domain-temp-evenly.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/domain-temp-evenly.svg -------------------------------------------------------------------------------- /doc/_static/domain-temp-evolve.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/domain-temp-evolve.svg -------------------------------------------------------------------------------- /doc/_static/domain-temp-moved.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/domain-temp-moved.svg -------------------------------------------------------------------------------- /doc/_static/domain-temp-strange.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/domain-temp-strange.svg -------------------------------------------------------------------------------- /doc/_static/gen-domain-temp-fig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/_static/gen-domain-temp-fig.py -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/conf.py -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/index.rst -------------------------------------------------------------------------------- /doc/intro.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/intro.rst -------------------------------------------------------------------------------- /doc/library.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/library.rst -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/make.bat -------------------------------------------------------------------------------- /doc/python.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/doc/python.rst -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | breathe 2 | -------------------------------------------------------------------------------- /examples/keywords/example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/keywords/example/frontera/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/keywords/example/frontera/common_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/example/frontera/common_settings.py -------------------------------------------------------------------------------- /examples/keywords/example/frontera/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/example/frontera/settings.py -------------------------------------------------------------------------------- /examples/keywords/example/frontera/web_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/example/frontera/web_settings.py -------------------------------------------------------------------------------- /examples/keywords/example/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/example/settings.py -------------------------------------------------------------------------------- /examples/keywords/example/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/keywords/example/spiders/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/example/spiders/example.py -------------------------------------------------------------------------------- /examples/keywords/keywords.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/keywords.txt -------------------------------------------------------------------------------- /examples/keywords/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.3.2 2 | scrapy 3 | frontera 4 | xxhash 5 | -------------------------------------------------------------------------------- /examples/keywords/scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/scorer.py -------------------------------------------------------------------------------- /examples/keywords/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/scrapy.cfg -------------------------------------------------------------------------------- /examples/keywords/seeds.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/seeds.txt -------------------------------------------------------------------------------- /examples/keywords/server-config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/keywords/server-config.py -------------------------------------------------------------------------------- /examples/locations/common_words.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/common_words.txt -------------------------------------------------------------------------------- /examples/locations/countryInfo.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/countryInfo.txt -------------------------------------------------------------------------------- /examples/locations/geonames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/geonames.py -------------------------------------------------------------------------------- /examples/locations/locations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/locations/locations/frontera/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/locations/locations/frontera/common_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/frontera/common_settings.py -------------------------------------------------------------------------------- /examples/locations/locations/frontera/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/frontera/settings.py -------------------------------------------------------------------------------- /examples/locations/locations/frontera/web_settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/frontera/web_settings.py -------------------------------------------------------------------------------- /examples/locations/locations/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/items.py -------------------------------------------------------------------------------- /examples/locations/locations/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/pipelines.py -------------------------------------------------------------------------------- /examples/locations/locations/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/settings.py -------------------------------------------------------------------------------- /examples/locations/locations/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/spiders/__init__.py -------------------------------------------------------------------------------- /examples/locations/locations/spiders/locations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/locations/spiders/locations.py -------------------------------------------------------------------------------- /examples/locations/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/requirements.txt -------------------------------------------------------------------------------- /examples/locations/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/scrapy.cfg -------------------------------------------------------------------------------- /examples/locations/seeds.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/seeds.txt -------------------------------------------------------------------------------- /examples/locations/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/examples/locations/tokenizer.py -------------------------------------------------------------------------------- /lib/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/CMakeLists.txt -------------------------------------------------------------------------------- /lib/lib/lmdb/COPYRIGHT: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/COPYRIGHT -------------------------------------------------------------------------------- /lib/lib/lmdb/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/LICENSE -------------------------------------------------------------------------------- /lib/lib/lmdb/lmdb.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/lmdb.h -------------------------------------------------------------------------------- /lib/lib/lmdb/mdb.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/mdb.c -------------------------------------------------------------------------------- /lib/lib/lmdb/midl.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/midl.c -------------------------------------------------------------------------------- /lib/lib/lmdb/midl.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/lmdb/midl.h -------------------------------------------------------------------------------- /lib/lib/mman.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/mman.c -------------------------------------------------------------------------------- /lib/lib/mman.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/mman.h -------------------------------------------------------------------------------- /lib/lib/smaz.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/smaz.c -------------------------------------------------------------------------------- /lib/lib/smaz.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/smaz.h -------------------------------------------------------------------------------- /lib/lib/xxhash.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/xxhash.c -------------------------------------------------------------------------------- /lib/lib/xxhash.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/lib/xxhash.h -------------------------------------------------------------------------------- /lib/src/bf_scheduler.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/bf_scheduler.c -------------------------------------------------------------------------------- /lib/src/bf_scheduler.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/bf_scheduler.h -------------------------------------------------------------------------------- /lib/src/bf_scheduler_reload.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/bf_scheduler_reload.c -------------------------------------------------------------------------------- /lib/src/domain_temp.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/domain_temp.c -------------------------------------------------------------------------------- /lib/src/domain_temp.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/domain_temp.h -------------------------------------------------------------------------------- /lib/src/freq_algo.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/freq_algo.c -------------------------------------------------------------------------------- /lib/src/freq_algo.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/freq_algo.h -------------------------------------------------------------------------------- /lib/src/freq_scheduler.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/freq_scheduler.c -------------------------------------------------------------------------------- /lib/src/freq_scheduler.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/freq_scheduler.h -------------------------------------------------------------------------------- /lib/src/freq_scheduler_dump.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/freq_scheduler_dump.c -------------------------------------------------------------------------------- /lib/src/hits.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/hits.c -------------------------------------------------------------------------------- /lib/src/hits.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/hits.h -------------------------------------------------------------------------------- /lib/src/hits_scorer.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/hits_scorer.c -------------------------------------------------------------------------------- /lib/src/hits_scorer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/hits_scorer.h -------------------------------------------------------------------------------- /lib/src/link_stream.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/link_stream.h -------------------------------------------------------------------------------- /lib/src/mmap_array.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/mmap_array.c -------------------------------------------------------------------------------- /lib/src/mmap_array.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/mmap_array.h -------------------------------------------------------------------------------- /lib/src/page_db.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db.c -------------------------------------------------------------------------------- /lib/src/page_db.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db.h -------------------------------------------------------------------------------- /lib/src/page_db_dump.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db_dump.c -------------------------------------------------------------------------------- /lib/src/page_db_find.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db_find.c -------------------------------------------------------------------------------- /lib/src/page_db_links.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db_links.c -------------------------------------------------------------------------------- /lib/src/page_db_path.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_db_path.c -------------------------------------------------------------------------------- /lib/src/page_rank.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_rank.c -------------------------------------------------------------------------------- /lib/src/page_rank.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_rank.h -------------------------------------------------------------------------------- /lib/src/page_rank_scorer.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_rank_scorer.c -------------------------------------------------------------------------------- /lib/src/page_rank_scorer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/page_rank_scorer.h -------------------------------------------------------------------------------- /lib/src/scheduler.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/scheduler.c -------------------------------------------------------------------------------- /lib/src/scheduler.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/scheduler.h -------------------------------------------------------------------------------- /lib/src/scorer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/scorer.h -------------------------------------------------------------------------------- /lib/src/txn_manager.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/txn_manager.c -------------------------------------------------------------------------------- /lib/src/txn_manager.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/txn_manager.h -------------------------------------------------------------------------------- /lib/src/util.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/util.c -------------------------------------------------------------------------------- /lib/src/util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/src/util.h -------------------------------------------------------------------------------- /lib/test/CuTest.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/CuTest.c -------------------------------------------------------------------------------- /lib/test/CuTest.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/CuTest.h -------------------------------------------------------------------------------- /lib/test/test.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test.c -------------------------------------------------------------------------------- /lib/test/test.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test.h -------------------------------------------------------------------------------- /lib/test/test_bf_scheduler.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_bf_scheduler.c -------------------------------------------------------------------------------- /lib/test/test_domain_temp.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_domain_temp.c -------------------------------------------------------------------------------- /lib/test/test_freq_scheduler.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_freq_scheduler.c -------------------------------------------------------------------------------- /lib/test/test_hits.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_hits.c -------------------------------------------------------------------------------- /lib/test/test_page_rank.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_page_rank.c -------------------------------------------------------------------------------- /lib/test/test_pagedb.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_pagedb.c -------------------------------------------------------------------------------- /lib/test/test_util.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/lib/test/test_util.c -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/setup.py -------------------------------------------------------------------------------- /test/links.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/test/links.txt.gz -------------------------------------------------------------------------------- /test/nodes.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/test/nodes.txt.gz -------------------------------------------------------------------------------- /test/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/test/requirements.txt -------------------------------------------------------------------------------- /test/test_crawl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/test/test_crawl.py -------------------------------------------------------------------------------- /test/test_freq_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/test/test_freq_scheduler.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scrapinghub/aduana/HEAD/tox.ini --------------------------------------------------------------------------------