├── .gitignore ├── .travis.yml ├── MANIFEST.in ├── README.rst ├── docs └── images │ └── logo_readme.jpg ├── generator.py ├── pytest.ini ├── requirements.txt ├── scrapy_eagle ├── __init__.py ├── dashboard │ ├── .babelrc │ ├── __init__.py │ ├── green_threads │ │ ├── __init__.py │ │ ├── executor.py │ │ ├── heartbeat.py │ │ └── stats.py │ ├── main.py │ ├── memory.py │ ├── package.json │ ├── react-src │ │ ├── components │ │ │ ├── App.jsx │ │ │ ├── App.scss │ │ │ ├── Home.jsx │ │ │ ├── List.jsx │ │ │ ├── ListItem.jsx │ │ │ ├── jobs │ │ │ │ ├── JobsConfig.jsx │ │ │ │ ├── JobsConfig.scss │ │ │ │ ├── JobsItem.jsx │ │ │ │ └── Root.jsx │ │ │ └── servers │ │ │ │ ├── Root.jsx │ │ │ │ ├── ServerNode.jsx │ │ │ │ ├── ServerSet.jsx │ │ │ │ └── ServerSubProcess.jsx │ │ ├── main.jsx │ │ ├── reducers │ │ │ ├── jobs.jsx │ │ │ └── servers.jsx │ │ └── services │ │ │ └── httpservice.js │ ├── settings.py │ ├── templates │ │ ├── index.html │ │ └── static │ │ │ ├── css │ │ │ ├── bundle.css │ │ │ ├── bundle.css.map │ │ │ └── main.css │ │ │ ├── img │ │ │ └── system-logo.jpg │ │ │ └── js │ │ │ ├── bundle.js │ │ │ └── vendor │ │ │ └── jquery.navgoco.min.js │ ├── utils │ │ ├── __init__.py │ │ ├── commandskit.py │ │ ├── ip.py │ │ ├── processkit.py │ │ └── spiderskit.py │ ├── views │ │ ├── __init__.py │ │ ├── jobs.py │ │ ├── processes.py │ │ ├── react_app.py │ │ ├── root.py │ │ └── servers.py │ ├── webpack.config.dev.js │ └── webpack.config.prod.js └── worker │ ├── __init__.py │ ├── connection.py │ ├── dupefilter.py │ ├── picklecompat.py │ ├── queue.py │ ├── scheduler.py │ └── spiders.py ├── setup.py ├── tests └── test_queue.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/.travis.yml -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/README.rst -------------------------------------------------------------------------------- /docs/images/logo_readme.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/docs/images/logo_readme.jpg -------------------------------------------------------------------------------- /generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/generator.py -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/pytest.ini -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/requirements.txt -------------------------------------------------------------------------------- /scrapy_eagle/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["react", "es2015"] 3 | } 4 | -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/green_threads/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/green_threads/__init__.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/green_threads/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/green_threads/executor.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/green_threads/heartbeat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/green_threads/heartbeat.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/green_threads/stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/green_threads/stats.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/main.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/memory.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/package.json -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/App.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/App.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/App.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/App.scss -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/Home.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/Home.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/List.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/List.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/ListItem.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/ListItem.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/jobs/JobsConfig.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/jobs/JobsConfig.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/jobs/JobsConfig.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/jobs/JobsConfig.scss -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/jobs/JobsItem.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/jobs/JobsItem.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/jobs/Root.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/jobs/Root.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/servers/Root.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/servers/Root.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/servers/ServerNode.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/servers/ServerNode.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/servers/ServerSet.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/servers/ServerSet.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/components/servers/ServerSubProcess.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/components/servers/ServerSubProcess.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/main.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/main.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/reducers/jobs.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/reducers/jobs.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/reducers/servers.jsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/reducers/servers.jsx -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/react-src/services/httpservice.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/react-src/services/httpservice.js -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/settings.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/index.html -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/css/bundle.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/css/bundle.css -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/css/bundle.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/css/bundle.css.map -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/css/main.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/css/main.css -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/img/system-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/img/system-logo.jpg -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/js/bundle.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/js/bundle.js -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/templates/static/js/vendor/jquery.navgoco.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/templates/static/js/vendor/jquery.navgoco.min.js -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/utils/__init__.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/utils/commandskit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/utils/commandskit.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/utils/ip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/utils/ip.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/utils/processkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/utils/processkit.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/utils/spiderskit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/utils/spiderskit.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/jobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/views/jobs.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/processes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/views/processes.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/react_app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/views/react_app.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/root.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/views/root.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/views/servers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/views/servers.py -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/webpack.config.dev.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/webpack.config.dev.js -------------------------------------------------------------------------------- /scrapy_eagle/dashboard/webpack.config.prod.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/dashboard/webpack.config.prod.js -------------------------------------------------------------------------------- /scrapy_eagle/worker/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy_eagle/worker/connection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/connection.py -------------------------------------------------------------------------------- /scrapy_eagle/worker/dupefilter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/dupefilter.py -------------------------------------------------------------------------------- /scrapy_eagle/worker/picklecompat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/picklecompat.py -------------------------------------------------------------------------------- /scrapy_eagle/worker/queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/queue.py -------------------------------------------------------------------------------- /scrapy_eagle/worker/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/scheduler.py -------------------------------------------------------------------------------- /scrapy_eagle/worker/spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/scrapy_eagle/worker/spiders.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/setup.py -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/tests/test_queue.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rafaelcapucho/scrapy-eagle/HEAD/tox.ini --------------------------------------------------------------------------------