├── .dockerignore
├── .drone.yml
├── .editorconfig
├── .gitattributes
├── .gitignore
├── .jshintrc
├── .travis.yml
├── CHANGES
├── Dockerfile
├── LICENSE
├── README.md
├── VERSION
├── Vagrantfile
├── bin
└── bump_version.py
├── data
└── projects
│ └── .gitkeep
├── docker-compose.yml
├── docker
├── compile-assets.sh
├── entry
├── nginx
│ ├── nginx.conf
│ ├── proxy_portia_server.conf
│ └── proxy_slyd.conf
├── portia.conf
├── provision.sh
├── qt_install.qs
├── restore-mtime.sh
└── run-tests.sh
├── docs
├── Makefile
├── _static
│ ├── getting-started-1.png
│ ├── portia-add-start-pages.png
│ ├── portia-annotation-creation.png
│ ├── portia-annotation.png
│ ├── portia-change-selection-mode.png
│ ├── portia-configuring-crawling.png
│ ├── portia-extracted-items.png
│ ├── portia-extractors.png
│ ├── portia-follow-patterns.png
│ ├── portia-goto-extractors.png
│ ├── portia-icon-add-repeat.png
│ ├── portia-icon-add.png
│ ├── portia-icon-pointer.png
│ ├── portia-icon-sub.png
│ ├── portia-icon-toggle-links.png
│ ├── portia-icon-wand.png
│ ├── portia-item-editor.png
│ ├── portia-landing-page.png
│ ├── portia-main-page.png
│ ├── portia-multi-last.png
│ ├── portia-multi-preview.png
│ ├── portia-new-project.png
│ ├── portia-new-spider.png
│ ├── portia-sample-multiple-fields.png
│ ├── portia-spider-link-crawling.png
│ ├── portia-spider-properties.png
│ └── portia-start-urls.png
├── conf.py
├── examples.rst
├── faq.rst
├── getting-started.rst
├── index.rst
├── installation.rst
├── items.rst
├── make.bat
├── projects.rst
├── samples.rst
└── spiders.rst
├── portia_server
├── db_repo
│ ├── __init__.py
│ ├── apps.py
│ ├── migrations
│ │ ├── 0001_initial.py
│ │ ├── __init__.py
│ │ └── slyd_to_django.sql
│ ├── models.py
│ └── repo.py
├── manage.py
├── portia_api
│ ├── __init__.py
│ ├── apps.py
│ ├── errors.py
│ ├── jsonapi
│ │ ├── __init__.py
│ │ ├── exceptions.py
│ │ ├── parsers.py
│ │ ├── registry.py
│ │ ├── relationships.py
│ │ ├── renderers.py
│ │ ├── response.py
│ │ ├── serializers.py
│ │ └── utils.py
│ ├── resources
│ │ ├── __init__.py
│ │ ├── annotations.py
│ │ ├── extractors.py
│ │ ├── fields.py
│ │ ├── items.py
│ │ ├── models.py
│ │ ├── projects.py
│ │ ├── response.py
│ │ ├── route.py
│ │ ├── samples.py
│ │ ├── schemas.py
│ │ ├── serializers.py
│ │ └── spiders.py
│ ├── routers.py
│ ├── tests
│ │ ├── __init__.py
│ │ └── test_routes.py
│ ├── urls.py
│ └── utils
│ │ ├── __init__.py
│ │ ├── annotations.py
│ │ ├── copy.py
│ │ ├── deploy
│ │ ├── base.py
│ │ ├── package.py
│ │ ├── scrapinghub.py
│ │ └── scrapyd.py
│ │ ├── download.py
│ │ ├── extract.py
│ │ ├── projects.py
│ │ └── spiders.py
├── portia_orm
│ ├── __init__.py
│ ├── apps.py
│ ├── base.py
│ ├── collection.py
│ ├── datastore.py
│ ├── decorators.py
│ ├── deletion.py
│ ├── exceptions.py
│ ├── fields.py
│ ├── middleware.py
│ ├── models.py
│ ├── registry.py
│ ├── relationships.py
│ ├── serializers.py
│ ├── snapshots.py
│ ├── tests
│ │ ├── __init__.py
│ │ ├── models.py
│ │ ├── test_basic.py
│ │ ├── test_collection.py
│ │ ├── test_model.py
│ │ ├── test_relationship.py
│ │ └── utils.py
│ ├── utils.py
│ └── validators.py
├── portia_server
│ ├── __init__.py
│ ├── backends.py
│ ├── models.py
│ ├── settings.py
│ ├── urls.py
│ ├── views.py
│ └── wsgi.py
├── requirements.txt
└── storage
│ ├── __init__.py
│ ├── apps.py
│ ├── backends.py
│ ├── jsondiff.py
│ ├── projecttemplates.py
│ └── repoman.py
├── portiaui
├── .bowerrc
├── .editorconfig
├── .ember-cli
├── .gitignore
├── .jshintrc
├── .watchmanconfig
├── app
│ ├── adapters
│ │ ├── application.js
│ │ └── project.js
│ ├── app.js
│ ├── components
│ │ ├── .gitkeep
│ │ ├── add-start-url-button.js
│ │ ├── animation-container.js
│ │ ├── annotation-options.js
│ │ ├── browser-iframe.js
│ │ ├── browser-url-blocked.js
│ │ ├── browser-url-failing.js
│ │ ├── browser-view-port.js
│ │ ├── buffered-input.js
│ │ ├── colored-badge.js
│ │ ├── colored-span.js
│ │ ├── combo-box.js
│ │ ├── create-project-button.js
│ │ ├── create-spider-button.js
│ │ ├── data-structure-annotations.js
│ │ ├── data-structure-listing.js
│ │ ├── dropdown-delete.js
│ │ ├── dropdown-divider.js
│ │ ├── dropdown-header.js
│ │ ├── dropdown-item.js
│ │ ├── dropdown-menu.js
│ │ ├── dropdown-widget.js
│ │ ├── edit-sample-button.js
│ │ ├── element-overlay.js
│ │ ├── element-rect-overlay.js
│ │ ├── extracted-item-table.js
│ │ ├── extracted-items-group.js
│ │ ├── extracted-items-json-panel.js
│ │ ├── extracted-items-json-value.js
│ │ ├── extracted-items-json.js
│ │ ├── extracted-items-panel.js
│ │ ├── extracted-items-status.js
│ │ ├── extracted-items-tab.js
│ │ ├── extractor-options.js
│ │ ├── feed-url-options.js
│ │ ├── field-options.js
│ │ ├── fragment-options.js
│ │ ├── generated-url-options.js
│ │ ├── help-icon.js
│ │ ├── icon-button.js
│ │ ├── indentation-spacer.js
│ │ ├── input-with-clear.js
│ │ ├── inspector-panel.js
│ │ ├── link-crawling-options.js
│ │ ├── list-item-add-annotation-menu.js
│ │ ├── list-item-annotation-field.js
│ │ ├── list-item-badge.js
│ │ ├── list-item-combo.js
│ │ ├── list-item-editable.js
│ │ ├── list-item-field-type.js
│ │ ├── list-item-icon-menu.js
│ │ ├── list-item-icon.js
│ │ ├── list-item-item-schema.js
│ │ ├── list-item-link-crawling.js
│ │ ├── list-item-relation-manager.js
│ │ ├── list-item-selectable.js
│ │ ├── list-item-text.js
│ │ ├── notification-container.js
│ │ ├── notification-message.js
│ │ ├── page-actions-editor.js
│ │ ├── project-list.js
│ │ ├── project-listing.js
│ │ ├── project-structure-listing.js
│ │ ├── project-structure-spider-feed-url.js
│ │ ├── project-structure-spider-generated-url.js
│ │ ├── project-structure-spider-url.js
│ │ ├── regex-pattern-list.js
│ │ ├── reorder-handler.js
│ │ ├── save-status.js
│ │ ├── schema-structure-listing.js
│ │ ├── scrapinghub-links.js
│ │ ├── select-box.js
│ │ ├── show-links-button.js
│ │ ├── show-links-legend.js
│ │ ├── sliding-main.js
│ │ ├── spider-indentation.js
│ │ ├── spider-message.js
│ │ ├── spider-options.js
│ │ ├── spider-row.js
│ │ ├── spider-structure-listing.js
│ │ ├── start-url-options.js
│ │ ├── tool-group.js
│ │ ├── tool-panel.js
│ │ ├── tool-tab.js
│ │ ├── tooltip-container.js
│ │ ├── tooltip-icon.js
│ │ ├── tree-list-item-row.js
│ │ ├── tree-list-item.js
│ │ ├── tree-list.js
│ │ └── url-bar.js
│ ├── controllers
│ │ ├── .gitkeep
│ │ └── projects
│ │ │ ├── project.js
│ │ │ └── project
│ │ │ ├── conflicts.js
│ │ │ ├── conflicts
│ │ │ └── conflict.js
│ │ │ ├── schema
│ │ │ └── field
│ │ │ │ └── options.js
│ │ │ ├── spider.js
│ │ │ └── spider
│ │ │ ├── link-options.js
│ │ │ ├── options.js
│ │ │ └── sample
│ │ │ ├── data.js
│ │ │ └── data
│ │ │ └── annotation
│ │ │ └── options.js
│ ├── helpers
│ │ ├── .gitkeep
│ │ ├── array-get.js
│ │ ├── attribute-annotation.js
│ │ ├── chain-actions.js
│ │ ├── guid.js
│ │ ├── includes.js
│ │ ├── indexed-object.js
│ │ ├── is-empty-object.js
│ │ ├── is-object-or-array.js
│ │ └── is-object.js
│ ├── index.html
│ ├── initializers
│ │ └── ui-state.js
│ ├── instance-initializers
│ │ └── error-handler.js
│ ├── mixins
│ │ ├── options-route.js
│ │ └── save-spider-mixin.js
│ ├── models
│ │ ├── .gitkeep
│ │ ├── annotation.js
│ │ ├── base-annotation.js
│ │ ├── base.js
│ │ ├── extractor.js
│ │ ├── field.js
│ │ ├── item.js
│ │ ├── project.js
│ │ ├── sample.js
│ │ ├── schema.js
│ │ ├── spider.js
│ │ └── start-url.js
│ ├── resolver.js
│ ├── router.js
│ ├── routes
│ │ ├── .gitkeep
│ │ ├── application.js
│ │ ├── browsers.js
│ │ ├── index.js
│ │ ├── projects.js
│ │ └── projects
│ │ │ ├── project.js
│ │ │ └── project
│ │ │ ├── compatibility.js
│ │ │ ├── conflicts.js
│ │ │ ├── conflicts
│ │ │ └── conflict.js
│ │ │ ├── schema.js
│ │ │ ├── schema
│ │ │ ├── field.js
│ │ │ └── field
│ │ │ │ └── options.js
│ │ │ ├── spider.js
│ │ │ └── spider
│ │ │ ├── link-options.js
│ │ │ ├── options.js
│ │ │ ├── sample.js
│ │ │ ├── sample
│ │ │ ├── data.js
│ │ │ ├── data
│ │ │ │ ├── annotation.js
│ │ │ │ ├── annotation
│ │ │ │ │ └── options.js
│ │ │ │ └── item.js
│ │ │ └── index.js
│ │ │ ├── start-url.js
│ │ │ └── start-url
│ │ │ └── options.js
│ ├── serializers
│ │ └── application.js
│ ├── services
│ │ ├── annotation-structure.js
│ │ ├── browser.js
│ │ ├── capabilities.js
│ │ ├── changes.js
│ │ ├── clock.js
│ │ ├── dispatcher.js
│ │ ├── extracted-items.js
│ │ ├── notification-manager.js
│ │ ├── overlays.js
│ │ ├── position-monitor.js
│ │ ├── saving-notification.js
│ │ ├── selector-matcher.js
│ │ ├── store.js
│ │ ├── ui-state.js
│ │ └── web-socket.js
│ ├── storages
│ │ ├── cookies.js
│ │ ├── page-loads.js
│ │ ├── ui-state-collapsed-panels.js
│ │ └── ui-state-selected-tools.js
│ ├── styles
│ │ ├── _animations.scss
│ │ ├── _bootstrap_overrides.scss
│ │ ├── _icons.scss
│ │ ├── _lib_config.scss
│ │ ├── _variables.scss
│ │ ├── app.scss
│ │ ├── components
│ │ │ ├── animation-container.scss
│ │ │ ├── browser-iframe.scss
│ │ │ ├── browser-view-port.scss
│ │ │ ├── combo-box.scss
│ │ │ ├── conflicts.scss
│ │ │ ├── dropdown-delete.scss
│ │ │ ├── dropdown-menu.scss
│ │ │ ├── dropdown-widget.scss
│ │ │ ├── extracted-item-table.scss
│ │ │ ├── extracted-items-json-panel.scss
│ │ │ ├── extractor-options.scss
│ │ │ ├── fragment-options.scss
│ │ │ ├── help-icon.scss
│ │ │ ├── icon-button.scss
│ │ │ ├── indentation-spacer.scss
│ │ │ ├── input-with-clear.scss
│ │ │ ├── inspector-panel.scss
│ │ │ ├── list-item-badge.scss
│ │ │ ├── list-item-combo.scss
│ │ │ ├── list-item-editable.scss
│ │ │ ├── list-item-icon.scss
│ │ │ ├── list-item-selectable.scss
│ │ │ ├── list-item-text.scss
│ │ │ ├── notifications.scss
│ │ │ ├── page-actions.scss
│ │ │ ├── project-structure-spider-generation-url.scss
│ │ │ ├── regex-pattern-list.scss
│ │ │ ├── save-status.scss
│ │ │ ├── select-box.scss
│ │ │ ├── show-links-legend.scss
│ │ │ ├── side-bar.scss
│ │ │ ├── sliding-main.scss
│ │ │ ├── start-url-options.scss
│ │ │ ├── tool-group.scss
│ │ │ ├── tool-panel.scss
│ │ │ ├── tooltip-container.scss
│ │ │ ├── top-bar.scss
│ │ │ ├── tree-list.scss
│ │ │ └── url-bar.scss
│ │ ├── document.scss
│ │ ├── droplet.scss
│ │ ├── generic.scss
│ │ ├── layout
│ │ │ ├── _clickable.scss
│ │ │ ├── _forms.scss
│ │ │ └── _full-page-content.scss
│ │ └── templates
│ │ │ ├── application.scss
│ │ │ ├── browsers.scss
│ │ │ └── projects.scss
│ ├── templates
│ │ ├── application.hbs
│ │ ├── branding.hbs
│ │ ├── browsers.hbs
│ │ ├── components
│ │ │ ├── .gitkeep
│ │ │ ├── add-start-url-button.hbs
│ │ │ ├── animation-container.hbs
│ │ │ ├── annotation-options.hbs
│ │ │ ├── browser-iframe.hbs
│ │ │ ├── browser-list.hbs
│ │ │ ├── browser-url-blocked.hbs
│ │ │ ├── browser-url-failing.hbs
│ │ │ ├── browser-view-port.hbs
│ │ │ ├── buffered-input.hbs
│ │ │ ├── colored-badge.hbs
│ │ │ ├── colored-span.hbs
│ │ │ ├── combo-box.hbs
│ │ │ ├── create-project-button.hbs
│ │ │ ├── create-spider-button.hbs
│ │ │ ├── data-structure-annotations.hbs
│ │ │ ├── data-structure-listing.hbs
│ │ │ ├── dropdown-delete.hbs
│ │ │ ├── dropdown-divider.hbs
│ │ │ ├── dropdown-header.hbs
│ │ │ ├── dropdown-item.hbs
│ │ │ ├── dropdown-menu.hbs
│ │ │ ├── dropdown-widget.hbs
│ │ │ ├── edit-sample-button.hbs
│ │ │ ├── element-overlay.hbs
│ │ │ ├── element-rect-overlay.hbs
│ │ │ ├── extracted-item-table.hbs
│ │ │ ├── extracted-items-group.hbs
│ │ │ ├── extracted-items-json-panel.hbs
│ │ │ ├── extracted-items-json-value.hbs
│ │ │ ├── extracted-items-json.hbs
│ │ │ ├── extracted-items-panel.hbs
│ │ │ ├── extracted-items-status.hbs
│ │ │ ├── extracted-items-tab.hbs
│ │ │ ├── extractor-options.hbs
│ │ │ ├── feed-url-options.hbs
│ │ │ ├── field-options.hbs
│ │ │ ├── fragment-options.hbs
│ │ │ ├── generated-url-options.hbs
│ │ │ ├── help-icon.hbs
│ │ │ ├── icon-button.hbs
│ │ │ ├── input-with-clear.hbs
│ │ │ ├── inspector-panel.hbs
│ │ │ ├── json-file-compare.hbs
│ │ │ ├── link-crawling-options.hbs
│ │ │ ├── list-item-add-annotation-menu.hbs
│ │ │ ├── list-item-annotation-field.hbs
│ │ │ ├── list-item-badge.hbs
│ │ │ ├── list-item-combo.hbs
│ │ │ ├── list-item-editable.hbs
│ │ │ ├── list-item-field-type.hbs
│ │ │ ├── list-item-icon-menu.hbs
│ │ │ ├── list-item-icon.hbs
│ │ │ ├── list-item-item-schema.hbs
│ │ │ ├── list-item-link-crawling.hbs
│ │ │ ├── list-item-relation-manager.hbs
│ │ │ ├── list-item-selectable.hbs
│ │ │ ├── list-item-text.hbs
│ │ │ ├── notification-container.hbs
│ │ │ ├── notification-message.hbs
│ │ │ ├── page-actions-editor.hbs
│ │ │ ├── project-list.hbs
│ │ │ ├── project-listing.hbs
│ │ │ ├── project-structure-listing.hbs
│ │ │ ├── project-structure-spider-feed-url.hbs
│ │ │ ├── project-structure-spider-generated-url.hbs
│ │ │ ├── project-structure-spider-url.hbs
│ │ │ ├── regex-pattern-list.hbs
│ │ │ ├── save-status.hbs
│ │ │ ├── schema-structure-listing.hbs
│ │ │ ├── scrapinghub-links.hbs
│ │ │ ├── select-box.hbs
│ │ │ ├── show-links-button.hbs
│ │ │ ├── show-links-legend.hbs
│ │ │ ├── sliding-main.hbs
│ │ │ ├── spider-indentation.hbs
│ │ │ ├── spider-message.hbs
│ │ │ ├── spider-options.hbs
│ │ │ ├── spider-row.hbs
│ │ │ ├── spider-structure-listing.hbs
│ │ │ ├── start-url-options.hbs
│ │ │ ├── tool-group.hbs
│ │ │ ├── tool-panel.hbs
│ │ │ ├── tool-tab.hbs
│ │ │ ├── tooltip-container.hbs
│ │ │ ├── tooltip-icon.hbs
│ │ │ ├── tree-list-item-row.hbs
│ │ │ ├── tree-list-item.hbs
│ │ │ ├── tree-list.hbs
│ │ │ └── url-bar.hbs
│ │ ├── options-panels.hbs
│ │ ├── projects.hbs
│ │ ├── projects
│ │ │ ├── project.hbs
│ │ │ └── project
│ │ │ │ ├── conflicts
│ │ │ │ ├── file-selector.hbs
│ │ │ │ ├── help.hbs
│ │ │ │ ├── resolver.hbs
│ │ │ │ └── topbar.hbs
│ │ │ │ ├── schema.hbs
│ │ │ │ ├── schema
│ │ │ │ ├── field.hbs
│ │ │ │ ├── field
│ │ │ │ │ └── options.hbs
│ │ │ │ └── structure.hbs
│ │ │ │ ├── spider.hbs
│ │ │ │ ├── spider
│ │ │ │ ├── link-options.hbs
│ │ │ │ ├── options.hbs
│ │ │ │ ├── overlays.hbs
│ │ │ │ ├── sample.hbs
│ │ │ │ ├── sample
│ │ │ │ │ ├── annotation
│ │ │ │ │ │ └── selection.hbs
│ │ │ │ │ ├── data.hbs
│ │ │ │ │ ├── data
│ │ │ │ │ │ ├── annotation.hbs
│ │ │ │ │ │ ├── annotation
│ │ │ │ │ │ │ └── options.hbs
│ │ │ │ │ │ ├── item.hbs
│ │ │ │ │ │ ├── overlays.hbs
│ │ │ │ │ │ ├── structure.hbs
│ │ │ │ │ │ ├── toolbar.hbs
│ │ │ │ │ │ └── tools.hbs
│ │ │ │ │ ├── item.hbs
│ │ │ │ │ ├── structure.hbs
│ │ │ │ │ └── toolbar.hbs
│ │ │ │ ├── start-url
│ │ │ │ │ └── options.hbs
│ │ │ │ ├── structure.hbs
│ │ │ │ ├── toolbar.hbs
│ │ │ │ └── tools.hbs
│ │ │ │ ├── structure.hbs
│ │ │ │ └── toolbar.hbs
│ │ └── tool-panels.hbs
│ ├── transforms
│ │ ├── array.js
│ │ ├── json.js
│ │ └── start-url.js
│ ├── utils
│ │ ├── attrs.js
│ │ ├── browser-features.js
│ │ ├── colors.js
│ │ ├── computed.js
│ │ ├── ensure-promise.js
│ │ ├── interaction-event.js
│ │ ├── promises.js
│ │ ├── selectors.js
│ │ ├── start-urls.js
│ │ ├── tree-mirror-delegate.js
│ │ ├── types.js
│ │ └── utils.js
│ ├── validations
│ │ ├── fixed-fragment.js
│ │ ├── list-fragment.js
│ │ └── range-fragment.js
│ └── validators
│ │ ├── range.js
│ │ └── whitespace.js
├── bower.json
├── config
│ ├── deprecation-workflow.js
│ ├── environment-development.js
│ ├── environment-production.js
│ ├── environment-test.js
│ └── environment.js
├── ember-cli-build.js
├── package-lock.json
├── package.json
├── public
│ ├── assets
│ │ └── images
│ │ │ ├── chrome-logo.jpg
│ │ │ ├── firefox-logo.png
│ │ │ └── portia-logo.svg
│ ├── crossdomain.xml
│ ├── empty-frame.html
│ ├── frames-not-supported.html
│ └── robots.txt
├── testem.js
├── tests
│ ├── .jshintrc
│ ├── helpers
│ │ ├── destroy-app.js
│ │ ├── module-for-acceptance.js
│ │ ├── resolver.js
│ │ └── start-app.js
│ ├── index.html
│ ├── test-helper.js
│ └── unit
│ │ ├── .gitkeep
│ │ ├── models
│ │ └── start-url-test.js
│ │ ├── utils
│ │ ├── selectors-test.js
│ │ └── start-urls-test.js
│ │ └── validators
│ │ ├── range-test.js
│ │ └── whitespace-test.js
└── vendor
│ ├── .gitkeep
│ ├── modernizr.js
│ ├── mutation-summary.js
│ └── tree-mirror.js
├── slybot
├── .gitignore
├── CHANGES
├── MANIFEST.in
├── Makefile.buildbot
├── README.rst
├── bin
│ ├── makedeb
│ ├── portiacrawl
│ └── slybot
├── debian
│ ├── changelog
│ ├── compat
│ ├── control
│ ├── copyright
│ ├── pyversions
│ └── rules
├── docs
│ ├── Makefile
│ ├── conf.py
│ ├── index.rst
│ ├── make.bat
│ ├── project.rst
│ └── spiderlets.rst
├── requirements-clustering.txt
├── requirements-test.txt
├── requirements.txt
├── scrapy.cfg
├── setup.py
├── slybot
│ ├── __init__.py
│ ├── baseurl.py
│ ├── closespider.py
│ ├── clustering.py
│ ├── dupefilter.py
│ ├── exporter.py
│ ├── extractors.py
│ ├── fieldtypes
│ │ ├── __init__.py
│ │ ├── date.py
│ │ ├── images.py
│ │ ├── number.py
│ │ ├── point.py
│ │ ├── price.py
│ │ ├── text.py
│ │ └── url.py
│ ├── generic_form.py
│ ├── item.py
│ ├── linkextractor
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── ecsv.py
│ │ ├── html.py
│ │ ├── pagination.py
│ │ ├── regex.py
│ │ └── xml.py
│ ├── meta.py
│ ├── pageactions.py
│ ├── plugins
│ │ ├── __init__.py
│ │ ├── scrapely_annotations
│ │ │ ├── __init__.py
│ │ │ ├── annotations.py
│ │ │ ├── builder.py
│ │ │ ├── exceptions.py
│ │ │ ├── extraction
│ │ │ │ ├── __init__.py
│ │ │ │ ├── container_extractors.py
│ │ │ │ ├── extractors.py
│ │ │ │ ├── pageparsing.py
│ │ │ │ ├── region_extractors.py
│ │ │ │ └── utils.py
│ │ │ ├── migration.py
│ │ │ ├── processors.py
│ │ │ └── utils.py
│ │ └── selectors
│ │ │ └── __init__.py
│ ├── settings.py
│ ├── spider.py
│ ├── spiderlets.py
│ ├── spidermanager.py
│ ├── splash.py
│ ├── starturls
│ │ ├── __init__.py
│ │ ├── feed_generator.py
│ │ ├── fragment_generator.py
│ │ ├── generated_url.py
│ │ └── generator.py
│ ├── tests
│ │ ├── __init__.py
│ │ ├── data
│ │ │ ├── SampleProject
│ │ │ │ ├── extractors.json
│ │ │ │ ├── items.json
│ │ │ │ ├── project.json
│ │ │ │ └── spiders
│ │ │ │ │ ├── allowed_domains.json
│ │ │ │ │ ├── any_allowed_domains.json
│ │ │ │ │ ├── books.toscrape.com.json
│ │ │ │ │ ├── books.toscrape.com
│ │ │ │ │ ├── 3617-44af-a2f0.json
│ │ │ │ │ ├── 3617-44af-a2f0
│ │ │ │ │ │ └── original_body.html
│ │ │ │ │ ├── 3652-4fa1-a912.json
│ │ │ │ │ ├── 4583-41b4-9edb.json
│ │ │ │ │ └── 4583-41b4-9edb
│ │ │ │ │ │ └── original_body.html
│ │ │ │ │ ├── books.toscrape.com_1.json
│ │ │ │ │ ├── cargurus.json
│ │ │ │ │ ├── ebay.json
│ │ │ │ │ ├── ebay2.json
│ │ │ │ │ ├── ebay3.json
│ │ │ │ │ ├── ebay4.json
│ │ │ │ │ ├── example.com.json
│ │ │ │ │ ├── example2.com.json
│ │ │ │ │ ├── example3.com.json
│ │ │ │ │ ├── example4.com.json
│ │ │ │ │ ├── networkhealth.com.json
│ │ │ │ │ ├── networkhealth.com
│ │ │ │ │ ├── networkhealthtemplate.json
│ │ │ │ │ └── networkhealthtemplate
│ │ │ │ │ │ ├── annotated_body.html
│ │ │ │ │ │ └── original_body.html
│ │ │ │ │ ├── pinterest.com.json
│ │ │ │ │ ├── seedsofchange.com.json
│ │ │ │ │ ├── seedsofchange.json
│ │ │ │ │ ├── seedsofchange2.json
│ │ │ │ │ └── sitemaps.json
│ │ │ ├── atom_sample.xml
│ │ │ ├── ebay_advanced_search.html
│ │ │ ├── pinterest.html
│ │ │ ├── rss_sample.xml
│ │ │ ├── sitemap_sample.xml
│ │ │ ├── templates
│ │ │ │ ├── 411_list.json
│ │ │ │ ├── autoevolution.html
│ │ │ │ ├── autoevolution.json
│ │ │ │ ├── autoevolution2.json
│ │ │ │ ├── cars.com.json
│ │ │ │ ├── cars.com_nested.json
│ │ │ │ ├── cs-cart.json
│ │ │ │ ├── daft_ie.html
│ │ │ │ ├── daft_list.json
│ │ │ │ ├── firmen.wko.at.html
│ │ │ │ ├── firmen.wko.at.json
│ │ │ │ ├── hn.html
│ │ │ │ ├── patchofland.html
│ │ │ │ ├── so_annotations.json
│ │ │ │ ├── stack_overflow.html
│ │ │ │ ├── stips.co.il.html
│ │ │ │ ├── stips.co.il.json
│ │ │ │ └── xceed.json
│ │ │ └── test_params.txt
│ │ ├── test_baseurl.py
│ │ ├── test_dropmeta.py
│ │ ├── test_dupefilter.py
│ │ ├── test_extraction_speed.py
│ │ ├── test_extractors.py
│ │ ├── test_fieldtypes.py
│ │ ├── test_fragment_generator.py
│ │ ├── test_generic_form.py
│ │ ├── test_linkextractors.py
│ │ ├── test_migration.py
│ │ ├── test_multiple_item_extraction.py
│ │ ├── test_page_actions.py
│ │ ├── test_schema_validation.py
│ │ ├── test_selectors.py
│ │ ├── test_spider.py
│ │ ├── test_starturls.py
│ │ ├── test_starturls_generator.py
│ │ └── utils.py
│ ├── utils.py
│ └── validation
│ │ ├── __init__.py
│ │ ├── schema.py
│ │ └── schemas.json
└── tox.ini
├── slyd
├── .gitignore
├── .jshintrc
├── README.md
├── bin
│ ├── init_mysql_db
│ ├── sh2sly
│ └── slyd
├── requirements.txt
├── setup.py
├── slybot
├── slyd
│ ├── __init__.py
│ ├── authmanager.py
│ ├── dummyauth.py
│ ├── errors.py
│ ├── gitstorage
│ │ ├── __init__.py
│ │ ├── jsondiff.py
│ │ ├── projects.py
│ │ └── projectspec.py
│ ├── html_utils.py
│ ├── projects.py
│ ├── projectspec.py
│ ├── resource.py
│ ├── server.py
│ ├── settings
│ │ ├── __init__.py
│ │ └── base.py
│ ├── specmanager.py
│ ├── splash
│ │ ├── __init__.py
│ │ ├── commands.py
│ │ ├── cookies.py
│ │ ├── css_utils.py
│ │ ├── ferry.py
│ │ ├── proxy.py
│ │ ├── qtutils.py
│ │ └── utils.py
│ └── tap.py
└── twisted
│ └── plugins
│ └── slyd_plugin.py
└── splash_utils
├── compile_slybot.sh
├── filters
└── easylist.txt
├── perform_actions.js
├── waitAsync.js
└── z_inject_this.js
/.dockerignore:
--------------------------------------------------------------------------------
1 | .git
2 | .vagrant
3 | docs
4 | */node_modules
5 | */bower_components
6 | */tests
7 | */tmp
8 | */db.sqlite3
9 | */.tox
10 | */.pyc
11 | */__pycache__
12 |
--------------------------------------------------------------------------------
/.drone.yml:
--------------------------------------------------------------------------------
1 | image: scrapinghub
2 |
3 | script:
4 | - echo "Portia is at:"`git show -s --pretty=%d HEAD`
5 | - git restore-mtime
6 | - shopt -s extglob
7 | - nvm install 10.16.0
8 | - nvm use 10.16.0
9 | - sudo mkdir -p ~/.npm ~/.node-gyp ~/.cache
10 | - sudo chown -R ubuntu ~/.npm ~/.node-gyp ~/.cache
11 | - npm install -g bower ember-cli@2.6.3 --cache-min 999999
12 | - docker/compile-assets.sh
13 | - build_docker_image
14 | - publish_to_dockerhub
15 |
16 | cache:
17 | - /home/ubuntu/.npm
18 | - /home/ubuntu/.node-gyp
19 | - /home/ubuntu/.cache
20 |
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | # EditorConfig helps developers define and maintain consistent
2 | # coding styles between different editors and IDEs
3 | # editorconfig.org
4 |
5 | root = true
6 |
7 |
8 | [*]
9 | end_of_line = lf
10 | charset = utf-8
11 | trim_trailing_whitespace = true
12 | insert_final_newline = true
13 | indent_style = space
14 | indent_size = 2
15 |
16 | [*.js]
17 | indent_style = space
18 | indent_size = 2
19 |
20 | [*.hbs]
21 | indent_style = space
22 | indent_size = 2
23 |
24 | [*.css]
25 | indent_style = space
26 | indent_size = 2
27 |
28 | [*.html]
29 | indent_style = space
30 | indent_size = 2
31 |
32 | [*.{diff,md}]
33 | trim_trailing_whitespace = false
34 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.sh eol=lf
2 | *.bat eol=crlf
3 | *.js text
4 | *.py text
5 | *.css text
6 | *.hbs text
7 | *.json text
8 | *.html text
9 | *.xml text
10 | *.yml text
11 | *.txt text
12 | *.rst text
13 | *.md text
14 | *.cfg text
15 | *.conf text
16 | Makefile* text
17 |
18 | *.png binary
19 | *.swf binary
20 | *.ttf binary
21 | *.woff binary
22 | *.woff2 binary
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Python compiled files
2 | *__pycache__/*
3 | *.pyc
4 |
5 | # Vagrant files
6 | .vagrant/
7 | /.idea/
8 |
9 | # Python build files
10 | *.egg-info
11 | slybot/dist
12 | slybot/build
13 | slyd/slyd/dist
14 | slyd/slyd/build
15 |
16 | # npm files
17 | node_modules/*
18 | slyd/bower_components/*
19 | slyd/tmp/*
20 | npm-debug.log
21 | slybot/slybot/splash-script-combined.js
22 |
23 | # Local Settings
24 | slyd/slyd/local_settings.py
25 | slybot/slybot/local_slybot_settings.py
26 |
27 | # Testing
28 | slybot/.tox
29 |
30 | # Docs build directory
31 | docs/_build
32 |
33 | # Development Databases
34 | *.sqlite*
35 |
36 | # Default Portia data directory
37 | slyd/slyd/data
38 | /data/
39 |
--------------------------------------------------------------------------------
/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "predef": [
3 | "document",
4 | "window",
5 | "-Promise"
6 | ],
7 | "browser": true,
8 | "boss": true,
9 | "curly": true,
10 | "debug": false,
11 | "devel": true,
12 | "eqeqeq": true,
13 | "evil": true,
14 | "forin": false,
15 | "immed": false,
16 | "laxbreak": false,
17 | "newcap": true,
18 | "noarg": true,
19 | "noempty": false,
20 | "nonew": false,
21 | "nomen": false,
22 | "onevar": false,
23 | "plusplus": false,
24 | "regexp": false,
25 | "undef": true,
26 | "sub": true,
27 | "strict": false,
28 | "white": false,
29 | "eqnull": true,
30 | "esnext": true,
31 | "unused": true
32 | }
33 |
--------------------------------------------------------------------------------
/VERSION:
--------------------------------------------------------------------------------
1 | 2.0.8
2 |
--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
1 | # vim:ft=ruby
2 |
3 | Vagrant.configure("2") do |config|
4 | config.vm.box = "ubuntu/trusty64"
5 | config.vm.host_name = "portia"
6 | config.vm.provision :shell, :path => 'docker/provision.sh', :args => [
7 | "install_deps", "install_splash", "install_python_deps", "configure_nginx", "configure_initctl", "migrate_django_db", "start_portia"
8 | ]
9 | config.vm.network "private_network", ip: "33.33.33.10"
10 | config.vm.network "forwarded_port", guest: 9001, host: 9001
11 | config.vm.provider "virtualbox" do |v|
12 | v.memory = 2048
13 | v.cpus = 2
14 | end
15 | end
16 |
17 |
--------------------------------------------------------------------------------
/data/projects/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/data/projects/.gitkeep
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | app:
4 | build: .
5 | command: /app/docker/entry start-dev
6 | volumes:
7 | - ./data/projects:/app/data/projects:rw
8 | - ./portiaui/dist:/app/portiaui/dist
9 | - ./slyd:/app/slyd
10 | - ./portia_server:/app/portia_server
11 | - ./slybot:/app/slybot
12 | ports:
13 | - 9001:9001
14 | restart: always
15 |
--------------------------------------------------------------------------------
/docker/compile-assets.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | cd portiaui
3 | npm install
4 | npm run build
5 |
--------------------------------------------------------------------------------
/docker/entry:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -x
3 | action=$1
4 | shift
5 |
6 | _run() {
7 | service nginx start
8 | _set_env
9 | echo $PYTHONPATH
10 | /app/slyd/bin/slyd -p 9002 -r /app/portiaui/dist &
11 | /app/portia_server/manage.py runserver
12 | }
13 |
14 | _set_env() {
15 | path='/app/portia_server:/app/slyd:/app/slybot'
16 | export PYTHONPATH="$path"
17 | }
18 |
19 | if [ -z "$action" ]; then
20 | _run
21 | else
22 | case $action in
23 | start-dev|start-prod)
24 | _run
25 | ;;
26 | start-webshell)
27 | _run_webshell "$@"
28 | ;;
29 | *)
30 | exec $action "$@"
31 | ;;
32 | esac
33 | fi
--------------------------------------------------------------------------------
/docker/nginx/proxy_portia_server.conf:
--------------------------------------------------------------------------------
1 | proxy_pass http://127.0.0.1:8000;
2 | proxy_redirect off;
3 | proxy_set_header Host $http_host;
4 | proxy_set_header X-Real-IP $remote_addr;
5 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
6 | proxy_set_header X-Forwarded-Host $server_name;
7 |
--------------------------------------------------------------------------------
/docker/nginx/proxy_slyd.conf:
--------------------------------------------------------------------------------
1 | proxy_pass http://127.0.0.1:9002;
2 | proxy_redirect off;
3 | proxy_set_header Host $host:9002;
4 | proxy_set_header X-Real-IP $remote_addr;
5 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
6 | proxy_set_header X-Forwarded-Host $server_name;
7 |
--------------------------------------------------------------------------------
/docker/portia.conf:
--------------------------------------------------------------------------------
1 | description "portia server"
2 | start on vagrant-mounted or filesystem
3 | stop on runlevel [!2345]
4 |
5 | script
6 | export PYTHONPATH='/vagrant/portia_server:/vagrant/slyd:/vagrant/slybot'
7 | /vagrant/slyd/bin/slyd -p 9002 -r /vagrant/portiaui/dist &
8 | /vagrant/portia_server/manage.py runserver
9 | end script
10 | respawn
11 |
--------------------------------------------------------------------------------
/docker/restore-mtime.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | commit=$(git rev-list -n 1 HEAD requirements.txt)
3 | mtime=$(git show --pretty=format:%ai --abbrev-commit $commit |head -n1)
4 | touch -d "$mtime" requirements.txt
5 |
--------------------------------------------------------------------------------
/docker/run-tests.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | export PYTHONPATH=`pwd`/slybot:`pwd`/slyd
4 | pip install tox
5 |
6 | cd /app/slyd
7 | python2.7 tests/testserver/server.py 2>&1 | grep -v 'HTTP/1.1" 200' &
8 | sleep 3
9 |
10 | cd /app/slybot
11 | tox
12 | cd /app/portia_server
13 | ./manage.py test portia_orm.tests
14 | ./manage.py test portia_api.tests
15 |
--------------------------------------------------------------------------------
/docs/_static/getting-started-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/getting-started-1.png
--------------------------------------------------------------------------------
/docs/_static/portia-add-start-pages.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-add-start-pages.png
--------------------------------------------------------------------------------
/docs/_static/portia-annotation-creation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-annotation-creation.png
--------------------------------------------------------------------------------
/docs/_static/portia-annotation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-annotation.png
--------------------------------------------------------------------------------
/docs/_static/portia-change-selection-mode.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-change-selection-mode.png
--------------------------------------------------------------------------------
/docs/_static/portia-configuring-crawling.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-configuring-crawling.png
--------------------------------------------------------------------------------
/docs/_static/portia-extracted-items.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-extracted-items.png
--------------------------------------------------------------------------------
/docs/_static/portia-extractors.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-extractors.png
--------------------------------------------------------------------------------
/docs/_static/portia-follow-patterns.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-follow-patterns.png
--------------------------------------------------------------------------------
/docs/_static/portia-goto-extractors.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-goto-extractors.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-add-repeat.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-add-repeat.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-add.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-add.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-pointer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-pointer.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-sub.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-sub.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-toggle-links.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-toggle-links.png
--------------------------------------------------------------------------------
/docs/_static/portia-icon-wand.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-icon-wand.png
--------------------------------------------------------------------------------
/docs/_static/portia-item-editor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-item-editor.png
--------------------------------------------------------------------------------
/docs/_static/portia-landing-page.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-landing-page.png
--------------------------------------------------------------------------------
/docs/_static/portia-main-page.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-main-page.png
--------------------------------------------------------------------------------
/docs/_static/portia-multi-last.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-multi-last.png
--------------------------------------------------------------------------------
/docs/_static/portia-multi-preview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-multi-preview.png
--------------------------------------------------------------------------------
/docs/_static/portia-new-project.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-new-project.png
--------------------------------------------------------------------------------
/docs/_static/portia-new-spider.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-new-spider.png
--------------------------------------------------------------------------------
/docs/_static/portia-sample-multiple-fields.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-sample-multiple-fields.png
--------------------------------------------------------------------------------
/docs/_static/portia-spider-link-crawling.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-spider-link-crawling.png
--------------------------------------------------------------------------------
/docs/_static/portia-spider-properties.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-spider-properties.png
--------------------------------------------------------------------------------
/docs/_static/portia-start-urls.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/docs/_static/portia-start-urls.png
--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
1 | Welcome to Portia's documentation!
2 | ==================================
3 |
4 | Contents:
5 |
6 | .. toctree::
7 | :maxdepth: 2
8 |
9 | installation
10 | getting-started
11 | examples
12 | projects
13 | spiders
14 | samples
15 | items
16 | faq
17 |
18 | Indices and tables
19 | ==================
20 |
21 | * :ref:`genindex`
22 | * :ref:`modindex`
23 | * :ref:`search`
24 |
--------------------------------------------------------------------------------
/portia_server/db_repo/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/db_repo/__init__.py
--------------------------------------------------------------------------------
/portia_server/db_repo/apps.py:
--------------------------------------------------------------------------------
1 | from __future__ import unicode_literals
2 |
3 | from django.apps import AppConfig
4 |
5 |
6 | class DbRepoConfig(AppConfig):
7 | name = 'db_repo'
8 |
--------------------------------------------------------------------------------
/portia_server/db_repo/migrations/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/db_repo/migrations/__init__.py
--------------------------------------------------------------------------------
/portia_server/db_repo/migrations/slyd_to_django.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE `objs` DROP PRIMARY KEY,
2 | ADD COLUMN `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
3 | ADD CONSTRAINT `objs_oid_feda89ac_uniq` UNIQUE (`oid`, `repo`),
4 | CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
5 | ALTER COLUMN `oid` DROP DEFAULT;
6 | DROP INDEX `type` ON `objs`;
7 | DROP INDEX `size` ON `objs`;
8 | CREATE INDEX `objs_599dcce2` ON `objs` (`type`);
9 | CREATE INDEX `objs_f7bd60b7` ON `objs` (`size`);
10 |
11 | ALTER TABLE `refs` DROP PRIMARY KEY,
12 | ADD COLUMN `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
13 | ADD CONSTRAINT `refs_ref_4a751775_uniq` UNIQUE (`ref`, `repo`),
14 | CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
15 | ALTER COLUMN `ref` DROP DEFAULT;
16 | DROP INDEX `value` ON `refs`;
17 | CREATE INDEX `refs_2063c160` ON `refs` (`value`);
--------------------------------------------------------------------------------
/portia_server/manage.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import os
3 | import sys
4 |
5 | if __name__ == "__main__":
6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portia_server.settings")
7 |
8 | from django.core.management import execute_from_command_line
9 |
10 | execute_from_command_line(sys.argv)
11 |
--------------------------------------------------------------------------------
/portia_server/portia_api/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_api/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_api/apps.py:
--------------------------------------------------------------------------------
1 | from __future__ import unicode_literals
2 |
3 | from django.apps import AppConfig
4 |
5 |
6 | class PortiaApiConfig(AppConfig):
7 | name = 'portia_api'
8 |
--------------------------------------------------------------------------------
/portia_server/portia_api/jsonapi/__init__.py:
--------------------------------------------------------------------------------
1 | from .response import JSONResponse
2 |
--------------------------------------------------------------------------------
/portia_server/portia_api/jsonapi/parsers.py:
--------------------------------------------------------------------------------
1 | from __future__ import unicode_literals
2 |
3 | from rest_framework.parsers import JSONParser
4 |
5 |
6 | class JSONApiParser(JSONParser):
7 | media_type = 'application/vnd.api+json'
8 |
--------------------------------------------------------------------------------
/portia_server/portia_api/jsonapi/registry.py:
--------------------------------------------------------------------------------
1 | from portia_orm.exceptions import ImproperlyConfigured
2 |
3 |
4 | __all__ = [
5 | 'schema',
6 | ]
7 |
8 | schemas = {}
9 |
10 |
11 | def get_schema(schema_type):
12 | try:
13 | return schemas[schema_type]
14 | except KeyError:
15 | raise ImproperlyConfigured(
16 | u"No schema for type '{}' exists".format(schema_type))
17 |
--------------------------------------------------------------------------------
/portia_server/portia_api/jsonapi/response.py:
--------------------------------------------------------------------------------
1 | from django.http import HttpResponse
2 | from rest_framework.renderers import JSONRenderer
3 |
4 |
5 | class JSONResponse(HttpResponse):
6 | """
7 | An HttpResponse that renders its content into JSON.
8 | """
9 | def __init__(self, data, **kwargs):
10 | content = JSONRenderer().render(data)
11 | kwargs['content_type'] = 'application/json'
12 | super(JSONResponse, self).__init__(content, **kwargs)
13 |
--------------------------------------------------------------------------------
/portia_server/portia_api/resources/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_api/resources/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_api/resources/extractors.py:
--------------------------------------------------------------------------------
1 | from .projects import BaseProjectModelRoute
2 | from portia_orm.models import Extractor
3 |
4 |
5 | class ExtractorRoute(BaseProjectModelRoute):
6 | lookup_url_kwarg = 'extractor_id'
7 | default_model = Extractor
8 |
9 | def get_instance(self):
10 | return self.get_collection()[self.kwargs.get('extractor_id')]
11 |
12 | def get_collection(self):
13 | return self.project.extractors
14 |
--------------------------------------------------------------------------------
/portia_server/portia_api/routers.py:
--------------------------------------------------------------------------------
1 | from rest_framework_nested.routers import SimpleRouter, NestedSimpleRouter
2 |
3 | __all__ = [
4 | 'Router',
5 | 'NestedRouter',
6 | ]
7 |
8 |
9 | class Router(SimpleRouter):
10 | def __init__(self, trailing_slash=False):
11 | super(Router, self).__init__(trailing_slash)
12 |
13 | def get_lookup_regex(self, viewset, lookup_prefix=''):
14 | return super(Router, self).get_lookup_regex(viewset, '')
15 |
16 |
17 | class NestedRouter(NestedSimpleRouter, Router):
18 | def __init__(self, parent_router, parent_prefix, trailing_slash=False,
19 | *args, **kwargs):
20 | super(NestedRouter, self).__init__(
21 | parent_router, parent_prefix, trailing_slash, *args, **kwargs)
22 |
--------------------------------------------------------------------------------
/portia_server/portia_api/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_api/tests/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_api/tests/test_routes.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from rest_framework.test import APIRequestFactory
4 |
5 | from portia_api.resources.route import JsonApiRoute
6 |
7 |
8 | class TestRoute(unittest.TestCase):
9 | def test_route_representation(self):
10 | factory = APIRequestFactory()
11 | request = factory.get('/projects/')
12 | route = JsonApiRoute(request=request)
13 | self.assertEqual(str(route), 'GET /projects/')
14 | self.assertEqual(repr(route), 'Route(GET /projects/)')
15 |
--------------------------------------------------------------------------------
/portia_server/portia_api/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_api/utils/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_api/utils/annotations.py:
--------------------------------------------------------------------------------
1 |
2 | DEFAULTS = {
3 | 'accept': 'url',
4 | 'align': 'number',
5 | 'code': 'url',
6 | 'codebase': 'url',
7 | 'coords': 'geopoint',
8 | 'data': 'url',
9 | 'datetime': 'date',
10 | 'download': 'url',
11 | 'high': 'number',
12 | 'href': 'url',
13 | 'icon': 'image',
14 | 'low': 'number',
15 | 'max': 'number',
16 | 'media': 'href',
17 | 'min': 'number',
18 | 'optimum': 'number',
19 | 'rel': 'href',
20 | 'rows': 'number',
21 | 'src': 'image',
22 | 'target': 'url',
23 | }
24 |
25 |
26 | def choose_field_type(annotation):
27 | attribute = annotation.attribute
28 | if attribute == 'content':
29 | return 'text'
30 | return DEFAULTS.get(attribute, 'text')
31 |
--------------------------------------------------------------------------------
/portia_server/portia_api/utils/deploy/base.py:
--------------------------------------------------------------------------------
1 | from portia_api.utils.download import ProjectArchiver
2 |
3 |
4 | class BaseDeploy(object):
5 | def __init__(self, project):
6 | self.project = project
7 | self.storage = project.storage
8 | self.config = self._get_config()
9 | self.config.version = self.project.version
10 |
11 | def build_archive(self):
12 | return ProjectArchiver(self.storage, project=self.project).archive(
13 | egg_info=True)
14 |
15 | def _get_config(self):
16 | raise NotImplementedError
17 |
18 | def deploy(self, target=None):
19 | raise NotImplementedError
20 |
21 | def schedule(self, spider, args=None, settings=None, target=None):
22 | raise NotImplementedError
23 |
--------------------------------------------------------------------------------
/portia_server/portia_api/utils/projects.py:
--------------------------------------------------------------------------------
1 | def unique_name(base_name, disallow=(), initial_suffix=''):
2 | disallow = set(disallow)
3 | suffix = initial_suffix
4 | while True:
5 | name = u'{}{}'.format(base_name, suffix)
6 | if name not in disallow:
7 | break
8 | try:
9 | suffix += 1
10 | except TypeError:
11 | suffix = 1
12 | return name
13 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_orm/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_orm/apps.py:
--------------------------------------------------------------------------------
1 | from __future__ import unicode_literals
2 |
3 | from django.apps import AppConfig
4 |
5 |
6 | class PortiaOrmConfig(AppConfig):
7 | name = 'portia_orm'
8 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/decorators.py:
--------------------------------------------------------------------------------
1 | from marshmallow.decorators import (validates, validates_schema,
2 | pre_dump, post_dump, pre_load, post_load)
3 |
4 | __all__ = [
5 | 'validates',
6 | 'validates_schema',
7 | 'pre_dump',
8 | 'post_dump',
9 | 'pre_load',
10 | 'post_load',
11 | ]
12 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/exceptions.py:
--------------------------------------------------------------------------------
1 | from marshmallow.exceptions import ValidationError
2 |
3 | __all__ = [
4 | 'ImproperlyConfigured',
5 | 'ValidationError',
6 | ]
7 |
8 |
9 | class ImproperlyConfigured(Exception):
10 | pass
11 |
12 |
13 | class PathResolutionError(Exception):
14 | pass
15 |
16 |
17 | class ProtectedError(Exception):
18 | pass
19 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/middleware.py:
--------------------------------------------------------------------------------
1 | from .datastore import data_store_context
2 |
3 |
4 | class ORMDataStoreMiddleware(object):
5 | def __init__(self, get_response=None):
6 | self.get_response = get_response
7 |
8 | def __call__(self, request):
9 | with data_store_context():
10 | return self.get_response(request)
11 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/registry.py:
--------------------------------------------------------------------------------
1 | from six import itervalues
2 |
3 | from .exceptions import ImproperlyConfigured
4 |
5 |
6 | __all__ = [
7 | 'get_model',
8 | 'get_polymorphic_model',
9 | ]
10 |
11 | models = {}
12 |
13 |
14 | def get_model(model_name):
15 | try:
16 | return models[model_name]
17 | except KeyError:
18 | raise ImproperlyConfigured(
19 | u"No model named '{}' exists".format(model_name))
20 |
21 |
22 | def get_polymorphic_model(data):
23 | for model in itervalues(models):
24 | polymorphic = model.opts.polymorphic
25 | if polymorphic:
26 | polymorphic_key = polymorphic
27 | if isinstance(polymorphic_key, bool):
28 | polymorphic_key = 'type'
29 | if data.get(polymorphic_key) == model.__name__:
30 | return model
31 | raise ImproperlyConfigured(
32 | u"No model found for data: {!r}".format(data))
33 |
--------------------------------------------------------------------------------
/portia_server/portia_orm/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_orm/tests/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_orm/validators.py:
--------------------------------------------------------------------------------
1 | from marshmallow.validate import (ContainsOnly, Range, Regexp, Predicate,
2 | NoneOf, OneOf)
3 |
4 | __all__ = [
5 | 'ContainsOnly',
6 | 'Range',
7 | 'Regexp',
8 | 'Predicate',
9 | 'NoneOf',
10 | 'OneOf'
11 | ]
12 |
--------------------------------------------------------------------------------
/portia_server/portia_server/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portia_server/portia_server/__init__.py
--------------------------------------------------------------------------------
/portia_server/portia_server/backends.py:
--------------------------------------------------------------------------------
1 | from .models import LocalUser
2 |
3 |
4 | class LocalAuthentication(object):
5 | def authenticate(self, request, **kwargs):
6 | return LocalUser(**kwargs), None
7 |
8 | def get_user(self, user_id):
9 | # fall through and let the middleware add the user again
10 | return None
11 |
--------------------------------------------------------------------------------
/portia_server/portia_server/urls.py:
--------------------------------------------------------------------------------
1 | from django.conf.urls import url, include
2 |
3 | from . import views
4 | from portia_api import urls
5 |
6 | urlpatterns = [
7 | url(r'^api/', include((urls, 'api'), namespace='api')),
8 | url(r'^server_capabilities$', views.capabilities),
9 | ]
10 |
--------------------------------------------------------------------------------
/portia_server/portia_server/views.py:
--------------------------------------------------------------------------------
1 | from django.conf import settings
2 | from portia_api.jsonapi import JSONResponse
3 |
4 |
5 | def capabilities(request):
6 | capabilities = {
7 | 'custom': settings.CUSTOM,
8 | 'username': request.user.username,
9 | 'capabilities': settings.CAPABILITIES,
10 | }
11 | return JSONResponse(capabilities)
12 |
--------------------------------------------------------------------------------
/portia_server/portia_server/wsgi.py:
--------------------------------------------------------------------------------
1 | """
2 | WSGI config for portia_server project.
3 |
4 | It exposes the WSGI callable as a module-level variable named ``application``.
5 |
6 | For more information on this file, see
7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/
8 | """
9 |
10 | import os
11 |
12 | from django.core.wsgi import get_wsgi_application
13 |
14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "portia_server.settings")
15 |
16 | application = get_wsgi_application()
17 |
--------------------------------------------------------------------------------
/portia_server/requirements.txt:
--------------------------------------------------------------------------------
1 | crochet==1.9.0
2 | django>=1.11.21
3 | django-cache-machine==1.0.0
4 | djangorestframework==3.7.7
5 | dj-database-url==0.5.0
6 | drf-nested-routers==0.11.1
7 | dulwich==0.18.6
8 | marshmallow==2.8.0
9 | marshmallow_jsonapi==0.10.0
10 | mysqlclient==1.3.12
11 | requests>=2.20.0
12 | toposort==1.5
13 | whitenoise==3.3.1
14 | portia2code==0.0.17
15 | shub==2.7.0
16 |
--------------------------------------------------------------------------------
/portia_server/storage/__init__.py:
--------------------------------------------------------------------------------
1 | from django.conf import settings
2 | from django.utils.module_loading import import_string
3 |
4 | __all__ = [
5 | 'get_storage_class',
6 | 'create_project_storage',
7 | ]
8 |
9 | storage_class = None
10 |
11 |
12 | def get_storage_class():
13 | global storage_class
14 | if storage_class is None and settings.PORTIA_STORAGE_BACKEND:
15 | storage_class = import_string(settings.PORTIA_STORAGE_BACKEND)
16 | storage_class.setup()
17 | return storage_class
18 |
19 |
20 | def create_project_storage(project_id, author=None, branch=None):
21 | storage_class = get_storage_class()
22 | return storage_class(project_id, author=author)
23 |
--------------------------------------------------------------------------------
/portia_server/storage/apps.py:
--------------------------------------------------------------------------------
1 | from django.apps import AppConfig
2 |
3 |
4 | class StorageConfig(AppConfig):
5 | name = 'storage'
6 |
--------------------------------------------------------------------------------
/portiaui/.bowerrc:
--------------------------------------------------------------------------------
1 | {
2 | "directory": "bower_components",
3 | "analytics": false
4 | }
5 |
--------------------------------------------------------------------------------
/portiaui/.editorconfig:
--------------------------------------------------------------------------------
1 | # EditorConfig helps developers define and maintain consistent
2 | # coding styles between different editors and IDEs
3 | # editorconfig.org
4 |
5 | root = true
6 |
7 |
8 | [*]
9 | end_of_line = lf
10 | charset = utf-8
11 | trim_trailing_whitespace = true
12 | insert_final_newline = true
13 | indent_style = space
14 | indent_size = 4
15 |
16 | [*.js]
17 | indent_style = space
18 | indent_size = 4
19 |
20 | [*.hbs]
21 | indent_style = space
22 | indent_size = 4
23 |
24 | [*.css]
25 | indent_style = space
26 | indent_size = 4
27 |
28 | [*.html]
29 | indent_style = space
30 | indent_size = 4
31 |
32 | [*.{diff,md}]
33 | trim_trailing_whitespace = false
34 |
--------------------------------------------------------------------------------
/portiaui/.ember-cli:
--------------------------------------------------------------------------------
1 | {
2 | "disableAnalytics": true
3 | }
4 |
--------------------------------------------------------------------------------
/portiaui/.gitignore:
--------------------------------------------------------------------------------
1 | # See http://help.github.com/ignore-files/ for more about ignoring files.
2 |
3 | # compiled output
4 | /dist
5 | /tmp
6 |
7 | # dependencies
8 | /node_modules
9 | /bower_components
10 |
11 | # misc
12 | /.sass-cache
13 | /connect.lock
14 | /coverage/*
15 | /libpeerconnection.log
16 | npm-debug.log
17 | testem.log
18 |
--------------------------------------------------------------------------------
/portiaui/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "predef": [
3 | "document",
4 | "window",
5 | "$",
6 | "cookie",
7 | "moment",
8 | "URI",
9 | "TreeMirror",
10 | "-Promise",
11 | "Raven",
12 | "Modernizr"
13 | ],
14 | "browser": true,
15 | "boss": true,
16 | "curly": true,
17 | "debug": false,
18 | "devel": true,
19 | "eqeqeq": true,
20 | "evil": true,
21 | "forin": false,
22 | "immed": false,
23 | "indent": 4,
24 | "laxbreak": false,
25 | "maxlen": 100,
26 | "newcap": true,
27 | "noarg": true,
28 | "noempty": false,
29 | "nonew": false,
30 | "nomen": false,
31 | "onevar": false,
32 | "plusplus": false,
33 | "regexp": false,
34 | "undef": true,
35 | "sub": true,
36 | "strict": false,
37 | "white": false,
38 | "eqnull": true,
39 | "esnext": true,
40 | "unused": true
41 | }
42 |
--------------------------------------------------------------------------------
/portiaui/.watchmanconfig:
--------------------------------------------------------------------------------
1 | {
2 | "ignore_dirs": ["tmp", "dist"]
3 | }
4 |
--------------------------------------------------------------------------------
/portiaui/app/adapters/project.js:
--------------------------------------------------------------------------------
1 | import ApplicationAdapter from './application';
2 |
3 | export default ApplicationAdapter.extend({
4 | urlTemplate: '{+host}/api/projects{/id}',
5 | findRecordUrlTemplate: '{+host}/api/projects{/id}',
6 | createRecordUrlTemplate: '{+host}/api/projects',
7 | shouldReloadRecord() { return true; }
8 | });
9 |
--------------------------------------------------------------------------------
/portiaui/app/app.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import Resolver from './resolver';
3 | import loadInitializers from 'ember-load-initializers';
4 | import config from './config/environment';
5 |
6 | let App;
7 |
8 | Ember.MODEL_FACTORY_INJECTIONS = true;
9 |
10 | App = Ember.Application.extend({
11 | modulePrefix: config.modulePrefix,
12 | podModulePrefix: config.podModulePrefix,
13 | Resolver,
14 |
15 | customEvents: {
16 | transitionend: 'transitionEnd'
17 | }
18 | });
19 |
20 | loadInitializers(App, config.modulePrefix);
21 |
22 | export default App;
23 |
--------------------------------------------------------------------------------
/portiaui/app/components/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/components/.gitkeep
--------------------------------------------------------------------------------
/portiaui/app/components/browser-url-blocked.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: 'span',
5 | });
6 |
--------------------------------------------------------------------------------
/portiaui/app/components/browser-url-failing.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { inject: { service } } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | tagName: 'span',
6 | browser: service(),
7 |
8 | actions: {
9 | reloadPage() {
10 | this.get('browser').reload();
11 | }
12 | }
13 | });
14 |
--------------------------------------------------------------------------------
/portiaui/app/components/colored-badge.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 |
6 | color: null,
7 | value: 0,
8 |
9 | badgeStyle: Ember.computed('color.main', function() {
10 | var color = this.get('color.main');
11 | return Ember.String.htmlSafe(color ? `background-color: ${color};` : '');
12 | })
13 | });
14 |
--------------------------------------------------------------------------------
/portiaui/app/components/colored-span.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | tagName: 'span',
6 | attributeBindings: ['colorStyle:style'],
7 | colorStyle: computed('color.main', function() {
8 | var color = this.get('color.main');
9 | return Ember.String.htmlSafe(color ? `color: ${color};` : '');
10 | })
11 | });
12 |
--------------------------------------------------------------------------------
/portiaui/app/components/create-project-button.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed, inject: { service } } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | dispatcher: service(),
6 | capabilities: service(),
7 | tagName: '',
8 |
9 | canCreateProjects: computed.readOnly('capabilities.capabilities.create_projects'),
10 | projectName: null,
11 |
12 | actions: {
13 | addProject() {
14 | this.get('dispatcher').addProject(this.get('projectName'), /* redirect = */true);
15 | }
16 | }
17 | });
18 |
--------------------------------------------------------------------------------
/portiaui/app/components/create-spider-button.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import {computedCanAddSpider} from '../services/dispatcher';
3 |
4 | export default Ember.Component.extend({
5 | browser: Ember.inject.service(),
6 | dispatcher: Ember.inject.service(),
7 |
8 | tagName: '',
9 |
10 | project: null,
11 |
12 | canAddSpider: computedCanAddSpider(),
13 |
14 | actions: {
15 | addSpider() {
16 | this.get('dispatcher').addSpider(this.get('project'), /* redirect = */true);
17 | }
18 | }
19 | });
20 |
--------------------------------------------------------------------------------
/portiaui/app/components/data-structure-listing.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | dispatcher: Ember.inject.service(),
5 |
6 | tagName: '',
7 |
8 | annotationColors: [],
9 |
10 | actions: {
11 | addItem(sample) {
12 | this.get('dispatcher').addItem(sample, /* redirect = */true);
13 | },
14 |
15 | removeItem(item) {
16 | this.get('dispatcher').removeItem(item);
17 | }
18 | }
19 | });
20 |
--------------------------------------------------------------------------------
/portiaui/app/components/dropdown-delete.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | tagName: 'li',
6 | classNames: ['dropdown-delete'],
7 | classNameBindings: ['isConfirmed'],
8 | isConfirmed: false,
9 |
10 | notConfirmed: computed.not('isConfirmed'),
11 |
12 | actions: {
13 | onDelete() {
14 | if (this.get('notConfirmed')) {
15 | this.set('isConfirmed', true);
16 | } else {
17 | this.get('onDelete')();
18 | }
19 | }
20 | }
21 | });
22 |
--------------------------------------------------------------------------------
/portiaui/app/components/dropdown-divider.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: 'li',
5 | classNames: ['divider'],
6 | attributeBindings: ['role'],
7 | role: 'separator'
8 | });
9 |
--------------------------------------------------------------------------------
/portiaui/app/components/dropdown-header.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: 'li',
5 | classNames: ['dropdown-header']
6 | });
7 |
--------------------------------------------------------------------------------
/portiaui/app/components/extracted-item-table.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 |
4 | export default Ember.Component.extend({
5 | tagName: 'table',
6 | classNames: ['extracted-item-table']
7 | });
8 |
--------------------------------------------------------------------------------
/portiaui/app/components/extracted-items-group.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: ''
5 | });
6 |
--------------------------------------------------------------------------------
/portiaui/app/components/extracted-items-json-panel.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | extractedItems: Ember.inject.service(),
5 |
6 | tagName: ''
7 | });
8 |
--------------------------------------------------------------------------------
/portiaui/app/components/extracted-items-panel.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { inject: { service }, computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | tagName: '',
6 |
7 | extractedItems: service(),
8 |
9 | isExtracting: computed.readOnly('extractedItems.isExtracting'),
10 | failedMsg: computed.readOnly('extractedItems.failedExtractionMsg'),
11 | failedExtraction: computed.readOnly('extractedItems.failedExtraction')
12 | });
13 |
--------------------------------------------------------------------------------
/portiaui/app/components/extracted-items-tab.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { inject: { service }, computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | extractedItems: service(),
6 |
7 | numItems: computed.readOnly('extractedItems.items.length'),
8 | isExtracting: computed.alias('extractedItems.isExtracting')
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/components/feed-url-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import { cleanUrl } from '../utils/utils';
3 |
4 | export default Ember.Component.extend({
5 | feedLink: 'http://files.scrapinghub.com/portia/urls.txt',
6 |
7 | didRender() {
8 | this._super(...arguments);
9 | this.$('.focus-control').focus();
10 | },
11 |
12 | actions: {
13 | saveFeedUrl() {
14 | const url = cleanUrl(this.get('startUrl.url'));
15 | this.set('startUrl.url', url);
16 | this.get('saveSpider').perform();
17 | }
18 | }
19 |
20 | });
21 |
--------------------------------------------------------------------------------
/portiaui/app/components/field-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 |
6 | field: null,
7 |
8 | actions: {
9 | save() {
10 | this.get('field').save();
11 | }
12 | }
13 | });
14 |
--------------------------------------------------------------------------------
/portiaui/app/components/help-icon.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 |
6 | tooltipClasses: null,
7 | tooltipContainer: 'body',
8 | placement: 'right',
9 | icon: 'help',
10 | classes: 'help-icon',
11 | });
12 |
--------------------------------------------------------------------------------
/portiaui/app/components/indentation-spacer.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | classNames: ['indentation-spacer'],
5 | classNameBindings: ['isSmall'],
6 | isSmall: false
7 | });
8 |
--------------------------------------------------------------------------------
/portiaui/app/components/input-with-clear.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | classNames: ['input-group', 'input-with-clear'],
5 |
6 | type: 'text',
7 | value: '',
8 |
9 | actions: {
10 | clear() {
11 | this.set('value', '');
12 | this.get('clear')();
13 | },
14 |
15 | keyUp() {
16 | this.update(this.get('value'));
17 | }
18 | }
19 | });
20 |
--------------------------------------------------------------------------------
/portiaui/app/components/link-crawling-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import SaveSpiderMixin from '../mixins/save-spider-mixin';
3 |
4 | export default Ember.Component.extend(SaveSpiderMixin,{
5 | tagName: '',
6 |
7 | spider: null,
8 |
9 | actions: {
10 | save() {
11 | this.saveSpider();
12 | }
13 | }
14 | });
15 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-add-annotation-menu.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import config from '../config/environment';
3 |
4 |
5 | export default Ember.Component.extend({
6 | dispatcher: Ember.inject.service(),
7 |
8 | tagName: '',
9 |
10 | item: null,
11 |
12 | allowNesting: config.APP.allow_nesting,
13 |
14 | actions: {
15 | addAnnotation() {
16 | const item = this.get('item');
17 | this.get('dispatcher').addAnnotation(item, undefined, undefined, /* redirect = */true);
18 | },
19 |
20 | addNestedItem() {
21 | const item = this.get('item');
22 | this.get('dispatcher').addNestedItem(item, /* redirect = */true);
23 | }
24 | }
25 | });
26 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-badge.js:
--------------------------------------------------------------------------------
1 | import ColoredBadge from './colored-badge';
2 |
3 | export default ColoredBadge.extend({
4 | });
5 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-combo.js:
--------------------------------------------------------------------------------
1 | import ListItemSelectable from './list-item-selectable';
2 |
3 | export default ListItemSelectable.extend({
4 | classNames: ['list-item-combo'],
5 |
6 | autoSelect: false
7 | });
8 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-editable.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | classNames: ['list-item-editable'],
5 | classNameBindings: ['editing'],
6 |
7 | editing: false,
8 | onChange: null,
9 | validate: null,
10 | spellcheck: true,
11 | value: null,
12 |
13 | click() {
14 | if (this.get('editing')) {
15 | return false;
16 | }
17 | },
18 |
19 | actions: {
20 | startEditing() {
21 | this.set('editing', true);
22 | }
23 | }
24 | });
25 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-field-type.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import { FIELD_TYPES } from '../models/field';
3 | import ensurePromise from '../utils/ensure-promise';
4 |
5 | export default Ember.Component.extend({
6 | tagName: '',
7 |
8 | field: null,
9 |
10 | types: FIELD_TYPES,
11 |
12 | actions: {
13 | saveField() {
14 | const field = this.get('field');
15 | ensurePromise(field).then(field => {
16 | if (!!field) {
17 | field.save();
18 | }
19 | });
20 | }
21 | }
22 | });
23 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-icon-menu.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 |
6 | icon: null,
7 |
8 | actions: {
9 | clickIcon() {
10 | const action = this.get('onClick');
11 | if (action) { action(); }
12 | }
13 | }
14 | });
15 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-icon.js:
--------------------------------------------------------------------------------
1 | import IconButton from './icon-button';
2 |
3 | export default IconButton.extend({
4 | classNames: ['list-item-icon'],
5 |
6 | beforeClick() {
7 | const action = this.get('onClick');
8 | if (action) { action(); }
9 | }
10 | });
11 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-item-schema.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | dispatcher: Ember.inject.service(),
5 |
6 | tagName: '',
7 |
8 | item: null,
9 | selecting: false,
10 |
11 | actions: {
12 | addSchema(name) {
13 | const item = this.get('item');
14 | const project = item.get('schema.project');
15 | this.get('dispatcher').addNamedSchema(
16 | project, name, /* redirect = */false).then((schema) => {
17 | item.set('schema', schema);
18 | item.save();
19 | });
20 | },
21 |
22 | changeSchema() {
23 | const item = this.get('item');
24 | item.get('schema').then(() => {
25 | item.set('schema', item.get('schema')); // Used to trigger updates
26 | item.save();
27 | });
28 | }
29 | }
30 | });
31 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-selectable.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | classNames: ['list-item-selectable'],
5 | classNameBindings: ['selecting'],
6 |
7 | change: null,
8 | choices: [],
9 | buttonClass: null,
10 | menuAlign: 'left',
11 | menuClass: null,
12 | menuContainer: null,
13 |
14 | selecting: false,
15 | value: null,
16 |
17 | actions: {
18 | startSelecting() {
19 | this.set('selecting', true);
20 | }
21 | }
22 | });
23 |
--------------------------------------------------------------------------------
/portiaui/app/components/list-item-text.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: 'span',
5 | classNames: ['list-item-text']
6 | });
7 |
--------------------------------------------------------------------------------
/portiaui/app/components/project-structure-spider-feed-url.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 | import { cleanUrl } from '../utils/utils';
4 |
5 | export default Ember.Component.extend({
6 | dispatcher: Ember.inject.service(),
7 |
8 | tagName: '',
9 |
10 | url: computed.alias('startUrl.url'),
11 | isEditing: computed.equal('url', ''),
12 |
13 | viewUrl: computed('url', {
14 | get() {
15 | return this.get('url');
16 | },
17 | set(key, value) {
18 | this.saveStartUrl(value);
19 | }
20 | }),
21 |
22 | saveStartUrl(url) {
23 | this.set('startUrl.url', cleanUrl(url));
24 | this.get('spider').save();
25 | }
26 | });
27 |
--------------------------------------------------------------------------------
/portiaui/app/components/project-structure-spider-generated-url.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | tagName: '',
6 |
7 | fragments: computed.alias('startUrl.fragments'),
8 | url: computed('startUrl.url', 'fragments.@each.type', 'fragments.@each.value', function() {
9 | return this.get('startUrl').show();
10 | })
11 | });
12 |
--------------------------------------------------------------------------------
/portiaui/app/components/reorder-handler.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | attributeBindings: ['draggable', 'style'],
5 | draggable: true,
6 | tagName: 'i',
7 | classNames: 'fa fa-icon fa-arrows reorder-handler',
8 | dragStart: function(event) {
9 | var dataTransfer = event.originalEvent.dataTransfer;
10 | dataTransfer.effectAllowed = "move";
11 | dataTransfer.setData('text/plain', "");
12 | var dragElement = this.$().parentsUntil('.reorderable-list').eq(-1);
13 | dataTransfer.addElement(dragElement[0]);
14 | dragElement.addClass('dragging').one("dragend", function(){
15 | dragElement.removeClass('dragging');
16 | });
17 | },
18 | });
19 |
20 |
--------------------------------------------------------------------------------
/portiaui/app/components/scrapinghub-links.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 | });
6 |
--------------------------------------------------------------------------------
/portiaui/app/components/show-links-button.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed, inject: { service } } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | browser: service(),
6 |
7 | disableLinks: computed.readOnly('browser.invalidUrl'),
8 | spider: null,
9 |
10 | actions: {
11 | toggleShowLinks() {
12 | const spider = this.get('spider');
13 | spider.toggleProperty('showLinks');
14 | spider.save();
15 | }
16 | }
17 | });
18 |
--------------------------------------------------------------------------------
/portiaui/app/components/show-links-legend.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import { NAMED_COLORS } from '../utils/colors';
3 |
4 | export default Ember.Component.extend({
5 | tagName: '',
6 |
7 | colors: NAMED_COLORS,
8 | followedLinks: 0,
9 | jsLinks: 0,
10 | ignoredLinks: 0
11 | });
12 |
--------------------------------------------------------------------------------
/portiaui/app/components/sliding-main.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | uiState: Ember.inject.service(),
5 |
6 | tagName: 'main',
7 | classNameBindings: ['slideRight'],
8 |
9 | slideRight: Ember.computed.bool('uiState.slideMain')
10 | });
11 |
--------------------------------------------------------------------------------
/portiaui/app/components/spider-indentation.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({ tagName: '' });
4 |
--------------------------------------------------------------------------------
/portiaui/app/components/spider-message.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 |
4 | export default Ember.Component.extend({
5 | api: Ember.inject.service(),
6 | notificationManager: Ember.inject.service(),
7 | hasSpider: computed.bool('currentSpider'),
8 |
9 | actions: {
10 | runSpider(spider) {
11 | this.get('api').post('schedule', {
12 | model: spider,
13 | jsonData: {data: {type: 'spiders', id: spider.id}}
14 | }).then(() => {
15 | this.get('notificationManager').showNotification(
16 | 'Your spider has been scheduled successfully');
17 | }, data => {
18 | let error = data.errors[0];
19 | if (error.status > 499) {
20 | throw data;
21 | }
22 | this.get('notificationManager').showNotification(error.title, error.detail);
23 | });
24 | }
25 | }
26 | });
--------------------------------------------------------------------------------
/portiaui/app/components/spider-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import SaveSpiderMixin from '../mixins/save-spider-mixin';
3 |
4 | export default Ember.Component.extend(SaveSpiderMixin, {
5 | tagName: '',
6 |
7 | spider: null,
8 |
9 | actions: {
10 | save() {
11 | this.saveSpider();
12 | }
13 | }
14 | });
15 |
--------------------------------------------------------------------------------
/portiaui/app/components/start-url-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { computed } = Ember;
3 | import { task, timeout } from 'ember-concurrency';
4 |
5 | const SPIDER_DEBOUNCE = 1000;
6 |
7 | export default Ember.Component.extend({
8 | startUrl: computed('spider.startUrls.[]', 'startUrlId', function() {
9 | return this.get('spider').get('startUrls').objectAt(this.get('startUrlId'));
10 | }),
11 |
12 | title: computed.alias('startUrl.optionsTitle'),
13 |
14 | saveSpider: task(function * () {
15 | yield timeout(SPIDER_DEBOUNCE);
16 | this.get('spider').save();
17 | }).restartable()
18 | });
19 |
--------------------------------------------------------------------------------
/portiaui/app/components/tool-panel.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import {computedPropertiesEqual} from '../utils/computed';
3 |
4 | export default Ember.Component.extend({
5 | classNames: ['tool-panel'],
6 | classNameBindings: ['active::hide'],
7 |
8 | active: computedPropertiesEqual('toolId', 'group.selected')
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/components/tool-tab.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import {computedPropertiesEqual} from '../utils/computed';
3 |
4 | export default Ember.Component.extend({
5 | tagName: 'li',
6 | classNameBindings: ['active'],
7 |
8 | active: computedPropertiesEqual('toolId', 'group.selected'),
9 |
10 | didInsertElement() {
11 | if (!this.$().prev().length) {
12 | Ember.run.schedule('afterRender', () => {
13 | if (!this.get('group.selected')) {
14 | this.send('selectTab');
15 | }
16 | });
17 | }
18 | },
19 |
20 | actions: {
21 | selectTab() {
22 | this.get('group').send('selectTab', this.get('toolId'));
23 | }
24 | }
25 | });
26 |
--------------------------------------------------------------------------------
/portiaui/app/components/tooltip-icon.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | tagName: '',
5 |
6 | actions: {
7 | onClick() {
8 | const action = this.get('onClick');
9 | if (action) {
10 | action();
11 | }
12 | }
13 | }
14 | });
15 |
--------------------------------------------------------------------------------
/portiaui/app/components/tree-list-item-row.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Component.extend({
4 | classNames: ['tree-list-item-row'],
5 |
6 | mouseEnter() {
7 | if (this.attrs.onMouseEnter && this.attrs.onMouseEnter.call) {
8 | this.attrs.onMouseEnter(...arguments);
9 | }
10 | },
11 |
12 | mouseLeave() {
13 | if (this.attrs.onMouseLeave && this.attrs.onMouseLeave.call) {
14 | this.attrs.onMouseLeave(...arguments);
15 | }
16 | }
17 | });
18 |
--------------------------------------------------------------------------------
/portiaui/app/components/tree-list-item.js:
--------------------------------------------------------------------------------
1 | import AnimationContainer from './animation-container';
2 |
3 | export default AnimationContainer.extend({
4 | tagName: 'li',
5 | classNames: ['tree-list-item'],
6 |
7 | setWidth: false,
8 | isCentered: false,
9 | hasChildren: false
10 | });
11 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/controllers/.gitkeep
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | browser: Ember.inject.service(),
5 |
6 | queryParams: ['url', 'baseurl'],
7 |
8 | url: Ember.computed.alias('browser.url'),
9 | baseurl: Ember.computed.alias('browser.baseurl'),
10 | clickHandler: null,
11 |
12 | setClickHandler(fn) {
13 | this.clickHandler = fn;
14 | },
15 |
16 | clearClickHandler() {
17 | this.clickHandler = null;
18 | },
19 |
20 | actions: {
21 | viewPortClick() {
22 | if (this.clickHandler) {
23 | this.clickHandler(...arguments);
24 | }
25 | }
26 | }
27 | });
28 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project/conflicts.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | projectController: Ember.inject.controller('projects.project'),
5 | currentFileName: null,
6 |
7 | conflictedKeyPaths: {},
8 |
9 | conflictedFiles: Ember.computed('model', function() {
10 | return Object.keys(this.get('model')).sort().map((name) => ({
11 | name: name,
12 | encodedName: btoa(name),
13 | }));
14 | }),
15 | });
16 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project/schema/field/options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | actions: {
5 | closeOptions() {
6 | this.send('close');
7 | }
8 | }
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project/spider/link-options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | actions: {
5 | closeOptions() {
6 | this.send('close');
7 | }
8 | }
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project/spider/options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | actions: {
5 | closeOptions() {
6 | this.send('close');
7 | }
8 | }
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/controllers/projects/project/spider/sample/data/annotation/options.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Controller.extend({
4 | actions: {
5 | closeOptions() {
6 | this.send('close');
7 | }
8 | }
9 | });
10 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/helpers/.gitkeep
--------------------------------------------------------------------------------
/portiaui/app/helpers/array-get.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Helper.extend({
4 | compute(params/*, hash*/) {
5 | this.setProperties({
6 | obj: params[0],
7 | index: params[1]
8 | });
9 |
10 | return this.get('content');
11 | },
12 |
13 | obj: null,
14 | index: null,
15 | content: Ember.computed('obj.[]', 'index', function() {
16 | return this.get('obj').get(this.get('index'));
17 | }),
18 |
19 | contentDidChange: Ember.observer('content', function () {
20 | this.recompute();
21 | })
22 | });
23 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/attribute-annotation.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default Ember.Helper.extend({
4 | compute([annotations, attribute]) {
5 | this.setProperties({
6 | annotations,
7 | attribute
8 | });
9 |
10 | return this.get('content');
11 | },
12 |
13 | annotations: null,
14 | attribute: null,
15 | content: Ember.computed('annotations.[]', 'attribute', function() {
16 | const attribute = this.get('attribute');
17 | return this.getWithDefault('annotations', []).find(annotation =>
18 | annotation.getWithDefault('attribute', null) === attribute) || {};
19 | }),
20 |
21 | contentDidChange: Ember.observer('content', function () {
22 | this.recompute();
23 | })
24 | });
25 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/chain-actions.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function chainActions(params/*, hash*/) {
4 | return function() {
5 | for (let action of params) {
6 | if (action.call) {
7 | action();
8 | }
9 | }
10 | };
11 | }
12 |
13 | export default Ember.Helper.helper(chainActions);
14 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/guid.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function guid([obj]/*, hash*/) {
4 | return Ember.guidFor(obj);
5 | }
6 |
7 | export default Ember.Helper.helper(guid);
8 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/includes.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function includes([list, value]) {
4 | return list && list.includes && list.includes(value);
5 | }
6 |
7 | export default Ember.Helper.helper(includes);
8 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/indexed-object.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function indexedObject([ param ] /*, hash*/) {
4 | let indexed = {}, i = 0;
5 | for (let key of Object.keys(param)) {
6 | indexed[key] = {
7 | index: i,
8 | value: param[key]
9 | };
10 | i += 1;
11 | }
12 | return indexed;
13 | }
14 |
15 | export default Ember.Helper.helper(indexedObject);
16 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/is-empty-object.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import {isObject} from './is-object';
3 |
4 | export function isEmptyObject(params) {
5 | return isObject(params) && !Object.keys(...params).length;
6 | }
7 |
8 | export default Ember.Helper.helper(isEmptyObject);
9 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/is-object-or-array.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import { isArrayHelper } from 'ember-truth-helpers/helpers/is-array';
3 | import { isObject } from './is-object';
4 |
5 | export function isObjectOrArray(params) {
6 | return isObject(params) || isArrayHelper(params);
7 | }
8 |
9 | export default Ember.Helper.helper(isObjectOrArray);
10 |
--------------------------------------------------------------------------------
/portiaui/app/helpers/is-object.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import { toType } from '../utils/types';
3 |
4 | export function isObject([object]) {
5 | return toType(object) === 'object';
6 | }
7 |
8 | export default Ember.Helper.helper(isObject);
9 |
--------------------------------------------------------------------------------
/portiaui/app/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Portia
7 |
2 | {{loading-slider isLoading=loading duration=200}}
3 |
4 |
7 |
8 | {{scrapinghub-links}}
9 |
10 |
11 | {{outlet 'top-bar'}}
12 |
13 |
14 |
15 |
16 | {{notification-container}}
17 | {{outlet 'side-bar'}}
18 | {{#sliding-main}}
19 | {{outlet 'options-panels'}}
20 | {{outlet 'main'}}
21 | {{outlet 'tool-panels'}}
22 | {{/sliding-main}}
23 |
24 |
25 |
--------------------------------------------------------------------------------
/portiaui/app/templates/branding.hbs:
--------------------------------------------------------------------------------
1 |
2 | Portia beta
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/browsers.hbs:
--------------------------------------------------------------------------------
1 |
2 |
Unfortunately your browser doesn't support some of the features required to give you a great experience with Portia.
3 |
Please try using an up-to-date version of one of these browsers, which are known to work well with Portia.
4 |
5 | {{browser-list browsers=model}}
6 |
6 | Add this page as a start page for your spider
7 |
8 | {{else}}
9 |
10 | Remove this page from your spider's start pages
11 |
12 | {{/if}}
13 | {{else}}
14 |
15 | {{icon-button icon='url'}} Start page
16 |
17 | {{/if}}
18 | {{/tooltip-container}}
19 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/animation-container.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{yield}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/browser-iframe.hbs:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/templates/components/browser-iframe.hbs
--------------------------------------------------------------------------------
/portiaui/app/templates/components/browser-list.hbs:
--------------------------------------------------------------------------------
1 |
9 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/browser-url-blocked.hbs:
--------------------------------------------------------------------------------
1 | Portia is having trouble loading this page at the moment. Try a different page or try again later.
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/browser-url-failing.hbs:
--------------------------------------------------------------------------------
1 | Portia is currently having trouble loading this page would you like to try again ?
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/buffered-input.hbs:
--------------------------------------------------------------------------------
1 | {{input id=inputId type=type class=(concat "form-control " (if focused "focused ") class) value=(mut displayedValue) enter=(if focused (action 'endEditing' 'enter')) escape-press=(if focused (action 'cancelEditing')) focus-in=(action 'startEditing') bubbles=true focus-out=(if focused (action 'endEditing' 'focus-out')) placeholder=placeholder autofocus=autofocus disabled=disabled spellcheck=spellcheck}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/colored-badge.hbs:
--------------------------------------------------------------------------------
1 | {{#if hasBlock}}{{yield}}{{else}}{{value}}{{/if}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/colored-span.hbs:
--------------------------------------------------------------------------------
1 | {{~ yield ~}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/create-project-button.hbs:
--------------------------------------------------------------------------------
1 | {{#if canCreateProjects}}
2 |
3 |
4 | {{input type="text" class="form-control" value=(mut projectName) placeholder="Create a new project"
5 | enter=(action 'addProject')}}
6 |
7 |
8 | {{icon-button icon='spider'}}
9 |
10 |
11 |
12 | {{#if projects}}
13 | OR
14 | {{/if}}
15 | {{/if}}
16 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/create-spider-button.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{icon-button icon='spider'}} New spider
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-delete.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{list-item-icon class="icon" icon="remove"}}{{if isConfirmed 'Are you sure?' text}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-divider.hbs:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/templates/components/dropdown-divider.hbs
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-header.hbs:
--------------------------------------------------------------------------------
1 | {{yield}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-item.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{#if hasBlock}}
3 | {{yield value}}
4 | {{else}}
5 | {{value}}
6 | {{/if}}
7 |
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-menu.hbs:
--------------------------------------------------------------------------------
1 | {{yield (hash menu=this header=(component 'dropdown-header') item=(component 'dropdown-item' menu=this) divider=(component 'dropdown-divider'))}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/dropdown-widget.hbs:
--------------------------------------------------------------------------------
1 | {{yield (hash section='widget' openMenu=(action 'openMenu') closeMenu=(action 'closeMenu') toggleMenu=(action 'toggleMenu') focusIn=(action 'focusIn') focusOut=(action 'focusOut') keyDown=(action 'keyDown'))}}
2 | {{#dropdown-menu class=menuClasses events=events keyNavigate=keyNavigate active=(mut active) focused=(mut focused) orderItemsForSearch=orderItemsForSearch valuesEqual=valuesEqual onFocusIn=(action 'focusIn') onFocusOut=(action 'focusOut') as |options|}}
3 | {{#if open}}
4 | {{yield (hash section='menu' menu=options.menu header=options.header item=options.item divider=options.divider openMenu=(action 'openMenu') closeMenu=(action 'closeMenu') toggleMenu=(action 'toggleMenu') focusIn=(action 'focusIn') focusOut=(action 'focusOut') keyDown=(action 'keyDown'))}}
5 | {{/if}}
6 | {{/dropdown-menu}}
7 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/element-overlay.hbs:
--------------------------------------------------------------------------------
1 | {{#each rects key="@index" as |rect index|}}
2 | {{element-rect-overlay index=index icon=icon color=color positionMode=positionMode class=class overlay=this}}
3 | {{/each}}
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/element-rect-overlay.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 | {{icon-button icon=icon}}
4 |
5 |
6 |
a
2 | {{#if selected ~}}
3 | {{extracted-items-json json=extractedItems.items position=0}}
4 | {{~/if}}
5 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/extracted-items-json-value.hbs:
--------------------------------------------------------------------------------
1 | {{#if fromArray}}{{depthSpaces}} {{/if}}
2 | {{escapedValue}}{{comma}}
3 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/extracted-items-panel.hbs:
--------------------------------------------------------------------------------
1 | {{#if isExtracting}}
2 |
3 |
5 | Extracting data...
6 | {{/if}}
7 |
8 | {{#if failedExtraction}}
9 |
10 |
12 |
13 | {{failedMsg}}
14 |
15 | {{/if}}
16 |
17 | {{#if (and selected (not isExtracting))}}
18 | {{#each extractedItems.items key='@index' as |item|}}
19 | {{extracted-item-table item=item}}
20 | {{/each}}
21 | {{/if}}
22 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/extracted-items-status.hbs:
--------------------------------------------------------------------------------
1 | {{#link-to changeInfo.path changeInfo.model bubbles=false}}
2 | {{#help-icon icon=icon placement='left'}}
3 | {{#if hasWarning}}
4 | {{changeInfo.text}}
5 | {{else}}
6 | Your sample is correctly configured for extraction
7 | {{/if}}
8 | {{/help-icon}}
9 | {{/link-to}}
10 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/extracted-items-tab.hbs:
--------------------------------------------------------------------------------
1 | {{yield}}
2 |
3 | {{#unless isExtracting}}
4 | {{numItems}}
5 | {{/unless}}
6 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/feed-url-options.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 | Enter a publicly available URL containing newline separated URLs
4 | like this.
5 |
6 |
9 | Feed URL
10 | {{input
11 | type="text"
12 | id='feedUrl'
13 | class="form-control focus-control"
14 | value=(mut startUrl.url)
15 | focus-out="saveFeedUrl"
16 | placeholder='https://gist.github.com/user/gist_id'
17 | }}
18 |
19 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/field-options.hbs:
--------------------------------------------------------------------------------
1 | Field
2 |
20 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/help-icon.hbs:
--------------------------------------------------------------------------------
1 | {{#tooltip-container tooltipClasses=tooltipClasses tooltipFor=(concat "help-icon-" elementId) tooltipContainer=tooltipContainer placement=placement as |tooltip|}}
2 | {{#if (eq tooltip.section 'tooltip')}}
3 | {{yield}}
4 | {{else}}
5 | {{icon-button id=(concat "help-icon-" elementId) class=classes icon=icon}}
6 | {{/if}}
7 | {{/tooltip-container}}
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/icon-button.hbs:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/templates/components/icon-button.hbs
--------------------------------------------------------------------------------
/portiaui/app/templates/components/input-with-clear.hbs:
--------------------------------------------------------------------------------
1 | {{input class="form-control" value=(mut value) placeholder=placeholder keyUp=(action "keyUp" on="key-up")}}
2 | {{icon-button class="clear-input" icon='close' action=(action 'clear')}}
3 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/link-crawling-options.hbs:
--------------------------------------------------------------------------------
1 | Crawling rules
2 |
20 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-add-annotation-menu.hbs:
--------------------------------------------------------------------------------
1 | {{#list-item-icon-menu icon='add-dropdown' as |options|}}
2 | {{#options.item value="Add annotation" action=(chain-actions (action 'addAnnotation') options.closeMenu) as |value|}}
3 | {{list-item-icon class="icon" icon="add"}}{{value}}
4 | {{/options.item}}
5 | {{#if allowNesting}}
6 | {{#options.item value="Add nested item" action=(chain-actions (action 'addNestedItem') options.closeMenu) as |value|}}
7 | {{list-item-icon class="icon" icon="add"}}{{value}}
8 | {{/options.item}}
9 | {{/if}}
10 | {{/list-item-icon-menu}}
11 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-annotation-field.hbs:
--------------------------------------------------------------------------------
1 | {{#list-item-relation-manager value=(mut annotation.field) choices=annotation.parent.schema.fields selecting=(mut selecting) onChange=(action 'changeField') validate=(action 'validateFieldName') create=(action 'addField') as |options|}}
2 | {{#if (eq options.section 'change-header')}}
3 | Type to change the field
4 | {{else if (eq options.section 'choices-header')}}
5 | Select an existing field
6 | {{else if (eq options.section 'choice')}}
7 | {{list-item-icon class="icon" icon=options.choice.type}}{{options.choice.name}}
8 | {{/if}}
9 | {{/list-item-relation-manager}}
10 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-badge.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 | {{#if hasBlock}}{{yield}}{{else}}{{value}}{{/if}}
4 |
5 |
6 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-editable.hbs:
--------------------------------------------------------------------------------
1 | {{#if editing}}
2 | {{buffered-input class="input-list-item" value=(mut value) focused=editing autoSelect=true spellcheck=spellcheck onChange=onChange validate=validate autofocus="autofocus"}}
3 | {{else}}
4 | {{value}}
5 | {{icon-button icon='edit' action=(action 'startEditing') bubbles=false}}
6 | {{/if}}
7 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-field-type.hbs:
--------------------------------------------------------------------------------
1 | {{#list-item-selectable value=(mut field.type) onChange=(action 'saveField') menuContainer="body" menuAlign="right" as |select|}}
2 | {{#if hasBlock}}
3 | {{#select.header}}
4 | {{yield}}
5 | {{/select.header}}
6 | {{/if}}
7 | {{#each types as |type|}}
8 | {{#select.item value=type action=(action select.setValueAndClose type)}}
9 | {{list-item-icon class="icon" icon=type}}{{type}}
10 | {{/select.item}}
11 | {{/each}}
12 | {{/list-item-selectable}}
13 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-icon-menu.hbs:
--------------------------------------------------------------------------------
1 | {{#dropdown-widget class="list-item-icon" menuContainer="body" menuAlign="right" as |options|}}
2 | {{#if (eq options.section 'widget')}}
3 | {{list-item-icon onClick=(action 'clickIcon') icon=icon action=options.toggleMenu tabindex=-1}}
4 | {{else if (eq options.section 'menu')}}
5 | {{yield options}}
6 | {{/if}}
7 | {{/dropdown-widget}}
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-icon.hbs:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/app/templates/components/list-item-icon.hbs
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-item-schema.hbs:
--------------------------------------------------------------------------------
1 | {{#list-item-relation-manager value=(mut item.schema) choices=item.ownerSample.spider.project.schemas selecting=(mut selecting) onChange=(action 'changeSchema') create=(action 'addSchema') as |options|}}
2 | {{#if (eq options.section 'change-header')}}
3 | Type to change the data format
4 | {{else if (eq options.section 'choices-header')}}
5 | Select an existing data format
6 | {{else if (eq options.section 'choice')}}
7 | {{list-item-icon class="icon" icon='schema'}}{{options.choice.name}}
8 | {{/if}}
9 | {{/list-item-relation-manager}}
10 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-link-crawling.hbs:
--------------------------------------------------------------------------------
1 | {{#list-item-selectable value=(mut linksToFollow) valueAttribute='label' onChange=(action 'saveSpider') menuContainer="body" menuAlign="left" as |select|}}
2 | {{#select.header}}
3 | Change how links are crawled
4 | {{/select.header}}
5 | {{#each followPatternOptions as |option|}}
6 | {{#select.item value=option action=(action select.setValueAndClose option)}}
7 | {{option.label}}
8 | {{/select.item}}
9 | {{/each}}
10 | {{/list-item-selectable}}
11 | {{#help-icon}}
12 | {{linksToFollow.description}}
13 | {{/help-icon}}
14 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-selectable.hbs:
--------------------------------------------------------------------------------
1 | {{#if selecting}}
2 | {{#if hasBlock}}
3 | {{#select-box choices=choices value=(mut value) valueAttribute=valueAttribute open=(mut selecting) onChange=onChange buttonClass=(concat "input-list-item " buttonClass) menuClass=menuClass menuAlign=menuAlign menuContainer=menuContainer as |options|}}
4 | {{yield options}}
5 | {{/select-box}}
6 | {{else}}
7 | {{select-box choices=choices value=(mut value) valueAttribute=valueAttribute open=(mut selecting) onChange=onChange buttonClass=(concat "input-list-item " buttonClass) menuClass=menuClass menuAlign=menuAlign menuContainer=menuContainer}}
8 | {{/if}}
9 | {{else}}
10 |
11 | {{#if valueAttribute}}
12 | {{get value valueAttribute}}
13 | {{else}}
14 | {{value}}
15 | {{/if}}
16 |
17 |
18 |
20 | {{/if}}
21 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/list-item-text.hbs:
--------------------------------------------------------------------------------
1 | {{yield}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/notification-container.hbs:
--------------------------------------------------------------------------------
1 | {{#each banners as |notification|}}
2 | {{notification-message notification=notification fade=notification.fading fadeAction=(action 'fadeBanner' notification)}}
3 | {{/each}}
4 |
5 | {{#each displayNotifications as |notification|}}
6 | {{notification-message notification=notification fade=notification.fading closeAction=(action 'dismissNotification' notification) fadeAction=(action 'fadeNotification' notification)}}
7 | {{/each}}
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/notification-message.hbs:
--------------------------------------------------------------------------------
1 | {{#if closeAction}}
2 |
3 | ×
4 |
5 | {{/if}}
6 | {{#if title}}
7 | {{title}}
8 | {{/if}}
9 | {{message}}
10 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/project-structure-spider-feed-url.hbs:
--------------------------------------------------------------------------------
1 | {{#tree-list-item as |options|}}
2 | {{#link-to 'projects.project.spider.start-url.options' index bubbles=false}}
3 | {{indentation-spacer}}
4 | {{list-item-icon icon='url-feed'}}
5 | {{#list-item-text class="txt-describe"}}{{url}}{{/list-item-text}}
6 | {{#link-to 'projects.project.spider' bubbles=false}}
7 | {{#link-to 'projects.project.spider.start-url.options' index
8 | bubbles=false
9 | class="ignore-active"
10 | }}
11 | {{list-item-icon icon='options'}}
12 | {{/link-to}}
13 | {{/link-to}}
14 | {{list-item-icon icon='remove' action=(action removeStartUrl) bubbles=false}}
15 | {{/link-to}}
16 | {{/tree-list-item}}
17 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/project-structure-spider-generated-url.hbs:
--------------------------------------------------------------------------------
1 | {{#tree-list-item as |options|}}
2 | {{#link-to 'projects.project.spider.start-url.options' index bubbles=false}}
3 | {{indentation-spacer}}
4 | {{list-item-icon icon='url-generated'}}
5 | {{#list-item-text class="generated-url"}}{{url}}{{/list-item-text}}
6 | {{#link-to 'projects.project.spider' bubbles=false}}
7 | {{#link-to 'projects.project.spider.start-url.options' index bubbles=false class="ignore-active"}}
8 | {{list-item-icon icon='options'}}
9 | {{/link-to}}
10 | {{/link-to}}
11 | {{list-item-icon icon='remove' action=(action removeStartUrl) bubbles=false}}
12 | {{/link-to}}
13 | {{/tree-list-item}}
14 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/project-structure-spider-url.hbs:
--------------------------------------------------------------------------------
1 | {{#tree-list-item as |options|}}
2 | {{#link-to 'projects.project.spider' spider (query-params url=url baseurl=null) active=false}}
3 | {{indentation-spacer}}
4 | {{list-item-icon icon='url'}}
5 | {{list-item-editable value=(mut viewUrl) editing=(mut urlAdded) spellcheck=false}}
6 | {{list-item-icon icon='remove' action=(action removeStartUrl) bubbles=false}}
7 | {{/link-to}}
8 | {{/tree-list-item}}
9 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/save-status.hbs:
--------------------------------------------------------------------------------
1 | {{#tooltip-container tooltipFor=(concat "label-" elementId) tooltipContainer='body' as |tooltip|}}
2 | {{#if (eq tooltip.section 'tooltip')}}
3 | Every change you make is automatically saved by Portia
4 | {{else}}
5 |
6 | {{#if isSaving}}
7 | Saving . . .
8 | {{else if timeSinceLastSave}}
9 | Last saved {{timeSinceLastSave}}
10 | {{else}}
11 | Changes are saved automatically
12 | {{/if}}
13 |
14 | {{/if}}
15 | {{/tooltip-container}}
16 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/scrapinghub-links.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 | Portia 2.0 Documentation
4 |
5 |
6 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/show-links-button.hbs:
--------------------------------------------------------------------------------
1 | {{#tooltip-container tooltipFor="show-links-button" text="Toggle link highlighting" tooltipContainer='body' as |options|}}
2 |
3 | {{icon-button icon='link'}}
4 |
5 | {{/tooltip-container}}
6 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/show-links-legend.hbs:
--------------------------------------------------------------------------------
1 | {{#tree-list}}
2 | {{#tree-list-item as |options|}}
3 | {{list-item-badge value=followedLinks color=colors.green}}
4 | Followed
5 | {{/tree-list-item}}
6 | {{#tree-list-item as |options|}}
7 | {{list-item-badge value=jsLinks color=colors.blue}}
8 | Followed when Javascript is enabled
9 | {{/tree-list-item}}
10 | {{#tree-list-item as |options|}}
11 | {{list-item-badge value=ignoredLinks color=colors.red}}
12 | Not Followed
13 | {{/tree-list-item}}
14 | {{/tree-list}}
15 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/sliding-main.hbs:
--------------------------------------------------------------------------------
1 | {{yield}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/spider-indentation.hbs:
--------------------------------------------------------------------------------
1 | {{indentation-spacer}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/spider-message.hbs:
--------------------------------------------------------------------------------
1 | {{#if hasSpider}}
2 | {{#tooltip-container tooltipFor='run-spider-button' text='Run this spider.' tooltipContainer='body'}}
3 | {{list-item-icon id='run-spider-button' icon='play' action=(action 'runSpider' currentSpider)}}
4 | {{/tooltip-container}}
5 | {{/if}}
--------------------------------------------------------------------------------
/portiaui/app/templates/components/start-url-options.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="fragments-options-group" collapsible=false onClose=(route-action 'closeOptions') as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="annotation-options"}}{{title}}{{/group.tab}}
4 | {{else if (eq group.section "panels")}}
5 | {{#group.panel class="extracted-items container-fluid" toolId="annotation-options" as |active|}}
6 | {{component startUrl.optionsComponentName
7 | spider=spider
8 | startUrl=startUrl
9 | saveSpider=saveSpider
10 | }}
11 | {{/group.panel}}
12 | {{/if}}
13 | {{/tool-group}}
14 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tool-group.hbs:
--------------------------------------------------------------------------------
1 |
18 |
19 | {{yield (hash section="panels" group=this tab=(component 'tool-tab' group=this) panel=(component 'tool-panel' group=this))}}
20 |
21 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tool-panel.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{yield active}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tool-tab.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{yield active}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tooltip-container.hbs:
--------------------------------------------------------------------------------
1 | {{yield (hash section='body')}}
2 |
3 |
4 |
5 | {{#if text}}
6 | {{text}}
7 | {{else}}
8 | {{yield (hash section='tooltip')}}
9 | {{/if}}
10 |
11 |
2 | {{yield}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tree-list-item.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{#tree-list-item-row
3 | isCentered=isCentered
4 | onMouseEnter=onMouseEnter
5 | onMouseLeave=onMouseLeave
6 | }}
7 | {{yield (hash section="item")}}
8 | {{/tree-list-item-row}}
9 | {{#if hasChildren}}
10 | {{#tree-list}}
11 | {{yield (hash section="subtrees")}}
12 | {{/tree-list}}
13 | {{/if}}
14 |
15 |
--------------------------------------------------------------------------------
/portiaui/app/templates/components/tree-list.hbs:
--------------------------------------------------------------------------------
1 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/options-panels.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{outlet 'options-panels'}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
What would you like to work on today?
5 | {{create-project-button projects=model}}
6 | {{#if model}}
7 |
Choose a project from the list below.
8 |
9 | {{project-list projects=model}}
10 | {{/if}}
11 |
2 | {{#if conflictedFiles}}
3 |
Aw, Snap!
4 |
5 |
Portia couldn't deploy the project because there are conflicts with another user's changes
6 |
Resolve the conflicts manually by selecting the conflicting files from the left panel to deploy the project.
7 | {{else}}
8 |
All done
9 |
10 |
11 | All conflicts are resolved, to continue, go back to the
12 | {{link-to "project page" 'projects.project' projectController.model}}
13 | and try to deploy again.
14 |
15 | {{/if}}
16 |
2 | {{json-file-compare json=model.contents update="updateConflict"}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/conflicts/topbar.hbs:
--------------------------------------------------------------------------------
1 |
2 | Resolving {{model.file}}
3 | {{#unless haveConflicts}}
4 |
5 |
8 | {{/unless}}
9 |
10 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/schema.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/schema/field.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/schema/field/options.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="field-options-group" collapsible=false onClose=(action 'closeOptions') as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="field-options"}}
4 | Field properties
5 | {{/group.tab}}
6 | {{else if (eq group.section "panels")}}
7 | {{#group.panel class="extracted-items container-fluid" toolId="field-options" as |active|}}
8 | {{field-options field=model}}
9 | {{/group.panel}}
10 | {{/if}}
11 | {{/tool-group}}
12 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/schema/structure.hbs:
--------------------------------------------------------------------------------
1 | {{schema-structure-listing schema=model}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/link-options.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="link-crawling-options-group" collapsible=false onClose=(action 'closeOptions') as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="link-crawling-options"}}
4 | Link crawling options
5 | {{/group.tab}}
6 | {{else if (eq group.section "panels")}}
7 | {{#group.panel class="extracted-items container-fluid" toolId="link-crawling-options" as |active|}}
8 | {{link-crawling-options spider=model}}
9 | {{/group.panel}}
10 | {{/if}}
11 | {{/tool-group}}
12 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/options.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="spider-options-group" collapsible=false onClose=(action 'closeOptions') as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="spider-options"}}
4 | Spider properties
5 | {{/group.tab}}
6 | {{else if (eq group.section "panels")}}
7 | {{#group.panel class="extracted-items container-fluid" toolId="spider-options" as |active|}}
8 | {{spider-options spider=model}}
9 | {{/group.panel}}
10 | {{/if}}
11 | {{/tool-group}}
12 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/overlays.hbs:
--------------------------------------------------------------------------------
1 | {{#if model.showLinks}}
2 |
3 | {{#each linkOverlayElements key="guid" as |overlay|}}
4 | {{element-overlay viewPortElement=overlay.element color=overlay.color}}
5 | {{/each}}
6 |
7 | {{/if}}
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/annotation/selection.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data/annotation.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data/annotation/options.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="annotation-options-group" collapsible=false onClose=(action 'closeOptions') as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="annotation-options"}}
4 | Annotation properties
5 | {{/group.tab}}
6 | {{else if (eq group.section "panels")}}
7 | {{#group.panel class="extracted-items container-fluid" toolId="annotation-options" as |active|}}
8 | {{annotation-options annotation=model}}
9 | {{field-options field=model.field.content}}
10 | {{extractor-options annotation=model}}
11 | {{/group.panel}}
12 | {{/if}}
13 | {{/tool-group}}
14 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data/item.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data/structure.hbs:
--------------------------------------------------------------------------------
1 | {{data-structure-listing sample=model annotationColors=annotationColors}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/data/tools.hbs:
--------------------------------------------------------------------------------
1 | {{#tool-group id="inspector-group" as |group|}}
2 | {{#if (eq group.section "tabs")}}
3 | {{#group.tab toolId="inspector"}}
4 | Inspector
5 | {{/group.tab}}
6 | {{else if (eq group.section "panels")}}
7 | {{#group.panel class="inspector container-fluid" toolId="inspector"}}
8 | {{inspector-panel sample=model annotationColors=annotationColors}}
9 | {{/group.panel}}
10 | {{/if}}
11 | {{/tool-group}}
12 |
13 | {{extracted-items-group}}
14 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/item.hbs:
--------------------------------------------------------------------------------
1 | {{outlet}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/structure.hbs:
--------------------------------------------------------------------------------
1 | {{outlet 'sample-structure'}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/sample/toolbar.hbs:
--------------------------------------------------------------------------------
1 | {{outlet 'browser-toolbar'}}
2 | {{#tooltip-container tooltipFor="sample-close-button-browser" text="Finish editing your sample so you can continue browsing and see how it works on other pages" tooltipContainer='body'}}
3 | {{#link-to 'projects.project.spider' class="btn btn-primary" activeClass="" id="sample-close-button-browser"}}
4 | {{icon-button icon='close'}} Close sample
5 | {{/link-to}}
6 | {{/tooltip-container}}
7 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/start-url/options.hbs:
--------------------------------------------------------------------------------
1 | {{start-url-options
2 | spider = model.spider
3 | startUrlId = model.startUrlId
4 | }}
5 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/structure.hbs:
--------------------------------------------------------------------------------
1 | {{spider-structure-listing
2 | project=model.project
3 | spider=model
4 | closeOptions=(route-action 'closeOptions')
5 | transitionToFragments=(route-action 'transitionToFragments')
6 | }}
7 | {{outlet 'spider-structure'}}
8 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/spider/toolbar.hbs:
--------------------------------------------------------------------------------
1 | {{show-links-button spider=model}}
2 | {{add-start-url-button spider=model}}
3 | {{edit-sample-button spider=model}}
4 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/structure.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 | {{save-status}}
4 | {{project-listing project=model reload="reload" conflict="conflict"}}
5 | {{project-structure-listing project=model}}
6 | {{outlet 'project-structure'}}
7 |
8 |
9 |
--------------------------------------------------------------------------------
/portiaui/app/templates/projects/project/toolbar.hbs:
--------------------------------------------------------------------------------
1 | {{create-spider-button project=model}}
2 |
--------------------------------------------------------------------------------
/portiaui/app/templates/tool-panels.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{outlet 'tool-panels'}}
3 |
4 |
--------------------------------------------------------------------------------
/portiaui/app/transforms/array.js:
--------------------------------------------------------------------------------
1 | import DS from 'ember-data';
2 |
3 | export default DS.Transform.extend({
4 | deserialize: function(serialized) {
5 | if (Array.isArray(serialized)) {
6 | return serialized;
7 | }
8 | return [];
9 | },
10 |
11 | serialize: function(deserialized) {
12 | if (Array.isArray(deserialized)) {
13 | return deserialized;
14 | }
15 | return [];
16 | }
17 | });
18 |
--------------------------------------------------------------------------------
/portiaui/app/transforms/json.js:
--------------------------------------------------------------------------------
1 | import DS from 'ember-data';
2 |
3 | export default DS.Transform.extend({
4 | deserialize: function(serialized) {
5 | return JSON.parse(serialized);
6 | },
7 |
8 | serialize: function(deserialized) {
9 | return JSON.stringify(deserialized);
10 | }
11 | });
12 |
--------------------------------------------------------------------------------
/portiaui/app/transforms/start-url.js:
--------------------------------------------------------------------------------
1 | import DS from 'ember-data';
2 | import buildStartUrl from '../models/start-url';
3 |
4 | export default DS.Transform.extend({
5 | deserialize: function(serialized) {
6 | if (Array.isArray(serialized)) {
7 | return serialized.map((url) => buildStartUrl(url));
8 | }
9 | return [];
10 | },
11 |
12 | serialize: function(deserialized) {
13 | if (Array.isArray(deserialized)) {
14 | return deserialized.map((startUrl) => startUrl.serialize());
15 | }
16 | return [];
17 | }
18 | });
19 |
--------------------------------------------------------------------------------
/portiaui/app/utils/attrs.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function attrValue(attr) {
4 | return (!Ember.isNone(attr) && typeof attr === 'object' && 'value' in attr) ? attr.value : attr;
5 | }
6 |
7 | export function attrChanged(oldAttrs, newAttrs, key) {
8 | return !oldAttrs || attrValue(oldAttrs[key]) !== attrValue(newAttrs[key]);
9 | }
10 |
11 | export function attrChangedTo(oldAttrs, newAttrs, key, value) {
12 | return attrChanged(oldAttrs, newAttrs, key) && attrValue(newAttrs[key]) === value;
13 | }
14 |
15 | export default {
16 | attrValue,
17 | attrChanged,
18 | attrChangedTo
19 | };
20 |
--------------------------------------------------------------------------------
/portiaui/app/utils/browser-features.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | const { RSVP } = Ember;
3 |
4 | export default function hasBrowserFeatures() {
5 | // generatedcontent: detection issue with zoom in chrome
6 | let features = [
7 | "eventlistener", "json", "postmessage", "queryselector", "requestanimationframe", "svg",
8 | "websockets", "cssanimations", "csscalc", "flexbox", "nthchild",
9 | "csspointerevents", "opacity", "csstransforms", "csstransitions", "cssvhunit",
10 | "classlist", "placeholder", "localstorage", "svgasimg", "datauri", "atobbtoa"
11 | ];
12 | let feature_promises = features.map((feature) => {
13 | return new RSVP.Promise((resolve) => {
14 | Modernizr.on(feature, (isFeatureActive) => { resolve(isFeatureActive); });
15 | });
16 | });
17 |
18 | return RSVP.all(feature_promises);
19 | }
20 |
--------------------------------------------------------------------------------
/portiaui/app/utils/computed.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function computedPropertiesEqual(a, b) {
4 | return Ember.computed(a, b, function() {
5 | return this.get(a) === this.get(b);
6 | });
7 | }
8 |
--------------------------------------------------------------------------------
/portiaui/app/utils/ensure-promise.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | // http://stackoverflow.com/questions/28247401/how-can-i-test-if-a-function-is-returning-a-promise-in-ember
4 | export default function ensurePromise(x) {
5 | return new Ember.RSVP.Promise(function(resolve) {
6 | resolve(x);
7 | });
8 | }
--------------------------------------------------------------------------------
/portiaui/app/utils/promises.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export function ensurePromise(valueOrPromise) {
4 | return new Ember.RSVP.Promise(function(resolve) {
5 | resolve(valueOrPromise);
6 | });
7 | }
8 |
9 | export default {
10 | ensurePromise
11 | };
12 |
--------------------------------------------------------------------------------
/portiaui/app/utils/types.js:
--------------------------------------------------------------------------------
1 | export function toType(obj) {
2 | return ({}).toString.call(obj).match(/\s([a-zA-Z]+)/)[1].toLowerCase();
3 | }
4 |
5 | export function isObject(obj) {
6 | return toType(obj) === 'object';
7 | }
8 |
9 | export function isArray(obj) {
10 | return Array.isArray(obj);
11 | }
12 |
13 |
--------------------------------------------------------------------------------
/portiaui/app/validations/fixed-fragment.js:
--------------------------------------------------------------------------------
1 | import { validatePresence } from 'ember-changeset-validations/validators';
2 | import validateWhitespace from '../validators/whitespace';
3 |
4 | export default {
5 | value: [
6 | validatePresence({ presence: true, message: 'Should not be empty.'}),
7 | validateWhitespace()
8 | ]
9 | };
10 |
--------------------------------------------------------------------------------
/portiaui/app/validations/list-fragment.js:
--------------------------------------------------------------------------------
1 | import { validatePresence } from 'ember-changeset-validations/validators';
2 |
3 | export default {
4 | value: validatePresence(true)
5 | };
6 |
--------------------------------------------------------------------------------
/portiaui/app/validations/range-fragment.js:
--------------------------------------------------------------------------------
1 | import validateRange from '../validators/range';
2 |
3 | export default {
4 | value: validateRange()
5 | };
6 |
--------------------------------------------------------------------------------
/portiaui/app/validators/whitespace.js:
--------------------------------------------------------------------------------
1 | export default function validateWhitespace() {
2 | return (key, newValue/*, oldValue, changes */) => {
3 | return newValue.match(/\s/g) ? 'Should not have whitespace' : true;
4 | };
5 | }
6 |
--------------------------------------------------------------------------------
/portiaui/bower.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "portia-ui",
3 | "ignore": [
4 | "*",
5 | "!bower.json"
6 | ],
7 | "dependencies": {
8 | "ember": "~2.6.0",
9 | "ember-cli-shims": "0.1.1",
10 | "ember-cli-test-loader": "0.2.2",
11 | "ember-qunit-notifications": "0.1.0",
12 | "jquery": "^2.2.0",
13 | "blob-polyfill": "~1.0.20150320",
14 | "cookie": "~1.1.0",
15 | "bootstrap-sass": "~3.3.6",
16 | "font-awesome": "~4.5.0",
17 | "jquery-color": "~2.1.2",
18 | "moment": "~2.11.2",
19 | "uri.js": "~1.16.0",
20 | "fetch": "~0.10.1",
21 | "es6-promise": "~3.0.2",
22 | "uri-templates": "~0.1.9",
23 | "css-escape": "~1.5.0",
24 | "animation-frame": "~0.2.4"
25 | },
26 | "private": true
27 | }
28 |
--------------------------------------------------------------------------------
/portiaui/config/deprecation-workflow.js:
--------------------------------------------------------------------------------
1 | window.deprecationWorkflow = window.deprecationWorkflow || {};
2 | window.deprecationWorkflow.config = {
3 | workflow: [
4 | { handler: "silence", matchMessage: /You modified .+ twice in a single render. This was unreliable in Ember 1.x and will be removed in Ember 3.0/ }
5 | ]
6 | };
7 |
--------------------------------------------------------------------------------
/portiaui/config/environment-development.js:
--------------------------------------------------------------------------------
1 | /* jshint node: true */
2 |
3 | module.exports = function(ENV) {
4 | ENV.APP.LOG_ACTIVE_GENERATION = true;
5 | ENV.APP.LOG_TRANSITIONS = true;
6 | ENV.APP.LOG_TRANSITIONS_INTERNAL = true;
7 | ENV.APP.LOG_VIEW_LOOKUPS = true;
8 | //ENV.APP.LOG_RESOLVER = true;
9 | return ENV;
10 | };
--------------------------------------------------------------------------------
/portiaui/config/environment-production.js:
--------------------------------------------------------------------------------
1 | /* jshint node: true */
2 |
3 | module.exports = function(ENV) {
4 | return ENV;
5 | };
--------------------------------------------------------------------------------
/portiaui/config/environment-test.js:
--------------------------------------------------------------------------------
1 | /* jshint node: true */
2 |
3 | module.exports = function(ENV) {
4 | // Testem prefers this...
5 | ENV.baseURL = '/';
6 | ENV.locationType = 'none';
7 |
8 | // keep test console output quieter
9 | ENV.APP.LOG_ACTIVE_GENERATION = false;
10 | ENV.APP.LOG_VIEW_LOOKUPS = false;
11 |
12 | ENV.APP.rootElement = '#ember-testing';
13 | return ENV;
14 | };
--------------------------------------------------------------------------------
/portiaui/public/assets/images/chrome-logo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/public/assets/images/chrome-logo.jpg
--------------------------------------------------------------------------------
/portiaui/public/assets/images/firefox-logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/public/assets/images/firefox-logo.png
--------------------------------------------------------------------------------
/portiaui/public/crossdomain.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
16 |
--------------------------------------------------------------------------------
/portiaui/public/empty-frame.html:
--------------------------------------------------------------------------------
1 |
2 |
6 |
--------------------------------------------------------------------------------
/portiaui/public/robots.txt:
--------------------------------------------------------------------------------
1 | # http://www.robotstxt.org
2 | User-agent: *
3 | Disallow:
4 |
--------------------------------------------------------------------------------
/portiaui/testem.js:
--------------------------------------------------------------------------------
1 | /*jshint node:true*/
2 | module.exports = {
3 | "framework": "qunit",
4 | "test_page": "tests/index.html?hidepassed",
5 | "disable_watching": true,
6 | "launch_in_ci": [
7 | "PhantomJS"
8 | ],
9 | "launch_in_dev": [
10 | "PhantomJS",
11 | "Chrome"
12 | ]
13 | };
14 |
--------------------------------------------------------------------------------
/portiaui/tests/helpers/destroy-app.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 |
3 | export default function destroyApp(application) {
4 | Ember.run(application, 'destroy');
5 | }
6 |
--------------------------------------------------------------------------------
/portiaui/tests/helpers/module-for-acceptance.js:
--------------------------------------------------------------------------------
1 | import { module } from 'qunit';
2 | import Ember from 'ember';
3 | import startApp from '../helpers/start-app';
4 | import destroyApp from '../helpers/destroy-app';
5 |
6 | const { RSVP: { Promise } } = Ember;
7 |
8 | export default function(name, options = {}) {
9 | module(name, {
10 | beforeEach() {
11 | this.application = startApp();
12 |
13 | if (options.beforeEach) {
14 | return options.beforeEach.apply(this, arguments);
15 | }
16 | },
17 |
18 | afterEach() {
19 | let afterEach = options.afterEach && options.afterEach.apply(this, arguments);
20 | return Promise.resolve(afterEach).then(() => destroyApp(this.application));
21 | }
22 | });
23 | }
24 |
--------------------------------------------------------------------------------
/portiaui/tests/helpers/resolver.js:
--------------------------------------------------------------------------------
1 | import Resolver from '../../resolver';
2 | import config from '../../config/environment';
3 |
4 | const resolver = Resolver.create();
5 |
6 | resolver.namespace = {
7 | modulePrefix: config.modulePrefix,
8 | podModulePrefix: config.podModulePrefix
9 | };
10 |
11 | export default resolver;
12 |
--------------------------------------------------------------------------------
/portiaui/tests/helpers/start-app.js:
--------------------------------------------------------------------------------
1 | import Ember from 'ember';
2 | import Application from '../../app';
3 | import config from '../../config/environment';
4 |
5 | export default function startApp(attrs) {
6 | let application;
7 |
8 | let attributes = Ember.merge({}, config.APP);
9 | attributes = Ember.merge(attributes, attrs); // use defaults, but you can override;
10 |
11 | Ember.run(() => {
12 | application = Application.create(attributes);
13 | application.setupForTesting();
14 | application.injectTestHelpers();
15 | });
16 |
17 | return application;
18 | }
19 |
--------------------------------------------------------------------------------
/portiaui/tests/test-helper.js:
--------------------------------------------------------------------------------
1 | import resolver from './helpers/resolver';
2 | import {
3 | setResolver
4 | } from 'ember-qunit';
5 |
6 | setResolver(resolver);
7 |
--------------------------------------------------------------------------------
/portiaui/tests/unit/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/tests/unit/.gitkeep
--------------------------------------------------------------------------------
/portiaui/tests/unit/utils/start-urls-test.js:
--------------------------------------------------------------------------------
1 | import { multiplicityFragment } from '../../../utils/start-urls';
2 | import { module, test } from 'qunit';
3 |
4 | module('Unit | Utility | startUrls');
5 |
6 | test('is correct for a one number range', function(assert) {
7 | const fragment = { type: 'range', value: '0-0' };
8 | assert.equal(multiplicityFragment(fragment), 1);
9 | });
10 |
11 | test('is correct for a large range', function(assert) {
12 | const fragment = { type: 'range', value: '0-99' };
13 | assert.equal(multiplicityFragment(fragment), 100);
14 | });
15 |
16 | test('is correct for a non-zero starting range', function(assert) {
17 | const fragment = { type: 'range', value: '51-100' };
18 | assert.equal(multiplicityFragment(fragment), 50);
19 | });
20 |
--------------------------------------------------------------------------------
/portiaui/tests/unit/validators/whitespace-test.js:
--------------------------------------------------------------------------------
1 | import validateWhitespace from '../../../validators/whitespace';
2 | import { module, test } from 'qunit';
3 |
4 | module('Unit | Validators | validateWhitespace');
5 |
6 | test('it should be true without whitespace', function(assert) {
7 | const key = 'value';
8 | const validator = validateWhitespace();
9 |
10 | assert.equal(validator(key, 'withoutspace'), true);
11 | });
12 |
13 | test('it should not have whitespace', function(assert) {
14 | const error = 'Should not have whitespace';
15 | const key = 'value';
16 | const validator = validateWhitespace();
17 |
18 | assert.equal(validator(key, 'with space'), error);
19 | assert.equal(validator(key, 'endspace '), error);
20 | assert.equal(validator(key, ' startspace'), error);
21 | });
22 |
--------------------------------------------------------------------------------
/portiaui/vendor/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/portiaui/vendor/.gitkeep
--------------------------------------------------------------------------------
/slybot/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | docs/_build
3 | slybot.egg-info/
4 |
--------------------------------------------------------------------------------
/slybot/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include README.rst
2 | include requirements.txt
3 | include slybot/validation/schemas.json
4 | include slybot/splash-script-combined.js
5 |
--------------------------------------------------------------------------------
/slybot/Makefile.buildbot:
--------------------------------------------------------------------------------
1 | build:
2 | bin/makedeb
3 |
--------------------------------------------------------------------------------
/slybot/README.rst:
--------------------------------------------------------------------------------
1 | ==============
2 | Slybot crawler
3 | ==============
4 |
5 | Slybot is a Python web crawler for doing web scraping. It's implemented on top of the
6 | `Scrapy`_ web crawling framework and the `Scrapely`_ extraction library.
7 |
8 | The documentation (including installation and usage) can be found at:
9 | http://slybot.readthedocs.org/
10 |
11 | .. _Scrapely: https://github.com/scrapy/scrapely
12 | .. _Scrapy: http://scrapy.org
13 |
--------------------------------------------------------------------------------
/slybot/bin/makedeb:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | version=$(python setup.py --version)-r$(git log --oneline | wc -l)+$(date +%Y%m%d%H%M)~$(git rev-parse --short HEAD)${BUILD_CODE:+~$BUILD_CODE}
4 | debchange -m -D unstable --force-distribution -v $version "Automatic build"
5 | debuild --no-lintian -us -uc -b
6 |
--------------------------------------------------------------------------------
/slybot/bin/slybot:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import os
3 | os.environ['SCRAPY_SETTINGS_MODULE'] = 'slybot.settings'
4 |
5 | from scrapy.cmdline import execute
6 | execute()
7 |
--------------------------------------------------------------------------------
/slybot/debian/changelog:
--------------------------------------------------------------------------------
1 | python-slybot (0.9) unstable; urgency=low
2 |
3 | * Initial release.
4 |
5 | -- Scrapinghub Team Wed, 31 Oct 2012 16:32:13 -0300
6 |
--------------------------------------------------------------------------------
/slybot/debian/compat:
--------------------------------------------------------------------------------
1 | 7
2 |
--------------------------------------------------------------------------------
/slybot/debian/control:
--------------------------------------------------------------------------------
1 | Source: python-slybot
2 | Section: python
3 | Priority: extra
4 | Maintainer: Scrapinghub Team
5 | Build-Depends: debhelper (>= 7), python (>=2.7)
6 | Standards-Version: 3.8.3
7 | Homepage: https://github.com/scrapinghub/portia
8 |
9 | Package: python-slybot
10 | Architecture: all
11 | Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends},
12 | scrapy (>= 1.0.3.post6),
13 | python-scrapely,
14 | python-loginform,
15 | python-lxml,
16 | python-dateparser,
17 | python-scrapy-splash,
18 | python-page-finder
19 | Description: A web crawler implemented in Python.
20 | Slybot is a Python web crawler for doing web scraping. It's implemented on top
21 | of the Scrapy web crawling framework and the Scrapely extraction library.
22 |
--------------------------------------------------------------------------------
/slybot/debian/copyright:
--------------------------------------------------------------------------------
1 | Copyright (C) 2011-2012 Scrapinghub
2 |
--------------------------------------------------------------------------------
/slybot/debian/pyversions:
--------------------------------------------------------------------------------
1 | 2.5-
2 |
--------------------------------------------------------------------------------
/slybot/debian/rules:
--------------------------------------------------------------------------------
1 | #!/usr/bin/make -f
2 | # -*- makefile -*-
3 | # Sample debian/rules that uses debhelper.
4 | # This file was originally written by Joey Hess and Craig Small.
5 | # As a special exception, when this file is copied by dh-make into a
6 | # dh-make output file, you may use that output file without restriction.
7 | # This special exception was added by Craig Small in version 0.37 of dh-make.
8 |
9 | # Uncomment this to turn on verbose mode.
10 | #export DH_VERBOSE=1
11 |
12 | %:
13 | dh $@
14 |
--------------------------------------------------------------------------------
/slybot/requirements-clustering.txt:
--------------------------------------------------------------------------------
1 | -r requirements.txt
2 | page_clustering==0.0.1
3 |
--------------------------------------------------------------------------------
/slybot/requirements-test.txt:
--------------------------------------------------------------------------------
1 | tox==3.12.1
2 | nose==1.3.7
3 | nose-timer==0.7.5
4 | doctest-ignore-unicode==0.1.2
5 | setuptools>=41.0.1
6 |
--------------------------------------------------------------------------------
/slybot/requirements.txt:
--------------------------------------------------------------------------------
1 | # Slybot requirements
2 | numpy==1.16.4
3 | Scrapy==1.6.0
4 | scrapely==0.13.5
5 | loginform==1.2.0
6 | lxml==4.3.4
7 | dateparser==0.7.1
8 | python-dateutil==2.8.0
9 | jsonschema==2.6.0
10 | six==1.12.0
11 | scrapy-splash==0.7.2
12 | page_finder==0.1.8
13 | chardet==3.0.4
14 |
--------------------------------------------------------------------------------
/slybot/scrapy.cfg:
--------------------------------------------------------------------------------
1 | [settings]
2 | default = slybot.settings
3 |
--------------------------------------------------------------------------------
/slybot/slybot/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.13.3'
2 |
--------------------------------------------------------------------------------
/slybot/slybot/exporter.py:
--------------------------------------------------------------------------------
1 | from scrapy.exporters import CsvItemExporter
2 | from scrapy.conf import settings
3 |
4 |
5 | class SlybotCSVItemExporter(CsvItemExporter):
6 | def __init__(self, *args, **kwargs):
7 | kwargs['fields_to_export'] = settings.getlist('CSV_EXPORT_FIELDS') or None
8 | super(SlybotCSVItemExporter, self).__init__(*args, **kwargs)
9 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/images.py:
--------------------------------------------------------------------------------
1 | """Images."""
2 | from scrapely.extractors import extract_image_url
3 | from slybot.fieldtypes.url import UrlFieldTypeProcessor
4 |
5 |
6 | class ImagesFieldTypeProcessor(UrlFieldTypeProcessor):
7 | name = 'image'
8 | description = 'extracts image URLs'
9 |
10 | def extract(self, text):
11 | if text is not None:
12 | return extract_image_url(text) or ''
13 | return ''
14 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/number.py:
--------------------------------------------------------------------------------
1 | """
2 | Numeric data extraction
3 | """
4 | from scrapely.extractors import contains_any_numbers, extract_number
5 |
6 | class NumberTypeProcessor(object):
7 | """NumberTypeProcessor
8 |
9 | Extracts a number from text
10 |
11 | >>> from scrapely.extractors import htmlregion
12 | >>> n = NumberTypeProcessor()
13 | >>> n.extract(htmlregion(u"there are no numbers here"))
14 | >>> n.extract(htmlregion(u"foo 34"))
15 | u'foo 34'
16 | >>> n.adapt(u"foo 34", None)
17 | u'34'
18 |
19 | If more than one number is present, nothing is extracted
20 | >>> n.adapt(u"34 48", None) is None
21 | True
22 | """
23 | name = 'number'
24 | description = 'extracts a single number in the text passed'
25 |
26 | def extract(self, htmlregion):
27 | """Only matches and extracts strings with at least one number"""
28 | return contains_any_numbers(htmlregion.text_content)
29 |
30 | def adapt(self, text, htmlpage=None):
31 | return extract_number(text)
32 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/point.py:
--------------------------------------------------------------------------------
1 |
2 | class GeoPointFieldTypeProcessor(object):
3 | """Renders point with tags"""
4 |
5 | name = 'geopoint'
6 | description = 'geo point'
7 | multivalue = True
8 |
9 | def extract(self, value):
10 | return value
11 |
12 | def adapt(self, value, htmlpage=None):
13 | return value
14 |
15 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/price.py:
--------------------------------------------------------------------------------
1 | """
2 | Price field types
3 | """
4 | from scrapely import extractors
5 |
6 | class PriceTypeProcessor(object):
7 | """Extracts price from text"""
8 | name = "price"
9 | description = "extracts a price decimal number in the text passed"
10 |
11 | def extract(self, htmlregion):
12 | return extractors.contains_any_numbers(htmlregion.text_content)
13 |
14 | def adapt(self, text, htmlpage=None):
15 | return extractors.extract_price(text)
16 |
17 |
--------------------------------------------------------------------------------
/slybot/slybot/plugins/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slybot/slybot/plugins/__init__.py
--------------------------------------------------------------------------------
/slybot/slybot/plugins/scrapely_annotations/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 |
3 | from .annotations import Annotations
4 |
5 | __all__ = [Annotations]
6 |
--------------------------------------------------------------------------------
/slybot/slybot/plugins/scrapely_annotations/exceptions.py:
--------------------------------------------------------------------------------
1 | class MissingRequiredError(Exception):
2 | pass
3 |
4 |
5 | class ItemNotValidError(Exception):
6 | pass
7 |
--------------------------------------------------------------------------------
/slybot/slybot/plugins/scrapely_annotations/extraction/__init__.py:
--------------------------------------------------------------------------------
1 | from .container_extractors import (
2 | BaseContainerExtractor, ContainerExtractor, RepeatedContainerExtractor,
3 | RepeatedFieldsExtractor
4 | )
5 | from .extractors import SlybotIBLExtractor, TemplatePageMultiItemExtractor
6 | from .pageparsing import (
7 | parse_template, SlybotTemplatePage, SlybotTemplatePageParser
8 | )
9 | from .region_extractors import BaseExtractor, SlybotRecordExtractor
10 |
--------------------------------------------------------------------------------
/slybot/slybot/starturls/feed_generator.py:
--------------------------------------------------------------------------------
1 | import re
2 | from scrapy import Request
3 | _NEWLINE_RE = re.compile('[\r\n]')
4 |
5 |
6 | class FeedGenerator(object):
7 | def __init__(self, callback):
8 | self.callback = callback
9 |
10 | def __call__(self, url):
11 | return Request(url, callback=self.parse_urls)
12 |
13 | def parse_urls(self, response):
14 | newline_urls = _NEWLINE_RE.split(response.text)
15 | urls = [url for url in newline_urls if url]
16 | for url in urls:
17 | yield Request(url, callback=self.callback)
18 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slybot/slybot/tests/__init__.py
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/extractors.json:
--------------------------------------------------------------------------------
1 | {
2 | "4fad3762688f920d76000000": {
3 | "regular_expression": "(\\d+)"
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/project.json:
--------------------------------------------------------------------------------
1 | {
2 | "version": "1",
3 | "name": "SampleProject"
4 | }
5 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/allowed_domains.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "allowed_domains": [
7 | "www.ebay.com",
8 | "www.yahoo.com"
9 | ],
10 | "exclude_patterns": [],
11 | "respect_nofollow": true,
12 | "follow_patterns": [],
13 | "links_to_follow": "none"
14 | }
15 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/any_allowed_domains.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/"
5 | ],
6 | "allowed_domains": null,
7 | "exclude_patterns": [],
8 | "respect_nofollow": true,
9 | "follow_patterns": [],
10 | "scrapes": "default",
11 | "links_to_follow": "none"
12 | }
13 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/books.toscrape.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "exclude_patterns": [],
3 | "follow_patterns": [
4 | "page-\\d+\\.html"
5 | ],
6 | "id": "f8ad-40cc-9916",
7 | "init_requests": [],
8 | "js_disable_patterns": [],
9 | "js_enable_patterns": [],
10 | "js_enabled": false,
11 | "links_to_follow": "patterns",
12 | "page_actions": [],
13 | "respect_nofollow": false,
14 | "start_urls": [
15 | "http://books.toscrape.com/"
16 | ],
17 | "version": "0.13.0b32"
18 | }
19 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/books.toscrape.com/3652-4fa1-a912.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/books.toscrape.com_1.json:
--------------------------------------------------------------------------------
1 | {
2 | "exclude_patterns": [],
3 | "follow_patterns": [],
4 | "id": "f8ad-40cc-9916",
5 | "init_requests": [],
6 | "js_disable_patterns": [],
7 | "js_enable_patterns": [],
8 | "js_enabled": false,
9 | "allowed_domains": null,
10 | "links_to_follow": "all",
11 | "page_actions": [],
12 | "respect_nofollow": false,
13 | "start_urls": [
14 | "http://books.toscrape.com/"
15 | ],
16 | "version": "0.13.0b34"
17 | }
18 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/cargurus.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [
3 | ],
4 | "start_urls": [
5 | "http://www.cargurus.com/Cars/sitemap.html"
6 | ],
7 | "exclude_patterns": [
8 | "-Pictures-",
9 | "-Specs-",
10 | "-Price-",
11 | "_v",
12 | "-Videos-"
13 | ],
14 | "follow_patterns": [
15 | "-Overview-",
16 | "-Reviews-",
17 | "/rss/"
18 | ],
19 | "links_to_follow": "patterns",
20 | "respect_nofollow": false
21 | }
22 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["Cars"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "links_to_follow": "none"
28 | }
29 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay3.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["{search_string}"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "scrapes": "default",
28 | "links_to_follow": "none"
29 | }
30 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay4.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["{search_string}"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "scrapes": "default",
28 | "links_to_follow": "none"
29 | }
30 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": ["http://www.example.com/index.html"],
4 | "init_requests": [
5 | {
6 | "type": "start",
7 | "url": "http://www.example.com/products.csv",
8 | "link_extractor": {
9 | "type": "column",
10 | "value": 1,
11 | "delimiter": ","
12 | }
13 | }
14 | ],
15 | "exclude_patterns": [],
16 | "follow_patterns": [],
17 | "links_to_follow": "patterns",
18 | "respect_nofollow": true
19 | }
20 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example2.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [],
4 | "init_requests": [
5 | {
6 | "type": "start",
7 | "url": "http://www.example.com/index.html"
8 | }
9 | ],
10 | "exclude_patterns": [],
11 | "follow_patterns": [],
12 | "links_to_follow": "patterns",
13 | "respect_nofollow": true
14 | }
15 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example3.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": ["http://www.example.com/index.html"],
4 | "exclude_patterns": [],
5 | "follow_patterns": [],
6 | "links_to_follow": "patterns",
7 | "respect_nofollow": true
8 | }
9 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example4.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": ["http://www.example.com/index.html"],
4 | "start_urls_type": "generated_urls",
5 | "generated_urls": [{
6 | "template": "http://www.example.com/{}",
7 | "paths": [{
8 | "type": "options",
9 | "values": ["about_us", "contact"]
10 | }],
11 | "params": []
12 | }, {
13 | "template": "http://www.example.com/{}/{}",
14 | "paths": [{
15 | "type": "default",
16 | "values": ["p"]
17 | }, {
18 | "type": "range",
19 | "values": [2, 5]
20 | }],
21 | "params": []
22 | }],
23 | "exclude_patterns": [],
24 | "follow_patterns": [],
25 | "links_to_follow": "patterns",
26 | "respect_nofollow": true
27 | }
28 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/networkhealth.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "template_names": [
3 | "networkhealthtemplate",
4 | "doesnotexist"
5 | ],
6 | "start_urls": [
7 | "http://www.networkhealth.com/network-health-plan-info/all/find-a-doctor/index.aspx"
8 | ],
9 | "exclude_patterns": [],
10 | "follow_patterns": [
11 | "provider-detail.aspx\\?Id="
12 | ],
13 | "links_to_follow": "patterns",
14 | "respect_nofollow": true
15 | }
16 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/networkhealth.com/networkhealthtemplate.json:
--------------------------------------------------------------------------------
1 | {
2 | "extractors": {},
3 | "url": "http://www.networkhealth.com/network-health-plan-info/all/find-a-doctor/provider-search-provider-detail.aspx?Id=P00138746&Network=HMO/POS",
4 | "scrapes": "doctor",
5 | "page_type": "item",
6 | "page_id": "50b67886d559307f097be904"
7 | }
8 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/pinterest.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://pinterest.com/popular/"
5 | ],
6 | "init_requests": [
7 | {
8 | "username": "test",
9 | "loginurl": "https://pinterest.com/login/",
10 | "password": "testpass",
11 | "type": "login"
12 | }
13 | ],
14 | "exclude_patterns": [],
15 | "respect_nofollow": true,
16 | "follow_patterns": [],
17 | "links_to_follow": "patterns"
18 | }
19 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/seedsofchange.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.seedsofchange.com/garden_center/browse_category.aspx?id=123"
5 | ],
6 | "exclude_patterns": [
7 | "/tellafriend.aspx.+"
8 | ],
9 | "follow_patterns": [
10 | "/garden_center/browse_category.aspx.+",
11 | "/garden_center/detailedCategoryDisplay.aspx.+",
12 | "/garden_center/product_details.aspx.+"
13 | ],
14 | "links_to_follow": "patterns",
15 | "respect_nofollow": true
16 | }
17 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/sitemaps.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [
3 | ],
4 | "start_urls": [
5 | ],
6 | "exclude_patterns": [
7 | ],
8 | "follow_patterns": [
9 | ],
10 | "allowed_domains": [
11 | "webupd8.org",
12 | "siliconrepublic.com"
13 | ],
14 | "links_to_follow": "patterns",
15 | "respect_nofollow": false
16 | }
17 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/atom_sample.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Webupd8 Posts
4 |
5 |
7 |
8 |
10 |
11 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/sitemap_sample.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | https://www.siliconrepublic.com/post-sitemap1.xml
6 | 2003-06-04T10:46:42+01:00
7 |
8 |
9 | https://www.siliconrepublic.com/post-sitemap2.xml
10 | 2004-02-11T12:29:08+00:00
11 |
12 |
13 | https://www.siliconrepublic.com/post-sitemap3.xml
14 | 2004-09-10T09:48:13+01:00
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/test_params.txt:
--------------------------------------------------------------------------------
1 | Cars
2 | Boats
--------------------------------------------------------------------------------
/slybot/slybot/validation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slybot/slybot/validation/__init__.py
--------------------------------------------------------------------------------
/slybot/tox.ini:
--------------------------------------------------------------------------------
1 | # Tox (http://tox.testrun.org/) is a tool for running tests
2 | # in multiple virtualenvs. This configuration file will run the
3 | # test suite on all supported python versions. To use it, "pip install tox"
4 | # and then run "tox" from this directory.
5 |
6 | [tox]
7 | envlist = py27,py37
8 |
9 | [testenv]
10 | deps =
11 | -r{toxinidir}/requirements-test.txt
12 | -r{toxinidir}/requirements.txt
13 | commands =
14 | nosetests \
15 | --with-doctest \
16 | --with-doctest-ignore-unicode \
17 | --doctest-options='+IGNORE_UNICODE'
18 |
19 |
--------------------------------------------------------------------------------
/slyd/.gitignore:
--------------------------------------------------------------------------------
1 | # python
2 | *.py[cod]
3 |
4 | # editor files
5 | *.orig
6 | *.bak
7 | *.swp
8 | *.project
9 | *.sublime-*
10 |
11 | # twisted
12 | dropin.cache
13 | twistd.log
14 | twistd.pid
15 | _trial_temp*
16 |
17 | # local data files
18 | data/*
19 |
20 | # npm files
21 | node_modules/*
22 | npm-debug.log
23 |
--------------------------------------------------------------------------------
/slyd/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "predef": [
3 | "document",
4 | "window",
5 | "$",
6 | "URI",
7 | "CanvasLoader",
8 | "TreeMirror",
9 | "Raven",
10 | "-Promise"
11 | ],
12 | "browser": true,
13 | "boss": false,
14 | "curly": true,
15 | "debug": false,
16 | "devel": true,
17 | "eqeqeq": true,
18 | "evil": true,
19 | "forin": false,
20 | "immed": false,
21 | "laxbreak": false,
22 | "newcap": true,
23 | "noarg": true,
24 | "noempty": false,
25 | "nonew": false,
26 | "nomen": false,
27 | "onevar": false,
28 | "plusplus": false,
29 | "regexp": false,
30 | "undef": true,
31 | "sub": true,
32 | "strict": false,
33 | "white": false,
34 | "eqnull": true,
35 | "esnext": true,
36 | "unused": true
37 | }
38 |
--------------------------------------------------------------------------------
/slyd/bin/init_mysql_db:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | import os
4 | import sys
5 |
6 | try:
7 | import slyd
8 | import slyd.settings as settings_module
9 | from scrapy.settings import Settings
10 | settings = Settings()
11 | settings.setmodule(settings_module)
12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings['DJANGO_SETTINGS'])
13 | except ImportError:
14 | sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
15 | try:
16 | import slyd
17 | except ImportError:
18 | sys.stderr.write("Error: Can't find the project package 'slyd'.\n")
19 | sys.exit(1)
20 |
21 | from portia_server.storage.repoman import Repoman
22 |
23 |
24 | def main():
25 | Repoman.setup(
26 | storage_backend='slyd.gitstorage.repo.MysqlRepo',
27 | location=os.environ.get('DB_URL'),
28 | )
29 | Repoman.pool._runWithConnection(Repoman.init_backend)
30 |
31 |
32 | if __name__ == '__main__':
33 | main()
34 |
--------------------------------------------------------------------------------
/slyd/bin/slyd:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | import splash.server
4 | import splash.defaults
5 | import argparse
6 |
7 |
8 | DEFAULT_PORTIA_PORT = 9001
9 | DEFAULT_PORTIA_ROOT = '../portiaui/dist'
10 | splash.defaults.SPLASH_PORT = DEFAULT_PORTIA_PORT
11 |
12 | def parse_args():
13 | op = argparse.ArgumentParser()
14 | op.add_argument('-p', '--port', default=DEFAULT_PORTIA_PORT, type=int)
15 | op.add_argument('-r', '--root', default=DEFAULT_PORTIA_ROOT,
16 | help='Location of Portia webserver assets')
17 | return op.parse_args()
18 |
19 |
20 | def make_server(*args, **kwargs):
21 | from slyd.tap import makeService
22 | from twisted.internet import reactor
23 | opts = parse_args()
24 | reactor.listenTCP(opts.port, makeService({'port': opts.port,
25 | 'docroot': opts.root}))
26 |
27 | if __name__ == '__main__':
28 | splash.server.main(server_factory=make_server, argv=[])
29 |
--------------------------------------------------------------------------------
/slyd/requirements.txt:
--------------------------------------------------------------------------------
1 | # Slyd requirements
2 | twisted==19.2.1
3 | pyOpenSSL==17.5.0
4 | service_identity==18.1.0
5 | requests>=2.20.0
6 | autobahn==18.3.1
7 | six==1.12.0
8 | chardet==3.0.4
9 | parse==1.8.2
10 | ndg-httpsclient==0.4.4
11 | retrying==1.3.3
12 | mock==2.0.0
13 |
14 | # Splash dev
15 | https://github.com/scrapinghub/splash/archive/3.2.x.tar.gz
16 |
--------------------------------------------------------------------------------
/slyd/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 | install_requires = ['Scrapy', 'scrapely', 'loginform', 'lxml', 'jsonschema',
4 | 'django', 'parse', 'marshmallow_jsonapi', 'chardet',
5 | 'autobahn', 'requests', 'service_identity',
6 | 'ndg-httpsclient']
7 | tests_requires = install_requires
8 |
9 | setup(name='slyd',
10 | license='BSD',
11 | description='Portia',
12 | author='Scrapinghub',
13 | url='http://github.com/scrapinghub/portia',
14 | packages=find_packages(),
15 | platforms=['Any'],
16 | scripts=['bin/sh2sly', 'bin/slyd', 'bin/init_mysql_db'],
17 | classifiers=[
18 | 'Development Status :: 4 - Beta',
19 | 'License :: OSI Approved :: BSD License',
20 | 'Operating System :: OS Independent',
21 | 'Programming Language :: Python',
22 | 'Programming Language :: Python :: 2',
23 | 'Programming Language :: Python :: 2.7'
24 | ])
25 |
--------------------------------------------------------------------------------
/slyd/slybot:
--------------------------------------------------------------------------------
1 | ../slybot
--------------------------------------------------------------------------------
/slyd/slyd/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slyd/slyd/__init__.py
--------------------------------------------------------------------------------
/slyd/slyd/authmanager.py:
--------------------------------------------------------------------------------
1 | from scrapy.utils.misc import load_object
2 |
3 |
4 | class AuthManager(object):
5 |
6 | def __init__(self, settings):
7 | self.settings = settings
8 | auth_settings = settings.get('AUTH_CONFIG', {})
9 | self.auth_method = load_object(
10 | auth_settings.get('CALLABLE', 'slyd.dummyauth.protectResource'))
11 | self.config = auth_settings.get('CONFIG', {})
12 |
13 | def protectResource(self, resource):
14 | return self.auth_method(resource, config=self.config)
15 |
--------------------------------------------------------------------------------
/slyd/slyd/dummyauth.py:
--------------------------------------------------------------------------------
1 | from twisted.web.resource import Resource
2 |
3 |
4 | def protectResource(resource, config):
5 | '''Dummy resource protector.'''
6 | return DummyAuthResource(resource)
7 |
8 |
9 | class DummyAuthResource(Resource):
10 | """A wrapper that injects dummy auth info to every passing request."""
11 |
12 | def __init__(self, resource):
13 | Resource.__init__(self)
14 | self.wrapped = resource
15 |
16 | def getChildWithDefault(self, path, request):
17 | request.auth_info = {
18 | 'username': 'defaultuser',
19 | }
20 | # Don't consume any segments.
21 | request.postpath.insert(0, request.prepath.pop())
22 | return self.wrapped
23 |
--------------------------------------------------------------------------------
/slyd/slyd/errors.py:
--------------------------------------------------------------------------------
1 | class BaseError(Exception):
2 | def __init__(self, status, title, body=''):
3 | self._status = status
4 | self._title = title
5 | self._body = body
6 |
7 | @property
8 | def title(self):
9 | return self._title
10 |
11 | @property
12 | def body(self):
13 | return self._body
14 |
15 | @property
16 | def status(self):
17 | return self._status
18 |
19 | def __repr__(self):
20 | return '%s(%s)' % (self.__class__.__name__, str(self))
21 |
22 | def __str__(self):
23 | return '%s: %s' % (self.status, self.title)
24 |
25 |
26 | class BaseHTTPError(BaseError):
27 | _status = 999
28 |
29 | def __init__(self, title, body=''):
30 | super(BaseHTTPError, self).__init__(self._status, title, body)
31 |
32 |
33 | class BadRequest(BaseHTTPError):
34 | _status = 400
35 |
--------------------------------------------------------------------------------
/slyd/slyd/gitstorage/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slyd/slyd/gitstorage/__init__.py
--------------------------------------------------------------------------------
/slyd/slyd/gitstorage/projectspec.py:
--------------------------------------------------------------------------------
1 | from slyd.projectspec import ProjectSpec
2 | from slyd.gitstorage.projects import GitProjectMixin
3 |
4 |
5 | class GitProjectSpec(GitProjectMixin, ProjectSpec):
6 | def _schedule_data(self, spider, args):
7 | branch = args.pop('branch', [None])[0]
8 | commit = args.pop('commit_id', [None])[0]
9 | project = self.project_name
10 | arg = {}
11 | if commit:
12 | arg['commit'] = commit
13 | elif branch:
14 | arg['branch'] = branch
15 | if not arg and self.storage.repo.has_branch(self.user):
16 | arg['branch'] = self.user
17 | self._checkout_commit_or_head(project, **arg)
18 | commit_id = self.storage._commit.id
19 | return {
20 | 'project': self.project_name,
21 | 'version': commit_id,
22 | 'spider': spider
23 | }
24 |
--------------------------------------------------------------------------------
/slyd/slyd/settings/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from .base import *
3 |
4 | try:
5 | from local_settings import *
6 | except ImportError:
7 | pass
8 |
--------------------------------------------------------------------------------
/slyd/slyd/splash/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scrapinghub/portia/606467d278eab2236afcb3d260cb03bf6fb906a0/slyd/slyd/splash/__init__.py
--------------------------------------------------------------------------------
/slyd/slyd/splash/qtutils.py:
--------------------------------------------------------------------------------
1 |
2 | try:
3 | from PyQt5.QtCore import QObject
4 | from PyQt5.QtCore import pyqtSlot
5 | from PyQt5.QtWebKit import QWebElement
6 | from PyQt5.QtNetwork import QNetworkRequest
7 | except ImportError:
8 | from PyQt4.QtCore import QObject
9 | from PyQt4.QtCore import pyqtSlot
10 | from PyQt4.QtWebKit import QWebElement
11 | from PyQt4.QtNetwork import QNetworkRequest
12 |
13 | def to_py(obj):
14 | if hasattr(obj, 'toPyObject'):
15 | return obj.toPyObject()
16 | return obj
17 |
18 |
--------------------------------------------------------------------------------
/slyd/twisted/plugins/slyd_plugin.py:
--------------------------------------------------------------------------------
1 | """Registers 'twistd slyd' command."""
2 | from twisted.application.service import ServiceMaker
3 |
4 | finger = ServiceMaker(
5 | 'slyd', 'slyd.tap', 'A server for creating scrapely spiders', 'slyd')
6 |
--------------------------------------------------------------------------------
/splash_utils/compile_slybot.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | {
4 | echo ";(function(){"
5 |
6 | cat '../portiaui/bower_components/es5-shim/es5-shim.js'
7 |
8 | # Page actions scripts
9 | cat 'waitAsync.js'
10 | cat 'perform_actions.js'
11 |
12 | echo '})();'
13 | } > ../slybot/slybot/splash-script-combined.js
14 |
15 |
--------------------------------------------------------------------------------
3 |
4 |