├── .babelrc ├── .circleci └── config.yml ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── CHANGES.rst ├── Gruntfile.js ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.rst ├── base.ini ├── buildout.cfg ├── candidate.cfg ├── circle-tests.sh ├── cloud-config-cluster.yml ├── cloud-config-elasticsearch.yml ├── cloud-config.yml ├── cloudwatchmon-requirements.txt ├── conf ├── elasticsearch.yml ├── jvm.options └── log4j2.properties ├── config.rb ├── demo.cfg ├── development.ini ├── docs ├── AWS.rst ├── advanced-search.rst ├── auth.rst ├── aws-deployment.rst ├── database.rst ├── indexer.rst ├── invalidation.rst ├── object-lifecycle.rst ├── overview.rst ├── rendering-overview.graffle ├── rendering-overview.pdf └── search.rst ├── etc ├── logging-apache.conf └── snovault-apache.conf ├── examples ├── s3cp.py └── submit_file.py ├── gulpfile.js ├── instance-dns.yml ├── jest └── environment.js ├── nginx.yml ├── node_shims ├── ckeditor │ ├── index.js │ └── package.json └── google-analytics │ ├── index.js │ └── package.json ├── package-lock.json ├── package.json ├── production.ini.in ├── pytest.ini ├── requirements.osx.txt ├── requirements.txt ├── scripts ├── LogToCsv.py ├── blackholes.py └── embeds.py ├── setup.cfg ├── setup.py ├── src ├── snovault │ ├── __init__.py │ ├── app.py │ ├── attachment.py │ ├── auditor.py │ ├── authentication.py │ ├── batchupgrade.py │ ├── cache.py │ ├── calculated.py │ ├── commands │ │ ├── __init__.py │ │ ├── check_rendering.py │ │ ├── es_index_data.py │ │ ├── jsonld_rdf.py │ │ ├── profile.py │ │ └── spreadsheet_to_json.py │ ├── config.py │ ├── connection.py │ ├── crud_views.py │ ├── dev_servers.py │ ├── elasticsearch │ │ ├── __init__.py │ │ ├── cached_views.py │ │ ├── create_mapping.py │ │ ├── es_index_listener.py │ │ ├── esstorage.py │ │ ├── indexer.py │ │ ├── indexer_state.py │ │ ├── interfaces.py │ │ ├── mpindexer.py │ │ ├── searches │ │ │ ├── __init__.py │ │ │ ├── configs.py │ │ │ ├── fields.py │ │ │ └── interfaces.py │ │ ├── simple_queue.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_esstorage.py │ │ │ ├── test_indexer_base.py │ │ │ ├── test_indexer_redis.py │ │ │ ├── test_indexer_simple.py │ │ │ └── test_simple_queue.py │ │ └── uuid_queue │ │ │ ├── __init__.py │ │ │ ├── adapter_queue.py │ │ │ ├── queues │ │ │ ├── base_queue.py │ │ │ └── redis_queues.py │ │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── test_adapter_queue.py │ │ │ ├── test_base_queue.py │ │ │ └── test_redis_queue.py │ ├── embed.py │ ├── etag.py │ ├── indexing_views.py │ ├── interfaces.py │ ├── invalidation.py │ ├── json_renderer.py │ ├── jsongraph.py │ ├── jsonld_context.py │ ├── local_storage.py │ ├── nginx-dev.conf │ ├── predicates.py │ ├── resource_views.py │ ├── resources.py │ ├── schema_graph.py │ ├── schema_utils.py │ ├── schema_views.py │ ├── snowflake_hash.py │ ├── stats.py │ ├── storage.py │ ├── tests │ │ ├── __init__.py │ │ ├── elasticsearch_fixture.py │ │ ├── indexfixtures.py │ │ ├── postgresql_fixture.py │ │ ├── pyramidfixtures.py │ │ ├── redis_storage_fixture.py │ │ ├── serverfixtures.py │ │ ├── test_attachment.py │ │ ├── test_auditor.py │ │ ├── test_authentication.py │ │ ├── test_embedding.py │ │ ├── test_indexer_state.py │ │ ├── test_indexing.py │ │ ├── test_key.py │ │ ├── test_link.py │ │ ├── test_post_put_patch.py │ │ ├── test_redis_store.py │ │ ├── test_schema_utils.py │ │ ├── test_searches_configs.py │ │ ├── test_searches_fields.py │ │ ├── test_skip_calculated.py │ │ ├── test_snowflake_hash.py │ │ ├── test_storage.py │ │ ├── test_upgrader.py │ │ ├── test_utils.py │ │ ├── testappfixtures.py │ │ ├── testing_auditor.py │ │ ├── testing_key.py │ │ ├── testing_upgrader.py │ │ ├── testing_views.py │ │ └── toolfixtures.py │ ├── typeinfo.py │ ├── upgrader.py │ ├── util.py │ ├── validation.py │ └── validators.py └── snowflakes │ ├── __init__.py │ ├── audit │ ├── __init__.py │ └── item.py │ ├── authorization.py │ ├── commands │ ├── __init__.py │ ├── check_rendering.py │ ├── create_admin_user.py │ ├── delete.py │ ├── deploy.py │ ├── es_index_data.py │ ├── import_data.py │ ├── jsonld_rdf.py │ ├── migrate_attachments_aws.py │ ├── profile.py │ └── spreadsheet_to_json.py │ ├── docs │ ├── making_audits.md │ └── schema-changes.md │ ├── loadxl.py │ ├── memlimit.py │ ├── renderers.py │ ├── root.py │ ├── schema_formats.py │ ├── schemas │ ├── access_key.json │ ├── award.json │ ├── changelogs │ │ ├── award.md │ │ └── example.md │ ├── image.json │ ├── lab.json │ ├── mixins.json │ ├── namespaces.json │ ├── page.json │ ├── snowball.json │ ├── snowflake.json │ ├── snowfort.json │ ├── snowset.json │ └── user.json │ ├── search_views.py │ ├── server_defaults.py │ ├── static │ ├── 404.html │ ├── browser.js │ ├── build │ │ └── .gitignore │ ├── components │ │ ├── ImpersonateUserSchema.js │ │ ├── JSONNode.js │ │ ├── StickyHeader.js │ │ ├── __tests__ │ │ │ ├── .jshintrc │ │ │ ├── server-render-test.js │ │ │ └── store-test.js │ │ ├── app.js │ │ ├── audit.js │ │ ├── blocks │ │ │ ├── fallback.js │ │ │ ├── index.js │ │ │ ├── item.js │ │ │ ├── richtext.js │ │ │ ├── search.js │ │ │ └── teaser.js │ │ ├── browserfeat.js │ │ ├── collection.js │ │ ├── doc.js │ │ ├── edit.js │ │ ├── errors.js │ │ ├── fetched.js │ │ ├── footer.js │ │ ├── form.js │ │ ├── globals.js │ │ ├── graph.js │ │ ├── home.js │ │ ├── image.js │ │ ├── index.js │ │ ├── inputs │ │ │ ├── file.js │ │ │ ├── index.js │ │ │ └── object.js │ │ ├── item.js │ │ ├── layout.js │ │ ├── lib │ │ │ ├── index.js │ │ │ └── store.js │ │ ├── mixins.js │ │ ├── navigation.js │ │ ├── objectutils.js │ │ ├── page.js │ │ ├── report.js │ │ ├── schema.js │ │ ├── search.js │ │ ├── sorttable.js │ │ ├── statuslabel.js │ │ ├── testdata │ │ │ ├── award.js │ │ │ ├── lab.js │ │ │ └── submitter.js │ │ ├── testing.js │ │ └── user.js │ ├── dev-robots.txt │ ├── font │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ └── fontawesome-webfont.woff │ ├── google63612883561ae8ff.html │ ├── img │ │ ├── asc.gif │ │ ├── bg.gif │ │ ├── checker.svg │ │ ├── close-icon.png │ │ ├── close-icon.svg │ │ ├── desc.gif │ │ ├── favicon.ico │ │ ├── file-broken.png │ │ ├── file-broken.svg │ │ ├── file-pdf.png │ │ ├── file-pdf.svg │ │ ├── file.png │ │ ├── file.svg │ │ ├── glyphicons-halflings-white.png │ │ ├── glyphicons-halflings.png │ │ ├── hiding-dots.svg │ │ ├── orientation-icons.png │ │ ├── som-logo-red.png │ │ ├── som-logo.png │ │ ├── spinner-orange-bg.gif │ │ ├── spinner1.gif │ │ ├── su-logo-white-2x.png │ │ ├── su-logo-white.png │ │ ├── su-logo.png │ │ ├── ucsc-logo-white-alt-2x.png │ │ ├── ucsc-logo-white-alt.png │ │ ├── ucsc-logo-white.png │ │ └── ucsc-logo.png │ ├── inline.js │ ├── libs │ │ ├── __tests__ │ │ │ ├── .jshintrc │ │ │ └── registry-test.js │ │ ├── bootstrap │ │ │ ├── button.js │ │ │ ├── dropdown-menu.js │ │ │ ├── navbar.js │ │ │ └── panel.js │ │ ├── closest.js │ │ ├── compat.js │ │ ├── jsonScriptEscape.js │ │ ├── noarg-memoize.js │ │ ├── offset.js │ │ ├── origin.js │ │ ├── react-middleware.js │ │ ├── react-patches.js │ │ ├── registry.js │ │ └── svg-icons.js │ ├── mime.types │ ├── robots.txt │ ├── scss │ │ ├── bootstrap │ │ │ ├── _alerts.scss │ │ │ ├── _badges.scss │ │ │ ├── _breadcrumbs.scss │ │ │ ├── _button-groups.scss │ │ │ ├── _buttons.scss │ │ │ ├── _carousel.scss │ │ │ ├── _close.scss │ │ │ ├── _code.scss │ │ │ ├── _component-animations.scss │ │ │ ├── _dropdowns.scss │ │ │ ├── _forms.scss │ │ │ ├── _glyphicons.scss │ │ │ ├── _grid.scss │ │ │ ├── _input-groups.scss │ │ │ ├── _jumbotron.scss │ │ │ ├── _labels.scss │ │ │ ├── _list-group.scss │ │ │ ├── _media.scss │ │ │ ├── _mixins.scss │ │ │ ├── _modals.scss │ │ │ ├── _navbar.scss │ │ │ ├── _navs.scss │ │ │ ├── _normalize.scss │ │ │ ├── _pager.scss │ │ │ ├── _pagination.scss │ │ │ ├── _panels.scss │ │ │ ├── _popovers.scss │ │ │ ├── _print.scss │ │ │ ├── _progress-bars.scss │ │ │ ├── _responsive-980px-1199px.scss │ │ │ ├── _responsive-utilities.scss │ │ │ ├── _scaffolding.scss │ │ │ ├── _tables.scss │ │ │ ├── _theme.scss │ │ │ ├── _thumbnails.scss │ │ │ ├── _tooltip.scss │ │ │ ├── _type.scss │ │ │ ├── _utilities.scss │ │ │ ├── _variables.scss │ │ │ └── _wells.scss │ │ ├── fontawesome │ │ │ ├── _bordered-pulled.scss │ │ │ ├── _core.scss │ │ │ ├── _extras.scss │ │ │ ├── _fixed-width.scss │ │ │ ├── _font-awesome.scss │ │ │ ├── _icons.scss │ │ │ ├── _larger.scss │ │ │ ├── _list.scss │ │ │ ├── _mixins.scss │ │ │ ├── _path.scss │ │ │ ├── _rotated-flipped.scss │ │ │ ├── _spinning.scss │ │ │ ├── _stacked.scss │ │ │ └── _variables.scss │ │ ├── react-forms │ │ │ ├── _CheckboxGroup.scss │ │ │ ├── _RadioButtonGroup.scss │ │ │ └── _index.scss │ │ ├── snowflakes │ │ │ ├── _base.scss │ │ │ ├── _bootstrap-lib.scss │ │ │ ├── _layout.scss │ │ │ ├── _mixins-custom.scss │ │ │ ├── _state.scss │ │ │ ├── _theme.scss │ │ │ └── modules │ │ │ │ ├── _audits.scss │ │ │ │ ├── _blocks.scss │ │ │ │ ├── _breadcrumbs.scss │ │ │ │ ├── _facet.scss │ │ │ │ ├── _forms.scss │ │ │ │ ├── _key-value-display.scss │ │ │ │ ├── _layout-editor.scss │ │ │ │ ├── _lightbox.scss │ │ │ │ ├── _lists.scss │ │ │ │ ├── _loading-spinner.scss │ │ │ │ ├── _modals.scss │ │ │ │ ├── _navbar.scss │ │ │ │ ├── _panels.scss │ │ │ │ ├── _search.scss │ │ │ │ ├── _signin-box.scss │ │ │ │ ├── _tables.scss │ │ │ │ └── _tooltip.scss │ │ └── style.scss │ └── server.js │ ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── data │ │ ├── documents │ │ │ ├── frowny_gel.png │ │ │ ├── jocelyn.jpg │ │ │ └── selma.jpg │ │ └── inserts │ │ │ ├── award.json │ │ │ ├── image.json │ │ │ ├── lab.json │ │ │ ├── page.json │ │ │ ├── small_db.tsv │ │ │ ├── snowball.json │ │ │ ├── snowflake.json │ │ │ ├── snowfort.json │ │ │ └── user.json │ ├── datafixtures.py │ ├── features │ │ ├── __init__.py │ │ ├── browsersteps.py │ │ ├── conftest.py │ │ ├── customsteps.py │ │ ├── forms.feature │ │ ├── generics.feature │ │ ├── page.feature │ │ ├── search.feature │ │ ├── test_admin_user.py │ │ ├── test_generics.py │ │ ├── test_nodata.py │ │ ├── test_submitter_user.py │ │ ├── test_workbook.py │ │ ├── title.feature │ │ ├── toolbar.feature │ │ └── user.feature │ ├── test_access_key.py │ ├── test_audit_item.py │ ├── test_batch_upgrade.py │ ├── test_create_mapping.py │ ├── test_download.py │ ├── test_fixtures.py │ ├── test_graph.py │ ├── test_permissions.py │ ├── test_renderers.py │ ├── test_schemas.py │ ├── test_searchv2.py │ ├── test_server_defaults.py │ ├── test_upgrade_award.py │ ├── test_views.py │ └── test_webuser_auth.py │ ├── typedsheets.py │ ├── types │ ├── __init__.py │ ├── access_key.py │ ├── base.py │ ├── image.py │ ├── page.py │ ├── snow.py │ └── user.py │ ├── upgrade │ ├── __init__.py │ ├── award.py │ ├── snowset.py │ └── user.py │ └── xlreader.py ├── test.cfg └── webpack.config.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | presets: [ 3 | '@babel/preset-env', 4 | '@babel/preset-react', 5 | '@babel/flow', 6 | ], 7 | plugins: [ 8 | '@babel/plugin-proposal-object-rest-spread', 9 | '@babel/plugin-transform-modules-commonjs', 10 | ] 11 | } -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | **/testdata 2 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "indent": [2, 4, {"SwitchCase": 1}], 4 | "linebreak-style": [2, "unix"], 5 | "semi": [2, "always"], 6 | "no-console": 0, 7 | "no-unused-vars": 0, 8 | "no-empty": 0 9 | }, 10 | "env": { 11 | "es6": true, 12 | "browser": true, 13 | "commonjs": true 14 | }, 15 | "extends": "eslint:recommended", 16 | "parserOptions": { 17 | "ecmaVersion": 6, 18 | "sourceType": "module", 19 | "ecmaFeatures": { 20 | "jsx": true 21 | } 22 | }, 23 | "plugins": [ 24 | "react" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | /.installed.cfg 3 | /.mr.developer.cfg 4 | /.cache/ 5 | /.sass-cache/ 6 | /annotations.json 7 | /aws-ip-ranges.json 8 | /bin/ 9 | /develop/ 10 | /develop-eggs/ 11 | /downloads/ 12 | /eggs/ 13 | /extends/ 14 | /node_modules/ 15 | /npm-shrinkwrap.json 16 | /ontology.json 17 | /parts/ 18 | /production.ini 19 | /session-secret.b64 20 | /dist/ 21 | /build/ 22 | /src/snowflakes/static/css/bootstrap.css 23 | /src/snowflakes/static/css/responsive.css 24 | /src/snowflakes/static/css/style.css 25 | /src/snowflakes/static/scss/_variables.original.scss 26 | 27 | *.DS_Store 28 | *.egg-info 29 | *.pyc 30 | +.project 31 | +.pydevproject 32 | *.log 33 | .python-version 34 | ~$* 35 | .~*# 36 | 37 | .python-version 38 | .vscode 39 | venv 40 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013 Stanford University 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include *.md 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | clean: 2 | rm -rf node_modules parts 3 | rm -rf .sass-cache 4 | rm -rf src/snowflakes/static/css 5 | -------------------------------------------------------------------------------- /buildout.cfg: -------------------------------------------------------------------------------- 1 | [buildout] 2 | parts = 3 | production-ini 4 | production 5 | production-indexer 6 | ckeditor 7 | npm-install 8 | compile-js 9 | compile-css 10 | cleanup 11 | 12 | [production-ini] 13 | recipe = collective.recipe.template 14 | input = ${buildout:directory}/production.ini.in 15 | output = ${buildout:directory}/production.ini 16 | accession_factory = snowflakes.server_defaults.test_accession 17 | file_upload_bucket = snowflakes-files-dev 18 | blob_bucket = snovault-blobs-dev 19 | indexer_processes = 20 | 21 | [production] 22 | recipe = collective.recipe.modwsgi 23 | config-file = ${buildout:directory}/production.ini 24 | 25 | [production-indexer] 26 | <= production 27 | app_name = indexer 28 | 29 | [ckeditor] 30 | recipe = collective.recipe.cmd 31 | on_install = true 32 | on_update = true 33 | # See http://stackoverflow.com/a/23108309/199100 34 | #TODO consider moving this to snovault-build 35 | cmds = 36 | curl https://s3-us-west-1.amazonaws.com/encoded-build/ckeditor/ckeditor_4.5.5_standard.zip | bsdtar -xf- -C src/snowflakes/static/build/ 37 | 38 | [npm-install] 39 | recipe = collective.recipe.cmd 40 | on_install = true 41 | on_update = true 42 | cmds = NODE_PATH="" npm_config_cache="" npm install 43 | 44 | [compile-js] 45 | recipe = collective.recipe.cmd 46 | on_install = true 47 | on_update = true 48 | cmds = NODE_PATH="" npm run build 49 | 50 | [compile-css] 51 | recipe = collective.recipe.cmd 52 | on_install = true 53 | on_update = true 54 | cmds = compass compile 55 | 56 | [cleanup] 57 | # Even if we don't need the bin or eggs dirs, buildout still creates them 58 | recipe = collective.recipe.cmd 59 | on_install = true 60 | on_update = true 61 | cmds = 62 | rm -rf bin eggs 63 | -------------------------------------------------------------------------------- /candidate.cfg: -------------------------------------------------------------------------------- 1 | [buildout] 2 | extends = buildout.cfg 3 | 4 | [production-ini] 5 | accession_factory = snowflakes.server_defaults.enc_accession 6 | file_upload_bucket = snowflake-files 7 | blob_bucket = snovault-blobs 8 | indexer_processes = 16 -------------------------------------------------------------------------------- /circle-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Helper to run tests locally using same commands as circle ci config 4 | # See: encoded/.circleci/config.yml 5 | # 6 | # Use Cases: No argument defaults to not bdd tests 7 | # $ circle-tests.sh bdd 8 | # $ circle-tests.sh npm 9 | # $ circle-tests.sh 10 | ## 11 | 12 | if [ "$1" == "bdd" ]; then 13 | pytest -v -v --timeout=400 -m "bdd" --tb=short --splinter-implicit-wait 10 --splinter-webdriver chrome --splinter-socket-timeout 300 --chrome-options "--headless --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-extensions --whitelisted-ips --window-size=1920,1080" 14 | exit 15 | fi 16 | 17 | if [ "$1" == "npm" ]; then 18 | npm test 19 | exit 20 | fi 21 | 22 | if [ -z "$1" ]; then 23 | pytest -v -v --timeout=400 -m "not bdd" 24 | exit 25 | fi 26 | -------------------------------------------------------------------------------- /cloudwatchmon-requirements.txt: -------------------------------------------------------------------------------- 1 | argparse==1.2.1 2 | boto==2.38.0 3 | cloudwatchmon==2.0.3 4 | wsgiref==0.1.2 5 | -------------------------------------------------------------------------------- /conf/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | cluster.name: elasticsearch_test_fixture 2 | discovery.type: single-node 3 | indices.query.bool.max_clause_count: 8192 -------------------------------------------------------------------------------- /conf/jvm.options: -------------------------------------------------------------------------------- 1 | -Xms2g 2 | -Xmx2g 3 | 4 | ## GC configuration 5 | -XX:+UseConcMarkSweepGC 6 | -XX:CMSInitiatingOccupancyFraction=75 7 | -XX:+UseCMSInitiatingOccupancyOnly 8 | 9 | # disable calls to System#gc 10 | -XX:+DisableExplicitGC 11 | 12 | # pre-touch memory pages used by the JVM during initialization 13 | -XX:+AlwaysPreTouch 14 | 15 | # force the server VM (remove on 32-bit client JVMs) 16 | -server 17 | 18 | # explicitly set the stack size (reduce to 320k on 32-bit client JVMs) 19 | -Xss1m 20 | 21 | # set to headless, just in case 22 | -Djava.awt.headless=true 23 | 24 | # ensure UTF-8 encoding by default (e.g. filenames) 25 | -Dfile.encoding=UTF-8 26 | 27 | # use our provided JNA always versus the system one 28 | -Djna.nosys=true 29 | 30 | # use old-style file permissions on JDK9 31 | -Djdk.io.permissionsUseCanonicalPath=true 32 | 33 | # flags to configure Netty 34 | -Dio.netty.noUnsafe=true 35 | -Dio.netty.noKeySetOptimization=true 36 | -Dio.netty.recycler.maxCapacityPerThread=0 37 | 38 | # log4j 2 39 | -Dlog4j.shutdownHookEnabled=false 40 | -Dlog4j2.disable.jmx=true 41 | -Dlog4j.skipJansi=true 42 | 43 | # generate a heap dump when an allocation from the Java heap fails 44 | # heap dumps are created in the working directory of the JVM 45 | -XX:+HeapDumpOnOutOfMemoryError -------------------------------------------------------------------------------- /config.rb: -------------------------------------------------------------------------------- 1 | # Require any additional compass plugins here. 2 | 3 | # Set this to the root of your project when deployed: 4 | http_path = "/" 5 | css_dir = "src/snowflakes/static/css" 6 | sass_dir = "src/snowflakes/static/scss" 7 | images_dir = "src/snowflakes/static/img" 8 | javascripts_dir = "src/snowflakes/static/modules" 9 | fonts_dir = "src/snowflakes/static/fonts" 10 | 11 | # To export minified css, uncomment :compress, and comments out :nested 12 | output_style = :compressed 13 | # output_style = :nested 14 | 15 | # To enable relative paths to assets via compass helper functions. Uncomment: 16 | # relative_assets = true 17 | 18 | # To disable debugging comments that display the original location of your selectors. Uncomment: 19 | # line_comments = false 20 | color_output = false 21 | 22 | 23 | # If you prefer the indented syntax, you might want to regenerate this 24 | # project again passing --syntax sass, or you can uncomment this: 25 | # preferred_syntax = :sass 26 | # and then run: 27 | # sass-convert -R --from scss --to sass src/snowflakes/static/sass scss && rm -rf sass && mv scss sass 28 | preferred_syntax = :scss -------------------------------------------------------------------------------- /demo.cfg: -------------------------------------------------------------------------------- 1 | [buildout] 2 | extends = buildout.cfg 3 | -------------------------------------------------------------------------------- /development.ini: -------------------------------------------------------------------------------- 1 | ### 2 | # app configuration 3 | # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html 4 | ### 5 | 6 | [app:app] 7 | use = config:base.ini#app 8 | sqlalchemy.url = postgresql://postgres@:5432/postgres?host=/tmp/snovault/pgdata 9 | snp_search.server = localhost:9200 10 | load_test_only = true 11 | local_tz = US/Pacific 12 | create_tables = true 13 | testing = true 14 | postgresql.statement_timeout = 20 15 | indexer.processes = 16 | 17 | pyramid.reload_templates = true 18 | pyramid.debug_authorization = false 19 | pyramid.debug_notfound = true 20 | pyramid.debug_routematch = false 21 | pyramid.default_locale_name = en 22 | 23 | snovault.load_test_data = snowflakes.loadxl:load_test_data 24 | # Local Storage: Settings must exist in... 25 | # snovault/tests/[testappsettings.py, test_key.py] 26 | # snowflakes/tests/conftest.py 27 | local_storage_host = localhost 28 | local_storage_port = 6378 29 | local_storage_redis_index = 1 30 | local_storage_timeout = 5 31 | 32 | [pipeline:debug] 33 | pipeline = 34 | egg:PasteDeploy#prefix 35 | egg:repoze.debug#pdbpm 36 | app 37 | set pyramid.includes = 38 | pyramid_translogger 39 | 40 | [composite:main] 41 | use = egg:rutter#urlmap 42 | / = debug 43 | /_indexer = indexer 44 | 45 | [composite:indexer] 46 | use = config:base.ini#indexer 47 | 48 | [composite:regionindexer] 49 | use = config:base.ini#regionindexer 50 | 51 | ### 52 | # wsgi server configuration 53 | ### 54 | 55 | [server:main] 56 | use = egg:waitress#main 57 | host = 0.0.0.0 58 | port = 6543 59 | threads = 1 60 | 61 | ### 62 | # logging configuration 63 | # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html 64 | ### 65 | 66 | [loggers] 67 | keys = root, wsgi, snovault 68 | 69 | [handlers] 70 | keys = console 71 | 72 | [formatters] 73 | keys = generic 74 | 75 | [logger_root] 76 | level = INFO 77 | handlers = console 78 | 79 | [logger_wsgi] 80 | level = DEBUG 81 | handlers = 82 | qualname = wsgi 83 | 84 | [logger_snovault] 85 | level = DEBUG 86 | handlers = 87 | qualname = snovault 88 | 89 | [handler_console] 90 | class = StreamHandler 91 | args = (sys.stderr,) 92 | level = NOTSET 93 | formatter = generic 94 | 95 | [formatter_generic] 96 | format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s 97 | -------------------------------------------------------------------------------- /docs/rendering-overview.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ENCODE-DCC/snovault/68336ee11fb1730dcc7af27796cdd93d52daf198/docs/rendering-overview.pdf -------------------------------------------------------------------------------- /docs/search.rst: -------------------------------------------------------------------------------- 1 | Search Documentation: 2 | ===================== 3 | 4 | **URIS** 5 | 6 | 1. http://{SERVER_NAME}/search/?searchTerm={term} 7 | Fetches all the documents which contain the text 'term'. 8 | The result set includes wild card searches and the 'term' should be atleast 3 characters long. 9 | 10 | - SERVER_NAME: ENCODE server 11 | - term: string that can be searched accross four item_types (i.e., experiment, biosample, antibody_approval, target) 12 | 13 | 2. http://{SERVER_NAME}/search/?type={item_type} 14 | Fetches all the documents of that particular 'item_type' 15 | 16 | - SERVER_NAME: ENCODE server 17 | - item_type: ENCODE item type (values can be: biosample, experiment, antibody_approval and target) 18 | 19 | 3. http://{SERVER_NAME}/search/?type={item_type}&{field_name}={text} 20 | Fetches and then filters all the documents of a particular item_type on that field 21 | 22 | - SERVER_NAME: ENCODE server 23 | - item_type: ENCODE item type (values can be: biosample, experiment, antibody_approval and target) 24 | - field_name: Any of the json property in the ENCODE 'item_type' schema 25 | -------------------------------------------------------------------------------- /etc/logging-apache.conf: -------------------------------------------------------------------------------- 1 | CustomLog ${APACHE_LOG_DIR}/access.log vhost_combined_stats 2 | -------------------------------------------------------------------------------- /examples/s3cp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: latin-1 -*- 3 | import requests, subprocess, shlex, urlparse, os, sys 4 | 5 | AUTHID='user'; AUTHPW='secret'; HEADERS = {'content-type': 'application/json'}; SERVER = 'https://www.encodeproject.org/' 6 | S3_SERVER='s3://encode-files/' 7 | 8 | #get all the file objects 9 | files = requests.get( 10 | 'https://www.encodeproject.org/search/?type=file&frame=embedded&limit=all', 11 | auth=(AUTHID,AUTHPW), headers=HEADERS).json()['@graph'] 12 | 13 | #select your file 14 | f_obj = files[123] 15 | 16 | #make the URL that will get redirected - get it from the file object's href property 17 | encode_url = urlparse.urljoin(SERVER,f_obj.get('href')) 18 | 19 | #stream=True avoids actually downloading the file, but it evaluates the redirection 20 | r = requests.get(encode_url, auth=(AUTHID,AUTHPW), headers=HEADERS, allow_redirects=True, stream=True) 21 | try: 22 | r.raise_for_status 23 | except: 24 | print '%s href does not resolve' %(f_obj.get('accession')) 25 | sys.exit() 26 | 27 | #this is the actual S3 https URL after redirection 28 | s3_url = r.url 29 | 30 | #release the connection 31 | r.close() 32 | 33 | #split up the url into components 34 | o = urlparse.urlparse(s3_url) 35 | 36 | #pull out the filename 37 | filename = os.path.basename(o.path) 38 | 39 | #hack together the s3 cp url (with the s3 method instead of https) 40 | bucket_url = S3_SERVER.rstrip('/') + o.path 41 | #print bucket_url 42 | 43 | #ls the file from the bucket 44 | s3ls_string = subprocess.check_output(shlex.split('aws s3 ls %s' %(bucket_url))) 45 | if s3ls_string.rstrip() == "": 46 | print >> sys.stderr, "%s not in bucket" %(bucket_url) 47 | else: 48 | print "%s %s" %(f_obj.get('accession'), s3ls_string.rstrip()) 49 | 50 | #do the actual s3 cp 51 | #return_value = subprocess.check_call(shlex.split('aws s3 cp %s %s' %(bucket_url, filename))) 52 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | const gulp = require('gulp'); 2 | const log = require('fancy-log'); 3 | const webpack = require('webpack'); 4 | 5 | const setProduction = function (cb) { 6 | process.env.NODE_ENV = 'production'; 7 | if (cb) { 8 | cb(); 9 | } 10 | }; 11 | 12 | const webpackOnBuild = function (done) { 13 | return function (err, stats) { 14 | if (err) { 15 | throw new log.error(err); 16 | } 17 | log(stats.toString({ 18 | colors: true 19 | })); 20 | if (done) { done(err); } 21 | }; 22 | }; 23 | 24 | const webpackSetup = function (cb) { 25 | var webpackConfig = require('./webpack.config.js'); 26 | webpack(webpackConfig).run(webpackOnBuild(cb)); 27 | }; 28 | 29 | const watch = function (cb) { 30 | var webpackConfig = require('./webpack.config.js'); 31 | webpack(webpackConfig).watch(300, webpackOnBuild(cb)); 32 | }; 33 | 34 | const series = gulp.series; 35 | 36 | gulp.task('default', series(webpackSetup, watch)); 37 | gulp.task('dev', series('default')); 38 | gulp.task('build', series(setProduction, webpackSetup)); -------------------------------------------------------------------------------- /jest/environment.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | jest.mock('scriptjs'); 3 | var jsdom = require('jsdom').jsdom; 4 | 5 | if (window.DOMParser === undefined) { 6 | // jsdom 7 | window.DOMParser = function DOMParser() {}; 8 | window.DOMParser.prototype.parseFromString = function parseFromString(markup, type) { 9 | var parsingMode = 'auto'; 10 | type = type || ''; 11 | if (type.indexOf('xml') >= 0) { 12 | parsingMode = 'xml'; 13 | } else if (type.indexOf('html') >= 0) { 14 | parsingMode = 'html'; 15 | } 16 | var doc = jsdom(markup, {parsingMode: parsingMode}); 17 | return doc; 18 | }; 19 | } 20 | -------------------------------------------------------------------------------- /node_shims/ckeditor/index.js: -------------------------------------------------------------------------------- 1 | module.exports = global.CKEDITOR; 2 | -------------------------------------------------------------------------------- /node_shims/ckeditor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ckeditor", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "MIT" 11 | } 12 | -------------------------------------------------------------------------------- /node_shims/google-analytics/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* global ga */ 3 | global.ga = global.ga || function () { 4 | (ga.q = ga.q || []).push(arguments); 5 | }; 6 | ga.l = +new Date(); 7 | module.exports = global.ga; 8 | -------------------------------------------------------------------------------- /node_shims/google-analytics/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "google-analytics", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "MIT" 11 | } 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "snowflakes", 3 | "version": "0.0.0", 4 | "description": "domready held back.", 5 | "scripts": { 6 | "test": "jest", 7 | "build": "gulp build", 8 | "dev": "gulp dev" 9 | }, 10 | "author": "", 11 | "license": "MIT", 12 | "files": [], 13 | "repository": "ENCODE-DCC/encoded", 14 | "jest": { 15 | "rootDir": "src/snowflakes/static", 16 | "setupFiles": [ 17 | "../../../jest/environment.js" 18 | ], 19 | "unmockedModulePathPatterns": [ 20 | "node_modules/react", 21 | "node_modules/underscore", 22 | "libs/react-patches", 23 | "jsdom" 24 | ] 25 | }, 26 | "devDependencies": { 27 | "@babel/core": "^7.6.0", 28 | "@babel/plugin-proposal-object-rest-spread": "^7.0.0", 29 | "@babel/plugin-transform-modules-commonjs": "^7.6.0", 30 | "@babel/plugin-transform-react-display-name": "^7.0.0", 31 | "@babel/plugin-transform-react-jsx": "^7.0.0", 32 | "@babel/polyfill": "^7.6.0", 33 | "@babel/preset-env": "^7.6.0", 34 | "@babel/preset-react": "^7.0.0", 35 | "@babel/register": "^7.6.0", 36 | "babel-jest": "^24.9.0", 37 | "babel-loader": "^8.0.6", 38 | "fancy-log": "^1.3.3", 39 | "gulp": "^4.0.2", 40 | "jest-cli": "^24.9.0", 41 | "jsdom": "^15.1.1", 42 | "json-loader": "^0.5.4", 43 | "string-replace-loader": "^2.2.0", 44 | "webpack": "^4.39.3" 45 | }, 46 | "dependencies": { 47 | "@babel/preset-flow": "^7.0.0", 48 | "babel-polyfill": "^6.7.4", 49 | "brace": "^0.3.0", 50 | "ckeditor": "file:node_shims/ckeditor", 51 | "color": "^0.10.1", 52 | "d3": "^3.4.6", 53 | "dagre-d3": "git+https://github.com/ENCODE-DCC/dagre-d3.git", 54 | "domready": "^1.0.8", 55 | "form-serialize": "^0.7.2", 56 | "google-analytics": "file:node_shims/google-analytics", 57 | "immutable": "^3.7.5", 58 | "js-cookie": "^2.2.1", 59 | "marked": "^0.7.0", 60 | "moment": "^2.8.2", 61 | "query-string": "^4.1.0", 62 | "react": "^0.12.2", 63 | "react-bootstrap": "^0.15.1", 64 | "react-forms": "git+https://github.com/lrowe/react-forms.git#3953a633b1f77640dffb5e3f1d5bbe78a98c3dfe", 65 | "scriptjs": "^2.5.7", 66 | "source-map-support": "^0.5.13", 67 | "subprocess-middleware": "^0.1.0", 68 | "terser-webpack-plugin": "^2.0.1", 69 | "underscore": "^1.8.3", 70 | "whatwg-fetch": "git+https://github.com/lrowe/fetch.git#bf5d58b738fb2ed6d60791b944d36075fee8604a" 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /production.ini.in: -------------------------------------------------------------------------------- 1 | [app:app] 2 | use = config:base.ini#app 3 | session.secret = %(here)s/session-secret.b64 4 | file_upload_bucket = ${file_upload_bucket} 5 | blob_bucket = ${blob_bucket} 6 | blob_store_profile_name = encoded-files-upload 7 | accession_factory = ${accession_factory} 8 | indexer.processes = ${indexer_processes} 9 | 10 | [composite:indexer] 11 | use = config:base.ini#indexer 12 | 13 | [composite:regionindexer] 14 | use = config:base.ini#regionindexer 15 | 16 | [pipeline:main] 17 | pipeline = 18 | config:base.ini#memlimit 19 | egg:PasteDeploy#prefix 20 | app 21 | 22 | [pipeline:debug] 23 | pipeline = 24 | egg:repoze.debug#pdbpm 25 | app 26 | set pyramid.includes = 27 | pyramid_translogger 28 | 29 | [server:main] 30 | use = egg:waitress#main 31 | host = 0.0.0.0 32 | port = 6543 33 | threads = 1 34 | 35 | [loggers] 36 | keys = root, snovault, snovault_listener 37 | 38 | [handlers] 39 | keys = console 40 | 41 | [formatters] 42 | keys = generic 43 | 44 | [logger_root] 45 | level = WARN 46 | handlers = console 47 | 48 | [logger_snovault] 49 | level = WARN 50 | handlers = console 51 | qualname = snovault 52 | propagate = 0 53 | 54 | [logger_snovault_listener] 55 | level = INFO 56 | handlers = console 57 | qualname = snovault.elasticsearch.es_index_listener 58 | propagate = 0 59 | 60 | [handler_console] 61 | class = StreamHandler 62 | args = (sys.stderr,) 63 | level = NOTSET 64 | formatter = generic 65 | 66 | [formatter_generic] 67 | format = %(levelname)s [%(name)s][%(threadName)s] %(message)s 68 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = 3 | --pyargs snowflakes.tests 4 | --pyargs snovault.tests 5 | --pyargs snovault.elasticsearch.tests 6 | # --pyargs snovault.elasticsearch.uuid_queue.tests 7 | -p snowflakes.tests 8 | --instafail 9 | --splinter-make-screenshot-on-failure=false 10 | --splinter-implicit-wait=5 11 | # Ignore warnings from splinter, we don't use browser.find_by_{href,link} directly 12 | filterwarnings = 13 | error 14 | ignore:browser\.find_link_by_href is deprecated\. Use browser\.links\.find_by_href instead\.:FutureWarning 15 | ignore:browser\.find_link_by_text is deprecated\. Use browser\.links\.find_by_text instead\.:FutureWarning 16 | markers = 17 | bdd: Encoded Scenario 18 | forms: Encoded Scenario 19 | generics: Encoded Scenario 20 | indexing: Encoded Scenario 21 | page: Encoded Scenario 22 | report: Encoded Scenario 23 | search: Encoded Scenario 24 | slow: Encoded Scenario 25 | storage: storage tests 26 | title: Encoded Scenario 27 | toolbar: Encoded Scenario 28 | -------------------------------------------------------------------------------- /requirements.osx.txt: -------------------------------------------------------------------------------- 1 | pip==20.0.2 2 | psycopg2==2.8.4 3 | redis==3.5.3 4 | redis-server==5.0.7 5 | setuptools==45.1.0 6 | zc.buildout==2.13.2 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pip==20.0.2 2 | psycopg2==2.8.4 3 | redis==3.5.3 4 | redis-server==5.0.7 5 | setuptools==45.1.0 6 | zc.buildout==2.13.2 7 | -------------------------------------------------------------------------------- /scripts/LogToCsv.py: -------------------------------------------------------------------------------- 1 | import re 2 | #regex = '([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) "-" (.*?) "(.*?)" "(.*?)"' 3 | pattern = re.compile('([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "-" "(.*?)" "(.*?)"') 4 | pattern_trim = re.compile('([(\d\.)]+) - (.*?) \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (.*?)') 5 | 6 | none_count = 0 7 | elif_count = 0 8 | 9 | file = open('encodeproject.org.log', 'r') 10 | 11 | outputFile = open('encodeProjectToCSV.csv', 'w') 12 | 13 | for line in file: 14 | result = re.search(pattern, line) 15 | result_trim = re.search(pattern_trim, line) 16 | 17 | if result: 18 | split_line = result.group(7) 19 | do_split = re.split(r'[-&?()]', split_line) 20 | completed_line = result.group(1,2,3,4,5,6), do_split 21 | 22 | outputFile.write(str(completed_line)) 23 | outputFile.write("\n") 24 | 25 | elif not result: 26 | elif_count = +1 27 | split_line = result_trim.group(7) 28 | do_split = re.split(r'[-&?()]', split_line) 29 | completed_line = result_trim.group(1,3,4,5,6,8), do_split 30 | 31 | outputFile.write(str(completed_line)) 32 | outputFile.write("\n") 33 | 34 | else: 35 | none_count = +1 36 | outputFile.write(line) 37 | 38 | print("None count: %d" % none_count) 39 | file.close() 40 | outputFile.close() -------------------------------------------------------------------------------- /scripts/blackholes.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Report on 'blackhole' objects, those which are embedded in over many other objects. 3 | 4 | Care should be taken with the reverse links from blackhole objects. 5 | ''' 6 | 7 | from collections import defaultdict 8 | import json 9 | 10 | 11 | rows = [json.loads(line) for line in open('embeds.txt')] 12 | uuid_row = {row['uuid']: row for row in rows} 13 | 14 | CUTOFF = 1000 15 | 16 | by_type = defaultdict(list) 17 | for row in rows: 18 | if row['embeds'] >= CUTOFF: 19 | by_type[row['item_type']].append(row) 20 | 21 | print(json.dumps({k: len(v) for k, v in by_type.items()}, sort_keys=True, indent=4)) 22 | 23 | ''' 24 | Report on number of transacations that invalidate many objects. 25 | 26 | $ sudo -u encoded psql -tAc "select row_to_json(transactions) from transactions where timestamp::date = '2016-01-20'::date;" > transactions.txt 27 | 28 | Beware that reverse link invalidations are entered into the transaction log, so any changes will not be reflected. 29 | ''' 30 | 31 | transactions = [json.loads(line) for line in open('transactions.txt')] 32 | 33 | sum_txn = [(sum(uuid_row[uuid]['embeds'] for uuid in txn['data']['updated']), txn) for txn in transactions] 34 | print('Transactions > {}'.format(CUTOFF), sum(s > CUTOFF for s, row in sum_txn)) 35 | -------------------------------------------------------------------------------- /scripts/embeds.py: -------------------------------------------------------------------------------- 1 | ''' 2 | For each object, count the number of objects in which it is embedded. 3 | 4 | Usage: python embeds.py > embeds.jsonlines 5 | ''' 6 | 7 | from elasticsearch import Elasticsearch 8 | from elasticsearch.helpers import scan 9 | import json 10 | 11 | es = Elasticsearch('localhost:9200') 12 | 13 | 14 | def embeds_uuid(es, uuid, item_type): 15 | query = { 16 | 'query': {'terms': {'embedded_uuids': [uuid]}}, 17 | 'aggregations': { 18 | 'item_type': {'terms': {'field': 'item_type'}}, 19 | }, 20 | } 21 | res = es.search(index='encoded', search_type='count', body=query) 22 | return { 23 | 'uuid': uuid, 24 | 'item_type': item_type, 25 | 'embeds': res['hits']['total'], 26 | 'buckets': res['aggregations']['item_type']['buckets'], 27 | } 28 | 29 | 30 | uuid_type = [(hit['_id'], hit['_type']) for hit in scan(es, query={'fields': []})] 31 | 32 | 33 | # rows = [embeds_uuid(es, uuid, item_type) for uuid, item_type in uuid_type] 34 | for uuid, item_type in uuid_type: 35 | data = embeds_uuid(es, uuid, item_type) 36 | print(json.dumps(data)) 37 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | -------------------------------------------------------------------------------- /src/snovault/cache.py: -------------------------------------------------------------------------------- 1 | from pyramid.threadlocal import manager 2 | from sqlalchemy.util import LRUCache 3 | import transaction.interfaces 4 | from zope.interface import implementer 5 | 6 | 7 | @implementer(transaction.interfaces.ISynchronizer) 8 | class ManagerLRUCache(object): 9 | """ Override capacity in settings. 10 | """ 11 | def __init__(self, name, default_capacity=100, threshold=.5): 12 | self.name = name 13 | self.default_capacity = default_capacity 14 | self.threshold = threshold 15 | transaction.manager.registerSynch(self) 16 | 17 | @property 18 | def cache(self): 19 | if not manager.stack: 20 | return None 21 | threadlocals = manager.stack[0] 22 | if self.name not in threadlocals: 23 | registry = threadlocals['registry'] 24 | capacity = int(registry.settings.get(self.name + '.capacity', self.default_capacity)) 25 | threadlocals[self.name] = LRUCache(capacity, self.threshold) 26 | return threadlocals[self.name] 27 | 28 | def get(self, key, default=None): 29 | cache = self.cache 30 | if cache is None: 31 | return default 32 | try: 33 | return cache[key] 34 | except KeyError: 35 | return default 36 | 37 | def __contains__(self, key): 38 | cache = self.cache 39 | if cache is None: 40 | return False 41 | return key in cache 42 | 43 | def __setitem__(self, key, value): 44 | cache = self.cache 45 | if cache is None: 46 | return 47 | self.cache[key] = value 48 | 49 | # ISynchronizer 50 | 51 | def beforeCompletion(self, transaction): 52 | pass 53 | 54 | def afterCompletion(self, transaction): 55 | # Ensure cache is cleared for retried transactions 56 | if manager.stack: 57 | threadlocals = manager.stack[0] 58 | threadlocals.pop(self.name, None) 59 | 60 | def newTransaction(self, transaction): 61 | pass 62 | -------------------------------------------------------------------------------- /src/snovault/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # package 2 | -------------------------------------------------------------------------------- /src/snovault/commands/es_index_data.py: -------------------------------------------------------------------------------- 1 | from pyramid.paster import get_app 2 | import logging 3 | from webtest import TestApp 4 | 5 | index = 'snovault' 6 | 7 | EPILOG = __doc__ 8 | 9 | 10 | def run(app, collections=None, record=False): 11 | environ = { 12 | 'HTTP_ACCEPT': 'application/json', 13 | 'REMOTE_USER': 'INDEXER', 14 | } 15 | testapp = TestApp(app, environ) 16 | testapp.post_json('/index', { 17 | 'last_xmin': None, 18 | 'types': collections, 19 | 'recovery': True 20 | } 21 | ) 22 | 23 | 24 | def main(): 25 | ''' Indexes app data loaded to elasticsearch ''' 26 | 27 | import argparse 28 | parser = argparse.ArgumentParser( 29 | description="Index data in Elastic Search", epilog=EPILOG, 30 | formatter_class=argparse.RawDescriptionHelpFormatter, 31 | ) 32 | parser.add_argument('--item-type', action='append', help="Item type") 33 | parser.add_argument('--record', default=False, action='store_true', help="Record the xmin in ES meta") 34 | parser.add_argument('--app-name', help="Pyramid app name in configfile") 35 | parser.add_argument('config_uri', help="path to configfile") 36 | args = parser.parse_args() 37 | 38 | logging.basicConfig() 39 | options = { 40 | 'embed_cache.capacity': '5000', 41 | 'indexer': 'true', 42 | } 43 | app = get_app(args.config_uri, args.app_name, options) 44 | 45 | # Loading app will have configured from config file. Reconfigure here: 46 | logging.getLogger('snovault').setLevel(logging.DEBUG) 47 | return run(app, args.item_type, args.record) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /src/snovault/commands/jsonld_rdf.py: -------------------------------------------------------------------------------- 1 | """\ 2 | Available formats: xml, n3, turtle, nt, pretty-xml, trix. 3 | Example. 4 | 5 | %(prog)s "https://YOUR.SNOWVAULT.RUL/search/?type=Item&frame=object" 6 | """ 7 | EPILOG = __doc__ 8 | 9 | import rdflib 10 | 11 | 12 | def run(sources, output, parser='json-ld', serializer='xml', base=None): 13 | g = rdflib.ConjunctiveGraph() 14 | for url in sources: 15 | g.parse(url, format=parser) 16 | g.serialize(output, format=serializer, base=base) 17 | 18 | 19 | def main(): 20 | import argparse 21 | import sys 22 | stdout = sys.stdout 23 | if sys.version_info.major > 2: 24 | stdout = stdout.buffer 25 | 26 | rdflib_parsers = sorted( 27 | p.name for p in rdflib.plugin.plugins(kind=rdflib.parser.Parser) 28 | if '/' not in p.name) 29 | rdflib_serializers = sorted( 30 | p.name for p in rdflib.plugin.plugins(kind=rdflib.serializer.Serializer) 31 | if '/' not in p.name) 32 | parser = argparse.ArgumentParser( 33 | description="Convert JSON-LD from source URLs to RDF", epilog=EPILOG, 34 | formatter_class=argparse.RawDescriptionHelpFormatter, 35 | ) 36 | parser.add_argument('sources', metavar='URL', nargs='+', help="URLs to convert") 37 | parser.add_argument( 38 | '-p', '--parser', default='json-ld', help=', '.join(rdflib_parsers)) 39 | parser.add_argument( 40 | '-s', '--serializer', default='xml', help=', '.join(rdflib_serializers)) 41 | parser.add_argument( 42 | '-b', '--base', default=None, help='Base URL') 43 | parser.add_argument( 44 | '-o', '--output', type=argparse.FileType('wb'), default=stdout, 45 | help="Output file.") 46 | args = parser.parse_args() 47 | run(args.sources, args.output, args.parser, args.serializer, args.base) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /src/snovault/commands/spreadsheet_to_json.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example: 3 | 4 | %(prog)s *.tsv 5 | 6 | """ 7 | 8 | from .. import loadxl 9 | import json 10 | import os.path 11 | 12 | EPILOG = __doc__ 13 | 14 | 15 | def rename_test_with_underscore(rows): 16 | for row in rows: 17 | if 'test' in row: 18 | if row['test'] != 'test': 19 | row['_test'] = row['test'] 20 | del row['test'] 21 | yield row 22 | 23 | 24 | def remove_empty(rows): 25 | for row in rows: 26 | if row: 27 | yield row 28 | 29 | 30 | def convert(filename, sheetname=None, outputdir=None, skip_blanks=False): 31 | if outputdir is None: 32 | outputdir = os.path.dirname(filename) 33 | source = loadxl.read_single_sheet(filename, sheetname) 34 | pipeline = [ 35 | loadxl.remove_keys_with_empty_value if skip_blanks else loadxl.noop, 36 | rename_test_with_underscore, 37 | remove_empty, 38 | ] 39 | data = list(loadxl.combine(source, pipeline)) 40 | if sheetname is None: 41 | sheetname, ext = os.path.splitext(os.path.basename(filename)) 42 | out = open(os.path.join(outputdir, sheetname + '.json'), 'w') 43 | json.dump(data, out, sort_keys=True, indent=4, separators=(',', ': ')) 44 | 45 | 46 | 47 | def main(): 48 | import argparse 49 | parser = argparse.ArgumentParser( 50 | description="Convert spreadsheet to json list", epilog=EPILOG, 51 | formatter_class=argparse.RawDescriptionHelpFormatter, 52 | ) 53 | parser.add_argument('filenames', metavar='FILE', nargs='+', help="Files to convert") 54 | parser.add_argument('--outputdir', help="Directory to write converted output") 55 | parser.add_argument('--sheetname', help="xlsx sheet name") 56 | parser.add_argument('--skip-blanks', help="Skip blank columns") 57 | args = parser.parse_args() 58 | 59 | for filename in args.filenames: 60 | convert(filename, args.sheetname, args.outputdir, args.skip_blanks) 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/cached_views.py: -------------------------------------------------------------------------------- 1 | """ Cached views used when model was pulled from elasticsearch. 2 | """ 3 | 4 | from itertools import chain 5 | from pyramid.httpexceptions import HTTPForbidden 6 | from pyramid.view import view_config 7 | from .interfaces import ICachedItem 8 | 9 | 10 | def includeme(config): 11 | config.scan(__name__) 12 | 13 | 14 | @view_config(context=ICachedItem, request_method='GET', name='embedded') 15 | def cached_view_embedded(context, request): 16 | source = context.model.source 17 | allowed = set(source['principals_allowed']['view']) 18 | if allowed.isdisjoint(request.effective_principals): 19 | raise HTTPForbidden() 20 | return source['embedded'] 21 | 22 | 23 | @view_config(context=ICachedItem, request_method='GET', name='object') 24 | def cached_view_object(context, request): 25 | source = context.model.source 26 | allowed = set(source['principals_allowed']['view']) 27 | if allowed.isdisjoint(request.effective_principals): 28 | raise HTTPForbidden() 29 | return source['object'] 30 | 31 | 32 | @view_config(context=ICachedItem, request_method='GET', name='audit') 33 | def cached_view_audit(context, request): 34 | source = context.model.source 35 | allowed = set(source['principals_allowed']['audit']) 36 | if allowed.isdisjoint(request.effective_principals): 37 | raise HTTPForbidden() 38 | return { 39 | '@id': source['object']['@id'], 40 | 'audit': source['audit'], 41 | } 42 | 43 | 44 | @view_config(context=ICachedItem, request_method='GET', name='audit-self') 45 | def cached_view_audit_self(context, request): 46 | source = context.model.source 47 | allowed = set(source['principals_allowed']['audit']) 48 | if allowed.isdisjoint(request.effective_principals): 49 | raise HTTPForbidden() 50 | path = source['object']['@id'] 51 | return { 52 | '@id': path, 53 | 'audit': [a for a in chain(*source['audit'].values()) if a['path'] == path], 54 | } 55 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/interfaces.py: -------------------------------------------------------------------------------- 1 | from zope.interface import Interface 2 | 3 | # Registry tool id 4 | APP_FACTORY = 'app_factory' 5 | ELASTIC_SEARCH = 'elasticsearch' 6 | SNP_SEARCH_ES = 'snp_search' 7 | INDEXER = 'indexer' 8 | RESOURCES_INDEX = 'snovault-resources' 9 | 10 | 11 | class ICachedItem(Interface): 12 | """ Marker for cached Item 13 | """ 14 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/searches/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ENCODE-DCC/snovault/68336ee11fb1730dcc7af27796cdd93d52daf198/src/snovault/elasticsearch/searches/__init__.py -------------------------------------------------------------------------------- /src/snovault/elasticsearch/searches/configs.py: -------------------------------------------------------------------------------- 1 | import venusian 2 | 3 | from snosearch.configs import SearchConfigRegistry 4 | from snovault.elasticsearch.searches.interfaces import SEARCH_CONFIG 5 | 6 | 7 | def includeme(config): 8 | registry = config.registry 9 | registry[SEARCH_CONFIG] = SearchConfigRegistry() 10 | config.add_directive('register_search_config', register_search_config) 11 | 12 | 13 | def register_search_config(config, name, factory): 14 | config.action( 15 | ('set-search-config', name), 16 | config.registry[SEARCH_CONFIG].register_from_func, 17 | args=( 18 | name, 19 | factory 20 | ) 21 | ) 22 | 23 | 24 | def search_config(name, **kwargs): 25 | ''' 26 | Register a custom search config by name. 27 | ''' 28 | def decorate(config): 29 | def callback(scanner, factory_name, factory): 30 | scanner.config.register_search_config(name, factory) 31 | venusian.attach(config, callback, category='pyramid') 32 | return config 33 | return decorate 34 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/searches/fields.py: -------------------------------------------------------------------------------- 1 | from snosearch.fields import ResponseField 2 | from snovault.elasticsearch.create_mapping import TEXT_FIELDS 3 | from snovault.elasticsearch.searches.interfaces import NON_SORTABLE 4 | 5 | 6 | class NonSortableResponseField(ResponseField): 7 | 8 | def __init__(self, *args, **kwargs): 9 | super().__init__(*args, **kwargs) 10 | 11 | def render(self, *args, **kwargs): 12 | self.parent = kwargs.get('parent') 13 | return { 14 | NON_SORTABLE: TEXT_FIELDS 15 | } 16 | 17 | 18 | class PassThroughResponseField(ResponseField): 19 | ''' 20 | Passes input values (dictionary) to output. 21 | ''' 22 | def __init__(self, *args, **kwargs): 23 | self.values_to_pass_through = kwargs.pop('values_to_pass_through', {}) 24 | super().__init__(*args, **kwargs) 25 | 26 | def render(self, *args, **kwargs): 27 | return self.values_to_pass_through 28 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/searches/interfaces.py: -------------------------------------------------------------------------------- 1 | NON_SORTABLE = 'non_sortable' 2 | SEARCH_CONFIG = 'search_config' 3 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ENCODE-DCC/snovault/68336ee11fb1730dcc7af27796cdd93d52daf198/src/snovault/elasticsearch/tests/__init__.py -------------------------------------------------------------------------------- /src/snovault/elasticsearch/uuid_queue/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Uuid Queue Module Adapter 3 | 4 | - QueueAdapter Class allows access to all queue types 5 | defined in QueueTypes through a set of standard methods. 6 | - All queues in ./queues should adhere to QueueAdapter standards. 7 | - Adapter queue has a server and a worker. 8 | - Another important object is the meta data needed to run the queue. 9 | """ 10 | from .adapter_queue import QueueAdapter 11 | from .adapter_queue import QueueTypes 12 | -------------------------------------------------------------------------------- /src/snovault/elasticsearch/uuid_queue/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ENCODE-DCC/snovault/68336ee11fb1730dcc7af27796cdd93d52daf198/src/snovault/elasticsearch/uuid_queue/tests/__init__.py -------------------------------------------------------------------------------- /src/snovault/interfaces.py: -------------------------------------------------------------------------------- 1 | # Tool names 2 | AUDITOR = 'auditor' 3 | BLOBS = 'blobs' 4 | CALCULATED_PROPERTIES = 'calculated_properties' 5 | COLLECTIONS = 'collections' 6 | CONNECTION = 'connection' 7 | DBSESSION = 'dbsession' 8 | STORAGE = 'storage' 9 | ROOT = 'root' 10 | TYPES = 'types' 11 | UPGRADER = 'upgrader' 12 | 13 | # Constants 14 | PHASE1_5_CONFIG = -15 15 | PHASE2_5_CONFIG = -5 16 | 17 | 18 | # Events 19 | class Created(object): 20 | def __init__(self, object, request): 21 | self.object = object 22 | self.request = request 23 | 24 | 25 | class BeforeModified(object): 26 | def __init__(self, object, request): 27 | self.object = object 28 | self.request = request 29 | 30 | 31 | class AfterModified(object): 32 | def __init__(self, object, request): 33 | self.object = object 34 | self.request = request 35 | 36 | 37 | class AfterUpgrade(object): 38 | def __init__(self, object): 39 | self.object = object 40 | -------------------------------------------------------------------------------- /src/snovault/json_renderer.py: -------------------------------------------------------------------------------- 1 | from pyramid.threadlocal import get_current_request 2 | import json 3 | import pyramid.renderers 4 | import uuid 5 | 6 | 7 | def includeme(config): 8 | config.add_renderer(None, json_renderer) 9 | 10 | 11 | class JSON(pyramid.renderers.JSON): 12 | '''Provide easier access to the configured serializer 13 | ''' 14 | def dumps(self, value): 15 | request = get_current_request() 16 | default = self._make_default(request) 17 | return json.dumps(value, default=default, **self.kw) 18 | 19 | 20 | class BinaryFromJSON: 21 | def __init__(self, app_iter): 22 | self.app_iter = app_iter 23 | 24 | def __len__(self): 25 | return len(self.app_iter) 26 | 27 | def __iter__(self): 28 | for s in self.app_iter: 29 | yield s.encode('utf-8') 30 | 31 | 32 | class JSONResult(object): 33 | def __init__(self): 34 | self.app_iter = [] 35 | self.write = self.app_iter.append 36 | 37 | @classmethod 38 | def serializer(cls, value, **kw): 39 | fp = cls() 40 | json.dump(value, fp, **kw) 41 | if str is bytes: 42 | return fp.app_iter 43 | else: 44 | return BinaryFromJSON(fp.app_iter) 45 | 46 | 47 | json_renderer = JSON(serializer=JSONResult.serializer) 48 | 49 | 50 | def uuid_adapter(obj, request): 51 | return str(obj) 52 | 53 | 54 | def listy_adapter(obj, request): 55 | return list(obj) 56 | 57 | 58 | json_renderer.add_adapter(uuid.UUID, uuid_adapter) 59 | json_renderer.add_adapter(set, listy_adapter) 60 | json_renderer.add_adapter(frozenset, listy_adapter) 61 | -------------------------------------------------------------------------------- /src/snovault/jsongraph.py: -------------------------------------------------------------------------------- 1 | from snovault import Item, Root, CONNECTION 2 | from snovault.elasticsearch.indexer import all_uuids 3 | from past.builtins import basestring 4 | from pyramid.view import view_config 5 | 6 | 7 | def includeme(config): 8 | config.scan(__name__) 9 | 10 | 11 | def uuid_to_ref(obj, path): 12 | if isinstance(path, basestring): 13 | path = path.split('.') 14 | if not path: 15 | return 16 | name = path[0] 17 | remaining = path[1:] 18 | value = obj.get(name, None) 19 | if value is None: 20 | return 21 | if remaining: 22 | if isinstance(value, list): 23 | for v in value: 24 | uuid_to_ref(v, remaining) 25 | else: 26 | uuid_to_ref(value, remaining) 27 | return 28 | if isinstance(value, list): 29 | obj[name] = [ 30 | {'$type': 'ref', 'value': ['uuid', v]} 31 | for v in value 32 | ] 33 | else: 34 | obj[name] = {'$type': 'ref', 'value': ['uuid', value]} 35 | 36 | 37 | def item_jsongraph(context, properties): 38 | properties = properties.copy() 39 | for path in context.type_info.schema_links: 40 | uuid_to_ref(properties, path) 41 | properties['uuid'] = str(context.uuid) 42 | properties['@type'] = context.type_info.name 43 | return properties 44 | 45 | 46 | @view_config(context=Root, request_method='GET', name='jsongraph') 47 | def jsongraph(context, request): 48 | conn = request.registry[CONNECTION] 49 | cache = { 50 | 'uuid': {}, 51 | } 52 | for uuid in all_uuids(request.registry): 53 | item = conn[uuid] 54 | properties = item.__json__(request) 55 | cache['uuid'][uuid] = item_jsongraph(item, properties) 56 | for k, values in item.unique_keys(properties).items(): 57 | for v in values: 58 | cache.setdefault(k, {})[v] = {'$type': 'ref', 'value': ['uuid', str(item.uuid)]} 59 | return cache 60 | -------------------------------------------------------------------------------- /src/snovault/local_storage.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | 3 | from os import urandom 4 | from datetime import datetime 5 | from pytz import timezone 6 | 7 | from redis import StrictRedis 8 | 9 | 10 | def base_result(local_store): 11 | local_dt = datetime.now(timezone(local_store.local_tz)) 12 | return { 13 | '@type': ['result'], 14 | 'utc_now': str(datetime.utcnow()), 15 | 'lcl_now': f"{local_store.local_tz}: {local_dt}", 16 | } 17 | 18 | 19 | class LocalStoreClient(): 20 | ''' 21 | Light redis wrapper and redis examples 22 | - get_tag function was added to return hex str 23 | - Can access client directly for full functionality 24 | ''' 25 | def __init__(self, **kwargs): 26 | self.local_tz = kwargs.get('local_tz', 'GMT') 27 | self.client = StrictRedis( 28 | encoding='utf-8', 29 | decode_responses=True, 30 | db=kwargs['db_index'], 31 | host=kwargs['host'], 32 | port=kwargs['port'], 33 | socket_timeout=kwargs['socket_timeout'], 34 | ) 35 | 36 | @staticmethod 37 | def get_tag(tag, num_bytes=2): 38 | ''' 39 | Tags are the tag plus a random hex bytes string 40 | - Bytes string length is 2 * num bytes 41 | ''' 42 | rand_hex_str = binascii.b2a_hex(urandom(num_bytes)).decode('utf-8') 43 | return f"{tag}:{rand_hex_str}" 44 | 45 | def ping(self): 46 | return self.client.ping() 47 | 48 | def dict_get(self, key): 49 | return self.client.hgetall(key) 50 | 51 | def dict_set(self, key, hash_dict): 52 | for k, v in hash_dict.items(): 53 | self.client.hset(name=key, key=k, value=v) 54 | 55 | def get_tag_keys(self, tag): 56 | return self.client.keys(f"{tag}:*") 57 | 58 | def item_get(self, key): 59 | return self.client.get(key) 60 | 61 | def item_set(self, key, item): 62 | return self.client.set(key, item) 63 | 64 | def list_add(self, key, item): 65 | return self.client.lpush(key, item) 66 | 67 | def list_get(self, key, start=0, stop=-1): 68 | return self.client.lrange(key, start, stop) 69 | -------------------------------------------------------------------------------- /src/snovault/nginx-dev.conf: -------------------------------------------------------------------------------- 1 | # Minimal nginx proxy for development 2 | # brew install nginx 3 | # nginx -p . nginx-dev.conf 4 | 5 | events { 6 | worker_connections 2048; 7 | } 8 | error_log stderr info; 9 | http { 10 | access_log /dev/stdout; 11 | 12 | resolver 8.8.8.8; 13 | upstream app { 14 | server 127.0.0.1:6543; 15 | keepalive 10; 16 | } 17 | 18 | map $http_x_forwarded_proto $forwarded_proto { 19 | default $http_x_forwarded_proto; 20 | '' $scheme; 21 | } 22 | 23 | server { 24 | listen 8000; 25 | location / { 26 | # Normalize duplicate slashes 27 | if ($request ~ ^(GET|HEAD)\s([^?]*)//(.*)\sHTTP/[0-9.]+$) { 28 | return 301 $2/$3; 29 | } 30 | proxy_set_header Host $http_host; 31 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 32 | proxy_set_header X-Forwarded-Proto $forwarded_proto; 33 | proxy_pass http://app; 34 | proxy_set_header Connection ""; 35 | } 36 | location ~ ^/_proxy/(.*)$ { 37 | internal; 38 | proxy_set_header Authorization ""; 39 | proxy_set_header Content-Type ""; 40 | proxy_buffering off; 41 | proxy_pass $1$is_args$args; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/snovault/predicates.py: -------------------------------------------------------------------------------- 1 | def includeme(config): 2 | config.add_view_predicate('subpath_segments', SubpathSegmentsPredicate) 3 | config.add_view_predicate('additional_permission', AdditionalPermissionPredicate) 4 | 5 | 6 | class SubpathSegmentsPredicate(object): 7 | def __init__(self, val, config): 8 | if isinstance(val, int): 9 | val = (val,) 10 | self.val = frozenset(val) 11 | 12 | def text(self): 13 | return 'subpath_segments in %r' % sorted(self.val) 14 | 15 | phash = text 16 | 17 | def __call__(self, context, request): 18 | return len(request.subpath) in self.val 19 | 20 | 21 | class AdditionalPermissionPredicate(object): 22 | def __init__(self, val, config): 23 | self.val = val 24 | 25 | def text(self): 26 | return 'additional_permission = %r' % self.val 27 | 28 | phash = text 29 | 30 | def __call__(self, context, request): 31 | return request.has_permission(self.val, context) 32 | -------------------------------------------------------------------------------- /src/snovault/snowflake_hash.py: -------------------------------------------------------------------------------- 1 | from base64 import ( 2 | b64decode, 3 | b64encode, 4 | ) 5 | from hashlib import sha384 6 | from passlib.registry import register_crypt_handler 7 | from passlib.utils import handlers as uh 8 | 9 | 10 | def includeme(config): 11 | register_crypt_handler(SNOWHash) 12 | 13 | 14 | class SNOWHash(uh.StaticHandler): 15 | """ a special snowflake of a password hashing scheme 16 | 17 | Cryptographic strength of the hashing function is less of a concern for 18 | randomly generated passwords. 19 | """ 20 | name = 'snowflake_hash' 21 | checksum_chars = uh.PADDED_BASE64_CHARS 22 | checksum_size = 64 23 | 24 | setting_kwds = ('salt_before', 'salt_after', 'salt_base') 25 | salt_before = b"186ED79BAEXzeusdioIsdklnw88e86cd73" 26 | salt_after = b"<*#$*(#)!DSDFOUIHLjksdf" 27 | salt_base = b64decode(b"""\ 28 | Kf8r/S37L/kh9yP1JfMn8TnvO+096z/pMecz5TXjN+EJ3wvdDdsP2QHXA9UF0wfRGc8bzR3LH8kR 29 | xxPFFcMXwWm/a71tu2+5YbdjtWWzZ7F5r3utfat/qXGnc6V1o3ehSZ9LnU2bT5lBl0OVRZNHkVmP 30 | W41di1+JUYdThVWDV4Gpf6t9rXuveaF3o3Wlc6dxuW+7bb1rv2mxZ7NltWO3YYlfi12NW49ZgVeD 31 | VYVTh1GZT5tNnUufSZFHk0WVQ5dB6T/rPe077znhN+M15TPnMfkv+y39K/8p8SfzJfUj9yHJH8sd 32 | zRvPGcEXwxXFE8cR2Q/bDd0L3wnRB9MF1QPXASn/K/0t+y/5Ifcj9SXzJ/E57zvtPes/6THnM+U1 33 | 4zfhCd8L3Q3bD9kB1wPVBdMH0RnPG80dyx/JEccTxRXDF8Fpv2u9bbtvuWG3Y7Vls2exea97rX2r 34 | f6lxp3OldaN3oUmfS51Nm0+ZQZdDlUWTR5FZj1uNXYtfiVGHU4VVg1eBqX+rfa17r3mhd6N1pXOn 35 | cblvu229a79psWezZbVjt2GJX4tdjVuPWYFXg1WFU4dRmU+bTZ1Ln0mRR5NFlUOXQek/6z3tO+85 36 | 4TfjNeUz5zH5L/st/Sv/KfEn8yX1I/chyR/LHc0bzxnBF8MVxRPHEdkP2w3dC98J0QfTBdUD1wE= 37 | """) 38 | 39 | def _calc_checksum(self, secret): 40 | if not isinstance(secret, bytes): 41 | secret = secret.encode('utf-8') 42 | salted = self.salt_before + secret + self.salt_after + b'\0' 43 | if len(salted) > len(self.salt_base): 44 | raise ValueError("Password too long") 45 | salted += self.salt_base[len(salted):] 46 | chk = sha384(salted).digest() 47 | return b64encode(chk).decode("ascii") 48 | -------------------------------------------------------------------------------- /src/snovault/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ENCODE-DCC/snovault/68336ee11fb1730dcc7af27796cdd93d52daf198/src/snovault/tests/__init__.py -------------------------------------------------------------------------------- /src/snovault/tests/pyramidfixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | # Fixtures for pyramid, embedding 4 | 5 | 6 | @pytest.yield_fixture 7 | def config(): 8 | from pyramid.testing import setUp, tearDown 9 | yield setUp() 10 | tearDown() 11 | 12 | 13 | @pytest.yield_fixture 14 | def threadlocals(request, dummy_request, registry): 15 | from pyramid.threadlocal import manager 16 | manager.push({'request': dummy_request, 'registry': registry}) 17 | yield manager.get() 18 | manager.pop() 19 | 20 | 21 | @pytest.fixture 22 | def dummy_request(root, registry, app): 23 | from pyramid.request import apply_request_extensions 24 | request = app.request_factory.blank('/dummy') 25 | request.root = root 26 | request.registry = registry 27 | request._stats = {} 28 | request.invoke_subrequest = app.invoke_subrequest 29 | apply_request_extensions(request) 30 | return request 31 | -------------------------------------------------------------------------------- /src/snovault/tests/test_indexer_state.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def test_heterogeneous_stream(): 5 | from snovault.elasticsearch.indexer_state import heterogeneous_stream 6 | gm = {'e': (x for x in [1, 2, 3, 4, 5])} 7 | assert list(heterogeneous_stream(gm)) == [1, 2, 3, 4, 5] 8 | gm = { 9 | 'e': (x for x in [1, 2, 3, 4, 5]), 10 | 'f': (x for x in ['a', 'b', 'c']) 11 | } 12 | assert list(heterogeneous_stream(gm)) == [1, 'a', 2, 'b', 3, 'c', 4, 5] 13 | gm = { 14 | 'e': (x for x in [1, 2, 3, 4, 5]), 15 | 'f': (x for x in ['a', 'b', 'c']), 16 | 'g': (x for x in (t for t in (None, True, False, None, None, True))) 17 | } 18 | assert list(heterogeneous_stream(gm)) == [ 19 | 1, 'a', 2, 'b', True, 3, 'c', False, 4, 5, True 20 | ] 21 | gm = { 22 | 'e': [1.0, 2.0, 3.0], 23 | 'f': [x for x in range(10)] 24 | } 25 | assert list(heterogeneous_stream(gm)) == [ 26 | 1.0, 0, 2.0, 1, 3.0, 2, 3, 4, 5, 6, 7, 8, 9 27 | ] 28 | -------------------------------------------------------------------------------- /src/snovault/tests/test_link.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture(autouse=True) 5 | def autouse_external_tx(external_tx): 6 | pass 7 | 8 | 9 | def test_links_add(targets, sources, posted_targets_and_sources, session): 10 | from snovault.storage import Link 11 | links = sorted([ 12 | (str(link.source_rid), link.rel, str(link.target_rid)) 13 | for link in session.query(Link).all() 14 | ]) 15 | expected = sorted([ 16 | (sources[0]['uuid'], u'target', targets[0]['uuid']), 17 | (sources[1]['uuid'], u'target', targets[1]['uuid']), 18 | ]) 19 | assert links == expected 20 | 21 | 22 | def test_links_update(targets, sources, posted_targets_and_sources, testapp, session): 23 | from snovault.storage import Link 24 | 25 | url = '/testing-link-sources/' + sources[1]['uuid'] 26 | new_item = {'name': 'B updated', 'target': targets[0]['name']} 27 | testapp.put_json(url, new_item, status=200) 28 | 29 | links = sorted([ 30 | (str(link.source_rid), link.rel, str(link.target_rid)) 31 | for link in session.query(Link).all() 32 | ]) 33 | expected = sorted([ 34 | (sources[0]['uuid'], u'target', targets[0]['uuid']), 35 | (sources[1]['uuid'], u'target', targets[0]['uuid']), 36 | ]) 37 | assert links == expected 38 | 39 | 40 | def test_links_reverse(targets, sources, posted_targets_and_sources, testapp, session): 41 | target = targets[0] 42 | res = testapp.get('/testing-link-targets/%s/?frame=object' % target['name']) 43 | assert res.json['reverse'] == ['/testing-link-sources/%s/' % sources[0]['uuid']] 44 | 45 | # DELETED sources are hidden from the list. 46 | target = targets[1] 47 | res = testapp.get('/testing-link-targets/%s/' % target['name']) 48 | assert res.json['reverse'] == [] 49 | 50 | 51 | def test_links_quoted_ids(posted_targets_and_sources, testapp, session): 52 | res = testapp.get('/testing-link-targets/quote:name/?frame=object') 53 | target = res.json 54 | source = {'name': 'C', 'target': target['@id']} 55 | testapp.post_json('/testing-link-sources/', source, status=201) 56 | -------------------------------------------------------------------------------- /src/snovault/tests/test_schema_utils.py: -------------------------------------------------------------------------------- 1 | from snovault.schema_utils import validate 2 | import pytest 3 | 4 | 5 | targets = [ 6 | {'name': 'one', 'uuid': '775795d3-4410-4114-836b-8eeecf1d0c2f'}, 7 | ] 8 | 9 | 10 | @pytest.fixture 11 | def content(testapp): 12 | url = '/testing-link-targets/' 13 | for item in targets: 14 | testapp.post_json(url, item, status=201) 15 | 16 | 17 | def test_uniqueItems_validates_normalized_links(content, threadlocals): 18 | schema = { 19 | 'uniqueItems': True, 20 | 'items': { 21 | 'linkTo': 'TestingLinkTarget', 22 | } 23 | } 24 | uuid = targets[0]['uuid'] 25 | data = [ 26 | uuid, 27 | '/testing-link-targets/{}'.format(uuid), 28 | ] 29 | validated, errors = validate(schema, data) 30 | assert len(errors) == 1 31 | assert ( 32 | errors[0].message == "['{}', '{}'] has non-unique elements".format( 33 | uuid, uuid) 34 | ) 35 | -------------------------------------------------------------------------------- /src/snovault/tests/test_searches_configs.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def included(config): 5 | def new_item_search_config(): 6 | return { 7 | 'facets': {'a': 'b'} 8 | } 9 | config.register_search_config( 10 | 'OtherConfigItem', new_item_search_config 11 | ) 12 | 13 | 14 | def test_searches_configs_search_config_decorator(config, dummy_request): 15 | from snovault.elasticsearch.searches.interfaces import SEARCH_CONFIG 16 | from snovault.elasticsearch.searches.configs import search_config 17 | assert dummy_request.registry[SEARCH_CONFIG].get('TestConfigItem').facets == {'a': 'b'} 18 | config.include('snovault.elasticsearch.searches.configs') 19 | config.include(included) 20 | config.commit() 21 | assert config.registry[SEARCH_CONFIG].registry.get('OtherConfigItem').facets == {'a': 'b'} 22 | config.register_search_config('OtherConfigItem', lambda: {'facets': {'c': 'd'}}) 23 | config.commit() 24 | assert config.registry[SEARCH_CONFIG].registry.get('OtherConfigItem').facets == {'c': 'd'} 25 | -------------------------------------------------------------------------------- /src/snovault/tests/test_searches_fields.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture() 5 | def dummy_parent(dummy_request): 6 | from pyramid.testing import DummyResource 7 | from pyramid.security import Allow 8 | from snosearch.parsers import ParamsParser 9 | from snosearch.queries import AbstractQueryFactory 10 | from snovault.elasticsearch.interfaces import ELASTIC_SEARCH 11 | from elasticsearch import Elasticsearch 12 | dummy_request.registry[ELASTIC_SEARCH] = Elasticsearch() 13 | dummy_request.context = DummyResource() 14 | dummy_request.context.__acl__ = lambda: [(Allow, 'group.submitter', 'search_audit')] 15 | class DummyParent(): 16 | def __init__(self): 17 | self._meta = {} 18 | self.response = {} 19 | dp = DummyParent() 20 | pp = ParamsParser(dummy_request) 21 | dp._meta = { 22 | 'params_parser': pp, 23 | 'query_builder': AbstractQueryFactory(pp) 24 | } 25 | return dp 26 | 27 | 28 | def test_searches_fields_non_sortable_response_field(dummy_parent): 29 | dummy_parent._meta['params_parser']._request.environ['QUERY_STRING'] = ( 30 | 'type=TestingSearchSchema' 31 | ) 32 | from snovault.elasticsearch.searches.fields import NonSortableResponseField 33 | nrf = NonSortableResponseField() 34 | r = nrf.render(parent=dummy_parent) 35 | assert r['non_sortable'] == ['pipeline_error_detail', 'description', 'notes'] 36 | 37 | 38 | def test_searches_fields_pass_through_response_field(): 39 | from snovault.elasticsearch.searches.fields import PassThroughResponseField 40 | ptrf = PassThroughResponseField( 41 | values_to_pass_through={ 42 | 'some': 'values', 43 | 'in': 'a', 44 | 'dictionary': [], 45 | } 46 | ) 47 | result = ptrf.render() 48 | assert result == { 49 | 'some': 'values', 50 | 'in': 'a', 51 | 'dictionary': [] 52 | } 53 | -------------------------------------------------------------------------------- /src/snovault/tests/test_snowflake_hash.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | TEST_HASHES = { 4 | "test": "Jnh+8wNnELksNFVbxkya8RDrxJNL13dUWTXhp5DCx/quTM2/cYn7azzl2Uk3I2zc", 5 | "test2": "sh33L5uQeLr//jJULb7mAnbVADkkWZrgcXx97DCacueGtEU5G2HtqUv73UTS0EI0", 6 | "testing100" * 10: "5rznDSIcDPd/9rjom6P/qkJGtJSV47y/u5+KlkILROaqQ6axhEyVIQTahuBYerLG", 7 | } 8 | 9 | 10 | @pytest.mark.parametrize(('password', 'pwhash'), TEST_HASHES.items()) 11 | def test_snowflake_hash(password, pwhash): 12 | from snovault.snowflake_hash import SNOWHash 13 | assert SNOWHash.hash(password) == pwhash 14 | -------------------------------------------------------------------------------- /src/snovault/tests/test_upgrader.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def step1(value, system): 5 | value['step1'] = True 6 | return value 7 | 8 | 9 | def step2(value, system): 10 | value['step2'] = True 11 | return value 12 | 13 | 14 | def finalizer(value, system, version): 15 | value['schema_version'] = version 16 | return value 17 | 18 | 19 | @pytest.fixture 20 | def schema_upgrader(): 21 | from snovault.upgrader import SchemaUpgrader 22 | schema_upgrader = SchemaUpgrader('test', '3') 23 | schema_upgrader.add_upgrade_step(step1, dest='2') 24 | schema_upgrader.add_upgrade_step(step2, source='2', dest='3') 25 | return schema_upgrader 26 | 27 | 28 | def test_upgrade(schema_upgrader): 29 | value = schema_upgrader.upgrade({}, '') 30 | assert value['step1'] 31 | assert value['step2'] 32 | 33 | 34 | def test_finalizer(schema_upgrader): 35 | schema_upgrader.finalizer = finalizer 36 | value = schema_upgrader.upgrade({}) 37 | assert value['schema_version'] == '3' 38 | 39 | 40 | def test_declarative_config(): 41 | from pyramid.config import Configurator 42 | from snovault.interfaces import UPGRADER 43 | config = Configurator() 44 | config.include('snovault.config') 45 | config.include('snovault.upgrader') 46 | config.include('.testing_upgrader') 47 | config.include('snovault.elasticsearch.searches.configs') 48 | config.commit() 49 | 50 | upgrader = config.registry[UPGRADER] 51 | value = upgrader.upgrade('testing_upgrader', {}, '') 52 | assert value['step1'] 53 | assert value['step2'] 54 | assert value['schema_version'] == '3' 55 | -------------------------------------------------------------------------------- /src/snovault/tests/testing_auditor.py: -------------------------------------------------------------------------------- 1 | from snovault.auditor import ( 2 | audit_checker, 3 | AuditFailure, 4 | ) 5 | 6 | 7 | def includeme(config): 8 | config.scan('.testing_views') 9 | config.scan(__name__) 10 | 11 | 12 | def has_condition1(value, system): 13 | return value.get('condition1') 14 | 15 | 16 | @audit_checker('testing_link_source', condition=has_condition1) 17 | def checker1(value, system): 18 | if not value.get('checker1'): 19 | return AuditFailure('testchecker', 'Missing checker1') 20 | 21 | 22 | @audit_checker('testing_link_target') 23 | def testing_link_target_status(value, system): 24 | if value.get('status') == 'CHECK': 25 | if not len(value['reverse']): 26 | return AuditFailure('status', 'Missing reverse items') 27 | -------------------------------------------------------------------------------- /src/snovault/tests/testing_key.py: -------------------------------------------------------------------------------- 1 | from snovault import ( 2 | Item, 3 | collection, 4 | ) 5 | 6 | # Test class for keys 7 | 8 | 9 | def includeme(config): 10 | config.scan(__name__) 11 | 12 | 13 | @collection( 14 | 'testing-keys', 15 | properties={ 16 | 'title': 'Test keys', 17 | 'description': 'Testing. Testing. 1, 2, 3.', 18 | }, 19 | unique_key='testing_alias', 20 | ) 21 | class TestingKey(Item): 22 | item_type = 'testing_key' 23 | schema = { 24 | 'type': 'object', 25 | 'properties': { 26 | 'name': { 27 | 'type': 'string', 28 | 'uniqueKey': True, 29 | }, 30 | 'alias': { 31 | 'type': 'string', 32 | 'uniqueKey': 'testing_alias', 33 | }, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/snovault/tests/testing_upgrader.py: -------------------------------------------------------------------------------- 1 | from snovault import ( 2 | Item, 3 | collection, 4 | ) 5 | from snovault.upgrader import ( 6 | upgrade_step, 7 | upgrade_finalizer, 8 | ) 9 | 10 | 11 | def includeme(config): 12 | config.scan(__name__) 13 | config.add_upgrade('testing_upgrader', '3') 14 | 15 | 16 | @collection('testing-upgrader') 17 | class TestingUpgrader(Item): 18 | item_type = 'testing_upgrader' 19 | 20 | 21 | @upgrade_step('testing_upgrader', '', '2') 22 | def step1(value, system): 23 | value['step1'] = True 24 | return value 25 | 26 | 27 | @upgrade_step('testing_upgrader', '2', '3') 28 | def step2(value, system): 29 | value['step2'] = True 30 | return value 31 | 32 | 33 | @upgrade_finalizer('testing_upgrader') 34 | def finalizer(value, system, version): 35 | value['schema_version'] = version 36 | return value 37 | -------------------------------------------------------------------------------- /src/snovault/tests/toolfixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | # Fixtures for app 5 | 6 | @pytest.fixture 7 | def registry(app): 8 | return app.registry 9 | 10 | 11 | @pytest.fixture 12 | def auditor(registry): 13 | import snovault.interfaces 14 | return registry[snovault.interfaces.AUDITOR] 15 | 16 | 17 | @pytest.fixture 18 | def blobs(registry): 19 | import snovault.interfaces 20 | return registry[snovault.interfaces.BLOBS] 21 | 22 | 23 | @pytest.fixture 24 | def calculated_properties(registry): 25 | import snovault.interfaces 26 | return registry[snovault.interfaces.CALCULATED_PROPERTIES] 27 | 28 | 29 | @pytest.fixture 30 | def collections(registry): 31 | import snovault.interfaces 32 | return registry[snovault.interfaces.COLLECTIONS] 33 | 34 | 35 | @pytest.fixture 36 | def connection(registry): 37 | import snovault.interfaces 38 | return registry[snovault.interfaces.CONNECTION] 39 | 40 | 41 | @pytest.fixture 42 | def elasticsearch(registry): 43 | from snovault.elasticsearch import ELASTIC_SEARCH 44 | return registry[ELASTIC_SEARCH] 45 | 46 | 47 | @pytest.fixture 48 | def storage(registry): 49 | import snovault.interfaces 50 | return registry[snovault.interfaces.STORAGE] 51 | 52 | 53 | @pytest.fixture 54 | def root(registry): 55 | import snovault.interfaces 56 | return registry[snovault.interfaces.ROOT] 57 | 58 | 59 | @pytest.fixture 60 | def types(registry): 61 | import snovault.interfaces 62 | return registry[snovault.interfaces.TYPES] 63 | 64 | 65 | @pytest.fixture 66 | def upgrader(registry): 67 | import snovault.interfaces 68 | return registry[snovault.interfaces.UPGRADER] 69 | -------------------------------------------------------------------------------- /src/snovault/validators.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | from .schema_utils import validate_request 3 | from .validation import ValidationFailure 4 | 5 | 6 | # No-validation validators 7 | 8 | 9 | def no_validate_item_content_post(context, request): 10 | data = request.json 11 | request.validated.update(data) 12 | 13 | 14 | def no_validate_item_content_put(context, request): 15 | data = request.json 16 | if 'uuid' in data: 17 | if UUID(data['uuid']) != context.uuid: 18 | msg = 'uuid may not be changed' 19 | raise ValidationFailure('body', ['uuid'], msg) 20 | request.validated.update(data) 21 | 22 | 23 | def no_validate_item_content_patch(context, request): 24 | data = context.properties.copy() 25 | data.update(request.json) 26 | if 'uuid' in data: 27 | if UUID(data['uuid']) != context.uuid: 28 | msg = 'uuid may not be changed' 29 | raise ValidationFailure('body', ['uuid'], msg) 30 | request.validated.update(data) 31 | 32 | 33 | # Schema checking validators 34 | 35 | 36 | def validate_item_content_post(context, request): 37 | data = request.json 38 | validate_request(context.type_info.schema, request, data) 39 | 40 | 41 | def validate_item_content_put(context, request): 42 | data = request.json 43 | schema = context.type_info.schema 44 | if 'uuid' in data and UUID(data['uuid']) != context.uuid: 45 | msg = 'uuid may not be changed' 46 | raise ValidationFailure('body', ['uuid'], msg) 47 | accession = context.properties.get('accession') 48 | if accession and accession != data.get('accession'): 49 | msg = 'must specify original accession' 50 | raise ValidationFailure('body', ['accession'], msg) 51 | current = context.upgrade_properties().copy() 52 | current['uuid'] = str(context.uuid) 53 | validate_request(schema, request, data, current) 54 | 55 | 56 | def validate_item_content_patch(context, request): 57 | data = context.upgrade_properties().copy() 58 | if 'schema_version' in data: 59 | del data['schema_version'] 60 | data.update(request.json) 61 | schema = context.type_info.schema 62 | if 'uuid' in data and UUID(data['uuid']) != context.uuid: 63 | msg = 'uuid may not be changed' 64 | raise ValidationFailure('body', ['uuid'], msg) 65 | current = context.upgrade_properties().copy() 66 | current['uuid'] = str(context.uuid) 67 | validate_request(schema, request, data, current) 68 | -------------------------------------------------------------------------------- /src/snowflakes/audit/__init__.py: -------------------------------------------------------------------------------- 1 | def includeme(config): 2 | config.scan() 3 | -------------------------------------------------------------------------------- /src/snowflakes/authorization.py: -------------------------------------------------------------------------------- 1 | from snovault import COLLECTIONS 2 | 3 | 4 | def groupfinder(login, request): 5 | if '.' not in login: 6 | return None 7 | namespace, localname = login.split('.', 1) 8 | user = None 9 | 10 | collections = request.registry[COLLECTIONS] 11 | 12 | if namespace == 'remoteuser': 13 | if localname in ['EMBED', 'INDEXER']: 14 | return [] 15 | elif localname in ['TEST', 'IMPORT', 'UPGRADE']: 16 | return ['group.admin'] 17 | elif localname in ['TEST_SUBMITTER']: 18 | return ['group.submitter'] 19 | elif localname in ['TEST_AUTHENTICATED']: 20 | return ['viewing_group.ENCODE'] 21 | 22 | if namespace in ('mailto', 'remoteuser', 'webuser'): 23 | users = collections.by_item_type['user'] 24 | try: 25 | user = users[localname] 26 | except KeyError: 27 | return None 28 | 29 | elif namespace == 'accesskey': 30 | access_keys = collections.by_item_type['access_key'] 31 | try: 32 | access_key = access_keys[localname] 33 | except KeyError: 34 | return None 35 | 36 | if access_key.properties.get('status') in ('deleted', 'disabled'): 37 | return None 38 | 39 | userid = access_key.properties['user'] 40 | user = collections.by_item_type['user'][userid] 41 | 42 | if user is None: 43 | return None 44 | 45 | user_properties = user.properties 46 | 47 | if user_properties.get('status') in ('deleted', 'disabled'): 48 | return None 49 | 50 | principals = ['userid.%s' % user.uuid] 51 | lab = user_properties.get('lab') 52 | if lab: 53 | principals.append('lab.%s' % lab) 54 | submits_for = user_properties.get('submits_for', []) 55 | principals.extend('lab.%s' % lab_uuid for lab_uuid in submits_for) 56 | principals.extend('submits_for.%s' % lab_uuid for lab_uuid in submits_for) 57 | if submits_for: 58 | principals.append('group.submitter') 59 | groups = user_properties.get('groups', []) 60 | principals.extend('group.%s' % group for group in groups) 61 | viewing_groups = user_properties.get('viewing_groups', []) 62 | principals.extend('viewing_group.%s' % group for group in viewing_groups) 63 | return principals 64 | -------------------------------------------------------------------------------- /src/snowflakes/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # package 2 | -------------------------------------------------------------------------------- /src/snowflakes/commands/create_admin_user.py: -------------------------------------------------------------------------------- 1 | from pyramid.paster import get_app 2 | import logging 3 | from webtest import TestApp 4 | 5 | EPILOG = __doc__ 6 | 7 | 8 | def run(app, first_name, last_name, email, lab): 9 | environ = { 10 | 'HTTP_ACCEPT': 'application/json', 11 | 'REMOTE_USER': 'TEST', 12 | } 13 | testapp = TestApp(app, environ) 14 | testapp.post_json('/user', { 15 | 'first_name': first_name, 16 | 'last_name': last_name, 17 | 'email': email, 18 | 'lab': lab, 19 | 'groups': ['admin'] 20 | }) 21 | 22 | 23 | def main(): 24 | ''' Creates an admin user ''' 25 | 26 | import argparse 27 | parser = argparse.ArgumentParser( 28 | description="Creates an admin user", epilog=EPILOG, 29 | formatter_class=argparse.RawDescriptionHelpFormatter, 30 | ) 31 | parser.add_argument('--first-name', default='Admin', help="First name") 32 | parser.add_argument('--last-name', default='Test', help="Last name") 33 | parser.add_argument('--email', default='admin_test@example.org', help="E-mail") 34 | parser.add_argument('--lab', default='/labs/j-michael-cherry/', help="Lab") 35 | parser.add_argument('--app-name', help="Pyramid app name in configfile") 36 | parser.add_argument('config_uri', help="path to configfile") 37 | args = parser.parse_args() 38 | 39 | logging.basicConfig() 40 | options = { 41 | 'embed_cache.capacity': '5000', 42 | 'indexer': 'true', 43 | } 44 | app = get_app(args.config_uri, args.app_name, options) 45 | 46 | logging.getLogger('encoded').setLevel(logging.DEBUG) 47 | return run(app, args.first_name, args.last_name, args.email, args.lab) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /src/snowflakes/commands/es_index_data.py: -------------------------------------------------------------------------------- 1 | from pyramid.paster import get_app 2 | import logging 3 | from webtest import TestApp 4 | 5 | index = 'snowflakes' 6 | 7 | EPILOG = __doc__ 8 | 9 | 10 | def run(app, collections=None, record=False): 11 | environ = { 12 | 'HTTP_ACCEPT': 'application/json', 13 | 'REMOTE_USER': 'INDEXER', 14 | } 15 | testapp = TestApp(app, environ) 16 | testapp.post_json('/index', { 17 | 'last_xmin': None, 18 | 'types': collections, 19 | 'recovery': True 20 | } 21 | ) 22 | 23 | 24 | def main(): 25 | ''' Indexes app data loaded to elasticsearch ''' 26 | 27 | import argparse 28 | parser = argparse.ArgumentParser( 29 | description="Index data in Elastic Search", epilog=EPILOG, 30 | formatter_class=argparse.RawDescriptionHelpFormatter, 31 | ) 32 | parser.add_argument('--item-type', action='append', help="Item type") 33 | parser.add_argument('--record', default=False, action='store_true', help="Record the xmin in ES meta") 34 | parser.add_argument('--app-name', help="Pyramid app name in configfile") 35 | parser.add_argument('config_uri', help="path to configfile") 36 | args = parser.parse_args() 37 | 38 | logging.basicConfig() 39 | options = { 40 | 'embed_cache.capacity': '5000', 41 | 'indexer': 'true', 42 | } 43 | app = get_app(args.config_uri, args.app_name, options) 44 | 45 | # Loading app will have configured from config file. Reconfigure here: 46 | logging.getLogger('snovault').setLevel(logging.DEBUG) 47 | return run(app, args.item_type, args.record) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /src/snowflakes/commands/jsonld_rdf.py: -------------------------------------------------------------------------------- 1 | """\ 2 | Available formats: xml, n3, turtle, nt, pretty-xml, trix. 3 | Example. 4 | 5 | %(prog)s "$SITE_URL/search/?type=Item&frame=object" 6 | """ 7 | EPILOG = __doc__ 8 | 9 | import rdflib 10 | 11 | 12 | def run(sources, output, parser='json-ld', serializer='xml', base=None): 13 | g = rdflib.ConjunctiveGraph() 14 | for url in sources: 15 | g.parse(url, format=parser) 16 | g.serialize(output, format=serializer, base=base) 17 | 18 | 19 | def main(): 20 | import argparse 21 | import sys 22 | stdout = sys.stdout 23 | if sys.version_info.major > 2: 24 | stdout = stdout.buffer 25 | 26 | rdflib_parsers = sorted( 27 | p.name for p in rdflib.plugin.plugins(kind=rdflib.parser.Parser) 28 | if '/' not in p.name) 29 | rdflib_serializers = sorted( 30 | p.name for p in rdflib.plugin.plugins(kind=rdflib.serializer.Serializer) 31 | if '/' not in p.name) 32 | parser = argparse.ArgumentParser( 33 | description="Convert JSON-LD from source URLs to RDF", epilog=EPILOG, 34 | formatter_class=argparse.RawDescriptionHelpFormatter, 35 | ) 36 | parser.add_argument('sources', metavar='URL', nargs='+', help="URLs to convert") 37 | parser.add_argument( 38 | '-p', '--parser', default='json-ld', help=', '.join(rdflib_parsers)) 39 | parser.add_argument( 40 | '-s', '--serializer', default='xml', help=', '.join(rdflib_serializers)) 41 | parser.add_argument( 42 | '-b', '--base', default=None, help='Base URL') 43 | parser.add_argument( 44 | '-o', '--output', type=argparse.FileType('wb'), default=stdout, 45 | help="Output file.") 46 | args = parser.parse_args() 47 | run(args.sources, args.output, args.parser, args.serializer, args.base) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /src/snowflakes/commands/spreadsheet_to_json.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example: 3 | 4 | %(prog)s *.tsv 5 | 6 | """ 7 | 8 | from .. import loadxl 9 | import json 10 | import os.path 11 | 12 | EPILOG = __doc__ 13 | 14 | 15 | def rename_test_with_underscore(rows): 16 | for row in rows: 17 | if 'test' in row: 18 | if row['test'] != 'test': 19 | row['_test'] = row['test'] 20 | del row['test'] 21 | yield row 22 | 23 | 24 | def remove_empty(rows): 25 | for row in rows: 26 | if row: 27 | yield row 28 | 29 | 30 | def convert(filename, sheetname=None, outputdir=None, skip_blanks=False): 31 | if outputdir is None: 32 | outputdir = os.path.dirname(filename) 33 | source = loadxl.read_single_sheet(filename, sheetname) 34 | pipeline = [ 35 | loadxl.remove_keys_with_empty_value if skip_blanks else loadxl.noop, 36 | rename_test_with_underscore, 37 | remove_empty, 38 | ] 39 | data = list(loadxl.combine(source, pipeline)) 40 | if sheetname is None: 41 | sheetname, ext = os.path.splitext(os.path.basename(filename)) 42 | out = open(os.path.join(outputdir, sheetname + '.json'), 'w') 43 | json.dump(data, out, sort_keys=True, indent=4, separators=(',', ': ')) 44 | 45 | 46 | 47 | def main(): 48 | import argparse 49 | parser = argparse.ArgumentParser( 50 | description="Convert spreadsheet to json list", epilog=EPILOG, 51 | formatter_class=argparse.RawDescriptionHelpFormatter, 52 | ) 53 | parser.add_argument('filenames', metavar='FILE', nargs='+', help="Files to convert") 54 | parser.add_argument('--outputdir', help="Directory to write converted output") 55 | parser.add_argument('--sheetname', help="xlsx sheet name") 56 | parser.add_argument('--skip-blanks', help="Skip blank columns") 57 | args = parser.parse_args() 58 | 59 | for filename in args.filenames: 60 | convert(filename, args.sheetname, args.outputdir, args.skip_blanks) 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /src/snowflakes/memlimit.py: -------------------------------------------------------------------------------- 1 | # https://code.google.com/p/modwsgi/wiki/RegisteringCleanupCode 2 | 3 | 4 | class Generator2: 5 | def __init__(self, iterable, callback, environ): 6 | self.__iterable = iterable 7 | self.__callback = callback 8 | self.__environ = environ 9 | 10 | def __iter__(self): 11 | for item in self.__iterable: 12 | yield item 13 | 14 | def close(self): 15 | try: 16 | if hasattr(self.__iterable, 'close'): 17 | self.__iterable.close() 18 | finally: 19 | self.__callback(self.__environ) 20 | 21 | 22 | class ExecuteOnCompletion2: 23 | def __init__(self, application, callback): 24 | self.__application = application 25 | self.__callback = callback 26 | 27 | def __call__(self, environ, start_response): 28 | try: 29 | result = self.__application(environ, start_response) 30 | except: 31 | self.__callback(environ) 32 | raise 33 | return Generator2(result, self.__callback, environ) 34 | 35 | 36 | import logging 37 | import psutil 38 | import humanfriendly 39 | 40 | 41 | def rss_checker(rss_limit=None): 42 | log = logging.getLogger(__name__) 43 | process = psutil.Process() 44 | 45 | def callback(environ): 46 | rss = process.memory_info().rss 47 | if rss_limit and rss > rss_limit: 48 | msg = "Restarting process. Memory usage exceeds limit of %d: %d" 49 | log.error(msg, rss_limit, rss) 50 | process.kill() 51 | 52 | return callback 53 | 54 | 55 | def filter_app(app, global_conf, rss_limit=None): 56 | if rss_limit is not None: 57 | rss_limit = humanfriendly.parse_size(rss_limit) 58 | 59 | callback = rss_checker(rss_limit) 60 | return ExecuteOnCompletion2(app, callback) 61 | -------------------------------------------------------------------------------- /src/snowflakes/schema_formats.py: -------------------------------------------------------------------------------- 1 | import re 2 | import rfc3987 3 | from jsonschema_serialize_fork import FormatChecker 4 | from pyramid.threadlocal import get_current_request 5 | from uuid import UUID 6 | 7 | accession_re = re.compile(r'^SNO(SS|FL)[0-9][0-9][0-9][A-Z][A-Z][A-Z]$') 8 | test_accession_re = re.compile(r'^TST(SS|FL)[0-9][0-9][0-9]([0-9][0-9][0-9]|[A-Z][A-Z][A-Z])$') 9 | uuid_re = re.compile(r'(?i)\{?(?:[0-9a-f]{4}-?){8}\}?') 10 | 11 | @FormatChecker.cls_checks("uuid") 12 | def is_uuid(instance): 13 | # Python's UUID ignores all dashes, whereas Postgres is more strict 14 | # http://www.postgresql.org/docs/9.2/static/datatype-uuid.html 15 | return bool(uuid_re.match(instance)) 16 | 17 | 18 | def is_accession(instance): 19 | ''' just a pattern checker ''' 20 | # Unfortunately we cannot access the accessionType here 21 | return ( 22 | accession_re.match(instance) is not None or 23 | test_accession_re.match(instance) is not None 24 | ) 25 | 26 | 27 | @FormatChecker.cls_checks("accession") 28 | def is_accession_for_server(instance): 29 | from .server_defaults import ( 30 | ACCESSION_FACTORY, 31 | test_accession, 32 | ) 33 | # Unfortunately we cannot access the accessionType here 34 | if accession_re.match(instance): 35 | return True 36 | request = get_current_request() 37 | if request.registry[ACCESSION_FACTORY] is test_accession: 38 | if test_accession_re.match(instance): 39 | return True 40 | return False 41 | 42 | 43 | @FormatChecker.cls_checks("uri", raises=ValueError) 44 | def is_uri(instance): 45 | if ':' not in instance: 46 | # We want only absolute uris 47 | return False 48 | return rfc3987.parse(instance, rule="URI_reference") 49 | -------------------------------------------------------------------------------- /src/snowflakes/schemas/access_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Admin access key", 3 | "id": "/profiles/access_key_admin.json", 4 | "$schema": "http://json-schema.org/draft-04/schema#", 5 | "required": [], 6 | "additionalProperties": false, 7 | "mixinProperties": [ 8 | { "$ref": "mixins.json#/schema_version" }, 9 | { "$ref": "mixins.json#/uuid" } 10 | ], 11 | "type": "object", 12 | "properties": { 13 | "schema_version": { 14 | "default": "2" 15 | }, 16 | "status": { 17 | "title": "Status", 18 | "type": "string", 19 | "default": "current", 20 | "enum" : [ 21 | "current", 22 | "deleted" 23 | ] 24 | }, 25 | "user": { 26 | "title": "User", 27 | "comment": "Only admins are allowed to set this value.", 28 | "type": "string", 29 | "linkTo": "User", 30 | "permission": "import_items" 31 | }, 32 | "description": { 33 | "title": "Description", 34 | "type": "string", 35 | "default": "" 36 | }, 37 | "access_key_id": { 38 | "title": "Access key ID", 39 | "comment": "Only admins are allowed to set this value.", 40 | "type": "string", 41 | "permission": "import_items", 42 | "uniqueKey": true 43 | }, 44 | "secret_access_key_hash": { 45 | "title": "Secret access key Hash", 46 | "comment": "Only admins are allowed to set this value.", 47 | "type": "string", 48 | "permission": "import_items" 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/snowflakes/schemas/changelogs/award.md: -------------------------------------------------------------------------------- 1 | Change log for award.json 2 | ========================= 3 | 4 | 5 | Schema version 2 6 | ---------------- 7 | 8 | * Default values of '' were removed. You can no longer submit a blank url (url='') 9 | 10 | * *status* was brought into line with other objects that are shared. Disabled grants with rfa in ['ENCODE2', 'ENCODE2-Mouse']: 11 | 12 | "enum" : [ 13 | "current", 14 | "deleted", 15 | "replaced", 16 | "disabled" 17 | ] 18 | -------------------------------------------------------------------------------- /src/snowflakes/schemas/changelogs/example.md: -------------------------------------------------------------------------------- 1 | ================================= 2 | Example Change Log 3 | ================================= 4 | 5 | Schema version 2 6 | ---------------- 7 | 8 | * *object_type had the some properties removed -------------------------------------------------------------------------------- /src/snowflakes/schemas/image.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Image", 3 | "description": "Schema for images embedded in page objects", 4 | "id": "/profiles/image.json", 5 | "$schema": "http://json-schema.org/draft-04/schema#", 6 | "type": "object", 7 | "required": [ "attachment" ], 8 | "identifyingProperties": ["uuid"], 9 | "additionalProperties": false, 10 | "mixinProperties": [ 11 | { "$ref": "mixins.json#/schema_version" }, 12 | { "$ref": "mixins.json#/uuid" }, 13 | { "$ref": "mixins.json#/attachment" }, 14 | { "$ref": "mixins.json#/submitted" }, 15 | { "$ref": "mixins.json#/standard_status"} 16 | ], 17 | "properties": { 18 | "schema_version": { 19 | "default": "1" 20 | }, 21 | "status": { 22 | "default": "released" 23 | }, 24 | "caption": { 25 | "title": "Caption", 26 | "type": "string" 27 | } 28 | }, 29 | "boost_values": { 30 | "caption": 1.0 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/snowflakes/schemas/namespaces.json: -------------------------------------------------------------------------------- 1 | { 2 | "_comment": "This should contain keys and URL references" 3 | } -------------------------------------------------------------------------------- /src/snowflakes/static/browser.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | // Entry point for browser 3 | require('./libs/react-patches'); 4 | var React = require('react'); 5 | var ReactMount = require('react/lib/ReactMount'); 6 | ReactMount.allowFullPageRender = true; 7 | 8 | var App = require('./components'); 9 | var domready = require('domready'); 10 | 11 | // Treat domready function as the entry point to the application. 12 | // Inside this function, kick-off all initialization, everything up to this 13 | // point should be definitions. 14 | if (!window.TEST_RUNNER) domready(function ready() { 15 | console.log('ready'); 16 | // Set class depending on browser features 17 | var BrowserFeat = require('./components/browserfeat').BrowserFeat; 18 | BrowserFeat.setHtmlFeatClass(); 19 | var props = App.getRenderedProps(document); 20 | var server_stats = require('querystring').parse(window.stats_cookie); 21 | App.recordServerStats(server_stats, 'html'); 22 | 23 | var app = React.render(, document); 24 | 25 | // Simplify debugging 26 | window.app = app; 27 | window.React = React; 28 | }); 29 | -------------------------------------------------------------------------------- /src/snowflakes/static/build/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/ImpersonateUserSchema.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var ReactForms = require('react-forms'); 3 | 4 | module.exports = ReactForms.schema.Mapping({}, { 5 | userid: ReactForms.schema.Scalar({ 6 | label: 'User', 7 | hint: 'Enter the email of the user you want to impersonate.', 8 | }), 9 | }); 10 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/JSONNode.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var ReactForms = require('react-forms'); 3 | 4 | class JSONNode extends ReactForms.schema.ScalarNode { 5 | serialize(value) { 6 | return JSON.stringify(value, null, 4); 7 | } 8 | deserialize(value) { 9 | return (typeof value === 'string') ? JSON.parse(value) : value; 10 | } 11 | } 12 | 13 | module.exports = JSONNode; 14 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/StickyHeader.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const React = require('react'); 3 | const offset = require('../libs/offset'); 4 | const cloneWithProps = require('react/lib/cloneWithProps'); 5 | 6 | module.exports = React.createClass({ 7 | render() { 8 | return React.Children.only(this.props.children); 9 | }, 10 | 11 | componentDidMount() { 12 | // Avoid shimming as ie8 does not support css transform 13 | if (window.getComputedStyle === undefined) return; 14 | this.stickyHeader(); 15 | window.addEventListener('scroll', this.stickyHeader); 16 | window.addEventListener('resize', this.stickyHeader); 17 | }, 18 | 19 | componentWillUnmount() { 20 | if (window.getComputedStyle === undefined) return; 21 | window.removeEventListener('scroll', this.stickyHeader); 22 | window.removeEventListener('resize', this.stickyHeader); 23 | }, 24 | 25 | stickyHeader() { 26 | // http://stackoverflow.com/a/6625189/199100 27 | // http://css-tricks.com/persistent-headers/ 28 | const header = this.getDOMNode(); 29 | const table = header.parentElement; 30 | const offsetTop = offset(table).top; 31 | const nb = document.querySelector('.navbar-fixed-top'); 32 | let nb_height = 0; 33 | 34 | if (window.getComputedStyle(nb).getPropertyValue('position') === 'fixed') { 35 | nb_height = nb.clientHeight; 36 | } 37 | const scrollTop = document.body.scrollTop + nb_height; 38 | let y = 0; 39 | 40 | if((scrollTop > offsetTop) && (scrollTop < (offsetTop + table.clientHeight))) { 41 | y = scrollTop - offsetTop - 3; // correction for borders 42 | } 43 | const transform = 'translate(0px,' + y + 'px)'; 44 | header.style.transform = transform; 45 | }, 46 | }); 47 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/__tests__/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "esnext": true, 3 | "node": true, 4 | "predef": [ 5 | "DOMParser", 6 | "afterEach", 7 | "beforeEach", 8 | "describe", 9 | "expect", 10 | "it", 11 | "jest", 12 | "pit", 13 | "xdescribe", 14 | "xit" 15 | ], 16 | "globalstrict": true, 17 | "newcap": false, 18 | "sub": true 19 | } 20 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/__tests__/server-render-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | jest.autoMockOff(); 4 | 5 | // Fixes https://github.com/facebook/jest/issues/78 6 | jest.dontMock('react'); 7 | jest.dontMock('underscore'); 8 | 9 | describe("Server rendering", function () { 10 | var React; 11 | var App; 12 | var document; 13 | var home_url = "http://localhost/"; 14 | var home = { 15 | "@id": "/", 16 | "@type": ["Portal"], 17 | "portal_title": "SNOWFLAKES", 18 | "title": "Home" 19 | }; 20 | 21 | beforeEach(function () { 22 | require('../../libs/react-patches'); 23 | React = require('react'); 24 | App = require('..'); 25 | var server_app = ; 26 | var markup = '\n' + React.renderToString(server_app); 27 | var parser = new DOMParser(); 28 | document = parser.parseFromString(markup, 'text/html'); 29 | window.location.href = home_url; 30 | }); 31 | 32 | it("renders the application to html", function () { 33 | expect(document.title).toBe(home.portal_title); 34 | }); 35 | 36 | it("react render http-equiv correctly", function () { 37 | var meta_http_equiv = document.querySelectorAll('meta[http-equiv]'); 38 | expect(meta_http_equiv.length).not.toBe(0); 39 | }); 40 | 41 | it("mounts the application over the rendered html", function () { 42 | var props = App.getRenderedProps(document); 43 | var app = React.render(, document); 44 | expect(app.getDOMNode()).toBe(document.documentElement); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /src/snowflakes/static/components/blocks/fallback.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var React = require('react'); 3 | var globals = require('../globals'); 4 | var item = require('../item'); 5 | var noarg_memoize = require('../../libs/noarg-memoize'); 6 | 7 | var FallbackBlockView = React.createClass({ 8 | render: function() { 9 | var Panel = item.Panel; 10 | return ( 11 |
12 |

{this.props.blocktype.label}

13 | 14 |
15 | ); 16 | } 17 | }); 18 | 19 | var FallbackBlockEdit = module.exports.FallbackBlockEdit = React.createClass({ 20 | render: function() { 21 | var ReactForms = require('react-forms'); 22 | return ; 23 | } 24 | }); 25 | 26 | 27 | // Use this as a fallback for any block we haven't registered 28 | globals.blocks.fallback = function (obj) { 29 | return { 30 | label: obj['@type'].join(','), 31 | schema: noarg_memoize(function() { 32 | var JSONNode = require('../JSONNode'); 33 | return JSONNode.create({ 34 | label: 'JSON', 35 | input: