├── .github └── workflows │ └── build-docker.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── README.md ├── autoapp.py ├── build_docker.sh ├── db.sqlite3 ├── dev-requirements.txt ├── docker-base ├── Dockerfile ├── build.sh └── requirements.txt ├── docker ├── README.md ├── build.sh ├── deploy.sh └── supervisord.conf ├── dump_from_old_taiga.py ├── flask ├── install_prereqs.sh ├── migrations ├── README ├── alembic.ini ├── env.py ├── script.py.mako └── versions │ ├── 028f07e95137_.py │ ├── 1cde9ea7a48f_.py │ ├── 2af60fdf97d1_update_started_log.py │ ├── 2d1729557360_replace_figshare_oauth_w_personal_token.py │ ├── 3b8cc444f691_add_figshare_update_article.py │ ├── 42d37b47160c_.py │ ├── 6bf0d302380c_add_compressed_field.py │ ├── 6d8d4536fb51_add_activity_log.py │ ├── 90837b08919f_.py │ ├── 92a10cd2cd72_add_virt_datasets.py │ ├── 944a0b71b403_virt_datafile_reorg.py │ ├── 946c898fcded_add_gcs_pointer.py │ ├── 9552e419c662_.py │ ├── 9c0d7f0a89e8_.py │ ├── a286182fdf59_.py │ ├── adb36ec6d16c_.py │ ├── b1e146e20467_add_email_subscriptions.py │ ├── b2d95ce03b09_.py │ ├── c1be0bfabe2e_.py │ ├── c605b801f114_.py │ ├── c9d2700c78d4_add_activity_log2.py │ ├── d091fc45fa8d_test_permaname_update.py │ ├── ddeac9a40a1c_.py │ ├── e04e7eb4fb16_add_figshare_integration.py │ └── f8856168298f_.py ├── pytype.cfg ├── react_frontend ├── package.json ├── run_webpack ├── src │ ├── @types │ │ ├── README │ │ ├── other_modules.d.ts │ │ ├── react-bootstrap-table.d.ts │ │ └── react-dropzone.d.ts │ ├── components │ │ ├── DatasetView.tsx │ │ ├── Dialogs.tsx │ │ ├── FolderView.tsx │ │ ├── GroupListView.tsx │ │ ├── GroupView.tsx │ │ ├── InfoIcon.tsx │ │ ├── LeftNav.tsx │ │ ├── NotFound.tsx │ │ ├── RecentlyViewed.tsx │ │ ├── Search.tsx │ │ ├── SearchView.tsx │ │ ├── Token.tsx │ │ ├── UploadTracker.tsx │ │ ├── dataset_view │ │ │ └── FigshareSection.tsx │ │ └── modals │ │ │ ├── EntryUsersPermissions.tsx │ │ │ ├── FigshareToken.tsx │ │ │ ├── FigshareUploadStatusTable.tsx │ │ │ ├── FigshareWYSIWYGEditor.tsx │ │ │ ├── UpdateFigshare.tsx │ │ │ ├── UpdateFigshareArticleIdStep.tsx │ │ │ ├── UpdateFigsharePollResultsStep.tsx │ │ │ ├── UpdateFigshareUpdateArticleStep.tsx │ │ │ ├── UploadForm.tsx │ │ │ ├── UploadTable.fixture.tsx │ │ │ ├── UploadTable.tsx │ │ │ └── UploadToFigshare.tsx │ ├── index.tsx │ ├── models │ │ ├── api.ts │ │ ├── figshare.ts │ │ └── models.ts │ ├── styles │ │ └── modals │ │ │ └── uploadtofigshare.css │ ├── utilities │ │ ├── common.ts │ │ ├── figshare.ts │ │ ├── formats.tsx │ │ ├── loading.tsx │ │ ├── r-clipboard.tsx │ │ └── route.ts │ └── version.ts ├── tsconfig.json ├── webpack.config.js └── yarn.lock ├── recreate_dev_db.sh ├── requirements.txt ├── settings.cfg.sample ├── setup.py ├── setup_env.sh ├── taiga2 ├── __init__.py ├── api_app.py ├── auth.py ├── celery.py ├── celery_init.py ├── commands.py ├── conf.py ├── controllers │ ├── __init__.py │ ├── endpoint.py │ ├── endpoint_validation.py │ └── models_controller.py ├── conv │ ├── __init__.py │ ├── columnar.py │ ├── exp.py │ ├── imp.py │ ├── sniff.py │ └── util.py ├── create_test_db_sqlalchemy.py ├── dataset_subscriptions.py ├── default_settings.py ├── extensions.py ├── manage.py ├── models.py ├── schemas.py ├── static │ ├── ikons │ │ └── folder.svg │ └── taiga3.png ├── swagger │ └── swagger.yaml ├── tasks.py ├── templates │ └── index.html ├── tests │ ├── __init__.py │ ├── blocked_conv_test.py │ ├── bounded_mem_conv_test.py │ ├── conftest.py │ ├── datafile_test.py │ ├── exp_conv_test.py │ ├── factories.py │ ├── fast_schemas_test.py │ ├── imp_conv_test.py │ ├── mock_s3.py │ ├── mock_s3_test.py │ ├── security_test.py │ ├── test_endpoint.py │ ├── test_files │ │ ├── hello.txt │ │ ├── non-utf8-table.csv │ │ ├── npcv1.csv │ │ ├── tall_matrix.csv │ │ ├── tall_matrix.gct │ │ ├── tall_table.csv │ │ ├── tiny_matrix.csv │ │ └── tiny_table.csv │ ├── test_models_controller.py │ └── test_utils.py ├── third_party_clients │ ├── aws.py │ ├── figshare.py │ └── gcs.py ├── ui.py └── utils │ ├── exception_reporter.py │ ├── fix_description_dataset_versions.py │ ├── migrate_provenance_taiga_to_taiga2.py │ └── migrate_taiga_to_taiga2.py ├── travis ├── login_docker.sh ├── push_docker.sh └── travis-docker-push-account.json.enc └── write_version.sh /.github/workflows/build-docker.yaml: -------------------------------------------------------------------------------- 1 | name: Build Taiga image 2 | 3 | on: 4 | push: 5 | branches: main 6 | 7 | env: 8 | IMAGE_TAG: us.gcr.io/cds-docker-containers/taiga:ga-build-${{ github.run_number }} 9 | FINAL_IMAGE_TAG: us.gcr.io/cds-docker-containers/taiga:latest 10 | 11 | jobs: 12 | build-docker: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: write SHA to file 17 | run: "bash write_version.sh ${{ github.sha }}" 18 | - name: Login to GCR 19 | uses: docker/login-action@v2 20 | with: 21 | registry: us.gcr.io 22 | username: _json_key 23 | password: ${{ secrets.DEPLOY_SVC_ACCT }} 24 | - name: Build Docker image 25 | run: bash ./build_docker.sh ${{ env.IMAGE_TAG }} 26 | - name: Run tests 27 | run: "docker run ${{ env.IMAGE_TAG }} pytest" 28 | - name: Push Docker image 29 | run: docker push ${{ env.IMAGE_TAG }} 30 | - name: Tag as latest 31 | run: docker tag ${{ env.IMAGE_TAG }} ${{ env.FINAL_IMAGE_TAG }} 32 | - name: Push Docker image 33 | run: docker push ${{ env.FINAL_IMAGE_TAG }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | node_modules 4 | dist 5 | typings 6 | *~ 7 | .* 8 | !.pre-commit-config.yaml 9 | 10 | 11 | # PyCharm zsh terminal 12 | /pycharm_rc 13 | 14 | # Ignore the generated by taiga2/create_test_db.py test.json 15 | test.json 16 | 17 | # Ignore the folders generated by webpack 18 | build/ 19 | bin/ 20 | taiga2/static/* 21 | frontend/**/*.js 22 | 23 | # Ignore the configuration files 24 | settings.*cfg 25 | 26 | # Ignore the Redis snapshots binary 27 | dump.rdb 28 | 29 | # Ignore the dump generated by...R? 30 | dump 31 | 32 | # Ignore the test database 33 | taiga2.db 34 | 35 | # Ignore the test log 36 | pytestdebug.log 37 | 38 | # Ignore the JSX since we use TSX 39 | *.jsx 40 | 41 | # For docker, ignore the taiga.tar.gz that could be built with 'build.sh' 42 | docker/taiga.tar.gz 43 | 44 | # Don't ignore travis.yml 45 | !.travis.yml 46 | 47 | # Ignore the migration data of taiga 48 | taiga2/tests/test_files/migration/* 49 | 50 | # Ignore the provenance folder in test_files 51 | taiga2/tests/test_files/provenance/* 52 | 53 | # Ignore some files in test_files (because of the size) 54 | taiga2/tests/test_files/large_numerical_matrix.csv 55 | taiga2/tests/test_files/large_table.csv 56 | taiga2/db.sqlite3 57 | pytype_output 58 | prod_settings.cfg 59 | depmap/static/js/react_frontend.js 60 | depmap/static/js/react_frontend.js.map 61 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: "19.3b0" 4 | hooks: 5 | - id: black 6 | - repo: https://github.com/pre-commit/mirrors-prettier 7 | rev: "v2.2.1" 8 | hooks: 9 | - id: prettier 10 | exclude: '^.*\.(json|d\.ts)$' 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM us.gcr.io/cds-docker-containers/taiga-base:v2 2 | 3 | COPY taiga2 /install/taiga/taiga2 4 | COPY requirements.txt setup.py /install/taiga/ 5 | WORKDIR /install/taiga 6 | 7 | RUN pip install pip==21.1.2 8 | RUN pip install -r requirements.txt 9 | 10 | COPY react_frontend /install/taiga/react_frontend/ 11 | # Install frontend javascript dependencies 12 | RUN cd /install/taiga/react_frontend && yarn install 13 | 14 | RUN cd /install/taiga/react_frontend && ./node_modules/.bin/webpack 15 | 16 | COPY flask setup_env.sh autoapp.py /install/taiga/ 17 | 18 | # Set celery as being able to run as root => Can find a better way 19 | ENV C_FORCE_ROOT=true 20 | # Set where celery can find the settings 21 | ENV LC_ALL=C.UTF-8 22 | ENV LANG=C.UTF-8 23 | 24 | # Configure supervisor 25 | COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf 26 | 27 | EXPOSE 8080 28 | 29 | CMD ["/usr/bin/supervisord"] 30 | 31 | -------------------------------------------------------------------------------- /autoapp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Create an application instance.""" 3 | import os 4 | 5 | from taiga2.api_app import create_app 6 | from taiga2.ui import create_app as ui_create_app 7 | from taiga2.celery_init import configure_celery, celery 8 | 9 | from werkzeug.wsgi import DispatcherMiddleware 10 | 11 | settings_file = os.getenv("TAIGASETTINGSFILE", "settings.cfg") 12 | 13 | print("Using settings from: {}".format(settings_file)) 14 | 15 | # Init Api/Backend app 16 | api_app, flask_api_app = create_app(settings_file=settings_file) 17 | configure_celery(flask_api_app) 18 | debug = flask_api_app.config["DEBUG"] 19 | 20 | # Init frontend app 21 | # ui_create_app uses also default_settings.py 22 | app = ui_create_app(settings_file=settings_file) 23 | 24 | prefix = flask_api_app.config["PREFIX"] 25 | assert prefix.startswith("/") 26 | prefix_with_api = os.path.join(prefix, "api") 27 | 28 | 29 | def _no_content_response(env, resp): 30 | resp(b"200 OK", [(b"Content-Type", b"text/plain")]) 31 | return [ 32 | "This url has no handler. Instead, try going to {}".format(prefix).encode( 33 | "utf8" 34 | ) 35 | ] 36 | 37 | 38 | app.wsgi_app = DispatcherMiddleware( 39 | _no_content_response, 40 | {prefix: app.wsgi_app, prefix_with_api: flask_api_app.wsgi_app}, 41 | ) 42 | -------------------------------------------------------------------------------- /build_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -evx 3 | 4 | IMAGE_TAG="$1" 5 | 6 | # TODO: Need to see how to tar from travis/ 7 | #tar --exclude='dist' \ 8 | #--exclude='*/taiga.tar.gz' \ 9 | #--exclude='*.pyc' \ 10 | #--exclude='*~' \ 11 | #--exclude='.idea' --exclude='node_modules' --exclude='frontend/node_modules' --exclude='.git' -zcvf docker/taiga.tar.gz * 12 | docker build -t ${IMAGE_TAG} . 13 | 14 | # TODO: We are missing multiple steps: 15 | # - Retrieve the settings.cfg or create it in the VM? 16 | -------------------------------------------------------------------------------- /db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/db.sqlite3 -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | alembic==1.0.11 3 | factory-boy==2.9.2 4 | Flask-Script==2.0.5 5 | flask-shell-ipython==0.4.1 6 | pre-commit==1.17.0 7 | pytest==3.0.5 8 | -------------------------------------------------------------------------------- /docker-base/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM ubuntu:xenial-20170619 3 | 4 | RUN apt-get update -y 5 | RUN apt-get install -y \ 6 | software-properties-common && \ 7 | add-apt-repository ppa:deadsnakes/ppa && \ 8 | apt-get update -y 9 | 10 | # Install R dependencies 11 | RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 51716619E084DAB9 && \ 12 | echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list && \ 13 | apt-get update -y && \ 14 | apt-get -y install r-base r-base-dev 15 | RUN echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.us.r-project.org'; options(repos = r);" > ~/.Rprofile && \ 16 | Rscript -e "source(\"https://bioconductor.org/biocLite.R\"); biocLite(\"rhdf5\")" 17 | 18 | # Preparing for yarn install 19 | RUN apt-get install -y curl apt-transport-https build-essential && \ 20 | curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ 21 | echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ 22 | apt-get update -y 23 | 24 | # Install Nodejs/npm 25 | RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && \ 26 | bash nodesource_setup.sh && \ 27 | apt-get install -y nodejs && rm nodesource_setup.sh 28 | 29 | # Install vim 30 | # Install python and its related packages 31 | # Install postgresql minimal set binaries and headers 32 | RUN apt-get install -y vim python3.6 libpq-dev python3.6-dev python3.6-venv supervisor redis-server yarn 33 | 34 | RUN mkdir /install && python3.6 -m venv /install/python 35 | ENV PATH=/install/python/bin:$PATH 36 | 37 | # Install supervisor 38 | RUN mkdir -p /var/log/supervisor && mkdir -p /var/run 39 | 40 | # Install base versions of all python libs 41 | COPY requirements.txt /install/taiga/ 42 | RUN pip install pip==21.1.2 43 | RUN pip install -r /install/taiga/requirements.txt 44 | -------------------------------------------------------------------------------- /docker-base/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | docker build . -t us.gcr.io/cds-docker-containers/taiga-base:v2 && \ 5 | docker push us.gcr.io/cds-docker-containers/taiga-base:v2 6 | -------------------------------------------------------------------------------- /docker-base/requirements.txt: -------------------------------------------------------------------------------- 1 | --index-url https://pypi.python.org/simple/ 2 | boto3==1.4.2 3 | botocore==1.4.93 4 | celery==4.1.1 5 | certifi==2019.6.16 6 | Click==7.0 7 | clickclick==1.2.2 8 | openapi-spec-validator==0.2.9 9 | connexion[swagger-ui]==2.3.0 10 | docutils==0.14 11 | Flask==1.0.3 12 | flask-marshmallow==0.7.0 13 | Flask-Migrate==2.0.3 14 | Flask-SQLAlchemy==2.1 15 | google-cloud-error-reporting==1.1.0 16 | google-cloud-storage==1.35.0 17 | h5py==2.6.0 18 | humanize==0.5.1 19 | jsonpointer==2.0 20 | jsonschema==2.7.0 21 | kombu==4.2.1 22 | mailjet_rest==1.3.3 23 | marshmallow==2.15.1 24 | marshmallow-enum==1.0 25 | marshmallow-oneofschema==1.0.3 26 | marshmallow-sqlalchemy==0.12.1 27 | numpy==1.18.1 28 | psutil==5.6.7 29 | PyYAML==5.4.1 30 | redis==2.10.5 31 | requests==2.22.0 32 | SQLAlchemy==1.3.0 33 | swagger-spec-validator==2.4.3 34 | typing-extensions==3.7.4.3 35 | Werkzeug==0.15.4 36 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Deploying Taiga as a docker container 2 | 3 | - Run `build.sh` outside of taiga/ to tar the code and build the docker image 4 | - Run `deploy.sh` to push the image to Amazon Registry Container (be careful to change the url of the repository) 5 | - Go on your server to pull the image 6 | - Execute the command `docker run -p 8888:8080 -t taiga` 7 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ev 3 | tar --exclude='dist' --exclude='*/taiga.tar.gz' --exclude='.idea' --exclude='node_modules' --exclude='frontend/node_modules' --exclude='.git' -zcvf docker/taiga.tar.gz * 4 | docker build -t taiga:latest docker/ -------------------------------------------------------------------------------- /docker/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # IMPORTANT: This script should not be used anymore. Now use ansible-config 3 | set -ev 4 | docker_image=784167841278.dkr.ecr.us-east-1.amazonaws.com/taiga:latest 5 | docker_login=`aws ecr get-login --region us-east-1 --no-include-email` 6 | ${docker_login} 7 | #docker tag taiga:latest ${docker_image} 8 | #docker push ${docker_image} 9 | ssh ubuntu@cds.team "${docker_login} && docker pull ${docker_image} && sudo systemctl restart taiga" -------------------------------------------------------------------------------- /docker/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | logfile=/var/log/supervisor/supervisord.log 4 | pidfile=/var/run/supervisord.pid 5 | childlogdir=/var/log/supervisor 6 | loglevel=debug 7 | 8 | [program:taiga] 9 | command=./flask run --port 8080 -h 0.0.0.0 10 | directory=/install/taiga 11 | 12 | [program:redis] 13 | command=/usr/bin/redis-server 14 | directory=/install/taiga 15 | 16 | [program:celery] 17 | command=./flask run-worker 18 | directory=/install/taiga 19 | -------------------------------------------------------------------------------- /dump_from_old_taiga.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sqlalchemy import create_engine 3 | 4 | cache_engine = create_engine( 5 | "sqlite:///" + os.path.abspath("summary_cache.db"), echo=True 6 | ) 7 | cache_conn = cache_engine.connect() 8 | 9 | engine = create_engine("sqlite:///" + os.path.abspath("metadata.sqlite3"), echo=True) 10 | conn = engine.connect() 11 | 12 | 13 | def get_summary(filename): 14 | key = "/xchip/datasci/data/taiga/" + filename 15 | row = cache_conn.execute( 16 | "select summary from summary where path = ?", [key] 17 | ).fetchone() 18 | if row is None: 19 | print("could not find", key) 20 | return "unknown" 21 | return row[0] 22 | 23 | 24 | import csv 25 | 26 | w_ds = open("datasets.csv", "wt") 27 | w_ds_csv = csv.writer(w_ds) 28 | 29 | w_dv = open("dataset_versions.csv", "wt") 30 | w_dv_csv = csv.writer(w_dv) 31 | 32 | 33 | def get_version_desc(permaname, version): 34 | print("fetching", permaname, version) 35 | return conn.execute( 36 | "select v.description, u.email from data_version v join named_data nd on nd.named_data_id = v.named_data_id left outer join user u on u.user_id = v.created_by_user_id where nd.permaname = ? and v.version = ?", 37 | [permaname, version], 38 | ).fetchone() 39 | 40 | 41 | w_ds_csv.writerow(["name", "permaname", "description", "folder"]) 42 | w_dv_csv.writerow( 43 | [ 44 | "permaname", 45 | "id", 46 | "version", 47 | "type", 48 | "short_desc", 49 | "created_by", 50 | "created_timestamp", 51 | "s3_location", 52 | ] 53 | ) 54 | 55 | for name, permaname, is_public, latest_version in conn.execute( 56 | "select name, permaname, is_public, latest_version from named_data" 57 | ).fetchall(): 58 | if len(permaname) == 0: 59 | continue 60 | # , "Dataset {} has no permaname".format(name) 61 | 62 | description, created_by = get_version_desc(permaname, latest_version) 63 | if permaname.startswith("achilles"): 64 | folder = "achilles" 65 | elif is_public: 66 | if permaname.startswith("ccle"): 67 | folder = "ccle" 68 | else: 69 | folder = "public" 70 | else: 71 | if created_by is None and permaname.startswith("avana"): 72 | folder = "achilles" 73 | else: 74 | if created_by is None: 75 | created_by = "pmontgom@broadinstitute.org" 76 | folder = "home({})".format(created_by) 77 | 78 | w_ds_csv.writerow([name, permaname, description, folder]) 79 | 80 | dv_count = 0 81 | for ( 82 | dataset_id, 83 | version, 84 | hdf5_path, 85 | columnar_path, 86 | created_by, 87 | created_timestamp, 88 | ) in conn.execute( 89 | "select dataset_id, version, hdf5_path, columnar_path, u.email, created_timestamp from data_version v join named_data nd on nd.named_data_id = v.named_data_id left outer join user u on u.user_id = v.created_by_user_id where nd.permaname = ? order by v.version", 90 | [permaname], 91 | ).fetchall(): 92 | if hdf5_path is not None: 93 | df_type = "hdf5" 94 | filename = dataset_id + ".hdf5" 95 | else: 96 | df_type = "columnar" 97 | filename = dataset_id + ".columnar" 98 | short_desc = get_summary(filename) 99 | if created_by is None: 100 | created_by = "unowned-from-taiga1@broadinstitute.org" 101 | w_dv_csv.writerow( 102 | [ 103 | permaname, 104 | dataset_id, 105 | version, 106 | df_type, 107 | short_desc, 108 | created_by, 109 | created_timestamp, 110 | "s3://taiga2/migrated/" + filename, 111 | ] 112 | ) 113 | dv_count += 1 114 | assert dv_count > 0, "{} has no versions".format(permaname) 115 | 116 | w_ds.close() 117 | w_dv.close() 118 | -------------------------------------------------------------------------------- /flask: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source setup_env.sh 3 | exec flask "$@" 4 | -------------------------------------------------------------------------------- /install_prereqs.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | pip install -r dev-requirements.txt 3 | python setup.py develop 4 | pre-commit install 5 | yarn install --cwd react_frontend -------------------------------------------------------------------------------- /migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. 2 | 3 | Use `manage.py -c settings.cfg db migrate` to generate the migration from your new model (models.py) 4 | then use `manage.py -c settings.cfg db upgrade` to apply these changes to the database in your settings.cfg 5 | (SQLALCHEMY_DATABASE_URI) 6 | -------------------------------------------------------------------------------- /migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | sqlalchemy.url = sqlite:///taiga2.db 12 | 13 | # Logging configuration 14 | [loggers] 15 | keys = root,sqlalchemy,alembic 16 | 17 | [handlers] 18 | keys = console 19 | 20 | [formatters] 21 | keys = generic 22 | 23 | [logger_root] 24 | level = WARN 25 | handlers = console 26 | qualname = 27 | 28 | [logger_sqlalchemy] 29 | level = WARN 30 | handlers = 31 | qualname = sqlalchemy.engine 32 | 33 | [logger_alembic] 34 | level = INFO 35 | handlers = 36 | qualname = alembic 37 | 38 | [handler_console] 39 | class = StreamHandler 40 | args = (sys.stderr,) 41 | level = NOTSET 42 | formatter = generic 43 | 44 | [formatter_generic] 45 | format = %(levelname)-5.5s [%(name)s] %(message)s 46 | datefmt = %H:%M:%S 47 | -------------------------------------------------------------------------------- /migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from alembic import context 3 | from sqlalchemy import engine_from_config, pool 4 | from logging.config import fileConfig 5 | import logging 6 | 7 | # this is the Alembic Config object, which provides 8 | # access to the values within the .ini file in use. 9 | config = context.config 10 | 11 | # Interpret the config file for Python logging. 12 | # This line sets up loggers basically. 13 | fileConfig(config.config_file_name) 14 | logger = logging.getLogger("alembic.env") 15 | 16 | # add your model's MetaData object here 17 | # for 'autogenerate' support 18 | # from myapp import mymodel 19 | # target_metadata = mymodel.Base.metadata 20 | from flask import current_app 21 | 22 | config.set_main_option( 23 | "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") 24 | ) 25 | target_metadata = current_app.extensions["migrate"].db.metadata 26 | 27 | # other values from the config, defined by the needs of env.py, 28 | # can be acquired: 29 | # my_important_option = config.get_main_option("my_important_option") 30 | # ... etc. 31 | 32 | 33 | def run_migrations_offline(): 34 | """Run migrations in 'offline' mode. 35 | 36 | This configures the context with just a URL 37 | and not an Engine, though an Engine is acceptable 38 | here as well. By skipping the Engine creation 39 | we don't even need a DBAPI to be available. 40 | 41 | Calls to context.execute() here emit the given string to the 42 | script output. 43 | 44 | """ 45 | url = config.get_main_option("sqlalchemy.url") 46 | context.configure(url=url) 47 | 48 | with context.begin_transaction(): 49 | context.run_migrations() 50 | 51 | 52 | def run_migrations_online(): 53 | """Run migrations in 'online' mode. 54 | 55 | In this scenario we need to create an Engine 56 | and associate a connection with the context. 57 | 58 | """ 59 | 60 | # this callback is used to prevent an auto-migration from being generated 61 | # when there are no changes to the schema 62 | # reference: http://alembic.readthedocs.org/en/latest/cookbook.html 63 | def process_revision_directives(context, revision, directives): 64 | if getattr(config.cmd_opts, "autogenerate", False): 65 | script = directives[0] 66 | if script.upgrade_ops.is_empty(): 67 | directives[:] = [] 68 | logger.info("No changes in schema detected.") 69 | 70 | engine = engine_from_config( 71 | config.get_section(config.config_ini_section), 72 | prefix="sqlalchemy.", 73 | poolclass=pool.NullPool, 74 | ) 75 | 76 | connection = engine.connect() 77 | context.configure( 78 | connection=connection, 79 | target_metadata=target_metadata, 80 | process_revision_directives=process_revision_directives, 81 | **current_app.extensions["migrate"].configure_args 82 | ) 83 | 84 | try: 85 | with context.begin_transaction(): 86 | context.run_migrations() 87 | finally: 88 | connection.close() 89 | 90 | 91 | if context.is_offline_mode(): 92 | run_migrations_offline() 93 | else: 94 | run_migrations_online() 95 | -------------------------------------------------------------------------------- /migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /migrations/versions/028f07e95137_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 028f07e95137 4 | Revises: adb36ec6d16c 5 | Create Date: 2017-07-19 13:50:47.429002 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "028f07e95137" 14 | down_revision = "adb36ec6d16c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_unique_constraint( 22 | op.f("uq_dataset_versions_dataset_id"), 23 | "dataset_versions", 24 | ["dataset_id", "version"], 25 | ) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_constraint( 32 | op.f("uq_dataset_versions_dataset_id"), "dataset_versions", type_="unique" 33 | ) 34 | # ### end Alembic commands ### 35 | -------------------------------------------------------------------------------- /migrations/versions/1cde9ea7a48f_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 1cde9ea7a48f 4 | Revises: 5 | Create Date: 2017-04-07 13:37:50.512285 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "1cde9ea7a48f" 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "user_logs", 23 | sa.Column("id", sa.String(length=80), nullable=False), 24 | sa.Column("user_id", sa.String(length=80), nullable=True), 25 | sa.Column("dataset_id", sa.String(length=80), nullable=True), 26 | sa.Column("last_access", sa.DateTime(), nullable=True), 27 | sa.ForeignKeyConstraint( 28 | ["dataset_id"], 29 | ["datasets.id"], 30 | name=op.f("fk_user_logs_dataset_id_datasets"), 31 | ), 32 | sa.ForeignKeyConstraint( 33 | ["user_id"], ["users.id"], name=op.f("fk_user_logs_user_id_users") 34 | ), 35 | sa.PrimaryKeyConstraint("id", name=op.f("pk_user_logs")), 36 | ) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade(): 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | op.drop_table("user_logs") 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /migrations/versions/2af60fdf97d1_update_started_log.py: -------------------------------------------------------------------------------- 1 | """update-started-log 2 | 3 | Revision ID: 2af60fdf97d1 4 | Revises: c9d2700c78d4 5 | Create Date: 2019-07-18 16:50:09.398289 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "2af60fdf97d1" 14 | down_revision = "c9d2700c78d4" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | # ### end Alembic commands ### 22 | op.execute("delete from activities") 23 | op.execute( 24 | """ 25 | INSERT into activities (id, user_id, dataset_id, type, dataset_name, dataset_description, dataset_version, timestamp) 26 | SELECT uuid_in(md5(random()::text || clock_timestamp()::text)::cstring), u.id, v.dataset_id, 'started_log', de.name, ve.description, v.version, CURRENT_TIMESTAMP 27 | FROM dataset_versions v 28 | JOIN datasets d ON v.dataset_id = d.id 29 | JOIN entries ve ON ve.id = v.id 30 | JOIN entries de on de.id = d.id, 31 | users u 32 | WHERE u.name = 'admin' 33 | """ 34 | ) 35 | op.execute("update entries set description = null where type = 'Dataset'") 36 | 37 | 38 | def downgrade(): 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | # ### end Alembic commands ### 41 | op.execute("delete from activities") 42 | -------------------------------------------------------------------------------- /migrations/versions/2d1729557360_replace_figshare_oauth_w_personal_token.py: -------------------------------------------------------------------------------- 1 | """replace figshare oauth w/ personal token 2 | 3 | Revision ID: 2d1729557360 4 | Revises: b1e146e20467 5 | Create Date: 2021-01-05 11:39:13.077740 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "2d1729557360" 14 | down_revision = "b1e146e20467" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_table("figshare_authorizations") 22 | op.add_column( 23 | "users", 24 | sa.Column("figshare_personal_token", sa.String(length=128), nullable=True), 25 | ) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_column("users", "figshare_personal_token") 32 | op.create_table( 33 | "figshare_authorizations", 34 | sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=False), 35 | sa.Column("user_id", sa.VARCHAR(length=80), autoincrement=False, nullable=True), 36 | sa.Column( 37 | "figshare_account_id", sa.INTEGER(), autoincrement=False, nullable=True 38 | ), 39 | sa.Column("token", sa.TEXT(), autoincrement=False, nullable=False), 40 | sa.Column("refresh_token", sa.TEXT(), autoincrement=False, nullable=False), 41 | sa.ForeignKeyConstraint( 42 | ["user_id"], ["users.id"], name="fk_figshare_authorizations_user_id_users" 43 | ), 44 | sa.PrimaryKeyConstraint("id", name="pk_figshare_authorizations"), 45 | ) 46 | # ### end Alembic commands ### 47 | -------------------------------------------------------------------------------- /migrations/versions/3b8cc444f691_add_figshare_update_article.py: -------------------------------------------------------------------------------- 1 | """add figshare update article 2 | 3 | Revision ID: 3b8cc444f691 4 | Revises: e04e7eb4fb16 5 | Create Date: 2020-04-29 11:56:21.180582 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "3b8cc444f691" 14 | down_revision = "e04e7eb4fb16" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column( 22 | "figshare_dataset_version_links", 23 | sa.Column("figshare_article_version", sa.Integer(), nullable=False), 24 | ) 25 | # ### end Alembic commands ### 26 | 27 | # There are currently no entries for this table, but if there were, we only had 28 | # article creation up until this point. 29 | op.execute("update figshare_dataset_version_links set figshare_article_version = 1") 30 | 31 | 32 | def downgrade(): 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column("figshare_dataset_version_links", "figshare_article_version") 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /migrations/versions/42d37b47160c_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 42d37b47160c 4 | Revises: 9c0d7f0a89e8 5 | Create Date: 2018-09-12 14:04:00.510194 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "42d37b47160c" 15 | down_revision = "9c0d7f0a89e8" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | dataset_version_state = postgresql.ENUM( 22 | "approved", "deprecated", "deleted", name="datasetversionstate" 23 | ) 24 | dataset_version_state.create(op.get_bind()) 25 | 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.add_column( 28 | "dataset_versions", sa.Column("reason_state", sa.Text(), nullable=True) 29 | ) 30 | op.add_column( 31 | "dataset_versions", 32 | sa.Column( 33 | "state", 34 | sa.Enum("approved", "deprecated", "deleted", name="datasetversionstate"), 35 | nullable=True, 36 | ), 37 | ) 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade(): 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.drop_column("dataset_versions", "state") 44 | op.drop_column("dataset_versions", "reason_state") 45 | # ### end Alembic commands ### 46 | dataset_version_state = postgresql.ENUM( 47 | "approved", "deprecated", "deleted", name="datasetversionstate" 48 | ) 49 | dataset_version_state.drop(op.get_bind()) 50 | -------------------------------------------------------------------------------- /migrations/versions/6bf0d302380c_add_compressed_field.py: -------------------------------------------------------------------------------- 1 | """add-compressed-field 2 | 3 | Revision ID: 6bf0d302380c 4 | Revises: 946c898fcded 5 | Create Date: 2020-02-12 10:58:23.610440 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "6bf0d302380c" 14 | down_revision = "946c898fcded" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column( 22 | "datafiles", sa.Column("column_types_as_json", sa.JSON(), nullable=True) 23 | ) 24 | op.add_column("datafiles", sa.Column("compressed_s3_key", sa.Text(), nullable=True)) 25 | op.add_column("datafiles", sa.Column("encoding", sa.Text(), nullable=True)) 26 | op.add_column("datafiles", sa.Column("original_file_md5", sa.Text(), nullable=True)) 27 | op.add_column( 28 | "upload_session_files", 29 | sa.Column("column_types_as_json", sa.JSON(), nullable=True), 30 | ) 31 | op.add_column( 32 | "upload_session_files", sa.Column("compressed_s3_key", sa.Text(), nullable=True) 33 | ) 34 | op.add_column( 35 | "upload_session_files", sa.Column("encoding", sa.Text(), nullable=True) 36 | ) 37 | op.add_column( 38 | "upload_session_files", sa.Column("original_file_md5", sa.Text(), nullable=True) 39 | ) 40 | # ### end Alembic commands ### 41 | 42 | 43 | def downgrade(): 44 | # ### commands auto generated by Alembic - please adjust! ### 45 | op.drop_column("upload_session_files", "original_file_md5") 46 | op.drop_column("upload_session_files", "encoding") 47 | op.drop_column("upload_session_files", "compressed_s3_key") 48 | op.drop_column("upload_session_files", "column_types_as_json") 49 | op.drop_column("datafiles", "original_file_md5") 50 | op.drop_column("datafiles", "encoding") 51 | op.drop_column("datafiles", "compressed_s3_key") 52 | op.drop_column("datafiles", "column_types_as_json") 53 | # ### end Alembic commands ### 54 | -------------------------------------------------------------------------------- /migrations/versions/6d8d4536fb51_add_activity_log.py: -------------------------------------------------------------------------------- 1 | """add-activity-log 2 | 3 | Revision ID: 6d8d4536fb51 4 | Revises: 944a0b71b403 5 | Create Date: 2019-07-17 16:35:31.967083 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "6d8d4536fb51" 14 | down_revision = "944a0b71b403" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_table("virtual_dataset_entries") 22 | op.drop_table("virtual_dataset_versions") 23 | op.drop_table("virtual_datasets") 24 | op.add_column( 25 | "activities", sa.Column("dataset_description", sa.Text(), nullable=True) 26 | ) 27 | op.add_column("activities", sa.Column("dataset_name", sa.Text(), nullable=True)) 28 | op.add_column( 29 | "activities", sa.Column("dataset_version", sa.Integer(), nullable=True) 30 | ) 31 | op.add_column("activities", sa.Column("timestamp", sa.DateTime(), nullable=True)) 32 | op.alter_column( 33 | "datafiles", 34 | "type", 35 | existing_type=postgresql.ENUM("s3", "virtual", name="datafiletype2"), 36 | nullable=True, 37 | ) 38 | # ### end Alembic commands ### 39 | op.execute("alter type activitytype rename to _activitytype") 40 | op.execute( 41 | "create type activitytype as enum ('created', 'changed_name', 'changed_description', 'added_version', 'started_log')" 42 | ) 43 | op.execute("alter table activities drop column type") 44 | op.execute("alter table activities add column type activitytype not null") 45 | op.execute( 46 | "update users set name = 'admin', email = 'no-reply@broadinstitute.org' where email = 'admin@broadinstitute.org'" 47 | ) 48 | op.execute( 49 | """ 50 | INSERT into activities (id, user_id, dataset_id, type, dataset_name, dataset_description) 51 | SELECT uuid_in(md5(random()::text || clock_timestamp()::text)::cstring), u.id, d.id, 'started_log', e.name, e.description 52 | FROM datasets d 53 | JOIN entries e ON e.id = d.id, 54 | users u 55 | WHERE u.name = 'admin' 56 | """ 57 | ) 58 | 59 | 60 | def downgrade(): 61 | # ### commands auto generated by Alembic - please adjust! ### 62 | op.alter_column( 63 | "datafiles", 64 | "type", 65 | existing_type=postgresql.ENUM("s3", "virtual", name="datafiletype2"), 66 | nullable=False, 67 | ) 68 | op.drop_column("activities", "timestamp") 69 | op.drop_column("activities", "dataset_version") 70 | op.drop_column("activities", "dataset_name") 71 | op.drop_column("activities", "dataset_description") 72 | op.create_table( 73 | "virtual_dataset_versions", 74 | sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=False), 75 | sa.Column( 76 | "virtual_dataset_id", 77 | sa.VARCHAR(length=80), 78 | autoincrement=False, 79 | nullable=True, 80 | ), 81 | sa.Column("version", sa.INTEGER(), autoincrement=False, nullable=True), 82 | sa.Column( 83 | "state", 84 | postgresql.ENUM( 85 | "approved", "deprecated", "deleted", name="v_datasetversionstate" 86 | ), 87 | autoincrement=False, 88 | nullable=True, 89 | ), 90 | sa.Column("reason_state", sa.TEXT(), autoincrement=False, nullable=True), 91 | sa.ForeignKeyConstraint( 92 | ["id"], ["entries.id"], name="fk_virtual_dataset_versions_id_entries" 93 | ), 94 | sa.ForeignKeyConstraint( 95 | ["virtual_dataset_id"], 96 | ["virtual_datasets.id"], 97 | name="fk_virtual_dataset_versions_virtual_dataset_id_virtual_datasets", 98 | ), 99 | sa.PrimaryKeyConstraint("id", name="pk_virtual_dataset_versions"), 100 | sa.UniqueConstraint( 101 | "virtual_dataset_id", 102 | "version", 103 | name="uq_virtual_dataset_versions_virtual_dataset_id", 104 | ), 105 | postgresql_ignore_search_path=False, 106 | ) 107 | op.create_table( 108 | "virtual_datasets", 109 | sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=False), 110 | sa.Column("permaname", sa.TEXT(), autoincrement=False, nullable=True), 111 | sa.ForeignKeyConstraint( 112 | ["id"], ["entries.id"], name="fk_virtual_datasets_id_entries" 113 | ), 114 | sa.PrimaryKeyConstraint("id", name="pk_virtual_datasets"), 115 | postgresql_ignore_search_path=False, 116 | ) 117 | op.create_table( 118 | "virtual_dataset_entries", 119 | sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=False), 120 | sa.Column("name", sa.VARCHAR(length=80), autoincrement=False, nullable=False), 121 | sa.Column( 122 | "virtual_dataset_version_id", 123 | sa.VARCHAR(length=80), 124 | autoincrement=False, 125 | nullable=True, 126 | ), 127 | sa.Column( 128 | "data_file_id", sa.VARCHAR(length=80), autoincrement=False, nullable=False 129 | ), 130 | sa.ForeignKeyConstraint( 131 | ["data_file_id"], 132 | ["datafiles.id"], 133 | name="fk_virtual_dataset_entries_data_file_id_datafiles", 134 | ), 135 | sa.ForeignKeyConstraint( 136 | ["virtual_dataset_version_id"], 137 | ["virtual_dataset_versions.id"], 138 | name="fk_virtual_dataset_entries_virtual_dataset_version_id_virtual_d", 139 | ), 140 | sa.PrimaryKeyConstraint("id", name="pk_virtual_dataset_entries"), 141 | sa.UniqueConstraint( 142 | "virtual_dataset_version_id", 143 | "name", 144 | name="uq_virtual_dataset_entries_virtual_dataset_version_id", 145 | ), 146 | ) 147 | # ### end Alembic commands ### 148 | -------------------------------------------------------------------------------- /migrations/versions/90837b08919f_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 90837b08919f 4 | Revises: 2d1729557360 5 | Create Date: 2022-06-02 11:46:11.207267 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "90837b08919f" 14 | down_revision = "2d1729557360" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "lock_table", 23 | sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), 24 | sa.Column("random", sa.Integer(), nullable=True), 25 | sa.PrimaryKeyConstraint("id", name=op.f("pk_lock_table")), 26 | ) 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.drop_table("lock_table") 33 | # ### end Alembic commands ### 34 | -------------------------------------------------------------------------------- /migrations/versions/92a10cd2cd72_add_virt_datasets.py: -------------------------------------------------------------------------------- 1 | """add-virt-datasets 2 | 3 | Revision ID: 92a10cd2cd72 4 | Revises: 42d37b47160c 5 | Create Date: 2019-04-01 09:00:55.366722 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "92a10cd2cd72" 14 | down_revision = "42d37b47160c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "virtual_datasets", 23 | sa.Column("id", sa.String(length=80), nullable=False), 24 | sa.Column("permaname", sa.Text(), nullable=True), 25 | sa.ForeignKeyConstraint( 26 | ["id"], ["entries.id"], name=op.f("fk_virtual_datasets_id_entries") 27 | ), 28 | sa.PrimaryKeyConstraint("id", name=op.f("pk_virtual_datasets")), 29 | ) 30 | op.create_table( 31 | "virtual_dataset_versions", 32 | sa.Column("id", sa.String(length=80), nullable=False), 33 | sa.Column("virtual_dataset_id", sa.String(length=80), nullable=True), 34 | sa.Column("version", sa.Integer(), nullable=True), 35 | sa.Column( 36 | "state", 37 | sa.Enum("approved", "deprecated", "deleted", name="v_datasetversionstate"), 38 | nullable=True, 39 | ), 40 | sa.Column("reason_state", sa.Text(), nullable=True), 41 | sa.ForeignKeyConstraint( 42 | ["id"], ["entries.id"], name=op.f("fk_virtual_dataset_versions_id_entries") 43 | ), 44 | sa.ForeignKeyConstraint( 45 | ["virtual_dataset_id"], 46 | ["virtual_datasets.id"], 47 | name=op.f( 48 | "fk_virtual_dataset_versions_virtual_dataset_id_virtual_datasets" 49 | ), 50 | ), 51 | sa.PrimaryKeyConstraint("id", name=op.f("pk_virtual_dataset_versions")), 52 | sa.UniqueConstraint( 53 | "virtual_dataset_id", 54 | "version", 55 | name=op.f("uq_virtual_dataset_versions_virtual_dataset_id"), 56 | ), 57 | ) 58 | op.create_table( 59 | "virtual_dataset_entries", 60 | sa.Column("id", sa.String(length=80), nullable=False), 61 | sa.Column("name", sa.String(length=80), nullable=False), 62 | sa.Column("virtual_dataset_version_id", sa.String(length=80), nullable=True), 63 | sa.Column("data_file_id", sa.String(length=80), nullable=False), 64 | sa.ForeignKeyConstraint( 65 | ["data_file_id"], 66 | ["datafiles.id"], 67 | name=op.f("fk_virtual_dataset_entries_data_file_id_datafiles"), 68 | ), 69 | sa.ForeignKeyConstraint( 70 | ["virtual_dataset_version_id"], 71 | ["virtual_dataset_versions.id"], 72 | name=op.f( 73 | "fk_virtual_dataset_entries_virtual_dataset_version_id_virtual_dataset_versions" 74 | ), 75 | ), 76 | sa.PrimaryKeyConstraint("id", name=op.f("pk_virtual_dataset_entries")), 77 | sa.UniqueConstraint( 78 | "virtual_dataset_version_id", 79 | "name", 80 | name=op.f("uq_virtual_dataset_entries_virtual_dataset_version_id"), 81 | ), 82 | ) 83 | # ### end Alembic commands ### 84 | 85 | 86 | def downgrade(): 87 | # ### commands auto generated by Alembic - please adjust! ### 88 | op.drop_table("virtual_datasets") 89 | op.drop_table("virtual_dataset_versions") 90 | op.drop_table("virtual_dataset_entries") 91 | # ### end Alembic commands ### 92 | -------------------------------------------------------------------------------- /migrations/versions/944a0b71b403_virt_datafile_reorg.py: -------------------------------------------------------------------------------- 1 | """virt-datafile-reorg 2 | 3 | Revision ID: 944a0b71b403 4 | Revises: 92a10cd2cd72 5 | Create Date: 2019-07-05 09:49:25.759335 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "944a0b71b403" 14 | down_revision = "92a10cd2cd72" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | datafiletype2 = sa.Enum("s3", "virtual", name="datafiletype2") 19 | datafileformat = sa.Enum("Raw", "HDF5", "Columnar", name="datafileformat") 20 | 21 | 22 | def upgrade(): 23 | op.execute( 24 | "insert into datasets (id, permaname) select id, permaname from virtual_datasets" 25 | ) 26 | op.execute( 27 | "INSERT INTO dataset_versions (id, dataset_id, version, reason_state, state) SELECT id, virtual_dataset_id, version, reason_state, CAST(CAST( state AS VARCHAR(20) ) AS datasetversionstate) FROM virtual_dataset_versions" 28 | ) 29 | op.execute("UPDATE entries SET type = 'Dataset' WHERE type = 'VirtualDataset'") 30 | op.execute( 31 | "UPDATE entries SET type = 'DatasetVersion' WHERE type = 'VirtualDatasetVersion'" 32 | ) 33 | datafileformat.create(op.get_bind(), checkfirst=False) 34 | op.add_column("datafiles", sa.Column("format", datafileformat, nullable=True)) 35 | op.execute( 36 | 'update datafiles set format = CAST ( CAST ( "type" as VARCHAR(80) ) as datafileformat) ' 37 | ) 38 | op.drop_column("datafiles", "type") 39 | datafiletype2.create(op.get_bind(), checkfirst=False) 40 | op.add_column("datafiles", sa.Column("type", datafiletype2, nullable=True)) 41 | op.execute("update datafiles set \"type\" = 's3'") 42 | op.alter_column("datafiles", "type", nullable=False) 43 | op.add_column( 44 | "datafiles", sa.Column("original_file_sha256", sa.Text(), nullable=True) 45 | ) 46 | op.add_column( 47 | "datafiles", 48 | sa.Column("underlying_data_file_id", sa.String(length=80), nullable=True), 49 | ) 50 | op.execute( 51 | "insert into datafiles (id, name, type, dataset_version_id, underlying_data_file_id) select id, name, 'virtual', virtual_dataset_version_id, data_file_id from virtual_dataset_entries" 52 | ) 53 | op.create_foreign_key( 54 | op.f("fk_datafiles_underlying_data_file_id_datafiles"), 55 | "datafiles", 56 | "datafiles", 57 | ["underlying_data_file_id"], 58 | ["id"], 59 | ) 60 | 61 | op.add_column( 62 | "upload_session_files", 63 | sa.Column("data_file_id", sa.String(length=80), nullable=True), 64 | ) 65 | op.add_column( 66 | "upload_session_files", 67 | sa.Column("original_file_sha256", sa.Text(), nullable=True), 68 | ) 69 | op.create_foreign_key( 70 | op.f("fk_upload_session_files_data_file_id_datafiles"), 71 | "upload_session_files", 72 | "datafiles", 73 | ["data_file_id"], 74 | ["id"], 75 | ) 76 | 77 | # ### end Alembic commands ### 78 | 79 | 80 | def downgrade(): 81 | raise Exception("unimplemented") 82 | # ### end Alembic commands ### 83 | -------------------------------------------------------------------------------- /migrations/versions/946c898fcded_add_gcs_pointer.py: -------------------------------------------------------------------------------- 1 | """add-gcs-pointer 2 | 3 | Revision ID: 946c898fcded 4 | Revises: d091fc45fa8d 5 | Create Date: 2019-08-13 13:25:26.636789 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "946c898fcded" 14 | down_revision = "d091fc45fa8d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column("datafiles", sa.Column("gcs_path", sa.Text(), nullable=True)) 22 | op.add_column("datafiles", sa.Column("generation_id", sa.Text(), nullable=True)) 23 | op.add_column( 24 | "dataset_versions", sa.Column("changes_description", sa.Text(), nullable=True) 25 | ) 26 | op.add_column( 27 | "upload_session_files", sa.Column("gcs_path", sa.Text(), nullable=True) 28 | ) 29 | op.add_column( 30 | "upload_session_files", sa.Column("generation_id", sa.Text(), nullable=True) 31 | ) 32 | # ### end Alembic commands ### 33 | op.execute("ALTER TABLE datafiles ALTER COLUMN type TYPE VARCHAR(20)") 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_column("upload_session_files", "generation_id") 39 | op.drop_column("upload_session_files", "gcs_path") 40 | op.drop_column("dataset_versions", "changes_description") 41 | op.drop_column("datafiles", "generation_id") 42 | op.drop_column("datafiles", "gcs_path") 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /migrations/versions/9552e419c662_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 9552e419c662 4 | Revises: 1cde9ea7a48f 5 | Create Date: 2017-04-20 16:23:31.868786 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "9552e419c662" 14 | down_revision = "1cde9ea7a48f" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.create_table( 21 | "provenance_graphs", 22 | sa.Column("graph_id", sa.String(length=80), nullable=False), 23 | sa.Column("permaname", sa.String(length=80), nullable=True), 24 | sa.Column("name", sa.Text(), nullable=True), 25 | sa.Column("created_by_user_id", sa.String(length=80), nullable=True), 26 | sa.Column("created_timestamp", sa.DateTime(), nullable=True), 27 | sa.ForeignKeyConstraint( 28 | ["created_by_user_id"], 29 | ["users.id"], 30 | name=op.f("fk_provenance_graphs_created_by_user_id_users"), 31 | ), 32 | sa.PrimaryKeyConstraint("graph_id", name=op.f("pk_provenance_graphs")), 33 | sa.UniqueConstraint("permaname", name=op.f("uq_provenance_graphs_permaname")), 34 | ) 35 | op.create_table( 36 | "provenance_nodes", 37 | sa.Column("node_id", sa.String(length=80), nullable=False), 38 | sa.Column("graph_id", sa.String(length=80), nullable=True), 39 | sa.Column("dataset_version_id", sa.String(length=80), nullable=True), 40 | sa.Column("label", sa.Text(), nullable=True), 41 | sa.Column( 42 | "type", 43 | sa.Enum("Dataset", "External", "Process", name="nodetype"), 44 | nullable=True, 45 | ), 46 | sa.ForeignKeyConstraint( 47 | ["dataset_version_id"], 48 | ["dataset_versions.id"], 49 | name=op.f("fk_provenance_nodes_dataset_version_id_dataset_versions"), 50 | ), 51 | sa.ForeignKeyConstraint( 52 | ["graph_id"], 53 | ["provenance_graphs.graph_id"], 54 | name=op.f("fk_provenance_nodes_graph_id_provenance_graphs"), 55 | ), 56 | sa.PrimaryKeyConstraint("node_id", name=op.f("pk_provenance_nodes")), 57 | ) 58 | # ### commands auto generated by Alembic - please adjust! ### 59 | op.create_table( 60 | "provenance_edges", 61 | sa.Column("edge_id", sa.String(length=80), nullable=False), 62 | sa.Column("from_node_id", sa.String(length=80), nullable=True), 63 | sa.Column("to_node_id", sa.String(length=80), nullable=True), 64 | sa.Column("label", sa.Text(), nullable=True), 65 | sa.ForeignKeyConstraint( 66 | ["from_node_id"], 67 | ["provenance_nodes.node_id"], 68 | name=op.f("fk_provenance_edges_from_node_id_provenance_nodes"), 69 | ), 70 | sa.ForeignKeyConstraint( 71 | ["to_node_id"], 72 | ["provenance_nodes.node_id"], 73 | name=op.f("fk_provenance_edges_to_node_id_provenance_nodes"), 74 | ), 75 | sa.PrimaryKeyConstraint("edge_id", name=op.f("pk_provenance_edges")), 76 | ) 77 | # ### end Alembic commands ### 78 | 79 | 80 | def downgrade(): 81 | # ### commands auto generated by Alembic - please adjust! ### 82 | op.drop_table("provenance_edges") 83 | op.drop_table("provenance_nodes") 84 | op.drop_table("provenance_graphs") 85 | # ### end Alembic commands ### 86 | -------------------------------------------------------------------------------- /migrations/versions/9c0d7f0a89e8_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 9c0d7f0a89e8 4 | Revises: f8856168298f 5 | Create Date: 2018-01-11 13:35:53.101336 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "9c0d7f0a89e8" 14 | down_revision = "f8856168298f" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "groups", 23 | sa.Column("id", sa.INTEGER(), nullable=False), 24 | sa.Column("name", sa.String(length=80), nullable=True), 25 | sa.PrimaryKeyConstraint("id", name=op.f("pk_groups")), 26 | ) 27 | 28 | op.create_table( 29 | "group_user_association", 30 | sa.Column("group_id", sa.INTEGER(), nullable=True), 31 | sa.Column("user_id", sa.String(length=80), nullable=True), 32 | sa.ForeignKeyConstraint( 33 | ["group_id"], 34 | ["groups.id"], 35 | name=op.f("fk_group_user_association_group_id_groups"), 36 | ), 37 | sa.ForeignKeyConstraint( 38 | ["user_id"], 39 | ["users.id"], 40 | name=op.f("fk_group_user_association_user_id_users"), 41 | ), 42 | ) 43 | # ### end Alembic commands ### 44 | 45 | 46 | def downgrade(): 47 | # ### commands auto generated by Alembic - please adjust! ### 48 | op.drop_table("group_user_association") 49 | op.drop_table("groups") 50 | # ### end Alembic commands ### 51 | -------------------------------------------------------------------------------- /migrations/versions/a286182fdf59_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: a286182fdf59 4 | Revises: c605b801f114 5 | Create Date: 2022-06-06 14:28:10.165108 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "a286182fdf59" 14 | down_revision = "c605b801f114" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.execute("INSERT INTO lock_table VALUES (0, 50)") 22 | op.create_table( 23 | "read_access_log", 24 | sa.Column("datafile_id", sa.String(), nullable=False), 25 | sa.Column("user_id", sa.String(), nullable=False), 26 | sa.Column("first_access", sa.DateTime(), nullable=True), 27 | sa.Column("last_access", sa.DateTime(), nullable=True), 28 | sa.Column("access_count", sa.Integer(), nullable=True), 29 | sa.PrimaryKeyConstraint( 30 | "datafile_id", "user_id", name=op.f("pk_read_access_log") 31 | ), 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade(): 37 | # ### commands auto generated by Alembic - please adjust! ### 38 | op.drop_table("read_access_log") 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /migrations/versions/adb36ec6d16c_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: adb36ec6d16c 4 | Revises: b2d95ce03b09 5 | Create Date: 2017-05-18 19:30:34.414632 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "adb36ec6d16c" 14 | down_revision = "b2d95ce03b09" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column( 22 | "user_logs", sa.Column("entry_id", sa.String(length=80), nullable=True) 23 | ) 24 | op.drop_constraint( 25 | "fk_user_logs_dataset_id_datasets", "user_logs", type_="foreignkey" 26 | ) 27 | op.create_foreign_key( 28 | op.f("fk_user_logs_entry_id_entries"), 29 | "user_logs", 30 | "entries", 31 | ["entry_id"], 32 | ["id"], 33 | ) 34 | op.drop_column("user_logs", "dataset_id") 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade(): 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.add_column( 41 | "user_logs", 42 | sa.Column( 43 | "dataset_id", sa.VARCHAR(length=80), autoincrement=False, nullable=True 44 | ), 45 | ) 46 | op.drop_constraint( 47 | op.f("fk_user_logs_entry_id_entries"), "user_logs", type_="foreignkey" 48 | ) 49 | op.create_foreign_key( 50 | "fk_user_logs_dataset_id_datasets", 51 | "user_logs", 52 | "datasets", 53 | ["dataset_id"], 54 | ["id"], 55 | ) 56 | op.drop_column("user_logs", "entry_id") 57 | # ### end Alembic commands ### 58 | -------------------------------------------------------------------------------- /migrations/versions/b1e146e20467_add_email_subscriptions.py: -------------------------------------------------------------------------------- 1 | """add email subscriptions 2 | 3 | Revision ID: b1e146e20467 4 | Revises: 3b8cc444f691 5 | Create Date: 2020-05-01 09:15:39.951412 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "b1e146e20467" 14 | down_revision = "3b8cc444f691" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "dataset_subscriptions", 23 | sa.Column("id", sa.String(length=80), nullable=False), 24 | sa.Column("user_id", sa.String(length=80), nullable=True), 25 | sa.Column("dataset_id", sa.String(length=80), nullable=True), 26 | sa.ForeignKeyConstraint( 27 | ["dataset_id"], 28 | ["datasets.id"], 29 | name=op.f("fk_dataset_subscriptions_dataset_id_datasets"), 30 | ), 31 | sa.ForeignKeyConstraint( 32 | ["user_id"], 33 | ["users.id"], 34 | name=op.f("fk_dataset_subscriptions_user_id_users"), 35 | ), 36 | sa.PrimaryKeyConstraint("id", name=op.f("pk_dataset_subscriptions")), 37 | ) 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade(): 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.drop_table("dataset_subscriptions") 44 | # ### end Alembic commands ### 45 | -------------------------------------------------------------------------------- /migrations/versions/b2d95ce03b09_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: b2d95ce03b09 4 | Revises: 9552e419c662 5 | Create Date: 2017-04-25 11:54:02.198800 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "b2d95ce03b09" 14 | down_revision = "9552e419c662" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column( 22 | "provenance_nodes", 23 | sa.Column("datafile_id", sa.String(length=80), nullable=True), 24 | ) 25 | op.drop_constraint( 26 | "fk_provenance_nodes_dataset_version_id_dataset_versions", 27 | "provenance_nodes", 28 | type_="foreignkey", 29 | ) 30 | op.create_foreign_key( 31 | op.f("fk_provenance_nodes_datafile_id_datafiles"), 32 | "provenance_nodes", 33 | "datafiles", 34 | ["datafile_id"], 35 | ["id"], 36 | ) 37 | op.drop_column("provenance_nodes", "dataset_version_id") 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade(): 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.add_column( 44 | "provenance_nodes", 45 | sa.Column( 46 | "dataset_version_id", 47 | sa.VARCHAR(length=80), 48 | autoincrement=False, 49 | nullable=True, 50 | ), 51 | ) 52 | op.drop_constraint( 53 | op.f("fk_provenance_nodes_datafile_id_datafiles"), 54 | "provenance_nodes", 55 | type_="foreignkey", 56 | ) 57 | op.create_foreign_key( 58 | "fk_provenance_nodes_dataset_version_id_dataset_versions", 59 | "provenance_nodes", 60 | "dataset_versions", 61 | ["dataset_version_id"], 62 | ["id"], 63 | ) 64 | op.drop_column("provenance_nodes", "datafile_id") 65 | # ### end Alembic commands ### 66 | -------------------------------------------------------------------------------- /migrations/versions/c1be0bfabe2e_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: c1be0bfabe2e 4 | Revises: a286182fdf59 5 | Create Date: 2023-03-06 11:16:22.434754 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c1be0bfabe2e" 14 | down_revision = "a286182fdf59" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.drop_table("tmp_datafiles_cleanup") 22 | op.add_column("datafiles", sa.Column("custom_metadata", sa.JSON(), nullable=True)) 23 | op.add_column( 24 | "upload_session_files", sa.Column("custom_metadata", sa.JSON(), nullable=True) 25 | ) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_column("upload_session_files", "custom_metadata") 32 | op.drop_column("datafiles", "custom_metadata") 33 | op.create_table( 34 | "tmp_datafiles_cleanup", 35 | sa.Column("id", sa.VARCHAR(length=80), autoincrement=False, nullable=True), 36 | sa.Column( 37 | "orig_underlying_data_file_id", 38 | sa.VARCHAR(length=80), 39 | autoincrement=False, 40 | nullable=True, 41 | ), 42 | sa.Column( 43 | "new_underlying_data_file_id", 44 | sa.VARCHAR(length=80), 45 | autoincrement=False, 46 | nullable=True, 47 | ), 48 | sa.Column("update_count", sa.INTEGER(), autoincrement=False, nullable=True), 49 | ) 50 | # ### end Alembic commands ### 51 | -------------------------------------------------------------------------------- /migrations/versions/c605b801f114_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: c605b801f114 4 | Revises: ddeac9a40a1c 5 | Create Date: 2022-06-06 11:24:24.010508 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c605b801f114" 14 | down_revision = "ddeac9a40a1c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | pass 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | pass 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /migrations/versions/c9d2700c78d4_add_activity_log2.py: -------------------------------------------------------------------------------- 1 | """add-activity-log2 2 | 3 | Revision ID: c9d2700c78d4 4 | Revises: 6d8d4536fb51 5 | Create Date: 2019-07-17 17:14:17.166376 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "c9d2700c78d4" 14 | down_revision = "6d8d4536fb51" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | # ### end Alembic commands ### 22 | op.execute("update activities set timestamp = (select CURRENT_TIMESTAMP)") 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | # ### end Alembic commands ### 28 | pass 29 | -------------------------------------------------------------------------------- /migrations/versions/d091fc45fa8d_test_permaname_update.py: -------------------------------------------------------------------------------- 1 | """test-permaname-update 2 | 3 | Revision ID: d091fc45fa8d 4 | Revises: 2af60fdf97d1 5 | Create Date: 2019-07-23 12:16:08.303441 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "d091fc45fa8d" 14 | down_revision = "2af60fdf97d1" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "dataset_permanames", 23 | sa.Column("permaname", sa.Text(), nullable=False), 24 | sa.Column("dataset_id", sa.String(length=80), nullable=True), 25 | sa.Column("creation_date", sa.DateTime(), nullable=True), 26 | sa.ForeignKeyConstraint( 27 | ["dataset_id"], 28 | ["datasets.id"], 29 | name=op.f("fk_dataset_permanames_dataset_id_datasets"), 30 | ), 31 | sa.PrimaryKeyConstraint("permaname", name=op.f("pk_dataset_permanames")), 32 | ) 33 | 34 | op.execute( 35 | """ 36 | INSERT into dataset_permanames (permaname, dataset_id, creation_date) 37 | SELECT d.permaname, d.id, e.creation_date 38 | FROM datasets d 39 | JOIN entries e ON e.id = d.id 40 | """ 41 | ) 42 | 43 | op.drop_column("datasets", "permaname") 44 | # ### end Alembic commands ### 45 | 46 | 47 | def downgrade(): 48 | # ### commands auto generated by Alembic - please adjust! ### 49 | op.add_column( 50 | "datasets", 51 | sa.Column("permaname", sa.TEXT(), autoincrement=False, nullable=True), 52 | ) 53 | 54 | op.execute( 55 | """ 56 | UPDATE 57 | datasets 58 | SET 59 | d.permaname = p.permaname 60 | FROM 61 | datasets d 62 | JOIN dataset_permanames p ON d.id = p.dataset_id 63 | """ 64 | ) 65 | 66 | op.drop_table("dataset_permanames") 67 | # ### end Alembic commands ### 68 | -------------------------------------------------------------------------------- /migrations/versions/ddeac9a40a1c_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: ddeac9a40a1c 4 | Revises: 90837b08919f 5 | Create Date: 2022-06-06 11:08:07.514116 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "ddeac9a40a1c" 14 | down_revision = "90837b08919f" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | pass 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | pass 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /migrations/versions/e04e7eb4fb16_add_figshare_integration.py: -------------------------------------------------------------------------------- 1 | """add figshare-integration 2 | 3 | Revision ID: e04e7eb4fb16 4 | Revises: 6bf0d302380c 5 | Create Date: 2020-03-11 11:46:54.150090 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "e04e7eb4fb16" 14 | down_revision = "6bf0d302380c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "figshare_authorizations", 23 | sa.Column("id", sa.String(length=80), nullable=False), 24 | sa.Column("user_id", sa.String(length=80), nullable=True), 25 | sa.Column("figshare_account_id", sa.Integer(), nullable=True), 26 | sa.Column("token", sa.Text(), nullable=False), 27 | sa.Column("refresh_token", sa.Text(), nullable=False), 28 | sa.ForeignKeyConstraint( 29 | ["user_id"], 30 | ["users.id"], 31 | name=op.f("fk_figshare_authorizations_user_id_users"), 32 | ), 33 | sa.PrimaryKeyConstraint("id", name=op.f("pk_figshare_authorizations")), 34 | ) 35 | op.create_table( 36 | "third_party_datafile_links", 37 | sa.Column("id", sa.String(length=80), nullable=False), 38 | sa.Column("type", sa.String(length=50), nullable=True), 39 | sa.PrimaryKeyConstraint("id", name=op.f("pk_third_party_datafile_links")), 40 | ) 41 | op.create_table( 42 | "third_party_dataset_version_links", 43 | sa.Column("id", sa.String(length=80), nullable=False), 44 | sa.Column("type", sa.String(length=50), nullable=True), 45 | sa.Column("creator_id", sa.String(length=80), nullable=True), 46 | sa.ForeignKeyConstraint( 47 | ["creator_id"], 48 | ["users.id"], 49 | name=op.f("fk_third_party_dataset_version_links_creator_id_users"), 50 | ), 51 | sa.PrimaryKeyConstraint( 52 | "id", name=op.f("pk_third_party_dataset_version_links") 53 | ), 54 | ) 55 | op.create_table( 56 | "figshare_dataset_version_links", 57 | sa.Column("id", sa.String(length=80), nullable=False), 58 | sa.Column("figshare_article_id", sa.Integer(), nullable=False), 59 | sa.Column("dataset_version_id", sa.String(length=80), nullable=True), 60 | sa.ForeignKeyConstraint( 61 | ["dataset_version_id"], 62 | ["dataset_versions.id"], 63 | name=op.f( 64 | "fk_figshare_dataset_version_links_dataset_version_id_dataset_versions" 65 | ), 66 | ), 67 | sa.ForeignKeyConstraint( 68 | ["id"], 69 | ["third_party_dataset_version_links.id"], 70 | name=op.f( 71 | "fk_figshare_dataset_version_links_id_third_party_dataset_version_links" 72 | ), 73 | ), 74 | sa.PrimaryKeyConstraint("id", name=op.f("pk_figshare_dataset_version_links")), 75 | ) 76 | op.create_table( 77 | "figshare_datafile_links", 78 | sa.Column("id", sa.String(length=80), nullable=False), 79 | sa.Column("figshare_file_id", sa.Integer(), nullable=False), 80 | sa.Column("datafile_id", sa.String(length=80), nullable=True), 81 | sa.Column( 82 | "figshare_dataset_version_link_id", sa.String(length=80), nullable=True 83 | ), 84 | sa.ForeignKeyConstraint( 85 | ["datafile_id"], 86 | ["datafiles.id"], 87 | name=op.f("fk_figshare_datafile_links_datafile_id_datafiles"), 88 | ), 89 | sa.ForeignKeyConstraint( 90 | ["figshare_dataset_version_link_id"], 91 | ["figshare_dataset_version_links.id"], 92 | name=op.f( 93 | "fk_figshare_datafile_links_figshare_dataset_version_link_id_figshare_dataset_version_links" 94 | ), 95 | ), 96 | sa.ForeignKeyConstraint( 97 | ["id"], 98 | ["third_party_datafile_links.id"], 99 | name=op.f("fk_figshare_datafile_links_id_third_party_datafile_links"), 100 | ), 101 | sa.PrimaryKeyConstraint("id", name=op.f("pk_figshare_datafile_links")), 102 | ) 103 | # ### end Alembic commands ### 104 | 105 | 106 | def downgrade(): 107 | # ### commands auto generated by Alembic - please adjust! ### 108 | op.drop_table("third_party_dataset_version_links") 109 | op.drop_table("third_party_datafile_links") 110 | op.drop_table("figshare_dataset_version_links") 111 | op.drop_table("figshare_datafile_links") 112 | op.drop_table("figshare_authorizations") 113 | # ### end Alembic commands ### 114 | -------------------------------------------------------------------------------- /migrations/versions/f8856168298f_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: f8856168298f 4 | Revises: 028f07e95137 5 | Create Date: 2017-07-19 16:20:47.984794 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "f8856168298f" 14 | down_revision = "028f07e95137" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_unique_constraint( 22 | op.f("uq_datafiles_dataset_version_id"), 23 | "datafiles", 24 | ["dataset_version_id", "name"], 25 | ) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade(): 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_constraint( 32 | op.f("uq_datafiles_dataset_version_id"), "datafiles", type_="unique" 33 | ) 34 | # ### end Alembic commands ### 35 | -------------------------------------------------------------------------------- /pytype.cfg: -------------------------------------------------------------------------------- 1 | # NOTE: All relative paths are relative to the location of this file. 2 | 3 | [pytype] 4 | 5 | # Paths to source code directories, separated by ':'. 6 | pythonpath = 7 | . 8 | 9 | # Comma separated list of error names to ignore. 10 | disable = 11 | pyi-error 12 | 13 | # Experimental: solve unknown types to label with structural types. 14 | protocols = False 15 | 16 | # Python version (major.minor) of the target code. 17 | python_version = 3.5 18 | 19 | # Space-separated list of files or directories to exclude. 20 | exclude = 21 | **/*_test.py 22 | **/test_*.py 23 | **/tests/*.py 24 | 25 | # Keep going past errors to analyze as many files as possible. 26 | keep_going = False 27 | 28 | # Space-separated list of files or directories to process. 29 | inputs = 30 | taiga2 31 | 32 | # Experimental: Only load submodules that are explicitly imported. 33 | strict_import = False 34 | 35 | # All pytype output goes here. 36 | output = pytype_output 37 | 38 | # Don't report errors. 39 | report_errors = True 40 | -------------------------------------------------------------------------------- /react_frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "taiga_frontend", 3 | "version": "1.0.0", 4 | "description": "Frontend Taiga dependencies", 5 | "main": "../taiga2/static/frontend.js", 6 | "author": "CDS Team ", 7 | "license": "MIT", 8 | "_comments": "", 9 | "dependencies": { 10 | "aws-sdk": "^2.7.17", 11 | "bootstrap": "^3.3.7", 12 | "clipboard": "^1.6.0", 13 | "css-loader": "^5.2.4", 14 | "d3-array": "^1.2.1", 15 | "d3-format": "^1.3.0", 16 | "d3-scale": "^2.1.0", 17 | "file-saver": "^2.0.2", 18 | "filesize": "^3.3.0", 19 | "immutability-helper": "2.7.1", 20 | "lodash": "^4.17.21", 21 | "react": "^17.0.0", 22 | "react-bootstrap": "^0.32.3", 23 | "react-bootstrap-table": "^4.3.1", 24 | "react-dom": "^17.0.0", 25 | "react-dropzone": "^3.7.3", 26 | "react-modal": "^3.8.1", 27 | "react-quill": "^1.3.5", 28 | "react-router": "^5.0.1", 29 | "react-router-dom": "^5.0.1", 30 | "react-select": "^2.4.3", 31 | "react-table": "6.7.3", 32 | "showdown": "^1.9.1" 33 | }, 34 | "devDependencies": { 35 | "@babel/core": "^7.0.0-beta.55", 36 | "@types/clipboard": "^1.5.31", 37 | "@types/file-saver": "^2.0.1", 38 | "@types/immutability-helper": "2.0.15", 39 | "@types/node": "^10.5.8", 40 | "@types/plotly.js": "1.28.10", 41 | "@types/react": "^17.0.0", 42 | "@types/react-bootstrap": "0.32.11", 43 | "@types/react-dom": "^17.0.0", 44 | "@types/react-dropzone": "^0.0.32", 45 | "@types/react-router": "^5.0.2", 46 | "@types/react-router-dom": "^4.3.4", 47 | "@types/react-select": "2.0.3", 48 | "@types/react-table": "6.7.3", 49 | "babel-core": "^6.26.3", 50 | "file-loader": "^3.0.1", 51 | "html-webpack-plugin": "^3.2.0", 52 | "source-map-loader": "^0.2.3", 53 | "style-loader": "^2.0.0", 54 | "ts-loader": "^8.0.14", 55 | "typescript": "^4.1.0", 56 | "url-loader": "^1.1.2", 57 | "webpack": "^4.27.0", 58 | "webpack-cli": "^4.3.1", 59 | "webpack-manifest-plugin": "^2.0.3" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /react_frontend/run_webpack: -------------------------------------------------------------------------------- 1 | ./node_modules/.bin/webpack "--watch" "--mode=development" 2 | -------------------------------------------------------------------------------- /react_frontend/src/@types/README: -------------------------------------------------------------------------------- 1 | This file is here to manage custom definition file -------------------------------------------------------------------------------- /react_frontend/src/@types/other_modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'react-bootstrap-table'; 2 | declare module 'showdown'; 3 | declare module 'react-modal'; 4 | declare module 'filesize'; -------------------------------------------------------------------------------- /react_frontend/src/@types/react-dropzone.d.ts: -------------------------------------------------------------------------------- 1 | // Type definitions for react-dropzone 2 | // Project: https://github.com/okonet/react-dropzone 3 | // Definitions by: Mathieu Larouche Dube , Ivo Jesus , Luís Rodrigues 4 | // Definitions: https://github.com/Vooban/DefinitelyTyped 5 | 6 | /// 7 | 8 | declare module "react-dropzone" { 9 | interface DropzoneProps { 10 | // Drop behavior 11 | onDrop?: Function, 12 | onDropAccepted?: Function, 13 | onDropRejected?: Function, 14 | 15 | // Drag behavior 16 | onDragStart?: Function, 17 | onDragEnter?: Function, 18 | onDragLeave?: Function, 19 | 20 | style?: Object, // CSS styles to apply 21 | activeStyle?: Object, // CSS styles to apply when drop will be accepted 22 | rejectStyle?: Object, // CSS styles to apply when drop will be rejected 23 | className?: string, // Optional className 24 | activeClassName?: string, // className for accepted state 25 | rejectClassName?: string, // className for rejected state 26 | 27 | disablePreview?: boolean, // Enable/disable preview generation 28 | disableClick?: boolean, // Disallow clicking on the dropzone container to open file dialog 29 | 30 | inputProps?: Object, // Pass additional attributes to the tag 31 | multiple?: boolean, // Allow dropping multiple files 32 | accept?: string, // Allow specific types of files. See https://github.com/okonet/attr-accept for more information 33 | name?: string, // name attribute for the input tag 34 | maxSize?: number, 35 | minSize?: number 36 | } 37 | 38 | let Dropzone: React.ClassicComponentClass; 39 | export = Dropzone; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /react_frontend/src/components/GroupListView.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { RouteComponentProps } from "react-router"; 3 | import { Link } from "react-router-dom"; 4 | 5 | import { BootstrapTable, TableHeaderColumn } from "react-bootstrap-table"; 6 | 7 | import { LeftNav } from "./LeftNav"; 8 | 9 | import { TaigaApi } from "../models/api"; 10 | import { Group } from "../models/models"; 11 | 12 | import { relativePath } from "../utilities/route"; 13 | 14 | export interface GroupListProps extends RouteComponentProps { 15 | tapi: TaigaApi; 16 | groups: Array>>; 17 | } 18 | 19 | export interface GroupListState {} 20 | 21 | export class GroupListView extends React.Component< 22 | GroupListProps, 23 | GroupListState 24 | > { 25 | groupLinkFormatter( 26 | cell: string, 27 | row: Pick> 28 | ) { 29 | return {cell}; 30 | } 31 | 32 | render() { 33 | let navItems: Array = []; 34 | return ( 35 |
36 | 37 | 38 |
39 |

Your User Groups

40 | 41 | 44 | 49 | Name 50 | 51 | 52 | Number of Users 53 | 54 | 55 |
56 |
57 | ); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /react_frontend/src/components/InfoIcon.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { Glyphicon, OverlayTrigger, Tooltip } from "react-bootstrap"; 3 | 4 | interface Props { 5 | tooltipId: string; 6 | message: React.ReactNode; 7 | } 8 | 9 | export default class InfoIcon extends React.Component { 10 | render() { 11 | const tooltip = ( 12 | 13 | {this.props.message} 14 | 15 | ); 16 | 17 | return ( 18 | 19 | 20 | 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /react_frontend/src/components/LeftNav.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | export interface MenuItem { 4 | label: string; 5 | action: () => void; 6 | } 7 | 8 | export interface LeftNavProps { 9 | items: MenuItem[]; 10 | } 11 | 12 | export class LeftNav extends React.Component { 13 | render() { 14 | let items = this.props.items.map((element, index) => { 15 | return ( 16 |
  • 17 | {element.label} 18 |
    • 19 | ); 20 | }); 21 | 22 | return ( 23 |
    24 |
      {items}
    25 |
    26 | ); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /react_frontend/src/components/NotFound.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | // import { Link } from 'react-router'; 3 | // import * as update from 'immutability-helper'; 4 | 5 | // import { LeftNav, MenuItem } from "./LeftNav"; 6 | // import * as Folder from "../models/models"; 7 | // import { TaigaApi } from "../models/api"; 8 | 9 | // import * as Dialogs from "./Dialogs"; 10 | // import { TreeView } from "./modals/TreeView"; 11 | 12 | // import { toLocalDateString } from "../utilities/formats"; 13 | // import { relativePath } from "../utilities/route"; 14 | // import { LoadingOverlay } from "../utilities/loading"; 15 | 16 | interface NotFoundProps { 17 | message?: string; 18 | } 19 | 20 | export class NotFound extends React.Component { 21 | render() { 22 | return ( 23 |
    24 |

    Not found :(

    25 |

    {this.props.message}

    26 |
    27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /react_frontend/src/components/RecentlyViewed.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { RouteComponentProps } from "react-router"; 3 | import { Link } from "react-router-dom"; 4 | 5 | import { Grid, Row } from "react-bootstrap"; 6 | 7 | import { 8 | BootstrapTable, 9 | TableHeaderColumn, 10 | SortOrder, 11 | } from "react-bootstrap-table"; 12 | 13 | import { LeftNav } from "./LeftNav"; 14 | 15 | import { TaigaApi } from "../models/api"; 16 | import { AccessLog } from "../models/models"; 17 | 18 | import { lastAccessFormatter } from "../utilities/formats"; 19 | 20 | export interface RecentlyViewedProps extends RouteComponentProps { 21 | tapi: TaigaApi; 22 | } 23 | 24 | interface RecentlyViewedState { 25 | accessLogs?: Array; 26 | } 27 | 28 | export class RecentlyViewed extends React.Component< 29 | RecentlyViewedProps, 30 | RecentlyViewedState 31 | > { 32 | constructor(props: any) { 33 | super(props); 34 | 35 | this.state = { 36 | accessLogs: [], 37 | }; 38 | } 39 | 40 | componentDidMount() { 41 | this.doFetch(); 42 | } 43 | 44 | doFetch() { 45 | return this.props.tapi 46 | .get_user_entry_access_log() 47 | .then((userAccessLogs) => { 48 | // TODO: Think about not using it as State because it does not change during the page lifecycle 49 | 50 | let mappedAL = userAccessLogs.map((userAccessLog) => { 51 | return new AccessLog(userAccessLog); 52 | }); 53 | 54 | this.setState({ 55 | accessLogs: mappedAL, 56 | }); 57 | }); 58 | } 59 | 60 | datasetFormatter(cell: any, row: any) { 61 | return {cell}; 62 | } 63 | 64 | render() { 65 | let navItems: Array = []; 66 | 67 | let displayAccessLogs = null; 68 | 69 | const options = { 70 | defaultSortName: "last_access", 71 | defaultSortOrder: "desc" as any, 72 | sizePerPageList: [25, 30, 50, 100], 73 | sizePerPage: 50, 74 | }; 75 | 76 | if (this.state.accessLogs.length != 0) { 77 | displayAccessLogs = ( 78 | 85 | 88 | 94 | Entry 95 | 96 | 103 | Last access 104 | 105 | 106 | ); 107 | } 108 | 109 | return ( 110 |
    111 | 112 | 113 |
    114 | 115 | 116 |

    Your dataset access history

    117 |     118 | {displayAccessLogs} 119 |     120 |
    121 |
    122 | ); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /react_frontend/src/components/Search.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { 4 | Glyphicon, 5 | Form, 6 | FormGroup, 7 | ControlLabel, 8 | FormControl, 9 | Button, 10 | } from "react-bootstrap"; 11 | 12 | interface SearchInputProps { 13 | onKeyPress: (event: any, searchQuery: any) => void; 14 | onClick: (searchQuery: any) => void; 15 | } 16 | 17 | interface SearchInputState { 18 | searchQuery: string; 19 | } 20 | 21 | export class SearchInput extends React.Component< 22 | SearchInputProps, 23 | SearchInputState 24 | > { 25 | state = { 26 | searchQuery: "", 27 | }; 28 | 29 | handleChangeSearchQuery(e: any) { 30 | this.setState({ 31 | searchQuery: e.target.value, 32 | }); 33 | } 34 | 35 | render() { 36 | return ( 37 |
    { 40 | e.preventDefault(); 41 | }} 42 | > 43 | 44 | this.handleChangeSearchQuery(event)} 49 | onKeyPress={(event) => 50 | this.props.onKeyPress(event, this.state.searchQuery) 51 | } 52 | /> 53 | {" "} 54 | 60 |
    61 | ); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /react_frontend/src/components/Token.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { RouteComponentProps } from "react-router"; 3 | 4 | import { Col, Grid, Row, OverlayTrigger, Tooltip } from "react-bootstrap"; 5 | import { InputGroup, FormGroup, FormControl } from "react-bootstrap"; 6 | 7 | import ClipboardButton from "../utilities/r-clipboard"; 8 | import { LeftNav, MenuItem } from "./LeftNav"; 9 | import FigshareToken from "./modals/FigshareToken"; 10 | 11 | import { TaigaApi } from "../models/api"; 12 | import { User } from "../models/models"; 13 | 14 | export interface TokenProps extends RouteComponentProps { 15 | tapi: TaigaApi; 16 | } 17 | 18 | export interface TokenState { 19 | token: string; 20 | showFigshareTokenModel: boolean; 21 | } 22 | 23 | const antiPadding = { 24 | paddingLeft: "0px", 25 | }; 26 | 27 | const clipboardButtonStyle = {}; 28 | 29 | export class Token extends React.Component { 30 | constructor(props: any) { 31 | super(props); 32 | 33 | this.state = { 34 | token: "", 35 | showFigshareTokenModel: false, 36 | }; 37 | } 38 | 39 | componentDidMount() { 40 | this.doFetch(); 41 | } 42 | 43 | doFetch() { 44 | this.props.tapi 45 | .get_user() 46 | .then((user: User) => { 47 | this.setState({ 48 | token: user.token, 49 | }); 50 | }) 51 | .catch((err: any) => { 52 | console.log(err); 53 | }); 54 | } 55 | 56 | render() { 57 | let navItems: Array = []; 58 | 59 | navItems.push({ 60 | label: "Connect to Figshare", 61 | action: () => { 62 | this.setState({ showFigshareTokenModel: true }); 63 | }, 64 | }); 65 | 66 | const tooltipToken = ( 67 | 68 | Copied! 69 | 70 | ); 71 | 72 | return ( 73 |
    74 | 75 | 76 |
    77 | 78 | 79 |

    Your token

    80 |     81 | 82 |

    83 | Please, place the string below in `~/.taiga/token`{" "} 84 | to use taigaR and taigaPy. 85 |

    86 |

    87 | 91 | More information. 92 | 93 |

    94 |     95 | 96 |
    97 | 98 | 99 | 100 | 106 | 107 | 113 | 118 | {/*TODO: Clipboard Button breaks Glyphicon by not adding ::before...find a way to counter this*/} 119 | Copy 120 | 121 | 122 | 123 | 124 | 125 | 126 |
    127 |     128 |     129 |
    130 | this.setState({ showFigshareTokenModel: false })} 134 | /> 135 |
    136 | ); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /react_frontend/src/components/dataset_view/FigshareSection.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { Button } from "react-bootstrap"; 3 | import { Link } from "react-router-dom"; 4 | import { saveAs } from "file-saver"; 5 | 6 | import { DatasetVersion } from "../../models/models"; 7 | import { relativePath } from "../../utilities/route"; 8 | import { TaigaApi } from "../../models/api"; 9 | import UpdateFigshare from "../modals/UpdateFigshare"; 10 | import UploadToFigshare from "../modals/UploadToFigshare"; 11 | 12 | type Props = { 13 | tapi: TaigaApi; 14 | datasetVersion: DatasetVersion; 15 | handleFigshareUploadComplete: () => Promise; 16 | userFigshareAccountLinked: boolean; 17 | figshareLinkedFiles: Map< 18 | string, 19 | { 20 | downloadLink: string; 21 | currentTaigaId: string; 22 | readableTaigaId?: string; 23 | } 24 | >; 25 | }; 26 | 27 | type State = { 28 | showUploadToFigshare: boolean; 29 | showUpdateFigshare: boolean; 30 | }; 31 | 32 | export default class FigshareSection extends React.Component { 33 | constructor(props: Props) { 34 | super(props); 35 | 36 | this.state = { 37 | showUploadToFigshare: false, 38 | showUpdateFigshare: false, 39 | }; 40 | } 41 | 42 | showUploadToFigshare = () => { 43 | this.setState({ showUploadToFigshare: true }); 44 | }; 45 | 46 | showUpdateFigshare = () => { 47 | this.setState({ showUpdateFigshare: true }); 48 | }; 49 | 50 | handleCloseUploadToFigshare = (uploadComplete: boolean) => { 51 | if (uploadComplete) { 52 | this.props.handleFigshareUploadComplete().then(() => { 53 | this.setState({ 54 | showUploadToFigshare: false, 55 | }); 56 | }); 57 | } else { 58 | this.setState({ 59 | showUploadToFigshare: false, 60 | }); 61 | } 62 | }; 63 | 64 | handleCloseUpdateFigshare = (uploadComplete: boolean) => { 65 | if (uploadComplete) { 66 | this.props.handleFigshareUploadComplete().then(() => { 67 | this.setState({ 68 | showUpdateFigshare: false, 69 | }); 70 | }); 71 | } else { 72 | this.setState({ 73 | showUpdateFigshare: false, 74 | }); 75 | } 76 | }; 77 | 78 | renderFigshareLinkedContent() { 79 | if ( 80 | !this.props.datasetVersion.figshare.is_public && 81 | !this.props.datasetVersion.figshare.url_private_html 82 | ) { 83 | return ( 84 |

    This dataset version is linked to a private article on Figshare.

    85 | ); 86 | } 87 | 88 | const articleLink = ( 89 |

    90 | This dataset version is linked to a{" "} 91 | 100 | {this.props.datasetVersion.figshare.is_public ? "public" : "private"}{" "} 101 | Figshare article 102 | 103 | . 104 |

    105 | ); 106 | return ( 107 | 108 | {articleLink} 109 | 110 | 126 | 127 | 128 | ); 129 | } 130 | 131 | renderFigshareNotLinkedContent() { 132 | const { userFigshareAccountLinked } = this.props; 133 | return ( 134 | 135 |

    This dataset version is not linked with any Figshare article.

    136 | {!userFigshareAccountLinked && ( 137 |

    138 | Link your Figshare account to upload this dataset version to 139 | Figshare through the sidebar of the{" "} 140 | My Token page. 141 |

    142 | )} 143 | 149 | 155 |     156 | ); 157 | } 158 | 159 | render() { 160 | const { userFigshareAccountLinked } = this.props; 161 | 162 | return ( 163 |
    164 |

    Link with Figshare

    165 | {this.props.datasetVersion && ( 166 | 167 | {this.props.datasetVersion.figshare 168 | ? this.renderFigshareLinkedContent() 169 | : this.renderFigshareNotLinkedContent()} 170 | 177 | 183 | 184 | )} 185 |
    186 | ); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/EntryUsersPermissions.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import * as Modal from "react-modal"; 3 | import { BootstrapTable, TableHeaderColumn } from "react-bootstrap-table"; 4 | 5 | import { modalStyles } from "../Dialogs"; 6 | 7 | import { TaigaApi } from "../../models/api"; 8 | 9 | import { AccessLog } from "../../models/models"; 10 | import { lastAccessFormatter } from "../../utilities/formats"; 11 | 12 | interface EntryUsersPermissionsProps { 13 | isVisible: boolean; 14 | cancel: () => void; 15 | 16 | entry_id: string; 17 | 18 | handleDeletedRow: (arrayUserIds: Array) => Promise; 19 | 20 | tapi: TaigaApi; 21 | } 22 | 23 | interface EntryUsersPermissionsState { 24 | accessLogs?: Array; 25 | } 26 | 27 | export class EntryUsersPermissions extends React.Component< 28 | EntryUsersPermissionsProps, 29 | EntryUsersPermissionsState 30 | > { 31 | componentWillMount() { 32 | this.setState({ 33 | accessLogs: [], 34 | }); 35 | } 36 | 37 | componentDidMount() { 38 | this.doFetch(); 39 | } 40 | 41 | componentWillReceiveProps(nextProps: any, nextState: any) { 42 | if (nextProps.entry_id != this.props.entry_id) { 43 | this.doFetch(); 44 | } 45 | } 46 | 47 | doFetch() { 48 | // Return access logs for this folder 49 | return this.props.tapi 50 | .get_entry_access_log(this.props.entry_id) 51 | .then((usersAccessLogs) => { 52 | // TODO: Think about not using it as State because it does not change during the page lifecycle 53 | let mappedAL = usersAccessLogs.map((userAccessLogs) => { 54 | return new AccessLog(userAccessLogs); 55 | }); 56 | 57 | this.setState({ 58 | accessLogs: mappedAL, 59 | }); 60 | }); 61 | } 62 | 63 | handleDeletedRow(rowKeys: any) { 64 | let state_accessLogs = this.state.accessLogs; 65 | let accessLogsToRemove = state_accessLogs.filter((accessLog) => { 66 | // TODO: Optimize this to not loop through the accessLogs array for each item 67 | for (let user_id of rowKeys) { 68 | if (user_id == accessLog.user_id) { 69 | return true; 70 | } 71 | } 72 | return false; 73 | }); 74 | this.props.handleDeletedRow(accessLogsToRemove).then(() => { 75 | // We update our list now 76 | this.doFetch(); 77 | }); 78 | } 79 | 80 | render() { 81 | const check_mode: any = "checkbox"; 82 | const selectRowProp = { 83 | mode: check_mode, 84 | }; 85 | 86 | const options = { 87 | afterDeleteRow: (rowKeys: any) => { 88 | this.handleDeletedRow(rowKeys); 89 | }, 90 | }; 91 | 92 | return ( 93 | 100 |
    101 |
    102 |

    Users Permissions

    103 |
    104 |
    105 | 112 | 115 | 116 | User Name 117 | 118 | 122 | Last Access 123 | 124 | 125 |
    126 |
    127 | 134 |
    135 |
    136 |     137 | ); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/FigshareToken.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { 3 | Modal, 4 | FormGroup, 5 | FormControlProps, 6 | ControlLabel, 7 | FormControl, 8 | Button, 9 | } from "react-bootstrap"; 10 | import { TaigaApi } from "../../models/api"; 11 | 12 | interface Props { 13 | tapi: TaigaApi; 14 | show: boolean; 15 | onHide: () => void; 16 | } 17 | 18 | interface State { 19 | token: string; 20 | tokenIsInvalid: boolean; 21 | success: boolean; 22 | } 23 | 24 | export default class FigshareToken extends React.Component { 25 | constructor(props: Props) { 26 | super(props); 27 | 28 | this.state = { 29 | token: null, 30 | tokenIsInvalid: false, 31 | success: false, 32 | }; 33 | } 34 | 35 | addToken = () => { 36 | this.props.tapi 37 | .authorize_figshare(this.state.token) 38 | .then(() => { 39 | this.setState({ success: true }); 40 | }) 41 | .catch((e) => { 42 | this.setState({ tokenIsInvalid: true }); 43 | }); 44 | }; 45 | 46 | renderForm() { 47 | return ( 48 | 49 | {" "} 50 | 51 |

    52 | Follow the instructions in Figshare's{" "} 53 | 58 | How to get a Personal Token 59 | {" "} 60 | article and enter the token in the box below. 61 |

    62 |
    63 | 67 | Personal token 68 | 73 | ) => 74 | this.setState({ 75 | token: event.currentTarget.value as string, 76 | tokenIsInvalid: false, 77 | }) 78 | } 79 | /> 80 | 81 | 82 |
    83 |     84 | 85 | 91 | 92 |     93 | ); 94 | } 95 | 96 | renderSuccess() { 97 | return ( 98 | 99 | 100 |

    Figshare personal token successfully added.

    101 |     102 | 103 | 118 | 119 |     120 | ); 121 | } 122 | 123 | render() { 124 | return ( 125 | 126 | 127 | Connect to Figshare 128 | 129 | {this.state.success ? this.renderSuccess() : this.renderForm()} 130 | 131 | ); 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/FigshareUploadStatusTable.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { Table, ProgressBar } from "react-bootstrap"; 3 | 4 | import { UploadFileStatus } from "../../models/figshare"; 5 | 6 | interface Props { 7 | datafileIdToName: Map; 8 | uploadResults: ReadonlyArray; 9 | } 10 | export default class FigshareUploadTable extends React.Component { 11 | render() { 12 | return ( 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {this.props.uploadResults.map((file, i) => { 22 | let progressIndicator = null; 23 | 24 | const state = file.taskStatus ? file.taskStatus.state : null; 25 | if (file.failure_reason) { 26 | progressIndicator = ( 27 | 32 | ); 33 | } else if (state == "PROGRESS") { 34 | progressIndicator = ( 35 | 39 | ); 40 | } else if (state == "SUCCESS") { 41 | progressIndicator = ; 42 | } else if (state == "FAILURE") { 43 | progressIndicator = ( 44 | 49 | ); 50 | } 51 | 52 | return ( 53 | 54 | 55 | 56 | 57 | ); 58 | })} 59 | 60 |
    DatafileUpload status
    {this.props.datafileIdToName.get(file.datafile_id)}{progressIndicator}
    61 | ); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/FigshareWYSIWYGEditor.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import ReactQuill, { Quill } from "react-quill"; 4 | let Inline = Quill.import("blots/inline"); 5 | 6 | class BoldBlot extends Inline {} 7 | BoldBlot.blotName = "bold"; 8 | BoldBlot.tagName = "b"; 9 | Quill.register("formats/bold", BoldBlot); 10 | 11 | class ItalicBlot extends Inline {} 12 | ItalicBlot.blotName = "italic"; 13 | ItalicBlot.tagName = "i"; 14 | Quill.register("formats/bold", ItalicBlot); 15 | 16 | const modules = { 17 | toolbar: [ 18 | "bold", 19 | "italic", 20 | "underline", 21 | { script: "sub" }, 22 | { script: "super" }, 23 | ], 24 | }; 25 | const formats = ["bold", "italic", "underline", "script"]; 26 | 27 | type Props = { 28 | value: string; 29 | onChange: (value: string) => void; 30 | }; 31 | 32 | export default class FigshareWYSIWYGEditor extends React.Component { 33 | render() { 34 | return ( 35 | 42 | ); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/UpdateFigshare.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { Modal } from "react-bootstrap"; 4 | import ArticleIdStep from "./UpdateFigshareArticleIdStep"; 5 | import UpdateArticleStep from "./UpdateFigshareUpdateArticleStep"; 6 | import PollResultsStep from "./UpdateFigsharePollResultsStep"; 7 | 8 | import { DatasetVersion } from "../../models/models"; 9 | import { TaigaApi } from "../../models/api"; 10 | import { 11 | ArticleInfo, 12 | UpdateArticleResponse, 13 | UploadFileStatus, 14 | } from "../../models/figshare"; 15 | 16 | import "../../styles/modals/uploadtofigshare.css"; 17 | 18 | type Props = { 19 | tapi: TaigaApi; 20 | handleClose: (uploadComplete: boolean, figsharePrivateUrl: string) => void; 21 | show: boolean; 22 | datasetVersion: DatasetVersion; 23 | }; 24 | 25 | interface State { 26 | figshareArticleInfo: ArticleInfo; 27 | uploadResults: ReadonlyArray; 28 | uploadComplete: boolean; 29 | } 30 | export default class UpdateFigshare extends React.Component { 31 | constructor(props: Props) { 32 | super(props); 33 | 34 | this.state = { 35 | figshareArticleInfo: null, 36 | uploadResults: null, 37 | uploadComplete: false, 38 | }; 39 | } 40 | 41 | componentDidUpdate(prevProps: Props) { 42 | if ( 43 | prevProps.show != this.props.show || 44 | prevProps.datasetVersion != this.props.datasetVersion 45 | ) { 46 | this.setState({ 47 | figshareArticleInfo: null, 48 | uploadResults: null, 49 | uploadComplete: false, 50 | }); 51 | } 52 | } 53 | 54 | handleFetchFigshareArticleSuccess = (figshareArticleInfo: ArticleInfo) => { 55 | this.setState({ figshareArticleInfo }); 56 | }; 57 | 58 | handleUpdateArticleResponse = ( 59 | updateArticleResponse: UpdateArticleResponse 60 | ) => { 61 | this.setState({ 62 | uploadResults: updateArticleResponse.files.map((r) => { 63 | if (!r.task_id && !r.failure_reason) { 64 | return { 65 | ...r, 66 | taskStatus: { 67 | id: null, 68 | state: "SUCCESS", 69 | message: null, 70 | current: 1, 71 | total: 1, 72 | s3Key: null, 73 | }, 74 | }; 75 | } 76 | return r; 77 | }), 78 | }); 79 | }; 80 | 81 | handleClose = () => { 82 | const { uploadComplete } = this.state; 83 | const figsharePrivateUrl = uploadComplete 84 | ? `https://figshare.com/account/articles/${this.state.figshareArticleInfo.id}` 85 | : null; 86 | 87 | this.setState( 88 | { 89 | figshareArticleInfo: null, 90 | uploadResults: null, 91 | uploadComplete: false, 92 | }, 93 | () => this.props.handleClose(uploadComplete, figsharePrivateUrl) 94 | ); 95 | }; 96 | 97 | render() { 98 | const { figshareArticleInfo, uploadResults, uploadComplete } = this.state; 99 | const figsharePrivateUrl = uploadComplete 100 | ? `https://figshare.com/account/articles/${figshareArticleInfo.id}` 101 | : null; 102 | 103 | return ( 104 | 109 | 110 | Update Figshare Article 111 | 112 | {!figshareArticleInfo ? ( 113 | 119 | ) : !uploadResults ? ( 120 | 126 | ) : !uploadComplete ? ( 127 | this.setState({ uploadComplete: true })} 132 | /> 133 | ) : ( 134 | 135 |

    136 | Files have finished uploading. Your Figshare article version is 137 | not yet published. Go to the Figshare website to publish it. 138 |

    139 |     140 | )} 141 |     142 | ); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/UpdateFigshareArticleIdStep.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { 3 | Modal, 4 | Button, 5 | FormGroup, 6 | FormControl, 7 | FormControlProps, 8 | ControlLabel, 9 | HelpBlock, 10 | } from "react-bootstrap"; 11 | 12 | import { TaigaApi } from "../../models/api"; 13 | import { ArticleInfo } from "../../models/figshare"; 14 | 15 | enum ArticleIdStepError { 16 | ArticleUpdateForbidden, 17 | ArticleDoesNotExist, 18 | } 19 | 20 | interface ArticleIdStepProps { 21 | tapi: TaigaApi; 22 | handleFetchFigshareArticleSuccess: (figshareArticleInfo: ArticleInfo) => void; 23 | } 24 | 25 | interface ArticleIdStepState { 26 | articleId: string; 27 | loading: boolean; 28 | error: ArticleIdStepError; 29 | } 30 | 31 | export default class ArticleIdStep extends React.Component< 32 | ArticleIdStepProps, 33 | ArticleIdStepState 34 | > { 35 | constructor(props: ArticleIdStepProps) { 36 | super(props); 37 | 38 | this.state = { 39 | articleId: "", 40 | loading: false, 41 | error: null, 42 | }; 43 | } 44 | 45 | handleArticleIdChange = ( 46 | e: React.FormEvent 47 | ) => { 48 | this.setState({ 49 | articleId: e.currentTarget.value as string, 50 | error: null, 51 | }); 52 | }; 53 | 54 | handleFetchFigshareArticle = () => { 55 | this.setState({ loading: true }, () => { 56 | this.props.tapi 57 | .get_figshare_article(parseInt(this.state.articleId)) 58 | .then(this.props.handleFetchFigshareArticleSuccess) 59 | .catch((reason: Error) => { 60 | if (reason.message == "NOT FOUND") { 61 | this.setState({ 62 | error: ArticleIdStepError.ArticleDoesNotExist, 63 | loading: false, 64 | }); 65 | } else if ("Forbidden" in reason) { 66 | this.setState({ 67 | error: ArticleIdStepError.ArticleUpdateForbidden, 68 | loading: false, 69 | }); 70 | } else { 71 | console.log(reason); 72 | } 73 | }); 74 | }); 75 | }; 76 | 77 | render() { 78 | return ( 79 | 80 | 84 | 88 | Figshare article ID 89 | 95 | 96 | {this.state.error == ArticleIdStepError.ArticleUpdateForbidden && ( 97 | 98 | The Figshare account linked to your Taiga account does not have 99 | permission to update this article. Please connect your Taiga 100 | account with Figshare account that does, or try a different 101 | article. 102 | 103 | )} 104 | {this.state.error == ArticleIdStepError.ArticleDoesNotExist && ( 105 | No editable article was found for this ID 106 | )} 107 | 108 | The number after the article name in the public Figshare URL. For 109 | example, 11384241 for 110 | https://figshare.com/articles/DepMap_19Q4_Public/11384241 111 | 112 | 113 | 114 | 115 | 122 | 123 | 124 | ); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/UpdateFigsharePollResultsStep.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { Modal, Button } from "react-bootstrap"; 3 | import update from "immutability-helper"; 4 | 5 | import FigshareUploadStatusTable from "./FigshareUploadStatusTable"; 6 | 7 | import { TaskStatus, DatasetVersionDatafiles } from "../../models/models"; 8 | import { TaigaApi } from "../../models/api"; 9 | import { UploadFileStatus } from "../../models/figshare"; 10 | 11 | import { relativePath } from "../../utilities/route"; 12 | 13 | interface Props { 14 | tapi: TaigaApi; 15 | datafiles: ReadonlyArray; 16 | initialUploadResults: ReadonlyArray; 17 | onUploadComplete: () => void; 18 | } 19 | 20 | interface State { 21 | uploadResults: ReadonlyArray; 22 | } 23 | 24 | export default class UpdateFigsharePollResultsStep extends React.Component< 25 | Props, 26 | State 27 | > { 28 | datafileIdToName: Map; 29 | constructor(props: Props) { 30 | super(props); 31 | 32 | if (this.isUploadComplete(props.initialUploadResults)) { 33 | props.onUploadComplete(); 34 | } 35 | 36 | this.state = { 37 | uploadResults: props.initialUploadResults, 38 | }; 39 | 40 | this.datafileIdToName = new Map( 41 | props.datafiles.map((datafile) => [datafile.id, datafile.name]) 42 | ); 43 | 44 | props.initialUploadResults.forEach((file, i) => { 45 | if (file.task_id) { 46 | this.pollUploadToFigshareFile(i, file.task_id); 47 | } 48 | }); 49 | } 50 | 51 | pollUploadToFigshareFile(i: number, taskId: string) { 52 | this.props.tapi.get_task_status(taskId).then((newStatus: TaskStatus) => { 53 | this.setState( 54 | { 55 | uploadResults: update(this.state.uploadResults, { 56 | [i]: { taskStatus: { $set: newStatus } }, 57 | }), 58 | }, 59 | () => { 60 | if (newStatus.state != "SUCCESS" && newStatus.state != "FAILURE") { 61 | setTimeout(() => this.pollUploadToFigshareFile(i, taskId), 1000); 62 | } else { 63 | if (this.isUploadComplete(this.state.uploadResults)) { 64 | this.props.onUploadComplete(); 65 | } 66 | } 67 | } 68 | ); 69 | }); 70 | } 71 | 72 | isUploadComplete(uploadResults: ReadonlyArray) { 73 | return uploadResults.every( 74 | (file) => 75 | !!file.failure_reason || 76 | (file.taskStatus && 77 | (file.taskStatus.state == "SUCCESS" || 78 | file.taskStatus.state == "FAILURE")) 79 | ); 80 | } 81 | 82 | render() { 83 | return ( 84 | 88 | 92 | 93 | ); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /react_frontend/src/components/modals/UploadTable.fixture.tsx: -------------------------------------------------------------------------------- 1 | import { UploadTable, UploadFile, UploadController } from "./UploadTable"; 2 | import { CreateVersionDialog } from "./UploadForm"; 3 | import { UploadFileType, UploadStatus } from "../UploadTracker"; 4 | 5 | import * as React from "react"; 6 | import * as Dropzone from "react-dropzone"; 7 | import { DataFileType } from "../../models/models"; 8 | 9 | let files = [ 10 | { 11 | name: "taigafile", 12 | fileType: UploadFileType.TaigaPath, 13 | size: "100", 14 | existingTaigaId: "original-123183.2/sample", 15 | }, 16 | ]; 17 | 18 | //let w = window as any; 19 | //w.update_hack = update; 20 | 21 | // const dropZoneStyle: any = { 22 | // height: '150px', 23 | // borderWidth: '2px', 24 | // borderColor: 'rgb(102, 102, 102)', 25 | // borderStyle: 'dashed', 26 | // borderRadius: '5px' 27 | // }; 28 | 29 | // export class UploadTableWrapper extends React.Component { 30 | // controller: UploadController 31 | 32 | // constructor(props: any) { 33 | // super(props); 34 | // this.state = { files: props.initialFiles } 35 | 36 | // this.controller = new UploadController(props.intialFiles, (files: any) => this.setState({ files: files })); 37 | // this.controller.files = props.initialFiles; 38 | // } 39 | 40 | // onDrop(acceptedFiles: Array, rejectedFiles: Array) { 41 | // console.log(acceptedFiles) 42 | // acceptedFiles.forEach((file) => this.controller.addUpload(file)) 43 | // } 44 | 45 | // render() { 46 | // return (
    47 | // 48 | // 49 | // 50 | // this.onDrop(acceptedFiles, rejectedFiles)} 51 | // /> 52 | 53 | // 54 | //
    ) 55 | // } 56 | // } 57 | 58 | // onFileUploadedAndConverted?: any; 59 | 60 | // title: string; 61 | // readOnlyName?: string; 62 | // readOnlyDescription?: string; 63 | 64 | // // Determines what is done when opening the modal/componentWillReceiveProps 65 | // onOpen?: Function; 66 | // // Parent gives the previous version files. If it exists, we display another Table 67 | // previousVersionFiles?: Array; 68 | // // Parent can give the previous version name. Need it if pass previousVersionFiles 69 | // // TODO: Only pass the previousVersion, so we can take the previous DataFiles from it too 70 | // previousVersionName?: string; 71 | // datasetPermaname?: string; 72 | // previousVersionNumber?: string; 73 | 74 | // // If we want to change the description, we can use this to pass the previous description 75 | // // It can't be compatible with readOnlyDescription 76 | // previousDescription?: string; 77 | 78 | // validationState?: string; 79 | // help?: string; 80 | 81 | // id: string; 82 | // name: string; 83 | // underlying_file_id: string; 84 | // url: string; 85 | // type: DataFileType; 86 | // allowed_conversion_type: Array; 87 | // short_summary: string; 88 | 89 | function mockUpload( 90 | files: any, 91 | params: any, 92 | uploadProgressCallback: (status: Array) => void 93 | ) { 94 | console.log("MockUploadStart"); 95 | return new Promise((resolve) => { 96 | let fileCount = files.length; 97 | let counter = 0; 98 | 99 | let nextCall = function () { 100 | counter += 5; 101 | console.log("counter=", counter); 102 | uploadProgressCallback([ 103 | { progress: counter, progressMessage: "Completed " + counter + "%" }, 104 | ]); 105 | 106 | if (counter >= 100) { 107 | resolve(); 108 | } else { 109 | setTimeout(nextCall, 1000); 110 | } 111 | }; 112 | 113 | nextCall(); 114 | }).then(() => { 115 | console.log("MockUploadComplete"); 116 | }); 117 | } 118 | 119 | export default [ 120 | { 121 | component: CreateVersionDialog, 122 | name: "dialog", 123 | props: { 124 | isVisible: true, 125 | isProcessing: true, 126 | upload: mockUpload, 127 | previousDescription: "prev Description", 128 | previousVersionNumber: "100", 129 | previousVersionFiles: [ 130 | { 131 | id: "id", 132 | name: "samplename", 133 | allowed_conversion_type: ["raw"], 134 | short_summary: "200x20", 135 | type: DataFileType.Raw, 136 | }, 137 | ], 138 | datasetPermaname: "permaname-1000", 139 | }, 140 | }, 141 | ]; 142 | -------------------------------------------------------------------------------- /react_frontend/src/models/figshare.ts: -------------------------------------------------------------------------------- 1 | import { TaskStatus } from "./models"; 2 | 3 | interface Author { 4 | full_name: string; 5 | id: number; 6 | is_active: boolean; 7 | } 8 | 9 | export interface File { 10 | computed_md5: string; 11 | download_url: string; 12 | id: number; 13 | is_link_only: boolean; 14 | name: string; 15 | size: number; 16 | supplied_md5: string; 17 | } 18 | 19 | export interface ArticleInfo { 20 | id: number; 21 | version: number; 22 | authors: Array; 23 | description: string; 24 | files: Array; 25 | } 26 | 27 | export interface FileToUpdate { 28 | figshare_file_id: number; 29 | action: "Add" | "Delete"; 30 | datafile_id: string; 31 | file_name: string; 32 | } 33 | 34 | export interface UploadFileStatus { 35 | datafile_id: string; 36 | file_name: string; 37 | failure_reason?: string; 38 | task_id?: string; 39 | taskStatus?: TaskStatus; 40 | } 41 | 42 | export interface UpdateArticleResponse { 43 | article_id: number; 44 | files: Array; 45 | } 46 | 47 | export interface UpdateArticleRemovedFigshareFile { 48 | figshareFileId: number; 49 | name: string; 50 | removeFile: boolean; 51 | } 52 | 53 | export interface UpdateArticleAdditionalTaigaDatafile { 54 | datafileId: string; 55 | name: string; 56 | addFile: boolean; 57 | datafileName: string; 58 | } 59 | 60 | export interface UpdateArticleUnchangedFile { 61 | figshareFileId: number; 62 | name: string; 63 | datafileId: string; 64 | datafileName: string; 65 | } 66 | -------------------------------------------------------------------------------- /react_frontend/src/styles/modals/uploadtofigshare.css: -------------------------------------------------------------------------------- 1 | .upload-to-figshare-modal { 2 | min-width: 66%; 3 | } 4 | 5 | .figshare-upload-files-table > tbody > tr > td { 6 | vertical-align: middle !important; 7 | } 8 | 9 | .textarea-lock-width { 10 | resize: vertical; 11 | } 12 | 13 | .figshare-modal-body { 14 | max-height: 66vh; 15 | overflow-y: auto; 16 | } 17 | 18 | .ql-editor { 19 | height: 200px; 20 | } 21 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/common.ts: -------------------------------------------------------------------------------- 1 | export const sortByName = (a: { name: string }, b: { name: string }) => { 2 | return a.name.localeCompare(b.name); 3 | }; 4 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/figshare.ts: -------------------------------------------------------------------------------- 1 | import { sortByName } from "./common"; 2 | 3 | import { 4 | DatasetVersionDatafiles as Datafile, 5 | DataFileType, 6 | } from "../models/models"; 7 | import { 8 | File, 9 | UpdateArticleRemovedFigshareFile, 10 | UpdateArticleAdditionalTaigaDatafile, 11 | UpdateArticleUnchangedFile, 12 | } from "../models/figshare"; 13 | 14 | export const getDefaultFilesToUpdate = ( 15 | figshareFiles: Array, 16 | datafiles: Array 17 | ): { 18 | removedFigshareFiles: Array; 19 | additionalTaigaDatafiles: Array; 20 | unchangedFiles: Array; 21 | } => { 22 | const getDefaultDatafileName = (datafile: Datafile) => 23 | datafile.type == DataFileType.Raw ? datafile.name : `${datafile.name}.csv`; 24 | 25 | const removedFigshareFiles: Array = []; 26 | const additionalTaigaDatafiles: Array = []; 27 | const unchangedFiles: Array = []; 28 | const matchedDatafiles = new Set(); 29 | 30 | figshareFiles.forEach((figshareFile) => { 31 | const matchingDataFileByMD5 = datafiles.find( 32 | (datafile) => 33 | !matchedDatafiles.has(datafile.id) && 34 | (datafile.original_file_md5 == figshareFile.supplied_md5 || 35 | datafile.original_file_md5 == figshareFile.computed_md5) 36 | ); 37 | if (matchingDataFileByMD5) { 38 | unchangedFiles.push({ 39 | figshareFileId: figshareFile.id, 40 | datafileId: matchingDataFileByMD5.id, 41 | name: figshareFile.name, 42 | datafileName: matchingDataFileByMD5.name, 43 | }); 44 | matchedDatafiles.add(matchingDataFileByMD5.id); 45 | return; 46 | } 47 | 48 | // Exclude file extension from figshareFile.name 49 | const figshareFileName = figshareFile.name.includes(".") 50 | ? figshareFile.name.substr(0, figshareFile.name.lastIndexOf(".")) 51 | : figshareFile.name; 52 | 53 | const matchingDataFileByName = datafiles.find( 54 | (datafile) => 55 | !matchedDatafiles.has(datafile.id) && datafile.name == figshareFileName 56 | ); 57 | if (matchingDataFileByName) { 58 | removedFigshareFiles.push({ 59 | figshareFileId: figshareFile.id, 60 | name: figshareFile.name, 61 | removeFile: true, 62 | }); 63 | additionalTaigaDatafiles.push({ 64 | datafileId: matchingDataFileByName.id, 65 | name: figshareFile.name, 66 | addFile: true, 67 | datafileName: matchingDataFileByName.name, 68 | }); 69 | matchedDatafiles.add(matchingDataFileByName.id); 70 | return; 71 | } 72 | 73 | removedFigshareFiles.push({ 74 | figshareFileId: figshareFile.id, 75 | name: figshareFile.name, 76 | removeFile: false, 77 | }); 78 | }); 79 | 80 | datafiles.forEach((datafile) => { 81 | if (matchedDatafiles.has(datafile.id)) { 82 | return; 83 | } 84 | additionalTaigaDatafiles.push({ 85 | name: getDefaultDatafileName(datafile), 86 | datafileId: datafile.id, 87 | addFile: false, 88 | datafileName: datafile.name, 89 | }); 90 | }); 91 | 92 | removedFigshareFiles.sort(sortByName); 93 | additionalTaigaDatafiles.sort(sortByName); 94 | unchangedFiles.sort(sortByName); 95 | return { 96 | removedFigshareFiles, 97 | additionalTaigaDatafiles, 98 | unchangedFiles, 99 | }; 100 | }; 101 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/formats.tsx: -------------------------------------------------------------------------------- 1 | import { InitialFileType } from "../models/models"; 2 | 3 | export function toLocalDateString(stringDate: string) { 4 | let _date = new Date(stringDate); 5 | var options = { 6 | month: "2-digit", 7 | day: "2-digit", 8 | year: "2-digit", 9 | }; 10 | return _date.toLocaleDateString("en-us", options); 11 | } 12 | 13 | export function getInitialFileTypeFromMimeType( 14 | mimeTypeOrEnum: string | InitialFileType 15 | ): InitialFileType { 16 | // Manage the case of receiving the Enum 17 | 18 | // Options values should also be changed in formats.tsx 19 | // TODO: Use the same text to print to the user between selection of option and print result in formats.tsx 20 | let formattedType: InitialFileType = InitialFileType.Raw; 21 | if (mimeTypeOrEnum in InitialFileType) { 22 | formattedType = mimeTypeOrEnum as InitialFileType; 23 | } else { 24 | if (mimeTypeOrEnum == "text/csv") { 25 | formattedType = InitialFileType.NumericMatrixCSV; 26 | } else { 27 | formattedType = InitialFileType.Raw; 28 | } 29 | } 30 | return formattedType; 31 | } 32 | 33 | export function lastAccessFormatter(cell: any, row: any) { 34 | // Formatter for Table Bootstrap 35 | let _date = new Date(cell); 36 | var options = { 37 | month: "2-digit", 38 | day: "2-digit", 39 | year: "2-digit", 40 | hour: "2-digit", 41 | minute: "2-digit", 42 | }; 43 | return _date.toLocaleTimeString("en-us", options); 44 | } 45 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/loading.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { Well } from "react-bootstrap"; 4 | 5 | interface LoadingOverlayProps { 6 | message?: string; 7 | } 8 | 9 | export class LoadingOverlay extends React.Component { 10 | render() { 11 | return ( 12 |
    13 | {this.props.message && {this.props.message}} 14 |
    15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/r-clipboard.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import * as clipboard from "clipboard"; 3 | 4 | export interface ClipboardButtonProps { 5 | options?: any; 6 | type?: string; 7 | className?: string; 8 | style?: any; 9 | component?: any; 10 | children?: any; 11 | onClick?: any; 12 | } 13 | 14 | export interface ClipboardButtonState {} 15 | 16 | export default class ClipboardButton extends React.Component< 17 | ClipboardButtonProps, 18 | ClipboardButtonState 19 | > { 20 | static defaultProps = { 21 | onClick: function () {}, 22 | }; 23 | 24 | constructor(props: any) { 25 | super(props); 26 | 27 | this.clipboard = clipboard; 28 | } 29 | 30 | refs: { 31 | element: any; 32 | }; 33 | 34 | clipboard: any; 35 | 36 | /* Returns a object with all props that fulfill a certain naming pattern 37 | * 38 | * @param {RegExp} regexp - Regular expression representing which pattern 39 | * you'll be searching for. 40 | * @param {Boolean} remove - Determines if the regular expression should be 41 | * removed when transmitting the key from the props 42 | * to the new object. 43 | * 44 | * e.g: 45 | * 46 | * // Considering: 47 | * // this.props = {option-foo: 1, onBar: 2, data-foobar: 3 data-baz: 4}; 48 | * 49 | * // *RegExps not using // so that this comment doesn't break up 50 | * this.propsWith(option-*, true); // returns {foo: 1} 51 | * this.propsWith(on*, true); // returns {Bar: 2} 52 | * this.propsWith(data-*); // returns {data-foobar: 1, data-baz: 4} 53 | */ 54 | propsWith(regexp: any, remove = false): Object { 55 | let object: any = Object; 56 | 57 | Object.keys(this.props).forEach(function (key) { 58 | if (key.search(regexp) !== -1) { 59 | const objectKey: any = remove ? key.replace(regexp, "") : key; 60 | object[objectKey] = this.props[key]; 61 | } 62 | }, this); 63 | 64 | return object; 65 | } 66 | 67 | componentWillUnmount() { 68 | this.clipboard && this.clipboard.destroy(); 69 | } 70 | 71 | componentDidMount() { 72 | // Support old API by trying to assign this.props.options first; 73 | const options = this.props.options || this.propsWith(/^option-/, true); 74 | const element = React.version.match(/0\.13(.*)/) 75 | ? this.refs.element.getDOMNode() 76 | : this.refs.element; 77 | const Clipboard = require("clipboard"); 78 | this.clipboard = new Clipboard(element, options); 79 | 80 | const callbacks = this.propsWith(/^on/, true); 81 | Object.keys(callbacks).forEach(function (callback) { 82 | this.clipboard.on(callback.toLowerCase(), this.props["on" + callback]); 83 | }, this); 84 | } 85 | 86 | render() { 87 | const attributes = { 88 | type: this.getType(), 89 | className: this.props.className || "", 90 | style: this.props.style || {}, 91 | ref: "element", 92 | onClick: this.props.onClick, 93 | ...this.propsWith(/^data-/), 94 | ...this.propsWith(/^button-/, true), 95 | }; 96 | 97 | // not sure when this is getting added, but we get a warning if we try to create the element with this attribute 98 | delete (attributes as any).Click; 99 | 100 | return React.createElement( 101 | this.getComponent(), 102 | attributes, 103 | this.props.children 104 | ); 105 | } 106 | 107 | getType() { 108 | if (this.getComponent() === "button" || this.getComponent() === "input") { 109 | return this.props.type || "button"; 110 | } else { 111 | return undefined; 112 | } 113 | } 114 | 115 | getComponent() { 116 | return this.props.component || "button"; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /react_frontend/src/utilities/route.ts: -------------------------------------------------------------------------------- 1 | declare let taigaPrefix: string; 2 | declare let taigaUserToken: string; 3 | 4 | export function getTaigaPrefix() { 5 | // If taigaPrefix exists in the global scope 6 | if (taigaPrefix) { 7 | return taigaPrefix; 8 | } else { 9 | return undefined; 10 | } 11 | } 12 | 13 | export function getUserToken() { 14 | return (window as any).taigaUserToken; 15 | } 16 | 17 | function pathJoin(parts: Array, sep?: string) { 18 | var separator = sep || "/"; 19 | var replace = new RegExp(separator + "{1,}", "g"); 20 | return parts.join(separator).replace(replace, separator); 21 | } 22 | 23 | // TODO: We could also create a component RelativeLink which could wrap a Component and manage the relativePath 24 | export function relativePath(relativePath: string) { 25 | return pathJoin([getTaigaPrefix(), relativePath]); 26 | } 27 | -------------------------------------------------------------------------------- /react_frontend/src/version.ts: -------------------------------------------------------------------------------- 1 | export const SHA = "DEV-80621ea96a6cf254703628dbf5ff7223b585"; 2 | -------------------------------------------------------------------------------- /react_frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "../taiga2/static/", 4 | "sourceMap": true, 5 | "noImplicitAny": true, 6 | "module": "commonjs", 7 | "target": "es6", 8 | "lib": [ 9 | "es6", 10 | "es2017", 11 | "dom" 12 | ], 13 | "jsx": "react" 14 | }, 15 | "files": [ 16 | "./src/index.tsx", 17 | "./src/@types/other_modules.d.ts" 18 | ] 19 | } -------------------------------------------------------------------------------- /react_frontend/webpack.config.js: -------------------------------------------------------------------------------- 1 | var webpack = require("webpack"); 2 | var ManifestPlugin = require("webpack-manifest-plugin"); 3 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 4 | const path = require("path"); 5 | 6 | module.exports = { 7 | mode: "development", 8 | entry: "./src/index.tsx", 9 | output: { 10 | filename: "react_frontend.js", 11 | path: __dirname + "./../taiga2/static/js", 12 | library: "Taiga", 13 | }, 14 | 15 | // Enable sourcemaps for debugging webpack's output. 16 | devtool: "source-map", 17 | 18 | resolve: { 19 | // Add '.ts' and '.tsx' as resolvable extensions. 20 | extensions: [".ts", ".tsx", ".js", ".json"], 21 | }, 22 | 23 | module: { 24 | rules: [ 25 | // All files with a '.ts' or '.tsx' extension will be handled by 'ts-loader'. 26 | { test: /\.tsx?$/, loader: "ts-loader" }, 27 | // All output '.js' files will have any sourcemaps re-processed by 'source-map-loader'. 28 | { 29 | enforce: "pre", 30 | test: /\.js$/, 31 | loader: "source-map-loader", 32 | exclude: ["/node_modules/"], 33 | }, 34 | // the following are only needed for processing css 35 | { test: /\.css$/, use: ["style-loader", "css-loader"] }, 36 | { 37 | test: /\.(ttf|otf|eot|svg)(\?v=[0-9]\.[0-9]\.[0-9])?|(jpg|gif)$/, 38 | loader: "file-loader", 39 | }, 40 | { 41 | test: /\.woff(2)?(\?v=[0-9]\.[0-9]\.[0-9])?$/, 42 | loader: "url-loader?limit=10000&mimetype=application/font-woff", 43 | }, 44 | ], 45 | }, 46 | }; 47 | -------------------------------------------------------------------------------- /recreate_dev_db.sh: -------------------------------------------------------------------------------- 1 | python taiga2/create_test_db_sqlalchemy.py settings.cfg 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | --index-url https://pypi.python.org/simple/ 2 | -e . 3 | boto3==1.4.2 4 | botocore==1.4.93 5 | celery==4.1.1 6 | certifi==2019.6.16 7 | Click==7.0 8 | clickclick==1.2.2 9 | connexion[swagger-ui]==2.3.0 10 | docutils==0.14 11 | Flask==1.0.3 12 | flask-marshmallow==0.7.0 13 | Flask-Migrate==2.0.3 14 | Flask-SQLAlchemy==2.1 15 | google-cloud-error-reporting==1.1.0 16 | google-cloud-storage==1.35.0 17 | h5py==2.6.0 18 | humanize==0.5.1 19 | jsonpointer==2.0 20 | jsonschema==2.6.0 21 | kombu==4.2.1 22 | mailjet_rest==1.3.3 23 | marshmallow==2.15.1 24 | marshmallow-enum==1.0 25 | marshmallow-oneofschema==1.0.3 26 | marshmallow-sqlalchemy==0.12.1 27 | numpy==1.18.1 28 | psutil==5.6.7 29 | PyYAML==5.4.1 30 | redis==2.10.5 31 | requests==2.22.0 32 | SQLAlchemy==1.3.0 33 | swagger-spec-validator==2.4.3 34 | typing-extensions==3.7.4.3 35 | Werkzeug==0.15.4 36 | openapi-spec-validator==0.2.9 -------------------------------------------------------------------------------- /settings.cfg.sample: -------------------------------------------------------------------------------- 1 | # Flask 2 | DEBUG = True 3 | ENV = 'dev' 4 | HOSTNAME = 'localhost' 5 | PORT = 8080 6 | USE_RELOADER = True 7 | USE_EVALEX = True 8 | PREFIX = '/taiga' 9 | 10 | # Amazon Web Services 11 | AWS_ACCESS_KEY_ID = '' 12 | AWS_SECRET_ACCESS_KEY = '' 13 | 14 | CLIENT_UPLOAD_AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID 15 | CLIENT_UPLOAD_AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY 16 | 17 | S3_BUCKET = '' 18 | 19 | # Database configuration 20 | SQLALCHEMY_DATABASE_URI = 'sqlite:///db.sqlite3' 21 | 22 | # Exception to StackDriver 23 | REPORT_EXCEPTIONS = False 24 | 25 | # Figshare 26 | FIGSHARE_CLIENT_ID = '' 27 | FIGSHARE_CLIENT_SECRET = '' 28 | 29 | # Mailjet 30 | MAILJET_EMAIL = '' 31 | MAILJET_API_KEY = '' 32 | MAILJET_SECRET = '' 33 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | import taiga2 4 | 5 | setup( 6 | name="taiga2", 7 | version=taiga2.__version__, 8 | packages=find_packages(), 9 | author="Remi Marenco", 10 | author_email="rmarenco@broadinstitute.org", 11 | ) 12 | -------------------------------------------------------------------------------- /setup_env.sh: -------------------------------------------------------------------------------- 1 | export FLASK_APP='autoapp.py' 2 | export FLASK_DEBUG=1 3 | -------------------------------------------------------------------------------- /taiga2/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __version__ = "0.1" 5 | -------------------------------------------------------------------------------- /taiga2/api_app.py: -------------------------------------------------------------------------------- 1 | import connexion 2 | import os 3 | import logging 4 | 5 | from taiga2.auth import init_backend_auth 6 | from taiga2.conf import load_config 7 | from taiga2.utils.exception_reporter import ExceptionReporter 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | exception_reporter = ExceptionReporter() 12 | 13 | 14 | def create_db(): 15 | """Create the database, based on the app configuration, 16 | if it does not exist already""" 17 | from taiga2.models import db as _db 18 | 19 | _db.create_all() 20 | 21 | 22 | def create_app(settings_override=None, settings_file=None): 23 | # create the flask app which handles api requests. If settings_override is set, then settings 24 | # are overriden using the values in the provided dictionary. Otherwise, the environment variable 25 | # TAIGA2_SETTINGS is used to look up a config file to use. 26 | 27 | from taiga2.models import db, migrate 28 | from taiga2.schemas import ma 29 | 30 | api_app = connexion.App(__name__, specification_dir="./swagger/") 31 | 32 | app = api_app.app 33 | 34 | load_config(app, settings_override=settings_override, settings_file=settings_file) 35 | 36 | api_app.add_api( 37 | "swagger.yaml", 38 | arguments={ 39 | "title": "No descripton provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)" 40 | }, 41 | ) 42 | 43 | # Init the database with the app 44 | db.init_app(app) 45 | 46 | # Init the migration with Alembic 47 | migrate.init_app(app, db) 48 | 49 | # Init the Serialization/Deserialization Schemas Marshmallow with the app 50 | # It needs to be done after the SQLAlchemy init 51 | ma.init_app(app) 52 | 53 | init_backend_auth(app) 54 | 55 | # Exception report with StackDriver 56 | exception_reporter.init_app(app=app, service_name="taiga-" + app.config["ENV"]) 57 | register_errorhandlers(app=app) 58 | 59 | return api_app, app 60 | 61 | 62 | def create_only_flask_app(settings_override=None, settings_file=None): 63 | api_app, app = create_app( 64 | settings_override=settings_override, settings_file=settings_file 65 | ) 66 | return app 67 | 68 | 69 | def register_errorhandlers(app): 70 | """Register error handlers.""" 71 | 72 | def render_error(error): 73 | """Render error template.""" 74 | # submit this exception to stackdriver if properly configured 75 | exception_reporter.report() 76 | # If a HTTPException, pull the `code` attribute; default to 500 77 | error_code = getattr(error, "code", 500) 78 | return error 79 | 80 | for errcode in [401, 403, 404, 500]: 81 | app.errorhandler(errcode)(render_error) 82 | return None 83 | -------------------------------------------------------------------------------- /taiga2/auth.py: -------------------------------------------------------------------------------- 1 | import flask 2 | import logging 3 | import re 4 | import uuid 5 | 6 | from sqlalchemy.orm.exc import NoResultFound 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def init_front_auth(app): 12 | """For each request to the front end Flask application, we process the current user, via headers""" 13 | app.before_request(set_current_user_from_x_forwarded) 14 | 15 | 16 | def set_current_user_from_x_forwarded(): 17 | """Check the headers X-Forwarded-User and X-Forwarded-Email from Oauth2 Proxy, and use them to load the user. 18 | If the user does not exist, create it. 19 | 20 | Important: If there is no header name, we load the default user from the configuration. Don't set it in Production. 21 | """ 22 | import taiga2.controllers.models_controller as mc 23 | 24 | request = flask.request 25 | config = flask.current_app.config 26 | user = None 27 | 28 | # Use for development environment 29 | default_user_email = config.get("DEFAULT_USER_EMAIL", None) 30 | 31 | # Use for production environment 32 | user_name_header_name = request.headers.get("X-Forwarded-User", None) 33 | user_email_header_name = request.headers.get("X-Forwarded-Email", None) 34 | 35 | if user_email_header_name is not None: 36 | try: 37 | user = mc.get_user_by_email(user_email_header_name) 38 | except NoResultFound: 39 | # User does not exists so we can create it 40 | username = user_email_header_name.split("@")[0] 41 | user = mc.add_user(name=username, email=user_email_header_name) 42 | log.debug( 43 | "We just created the user {} with email {}".format( 44 | username, user_email_header_name 45 | ) 46 | ) 47 | log.debug( 48 | "Check of the user name ({}) and email ({})".format( 49 | user.name, user.email 50 | ) 51 | ) 52 | user_id = user.id 53 | log.debug( 54 | "Looked up header user_email %s to find username: %s", 55 | user_email_header_name, 56 | user_id, 57 | ) 58 | elif user_name_header_name is not None: 59 | user = mc.get_user_by_name(user_name_header_name) 60 | log.debug( 61 | f"Looked up header user_name {user_name_header_name} to find user with id: {user.id}" 62 | ) 63 | 64 | if user is None and default_user_email is not None: 65 | print( 66 | "We did not find the user from the headers, loading the default user by its email {}".format( 67 | default_user_email 68 | ) 69 | ) 70 | 71 | try: 72 | user = mc.get_user_by_email(default_user_email) 73 | except NoResultFound: 74 | user = mc.add_user(name=str(uuid.uuid4()), email=default_user_email) 75 | 76 | flask.g.current_user = user 77 | return None 78 | 79 | 80 | def init_backend_auth(app): 81 | """For each request to the backend end Flask application, we process the current user, via Authorization header""" 82 | app.before_request(set_current_user_from_bearer_token) 83 | 84 | 85 | def set_current_user_from_bearer_token(): 86 | """Use the header Authorization to authenticate the user. If we don't find it, we create it. 87 | The token is a UUID generated by the first Flask app to receive a new user. 88 | 89 | Important: If no Authorization header is passed, we use the DEFAULT_USER_EMAIL from the configuration settings""" 90 | import taiga2.controllers.models_controller as mc 91 | 92 | request = flask.request 93 | config = flask.current_app.config 94 | user = None 95 | bearer_token = request.headers.get("Authorization", None) 96 | default_user_email = config.get("DEFAULT_USER_EMAIL", None) 97 | 98 | if user is None and bearer_token is not None: 99 | m = re.match("Bearer (\\S+)", bearer_token) 100 | if m is not None: 101 | token = m.group(1) 102 | user = bearer_token_lookup(token) 103 | if not user: 104 | # If we did not find the user, we return unauthorized 105 | flask.abort(401) 106 | log.debug("Got token %s which mapped to user %s", token, user.email) 107 | else: 108 | log.warning("Authorization header malformed: %s", bearer_token) 109 | else: 110 | # TODO: Should ask for returning a "Not authenticated" page/response number 111 | if default_user_email is not None: 112 | log.critical( 113 | "DEFAULT_USER_EMAIL is set in config, using that when accessing API" 114 | ) 115 | try: 116 | user = mc.get_user_by_email(default_user_email) 117 | except NoResultFound: 118 | user = mc.add_user(name=str(uuid.uuid4()), email=default_user_email) 119 | else: 120 | log.critical( 121 | "A request without authentication has been received. Rejecting." 122 | ) 123 | # raise Exception("No user passed") 124 | flask.abort(403) 125 | flask.g.current_user = user 126 | 127 | 128 | def bearer_token_lookup(token): 129 | """Ask the controller to return the user given the token""" 130 | import taiga2.controllers.models_controller as mc 131 | 132 | user = mc.get_user_by_token(token) 133 | return user 134 | -------------------------------------------------------------------------------- /taiga2/celery.py: -------------------------------------------------------------------------------- 1 | from taiga2 import tasks 2 | from flask import current_app 3 | 4 | app = tasks.celery 5 | -------------------------------------------------------------------------------- /taiga2/celery_init.py: -------------------------------------------------------------------------------- 1 | from taiga2.api_app import exception_reporter 2 | from taiga2.tasks import celery 3 | 4 | 5 | def configure_celery(app): 6 | """Loads the configuration of CELERY from the Flask config and attaches celery to the app context. 7 | 8 | Returns the celery object""" 9 | celery.config_from_object(app.config) 10 | 11 | # only relevant for the worker process 12 | TaskBase = celery.Task 13 | 14 | class TaskWithStackdriverLogging(TaskBase): 15 | def on_failure(self, exc, task_id, args, kwargs, einfo): 16 | exception_reporter.report(with_request_context=False) 17 | super().on_failure(exc, task_id, args, kwargs, einfo) 18 | 19 | celery.Task = TaskWithStackdriverLogging 20 | 21 | return celery 22 | -------------------------------------------------------------------------------- /taiga2/commands.py: -------------------------------------------------------------------------------- 1 | from celery.bin.celery import main 2 | import click 3 | import subprocess 4 | from flask.cli import with_appcontext 5 | 6 | from .create_test_db_sqlalchemy import recreate_dev_db as _recreate_dev_db 7 | 8 | 9 | @click.command() 10 | @with_appcontext 11 | def recreate_dev_db(): 12 | _recreate_dev_db() 13 | 14 | 15 | @click.command() 16 | def webpack(): 17 | subprocess.call( 18 | ["./node_modules/.bin/webpack", "--watch", "--mode=development"], 19 | cwd="react_frontend", 20 | ) 21 | 22 | 23 | @click.command() 24 | @with_appcontext 25 | def run_worker(): 26 | """Starts a celery worker which will """ 27 | from flask import current_app 28 | 29 | print("config", current_app.config) 30 | 31 | main( 32 | [ 33 | "", 34 | "-A", 35 | "taiga2.celery:app", 36 | "worker", 37 | "-l", 38 | "info", 39 | "-E", 40 | "-n", 41 | "worker1@%h", 42 | "--max-memory-per-child", 43 | "200000", 44 | ] 45 | ) 46 | -------------------------------------------------------------------------------- /taiga2/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | def load_config(app, settings_override=None, settings_file=None): 8 | """Loads the configuration for the Flask application `app`. Uses taiga2.default_settings by default, and 9 | override (if applicable) with the `settings_override` dictionary or `settings_file` filepath""" 10 | app.config.from_object("taiga2.default_settings") 11 | if settings_override is not None: 12 | app.config.update(settings_override) 13 | elif settings_file is not None: 14 | if os.path.exists(settings_file): 15 | settings_file = os.path.abspath(settings_file) 16 | log.warning("Loading settings from %s", settings_file) 17 | app.config.from_pyfile(settings_file) 18 | else: 19 | if "TAIGA2_SETTINGS" in os.environ: 20 | settings_file = os.path.abspath(os.environ["TAIGA2_SETTINGS"]) 21 | log.warning( 22 | "Loading settings from (envvar TAIGA2_SETTINGS): %s", settings_file 23 | ) 24 | app.config.from_pyfile(settings_file) 25 | -------------------------------------------------------------------------------- /taiga2/controllers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/taiga2/controllers/__init__.py -------------------------------------------------------------------------------- /taiga2/controllers/endpoint_validation.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import jsonschema 3 | import jsonpointer 4 | from functools import wraps 5 | import os 6 | import json 7 | import logging 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | CACHED_SWAGGER_SPEC = None 12 | 13 | 14 | def expand_refs(obj, root): 15 | if isinstance(obj, list): 16 | return [expand_refs(x, root) for x in obj] 17 | elif isinstance(obj, dict): 18 | if len(obj) == 1 and "$ref" in obj: 19 | ptr_str = obj["$ref"] 20 | assert ptr_str.startswith("#") 21 | ref = jsonpointer.JsonPointer(ptr_str[1:]) 22 | return expand_refs(ref.get(root), root) 23 | else: 24 | return dict([(k, expand_refs(v, root)) for k, v in obj.items()]) 25 | return obj 26 | 27 | 28 | def load_endpoint_per_operation(filename): 29 | with open(filename) as fd: 30 | spec = yaml.load(fd, Loader=yaml.SafeLoader) 31 | spec = expand_refs(spec, spec) 32 | 33 | endpoint_per_operation = {} 34 | 35 | for path, path_obj in spec["paths"].items(): 36 | for method, endpoint in path_obj.items(): 37 | endpoint_per_operation[endpoint["operationId"]] = endpoint 38 | 39 | return endpoint_per_operation 40 | 41 | 42 | def get_body_parameter(endpoint): 43 | body_params = [x for x in endpoint.get("parameters", []) if x["in"] == "body"] 44 | 45 | if len(body_params) == 0: 46 | return None 47 | 48 | assert len(body_params) == 1 49 | return body_params[0]["name"], body_params[0]["schema"] 50 | 51 | 52 | def get_response_schema(endpoint, status_code: int): 53 | return endpoint["responses"][status_code].get("schema") 54 | 55 | 56 | def get_endpoint_for(operation_id): 57 | global CACHED_SWAGGER_SPEC 58 | if CACHED_SWAGGER_SPEC is None: 59 | filename = os.path.join(os.path.dirname(__file__), "../swagger/swagger.yaml") 60 | CACHED_SWAGGER_SPEC = load_endpoint_per_operation(filename) 61 | return CACHED_SWAGGER_SPEC[operation_id] 62 | 63 | 64 | import inspect 65 | 66 | 67 | def validate(endpoint_func): 68 | @wraps(endpoint_func) 69 | def execute_with_validation(*args, **kwargs): 70 | if len(args) > 0: 71 | positional_arg_names = list( 72 | inspect.signature(endpoint_func).parameters.keys() 73 | ) 74 | for name, arg in zip(positional_arg_names, args): 75 | kwargs[name] = arg 76 | 77 | endpoint = get_endpoint_for(endpoint_func.__name__) 78 | p = get_body_parameter(endpoint) 79 | if p is not None: 80 | name, schema = p 81 | instance = kwargs[name] 82 | jsonschema.validate(instance=instance, schema=schema) 83 | 84 | result = endpoint_func(**kwargs) 85 | 86 | # if result.content_type == "application/json" 87 | parsed_result = json.loads(result.data.decode("utf8")) 88 | 89 | response_schema = get_response_schema(endpoint, result.status_code) 90 | # assert response_schema is not None, "No response schema for {}".format(endpoint_func.__name__) 91 | if response_schema is not None: 92 | try: 93 | jsonschema.validate(instance=parsed_result, schema=response_schema) 94 | except: 95 | log.error( 96 | "Got error trying to validate {}".format(endpoint_func.__name__) 97 | ) 98 | raise 99 | 100 | return result 101 | 102 | return execute_with_validation 103 | -------------------------------------------------------------------------------- /taiga2/conv/__init__.py: -------------------------------------------------------------------------------- 1 | BYTES_PER_STR_OBJECT = 60 2 | MAX_MB_PER_CHUNK = 50 3 | 4 | # These three must all have the same signature: ( input_file, temp_file_generator: "() -> str" ) -> list of files 5 | from taiga2.conv.columnar import columnar_to_rds, read_column_definitions 6 | 7 | from taiga2.conv.imp import csv_to_hdf5 8 | from taiga2.conv.exp import hdf5_to_rds, hdf5_to_csv, hdf5_to_tsv, hdf5_to_gct 9 | 10 | from taiga2.conv import columnar 11 | 12 | 13 | def csv_to_columnar(progress, src, dst, encoding="iso-8859-1", **kwargs): 14 | return columnar.convert_csv_to_tabular(src, dst, ",", encoding, **kwargs) 15 | 16 | 17 | def columnar_to_csv(progress, src, temp_file_generator, encoding="iso-8859-1"): 18 | dst = temp_file_generator() 19 | columnar.convert_tabular_to_csv(src, dst, ",", encoding) 20 | return [dst] 21 | 22 | 23 | def columnar_to_tsv(progress, src, temp_file_generator, encoding="iso-8859-1"): 24 | dst = temp_file_generator() 25 | columnar.convert_tabular_to_csv(src, dst, "\t", encoding) 26 | return [dst] 27 | 28 | 29 | # text formats 30 | CSV_FORMAT = "csv" 31 | TSV_FORMAT = "tsv" 32 | GCT_FORMAT = "gct" 33 | 34 | RAW_FORMAT = "raw" 35 | COMPRESSED_FORMAT = "raw_test" 36 | 37 | # R format 38 | RDS_FORMAT = "rds" 39 | 40 | # Canonical format 41 | HDF5_FORMAT = "hdf5" 42 | COLUMNAR_FORMAT = "columnar" 43 | -------------------------------------------------------------------------------- /taiga2/conv/sniff.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from collections import namedtuple 3 | 4 | Column = namedtuple("Column", ["name", "type"]) 5 | 6 | 7 | class TypeAggregator: 8 | def __init__(self): 9 | self.couldBeFloat = True 10 | self.couldBeInt = True 11 | 12 | def add(self, v): 13 | if self.couldBeInt: 14 | try: 15 | int(v) 16 | except ValueError: 17 | self.couldBeInt = False 18 | 19 | if not self.couldBeInt and self.couldBeFloat: 20 | try: 21 | float(v) 22 | except ValueError: 23 | self.couldBeFloat = False 24 | 25 | def get_type(self): 26 | if self.couldBeInt: 27 | return int 28 | 29 | if self.couldBeFloat: 30 | return float 31 | 32 | return str 33 | 34 | 35 | def sniff(filename, encoding, rows_to_check=None, delimiter="\t"): 36 | with open(filename, "rU", encoding=encoding) as fd: 37 | r = csv.reader(fd, delimiter=delimiter) 38 | col_header = next(r) 39 | row = next(r) 40 | if len(col_header) == len(row): 41 | hasRowNames = False 42 | elif len(col_header) == (len(row) - 1): 43 | hasRowNames = True 44 | else: 45 | raise Exception("First and second rows have different numbers of columns") 46 | 47 | columnTypes = [TypeAggregator() for x in row] 48 | row_count = 0 49 | while rows_to_check is None or row_count < rows_to_check: 50 | for i, x in enumerate(row): 51 | columnTypes[i].add(x) 52 | 53 | try: 54 | row = next(r) 55 | row_count += 1 56 | except StopIteration: 57 | break 58 | 59 | if hasRowNames: 60 | del columnTypes[0] 61 | 62 | columns = [ 63 | Column(col_header[i], columnTypes[i].get_type()) 64 | for i in range(len(columnTypes)) 65 | ] 66 | 67 | return hasRowNames, columns 68 | 69 | 70 | # Taken from pandas.read_csv's list of na_values 71 | NA_STRINGS = [ 72 | "", 73 | "#N/A", 74 | "#N/A N/A", 75 | "#NA", 76 | "-1.#IND", 77 | "-1.#QNAN", 78 | "-NaN", 79 | "-nan", 80 | "1.#IND", 81 | "1.#QNAN", 82 | "", 83 | "N/A", 84 | "NA", 85 | "NULL", 86 | "NaN", 87 | "n/a", 88 | "nan", 89 | "null", 90 | ] 91 | 92 | 93 | class TypeAggregator2: 94 | def __init__(self): 95 | self.couldBeFloat = True 96 | 97 | def add(self, v): 98 | if v in NA_STRINGS: 99 | return 100 | 101 | if self.couldBeFloat: 102 | try: 103 | float(v) 104 | except ValueError: 105 | self.couldBeFloat = False 106 | 107 | def get_type(self): 108 | if self.couldBeFloat: 109 | return "float" 110 | 111 | return "str" 112 | -------------------------------------------------------------------------------- /taiga2/conv/util.py: -------------------------------------------------------------------------------- 1 | import math 2 | import contextlib 3 | import os 4 | import tempfile 5 | from typing import Tuple 6 | 7 | 8 | @contextlib.contextmanager 9 | def make_temp_file_generator(): 10 | filenames = [] 11 | 12 | def temp_file_generator(): 13 | fd = tempfile.NamedTemporaryFile(delete=False) 14 | filename = fd.name 15 | fd.close() 16 | filenames.append(filename) 17 | print("Returning new file", filename) 18 | return filename 19 | 20 | yield temp_file_generator 21 | 22 | for filename in filenames: 23 | os.unlink(filename) 24 | 25 | 26 | class Progress: 27 | def __init__(self, celery_instance): 28 | self.celery_instance = celery_instance 29 | 30 | def failed(self, message, filename=None): 31 | self.celery_instance.update_state( 32 | state="FAILURE", 33 | meta={"current": 0, "total": "0", "message": message, "fileName": filename}, 34 | ) 35 | 36 | def progress(self, message, filename=None, current=0, total=1): 37 | self.celery_instance.update_state( 38 | state="PROGRESS", 39 | meta={ 40 | "current": current, 41 | "total": total, 42 | "message": message, 43 | "fileName": filename, 44 | }, 45 | ) 46 | 47 | 48 | def _to_string_with_nan_mask(x): 49 | if math.isnan(x): 50 | return "NA" 51 | else: 52 | return str(x) 53 | 54 | 55 | r_escape_str = lambda x: '"' + x.replace('"', '\\"') + '"' 56 | 57 | 58 | def shortened_list(l): 59 | if len(l) > 30: 60 | return ", ".join(l[:15]) + " ... " + ", ".join(l[15:]) 61 | else: 62 | return ", ".join(l) 63 | 64 | 65 | import hashlib 66 | 67 | from collections import namedtuple 68 | 69 | ImportResult = namedtuple("ImportResult", "sha256 md5 short_summary long_summary") 70 | 71 | 72 | def get_file_sha256(filename): 73 | hash = hashlib.sha256() 74 | with open(filename, "rb") as fd: 75 | while True: 76 | buffer = fd.read(1024 * 1024) 77 | if len(buffer) == 0: 78 | break 79 | hash.update(buffer) 80 | return hash.hexdigest() 81 | 82 | 83 | def get_file_hashes(filename: str) -> Tuple[str, str]: 84 | """Returns the sha256 and md5 hashes for a file.""" 85 | sha256 = hashlib.sha256() 86 | md5 = hashlib.md5() 87 | with open(filename, "rb") as fd: 88 | while True: 89 | buffer = fd.read(1024 * 1024) 90 | if len(buffer) == 0: 91 | break 92 | sha256.update(buffer) 93 | md5.update(buffer) 94 | return sha256.hexdigest(), md5.hexdigest() 95 | -------------------------------------------------------------------------------- /taiga2/create_test_db_sqlalchemy.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | from flask import current_app 4 | 5 | from taiga2.api_app import create_app, create_db 6 | import taiga2.controllers.models_controller as models_controller 7 | import taiga2.models as models 8 | 9 | import flask 10 | import os 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | # Create the Admin user 16 | # Create the origin data in Home folder 17 | # Create Folder A 18 | # Create Folder B inside Folder A 19 | # Create Data inside Folder B 20 | # Create A1 Data/A2 Data/A3 Data inside Folder A 21 | 22 | ## NEVER USE IN PRODUCTION 23 | 24 | # TODO: Should use the settings.cfg for the bucket name 25 | bucket_name = "broadtaiga2prototype" 26 | 27 | 28 | def get_latest_version_datafiles_from_dataset(dataset_id): 29 | dataset = models_controller.get_dataset(dataset_id) 30 | 31 | latest_dataset_version = dataset.dataset_versions[-1] 32 | 33 | return latest_dataset_version.datafiles 34 | 35 | 36 | def create_sample_dataset( 37 | name="dataset", 38 | description="Sample dataset description", 39 | filename="data", 40 | folder_id="public", 41 | forced_permaname=None, 42 | ): 43 | upload_session_data = models_controller.add_new_upload_session() 44 | upload_session_file_data = models_controller.add_upload_session_s3_file( 45 | session_id=upload_session_data.id, 46 | filename=filename, 47 | s3_bucket=bucket_name, 48 | initial_file_type=models.InitialFileType.Raw, 49 | initial_s3_key="y", 50 | encoding="UTF-8", 51 | ) 52 | 53 | data = models_controller.add_dataset_from_session( 54 | session_id=upload_session_data.id, 55 | dataset_name=name, 56 | dataset_description=description, 57 | current_folder_id=folder_id, 58 | ) 59 | 60 | if forced_permaname: 61 | models_controller.update_permaname(data.id, forced_permaname) 62 | 63 | 64 | def create_db_and_populate(): 65 | create_db() 66 | 67 | admin_group = models_controller.get_group_by_name("Admin") 68 | 69 | # Create the Admin user 70 | admin_user = models_controller.add_user( 71 | name="admin", email="admin@broadinstitute.org", token="test-token" 72 | ) 73 | admin_group.users.append(admin_user) 74 | home_folder_admin = admin_user.home_folder 75 | 76 | # Setting up the flask user 77 | flask.g.current_user = admin_user 78 | 79 | # Create a session where all this is happening 80 | upload_session_origin = models_controller.add_new_upload_session() 81 | 82 | # Create the origin data 83 | upload_session_file_origin = models_controller.add_upload_session_s3_file( 84 | session_id=upload_session_origin.id, 85 | filename="origin", 86 | s3_bucket=bucket_name, 87 | initial_file_type=models.InitialFileType.Raw, 88 | initial_s3_key="x", 89 | encoding="UTF-8", 90 | ) 91 | 92 | origin_dataset = models_controller.add_dataset_from_session( 93 | session_id=upload_session_origin.id, 94 | dataset_name="origin", 95 | dataset_description="No description", 96 | current_folder_id=home_folder_admin.id, 97 | ) 98 | 99 | # Create the Folder A folder 100 | folderA = models_controller.add_folder( 101 | name="Folder A", folder_type=models.Folder.FolderType.folder, description="desc" 102 | ) 103 | models_controller.add_folder_entry( 104 | folder_id=home_folder_admin.id, entry_id=folderA.id 105 | ) 106 | 107 | # Create Folder B inside Folder A 108 | folderB = models_controller.add_folder( 109 | name="Folder B", folder_type=models.Folder.FolderType.folder, description="" 110 | ) 111 | models_controller.add_folder_entry(folder_id=folderA.id, entry_id=folderB.id) 112 | 113 | # Create Data inside Folder B 114 | upload_session_data = models_controller.add_new_upload_session() 115 | upload_session_file_data = models_controller.add_upload_session_s3_file( 116 | session_id=upload_session_data.id, 117 | filename="Data", 118 | s3_bucket=bucket_name, 119 | initial_file_type=models.InitialFileType.Raw, 120 | initial_s3_key="y", 121 | encoding="UTF-8", 122 | ) 123 | 124 | data = models_controller.add_dataset_from_session( 125 | session_id=upload_session_data.id, 126 | dataset_name="Data", 127 | dataset_description="No description", 128 | current_folder_id=folderB.id, 129 | ) 130 | 131 | data_datafiles = get_latest_version_datafiles_from_dataset(data.id) 132 | 133 | temp_data_datafiles = copy.copy(data_datafiles) 134 | 135 | # Create A1 Data/A2 Data/A3 Data inside Folder A 136 | for i in range(1, 4): 137 | name = "".join(["A", str(i), " DatasetVersion"]) 138 | 139 | # We need now to generate new datafiles 140 | if i >= 1: 141 | loop_datafiles = [] 142 | for datafile in temp_data_datafiles: 143 | loop_datafile = models_controller.add_s3_datafile( 144 | name=datafile.name + "v" + str(i), 145 | s3_bucket=bucket_name, 146 | s3_key=models_controller.generate_convert_key(), 147 | compressed_s3_key=models_controller.generate_compressed_key(), 148 | type=datafile.format, 149 | encoding="UTF-8", 150 | short_summary="short summary", 151 | long_summary="long_summary", 152 | ) 153 | loop_datafiles.append(loop_datafile) 154 | temp_data_datafiles = loop_datafiles 155 | datafiles_id = [datafile.id for datafile in temp_data_datafiles] 156 | dataAX = models_controller.add_dataset_version( 157 | dataset_id=origin_dataset.id, datafiles_ids=datafiles_id 158 | ) 159 | 160 | # create a sample dataset in a known location with a known permaname 161 | create_sample_dataset(forced_permaname="sample-1", folder_id="public") 162 | 163 | 164 | def recreate_dev_db(): 165 | database_uri = current_app.config["SQLALCHEMY_DATABASE_URI"] 166 | assert database_uri.startswith("sqlite:///") 167 | database_path = database_uri[len("sqlite:///") :] 168 | database_path = os.path.join("taiga2", database_path) 169 | if os.path.exists(database_path): 170 | print("deleting existing DB before recreating it: {}".format(database_path)) 171 | os.unlink(database_path) 172 | create_db_and_populate() 173 | -------------------------------------------------------------------------------- /taiga2/dataset_subscriptions.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from flask import current_app, url_for 4 | from mailjet_rest import Client 5 | 6 | from taiga2.controllers import models_controller 7 | from taiga2.models import DatasetSubscription 8 | 9 | 10 | def send_emails_for_dataset(dataset_id: str, version_author_id: str): 11 | dataset = models_controller.get_dataset(dataset_id, True) 12 | dataset_subscriptions = ( 13 | dataset.dataset_subscriptions 14 | ) # type: List[DatasetSubscription] 15 | 16 | # Don't email the person who just updated the dataset 17 | dataset_subscribers = [ 18 | ds.user 19 | for ds in dataset_subscriptions 20 | if ds.user.email is not None and ds.user.id != version_author_id 21 | ] 22 | if len(dataset_subscribers) == 0: 23 | return 24 | 25 | mailjet = Client( 26 | auth=( 27 | current_app.config["MAILJET_API_KEY"], 28 | current_app.config["MAILJET_SECRET"], 29 | ), 30 | version="v3.1", 31 | ) 32 | 33 | dataset_url = "https://cds.team/taiga/dataset/{}".format(dataset.permaname) 34 | 35 | data = { 36 | "Messages": [ 37 | { 38 | "From": {"Email": current_app.config["MAILJET_EMAIL"]}, 39 | "To": [{"Email": user.email}], 40 | "Subject": "Taiga Dataset Updated - {}".format(dataset.name), 41 | "TextPart": "Hello,\n" 42 | + "Someone has created a new version of the Taiga dataset {}. See the new version here: {}\n".format( 43 | dataset.name, dataset_url 44 | ) 45 | + 'To unsubscribe from updates about this dataset, click the "unsubscribe" button on the page linked above.', 46 | } 47 | for user in dataset_subscribers 48 | ] 49 | } 50 | result = mailjet.send.create(data=data) 51 | print(result.status_code) 52 | print(result.json()) 53 | return result 54 | -------------------------------------------------------------------------------- /taiga2/default_settings.py: -------------------------------------------------------------------------------- 1 | # TODO: Move this into setting.cfg.sample so we can use the sample by default and it can stay iso with the required configuration 2 | 3 | SQLALCHEMY_DATABASE_URI = "sqlite:///taiga2.db" 4 | SQLALCHEMY_ECHO = False 5 | SQLALCHEMY_TRACK_MODIFICATIONS = True 6 | 7 | # celery settings 8 | BROKER_URL = "redis://localhost:6379" 9 | CELERY_RESULT_BACKEND = "redis://localhost:6379" 10 | CELERYD_MAX_TASKS_PER_CHILD = 5 # Each task can have a lot of memory used 11 | S3_PREFIX = "upload/" 12 | PREFIX = "/taiga2" 13 | # Frontend auth 14 | DEFAULT_USER_EMAIL = "admin@broadinstitute.org" 15 | 16 | # S3 settings 17 | CLIENT_UPLOAD_TOKEN_EXPIRY = 86400 # A day 18 | -------------------------------------------------------------------------------- /taiga2/extensions.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import MetaData 2 | from flask_sqlalchemy import SQLAlchemy 3 | from flask_migrate import Migrate 4 | 5 | convention = { 6 | "ix": "ix_%(column_0_label)s", 7 | "uq": "uq_%(table_name)s_%(column_0_name)s", 8 | "ck": "ck_%(table_name)s_%(constraint_name)s", 9 | "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", 10 | "pk": "pk_%(table_name)s", 11 | } 12 | metadata = MetaData(naming_convention=convention) 13 | db = SQLAlchemy(metadata=metadata) 14 | migrate = Migrate() 15 | -------------------------------------------------------------------------------- /taiga2/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from flask_script import Manager, Command 4 | from flask_migrate import MigrateCommand 5 | 6 | from taiga2.api_app import create_only_flask_app 7 | 8 | """Uses Flask Manager to handle some application managements: 9 | 10 | - db init/migrate/upgrade (for Alembic, more infos with db --help 11 | """ 12 | manager = Manager(create_only_flask_app) 13 | manager.add_option("-c", "--config", dest="settings_file", required=False) 14 | manager.add_command("db", MigrateCommand) 15 | 16 | if __name__ == "__main__": 17 | manager.run() 18 | -------------------------------------------------------------------------------- /taiga2/static/ikons/folder.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /taiga2/static/taiga3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/taiga2/static/taiga3.png -------------------------------------------------------------------------------- /taiga2/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/taiga2/tests/__init__.py -------------------------------------------------------------------------------- /taiga2/tests/blocked_conv_test.py: -------------------------------------------------------------------------------- 1 | from taiga2.tests.datafile_test import StubProgress 2 | from taiga2.conv import ( 3 | hdf5_to_csv, 4 | csv_to_hdf5, 5 | csv_to_columnar, 6 | columnar_to_csv, 7 | hdf5_to_gct, 8 | columnar_to_rds, 9 | ) 10 | import os 11 | import sys 12 | 13 | # tests to make sure processing in chunks/blocks is working correctly by roundtripping data through converters 14 | 15 | tall_matrix_filename = "tall_matrix.csv" 16 | tall_matrix_file_path = os.path.join( 17 | os.path.dirname(sys.modules[__name__].__file__), "test_files", tall_matrix_filename 18 | ) 19 | 20 | tall_table_filename = "tall_table.csv" 21 | tall_table_file_path = os.path.join( 22 | os.path.dirname(sys.modules[__name__].__file__), "test_files", tall_table_filename 23 | ) 24 | 25 | tall_gct_filename = "tall_matrix.gct" 26 | tall_gct_file_path = os.path.join( 27 | os.path.dirname(sys.modules[__name__].__file__), "test_files", tall_gct_filename 28 | ) 29 | 30 | 31 | def test_csv_to_hdf5(tmpdir): 32 | dst_hdf5_file = str(tmpdir.join("out.hdf5")) 33 | dest_csv = str(tmpdir.join("dest.csv")) 34 | 35 | # convert to hdf5 and back, two rows at a time 36 | csv_to_hdf5(StubProgress(), tall_matrix_file_path, dst_hdf5_file, rows_per_block=2) 37 | hdf5_to_csv(StubProgress(), dst_hdf5_file, lambda: dest_csv) 38 | 39 | assert open(tall_matrix_file_path, "rt").read() == open(dest_csv, "rt").read() 40 | 41 | 42 | def test_csv_to_columnar(tmpdir): 43 | dst_hdf5_file = str(tmpdir.join("out.hdf5")) 44 | dest_csv = str(tmpdir.join("dest.csv")) 45 | 46 | # convert to hdf5 and back, two rows at a time 47 | csv_to_columnar( 48 | StubProgress(), tall_table_file_path, dst_hdf5_file, rows_per_block=2 49 | ) 50 | columnar_to_csv(StubProgress(), dst_hdf5_file, lambda: dest_csv) 51 | 52 | assert open(tall_table_file_path, "rt").read() == open(dest_csv, "rt").read() 53 | 54 | 55 | def test_large_columnar_to_rds(tmpdir): 56 | import csv 57 | 58 | src_csv = str(tmpdir.join("source.csv")) 59 | dst_columnar = str(tmpdir.join("source.columnar")) 60 | 61 | # make sizable source file 62 | with open(src_csv, "wt") as fd: 63 | w = csv.writer(fd) 64 | w.writerow(["X" + str(i) for i in range(100)]) 65 | for i in range(500): 66 | w.writerow(["V"] * 100) 67 | 68 | filename_count = [0] 69 | 70 | def temp_file_generator(): 71 | f = str(tmpdir.join("t" + str(filename_count[0]))) 72 | filename_count[0] += 1 73 | return f 74 | 75 | csv_to_columnar(StubProgress(), src_csv, dst_columnar) 76 | # run, with max bytes set to approximate that we should write out 4 files 77 | files = columnar_to_rds( 78 | StubProgress(), dst_columnar, temp_file_generator, max_bytes=200 * 500 / 3 79 | ) 80 | assert len(files) == 4 81 | -------------------------------------------------------------------------------- /taiga2/tests/bounded_mem_conv_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/taiga2/tests/bounded_mem_conv_test.py -------------------------------------------------------------------------------- /taiga2/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from flask import g 4 | import flask 5 | 6 | from taiga2.api_app import create_app 7 | from taiga2.celery_init import configure_celery 8 | from taiga2 import tasks 9 | 10 | from taiga2.tests.mock_s3 import MockS3, MockSTS, MockS3Client 11 | 12 | from taiga2.models import db as _db 13 | from taiga2.controllers import models_controller as mc 14 | 15 | import os 16 | import logging 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | TEST_USER_NAME = "username" 21 | TEST_USER_EMAIL = "username@broadinstitute.org" 22 | AUTH_HEADERS = { 23 | "X-Forwarded-User": TEST_USER_NAME, 24 | "X-Forwarded-Email": TEST_USER_EMAIL, 25 | } 26 | 27 | 28 | @pytest.fixture(scope="function") 29 | def mock_s3(tmpdir): 30 | s3 = MockS3(str(tmpdir)) 31 | return s3 32 | 33 | 34 | @pytest.fixture(scope="function") 35 | def mock_sts(): 36 | sts = MockSTS() 37 | return sts 38 | 39 | 40 | @pytest.fixture(scope="function") 41 | def app(request, mock_s3, mock_sts, tmpdir): 42 | print("creating app...") 43 | db_path = str(tmpdir.join("db.sqlite")) 44 | """Session-wide test `Flask` application.""" 45 | settings_override = { 46 | "TESTING": True, 47 | "SQLALCHEMY_DATABASE_URI": "sqlite:///" + db_path, 48 | "SQLALCHEMY_TRACK_MODIFICATIONS": True, 49 | "SQLALCHEMY_ECHO": False, 50 | "BROKER_URL": None, 51 | "CELERY_RESULT_BACKEND": None, 52 | # TODO: Change this. http://docs.celeryproject.org/en/latest/userguide/testing.html 53 | "CELERY_ALWAYS_EAGER": True, 54 | "S3_BUCKET": "Test_Bucket", 55 | "ENV": "Test", 56 | "REPORT_EXCEPTIONS": False, 57 | } 58 | api_app, _app = create_app(settings_override) 59 | 60 | # create the celery app 61 | configure_celery(_app) 62 | 63 | # the celery worker uses this instance 64 | celery = tasks.celery 65 | 66 | # Establish an application context before running the tests. 67 | ctx = _app.test_request_context() 68 | ctx.push() 69 | 70 | # Monkey patch S3 71 | g._s3_resource = mock_s3 72 | 73 | g._s3_client = MockS3Client() 74 | 75 | g._sts_client = mock_sts 76 | 77 | # Celery of the app 78 | g._celery_instance = celery 79 | 80 | # Return _app and teardown 81 | yield _app 82 | ctx.pop() 83 | 84 | 85 | @pytest.fixture(scope="function") 86 | def db(app, request): 87 | """Session-wide test database.""" 88 | print("creating db...") 89 | _db.create_all() 90 | 91 | # Return db and teardown 92 | yield _db 93 | # _db.drop_all() 94 | 95 | 96 | # Note: this is pretty much completely useless until _all_ calls to db.commit are removed from model_controller. 97 | @pytest.fixture(scope="function") 98 | def session(db, request): 99 | """Creates a new database session for a test.""" 100 | print("Begin session") 101 | flask.current_app.preprocess_request() 102 | yield db.session 103 | 104 | # connection = db.engine.connect() 105 | # transaction = connection.begin() 106 | # 107 | # options = dict(bind=connection, binds={}) 108 | # _session = db.create_scoped_session(options=options) 109 | # db.session = _session 110 | # print(db.session) 111 | # 112 | # # We call before_request 113 | # flask.current_app.preprocess_request() 114 | # 115 | # # Return db and teardown 116 | # yield _session 117 | # _session.remove() 118 | # transaction.rollback() 119 | # connection.close() 120 | 121 | 122 | @pytest.fixture(scope="function") 123 | def user_id(db): 124 | u = mc.add_user(TEST_USER_NAME, TEST_USER_EMAIL) 125 | return u.id 126 | -------------------------------------------------------------------------------- /taiga2/tests/exp_conv_test.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from taiga2.conv import exp 4 | from taiga2 import conv 5 | from taiga2.conv.util import r_escape_str 6 | import csv 7 | 8 | import pytest 9 | from taiga2.conv.util import make_temp_file_generator 10 | 11 | 12 | def rds_to_csv(sources, dest): 13 | handle = subprocess.Popen( 14 | ["R", "--vanilla"], 15 | stdin=subprocess.PIPE, 16 | stdout=subprocess.PIPE, 17 | stderr=subprocess.PIPE, 18 | ) 19 | script_buf = [] 20 | for source in sources: 21 | if len(script_buf) > 0: 22 | script_buf.append( 23 | "data <- rbind(data, readRDS(%s));\n" % r_escape_str(source) 24 | ) 25 | else: 26 | script_buf.append("data <- readRDS(%s);\n" % r_escape_str(source)) 27 | script_buf.append("write.table(data, file=%s, sep=',')" % r_escape_str(dest)) 28 | stdout, stderr = handle.communicate("".join(script_buf).encode("utf8")) 29 | if handle.returncode != 0: 30 | raise Exception("R process failed: %s\n%s" % (stdout, stderr)) 31 | 32 | 33 | def write_sample_csv(csv_file, rows, columns): 34 | with open(csv_file, "wt") as fd: 35 | w = csv.writer(fd) 36 | w.writerow([""] + ["c" + str(i) for i in range(columns)]) 37 | next_value = 0 38 | for row_i in range(rows): 39 | row = ["r" + str(row_i)] 40 | for i in range(columns): 41 | row.append(next_value) 42 | next_value += 1 43 | next_value += 100 44 | w.writerow(row) 45 | fd.close() 46 | 47 | 48 | def write_sample_table(csv_file, rows, columns): 49 | with open(csv_file, "wt") as fd: 50 | w = csv.writer(fd) 51 | w.writerow(["c" + str(i) for i in range(columns)]) 52 | next_value = 0 53 | for row_i in range(rows): 54 | row = [] 55 | for i in range(columns): 56 | row.append(next_value) 57 | next_value += 1 58 | next_value += 100 59 | w.writerow(row) 60 | fd.close() 61 | 62 | 63 | class ProgressStub: 64 | def progress(self, *args, **kwargs): 65 | print("progress", args, kwargs) 66 | 67 | def failed(self, *args, **kwargs): 68 | print("failed", args, kwargs) 69 | 70 | 71 | @pytest.mark.parametrize( 72 | "max_elements_per_block, expected_file_count", [(10000, 1), (5 * 5, 2)] 73 | ) 74 | def test_hdf5_to_rds(tmpdir, max_elements_per_block, expected_file_count): 75 | # actually test this by round-tripping from csv to hdf5 to rds to csv to make sure that we can recapitulate what was 76 | # originally submitted 77 | 78 | original_csv = str(tmpdir.join("t.csv")) 79 | hdf5_file = str(tmpdir.join("t.hdf5")) 80 | final_csv = str(tmpdir.join("final.csv")) 81 | 82 | write_sample_csv(original_csv, 10, 5) 83 | 84 | progress = ProgressStub() 85 | with make_temp_file_generator() as temp_file_generator: 86 | conv.csv_to_hdf5(progress, original_csv, hdf5_file) 87 | files = exp.hdf5_to_rds( 88 | progress, 89 | hdf5_file, 90 | temp_file_generator, 91 | max_elements_per_block=max_elements_per_block, 92 | ) 93 | assert len(files) == expected_file_count 94 | rds_to_csv(files, final_csv) 95 | 96 | 97 | @pytest.mark.parametrize("max_rows, expected_file_count", [(None, 1), (5, 2)]) 98 | def test_columnar_to_rds(tmpdir, max_rows, expected_file_count): 99 | # actually test this by round-tripping from csv to columnar to rds to csv to make sure that we can recapitulate what was 100 | # originally submitted 101 | 102 | original_csv = str(tmpdir.join("t.csv")) 103 | columnar_file = str(tmpdir.join("t.columnar")) 104 | final_csv = str(tmpdir.join("final.csv")) 105 | 106 | write_sample_table(original_csv, 10, 5) 107 | 108 | progress = ProgressStub() 109 | with make_temp_file_generator() as temp_file_generator: 110 | conv.csv_to_columnar(progress, original_csv, columnar_file) 111 | files = exp.columnar_to_rds( 112 | progress, columnar_file, temp_file_generator, max_rows=max_rows 113 | ) 114 | assert len(files) == expected_file_count 115 | rds_to_csv(files, final_csv) 116 | -------------------------------------------------------------------------------- /taiga2/tests/factories.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import factory 4 | 5 | from taiga2.models import db as _db 6 | from taiga2.models import Group, User, Folder, Entry 7 | from taiga2.models import generate_uuid, generate_str_uuid 8 | 9 | 10 | class UserFactory(factory.alchemy.SQLAlchemyModelFactory): 11 | class Meta: 12 | model = User 13 | sqlalchemy_session = _db.session 14 | 15 | id = factory.LazyFunction(generate_uuid) 16 | 17 | name = factory.Faker("name") 18 | 19 | email = factory.Faker("email") 20 | token = factory.LazyFunction(generate_str_uuid) 21 | 22 | # TODO: Home folder and Trash folder 23 | home_folder = factory.RelatedFactory("taiga2.tests.factories.FolderFactory") 24 | trash_folder = factory.RelatedFactory("taiga2.tests.factories.FolderFactory") 25 | 26 | 27 | class GroupFactory(factory.alchemy.SQLAlchemyModelFactory): 28 | class Meta: 29 | model = Group 30 | sqlalchemy_session = _db.session 31 | 32 | name = factory.Sequence(lambda n: "Group {}".format(n)) 33 | 34 | users = factory.List([factory.SubFactory(UserFactory)]) 35 | 36 | 37 | class EntryFactory(factory.alchemy.SQLAlchemyModelFactory): 38 | class Meta: 39 | model = Entry 40 | sqlalchemy_session = _db.session 41 | sqlalchemy_session_persistence = "flush" 42 | 43 | id = factory.LazyFunction(generate_uuid) 44 | name = factory.Faker("name") 45 | 46 | creation_date = factory.LazyFunction(datetime.datetime.utcnow) 47 | 48 | creator = factory.SubFactory(UserFactory) 49 | 50 | description = factory.Faker("text") 51 | 52 | 53 | class FolderFactory(EntryFactory): 54 | class Meta: 55 | model = Folder 56 | sqlalchemy_session = _db.session 57 | 58 | type = Folder.__name__ 59 | 60 | folder_type = Folder.FolderType.folder 61 | 62 | # TODO: Might be a problem here as we are giving a unique Entry instead of a list 63 | entries = factory.List([factory.SubFactory(EntryFactory)]) 64 | -------------------------------------------------------------------------------- /taiga2/tests/fast_schemas_test.py: -------------------------------------------------------------------------------- 1 | # TODO: To replace with true tests 2 | # from taiga2.api_app import * 3 | # from taiga2.models import * 4 | # from taiga2.schemas import * 5 | # 6 | # if __name__ == "__main__": 7 | # with .app_context(): 8 | # # Test Folder Schema 9 | # folder_schema = FolderSchema() 10 | # folderA = db.session.query(Folder).filter(Folder.name == "Home").one() 11 | # print("Home: {}".format(folderA)) 12 | # data_folder_a = folder_schema.dump(folderA).data 13 | # print("") 14 | # print("Data of Home: {}".format(data_folder_a)) 15 | # 16 | # # Test User Schema 17 | # user_schema = UserSchema() 18 | # admin = db.session.query(User).filter(User.name == "Admin").one() 19 | # data_admin_user = user_schema.dump(admin).data 20 | # print("") 21 | # print("Data of Admin user: {}".format(data_admin_user)) 22 | # 23 | # # Test Dataset Schema 24 | # dataset_schema = DatasetSchema() 25 | # dataset_origin = db.session.query(Dataset).filter(Dataset.name == "origin").one() 26 | # data_dataset_origin = dataset_schema.dump(dataset_origin).data 27 | # print("") 28 | # print("Data of Dataset first: {}".format(data_dataset_origin)) 29 | # 30 | # # Test Datafile Schema 31 | # datafile_schema = DataFileSummarySchema() 32 | # datafile_origin = db.session.query(DataFile).filter(DataFile.name == "Origin Datafile").one() 33 | # data_datafile_origin = datafile_schema.dump(datafile_origin).data 34 | # print("") 35 | # print("Data of DataFile Origin: {}".format(data_datafile_origin)) 36 | # 37 | # # Test DatasetVersion Schema 38 | # dataset_version_schema = DatasetVersionSchema() 39 | # dataset_version_origin = db.session.query(DatasetVersion).filter(DatasetVersion.dataset_id == dataset_origin.id).first() 40 | # data_dataset_version_origin = dataset_version_schema.dump(dataset_version_origin).data 41 | # print("") 42 | # print("Data of DatasetVersion origin first: {}".format(data_dataset_version_origin)) 43 | -------------------------------------------------------------------------------- /taiga2/tests/imp_conv_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | import csv 5 | from taiga2.tests.exp_conv_test import ProgressStub 6 | 7 | from flask_sqlalchemy import SessionBase 8 | 9 | from taiga2.third_party_clients.aws import aws 10 | from taiga2.models import InitialFileType 11 | from taiga2.controllers import models_controller 12 | from taiga2.tasks import background_process_new_upload_session_file 13 | from taiga2.conv.imp import _get_csv_dims 14 | from taiga2.conv import ( 15 | csv_to_columnar, 16 | columnar_to_csv, 17 | columnar_to_rds, 18 | csv_to_hdf5, 19 | hdf5_to_csv, 20 | ) 21 | 22 | test_files_folder_path = "taiga2/tests/test_files" 23 | 24 | raw_file_name = "hello.txt" 25 | raw_file_path = os.path.join(test_files_folder_path, raw_file_name) 26 | 27 | csv_file_name = "tiny_matrix.csv" 28 | csv_file_path = os.path.join(test_files_folder_path, csv_file_name) 29 | 30 | tiny_table_name = "tiny_table.csv" 31 | tiny_table_path = os.path.join(test_files_folder_path, tiny_table_name) 32 | 33 | nonutf8_file_name = "non-utf8-table.csv" 34 | nonutf8_file_path = os.path.join(test_files_folder_path, nonutf8_file_name) 35 | 36 | # Not included in the git repository for space reasons. Add your own large numerical matrix in `test_files_folder_path` 37 | # And uncomment where this file is used in tests 38 | large_numerical_matrix_name = "large_numerical_matrix.csv" 39 | large_numerical_matrix_path = os.path.join( 40 | test_files_folder_path, large_numerical_matrix_name 41 | ) 42 | 43 | large_table_name = "large_table.csv" 44 | large_table_path = os.path.join(test_files_folder_path, large_table_name) 45 | 46 | 47 | @pytest.mark.parametrize( 48 | "filename, initial_file_type", 49 | [ 50 | (raw_file_path, InitialFileType.Raw.value), 51 | (csv_file_path, InitialFileType.NumericMatrixCSV.value), 52 | (tiny_table_path, InitialFileType.TableCSV.value), 53 | # (large_numerical_matrix_path, InitialFileType.NumericMatrixCSV.value), 54 | # (large_table_path, InitialFileType.TableCSV.value) 55 | ], 56 | ) 57 | def test_upload_session_file( 58 | filename, initial_file_type, session: SessionBase, user_id 59 | ): 60 | print("initial_file_type", initial_file_type, filename) 61 | 62 | new_bucket = aws.s3.Bucket("bucket") 63 | new_bucket.create() 64 | 65 | converted_s3_key = models_controller.generate_convert_key() 66 | compressed_s3_key = models_controller.generate_compressed_key() 67 | 68 | with open(filename, "rb") as data: 69 | aws.s3.Bucket(new_bucket.name).put_object(Key=converted_s3_key, Body=data) 70 | s3_raw_uploaded_file = aws.s3.Object(new_bucket.name, converted_s3_key) 71 | 72 | bucket_name = "bucket" 73 | initial_s3_key = s3_raw_uploaded_file.key 74 | 75 | session = models_controller.add_new_upload_session() 76 | upload_session_file = models_controller.add_upload_session_s3_file( 77 | session.id, 78 | os.path.basename(filename), 79 | initial_file_type, 80 | initial_s3_key, 81 | bucket_name, 82 | "UTF-8", 83 | ) 84 | 85 | background_process_new_upload_session_file.delay( 86 | upload_session_file.id, 87 | initial_s3_key, 88 | initial_file_type, 89 | bucket_name, 90 | converted_s3_key, 91 | compressed_s3_key, 92 | upload_session_file.encoding, 93 | ).wait() 94 | 95 | # confirm the converted object was published back to s3 96 | assert aws.s3.Object(bucket_name, converted_s3_key).download_as_bytes() is not None 97 | 98 | # Check updated UploadSessionFile 99 | updated_upload_session_file = models_controller.get_upload_session_file( 100 | upload_session_file.id 101 | ) 102 | 103 | assert updated_upload_session_file.column_types_as_json is None 104 | 105 | 106 | def test_get_csv_dims(tmpdir): 107 | filename = tmpdir.join("sample") 108 | filename.write_binary(b"a,b,c\nd,1,2,3\n") 109 | row_count, col_count, sha256, md5 = _get_csv_dims( 110 | ProgressStub(), str(filename), csv.excel, "utf-8" 111 | ) 112 | assert row_count == 1 113 | assert col_count == 3 114 | assert sha256 == "629910bba467f4d6f518d309b3d2a99e316d7d5ef1faa744a7c5a6a084219255" 115 | assert md5 == "28ed28fa14570cc1409563f848a4c962" 116 | 117 | 118 | def test_get_large_csv_dims_wrong_delimeter(tmpdir): 119 | filename = tmpdir.join("sample") 120 | file_contents = ",".join(str(i) for i in range(10000000)) 121 | file_contents = "\n".join(file_contents for _ in range(3)) 122 | file_contents = file_contents.encode("ascii") 123 | filename.write_binary(file_contents) 124 | 125 | with pytest.raises(Exception, match=r".*field larger than field limit.*"): 126 | row_count, col_count, sha256, md5 = _get_csv_dims( 127 | ProgressStub(), str(filename), csv.excel_tab, "utf-8" 128 | ) 129 | 130 | 131 | def test_non_utf8(tmpdir): 132 | dest = str(tmpdir.join("dest.columnar")) 133 | final = str(tmpdir.join("final.csv")) 134 | rds_dest = str(tmpdir.join("final.rds")) 135 | 136 | csv_to_columnar(None, nonutf8_file_path, dest) 137 | columnar_to_csv(None, dest, lambda: final) 138 | import csv 139 | 140 | with open(final, "rU") as fd: 141 | r = csv.DictReader(fd) 142 | row1 = next(r) 143 | assert row1["row"] == "1" 144 | assert len(row1["value"]) == 1 145 | row2 = next(r) 146 | assert row2["row"] == "2" 147 | assert row2["value"] == "R" 148 | 149 | # lastly, make sure we don't get an exception when converting to rds because R has its own ideas about encoding 150 | columnar_to_rds(None, dest, lambda: rds_dest) 151 | 152 | 153 | def test_matrix_with_full_header_import(tmpdir): 154 | r_filename = tmpdir.join("r_style") 155 | r_dest = tmpdir.join("r.hdf5") 156 | r_final = tmpdir.join("r_final.csv") 157 | r_filename.write_binary(b"a,b,c\nd,1,2,3\n") 158 | csv_to_hdf5(ProgressStub(), str(r_filename), str(r_dest)) 159 | hdf5_to_csv(ProgressStub(), str(r_dest), lambda: str(r_final)) 160 | 161 | pandas_filename = tmpdir.join("pandas_style") 162 | pandas_final = tmpdir.join("pandas_final.csv") 163 | pandas_dest = tmpdir.join("pandas.hdf5") 164 | pandas_filename.write_binary(b"i,a,b,c\nd,1,2,3\n") 165 | csv_to_hdf5(ProgressStub(), str(pandas_filename), str(pandas_dest)) 166 | hdf5_to_csv(ProgressStub(), str(pandas_dest), lambda: str(pandas_final)) 167 | 168 | assert r_final.read_binary() == pandas_final.read_binary() 169 | -------------------------------------------------------------------------------- /taiga2/tests/mock_s3.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import datetime 3 | import os 4 | import io 5 | import tempfile 6 | import re 7 | 8 | # 9 | class MockS3: 10 | def __init__(self, tmpdir): 11 | self.file_per_key = {} 12 | self.tmpdir = tmpdir 13 | 14 | def _get_unique_filename(self): 15 | fd = tempfile.NamedTemporaryFile(dir=self.tmpdir, delete=False) 16 | fd.close() 17 | return fd.name 18 | 19 | def Bucket(self, bucket_name): 20 | return MockBucket(self, bucket_name) 21 | 22 | def Object(self, bucket_name, key): 23 | bucket = self.Bucket(bucket_name) 24 | return MockS3Object(bucket, key) 25 | 26 | def generate_presigned_url(self, ClientMethod, Params): 27 | assert ClientMethod == "get_object" 28 | return "s3://" + Params["Bucket"] + "/" + Params["Key"] + "?signed" 29 | 30 | 31 | class MockBucket: 32 | def __init__(self, s3, name): 33 | self.s3 = s3 34 | self.name = name 35 | 36 | def create(self): 37 | return self 38 | 39 | # TODO: I am adding back the capitalized arguments because this is how it is done in the Boto3 doc: http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Bucket.put_object 40 | def put_object(self, Key, Body): 41 | new_object = MockS3Object(self, Key) 42 | new_object.upload_fileobj(Body) 43 | return new_object 44 | 45 | def copy(self, copy_source, key): 46 | bucket_source_name = copy_source["Bucket"] 47 | key_source_name = copy_source["Key"] 48 | 49 | path_source = self.s3.file_per_key[(bucket_source_name, key_source_name)] 50 | # obj_source = self.s3.buckets[bucket_source_name].objects[key_source_name] 51 | 52 | with open(path_source, "rb") as data_copy_source: 53 | self.put_object(key, data_copy_source) 54 | 55 | def Object(self, key): 56 | return MockS3Object(self, key) 57 | 58 | def __call__(self, name): 59 | return self 60 | 61 | 62 | class MockS3Object: 63 | def __init__(self, bucket, key): 64 | self.bucket = bucket 65 | self.key = key 66 | self.content_length = 10 67 | 68 | def download_fileobj(self, writer): 69 | full_path = self.bucket.s3.file_per_key[(self.bucket.name, self.key)] 70 | 71 | with open(full_path, "rb") as f: 72 | writer.write(f.read()) 73 | 74 | def upload_fileobj(self, fileobj, ExtraArgs=None): 75 | full_path = self.bucket.s3._get_unique_filename() 76 | 77 | with open(full_path, "w+b") as f: 78 | f.write(fileobj.read()) 79 | 80 | self.bucket.s3.file_per_key[(self.bucket.name, self.key)] = full_path 81 | 82 | def upload_file(self, path, ExtraArgs=None): 83 | with open(path, "rb") as data: 84 | self.upload_fileobj(data) 85 | 86 | def upload_bytes(self, b): 87 | """Not a real boto3 method. Just convience for testing""" 88 | assert isinstance(b, bytes) 89 | self.upload_fileobj(io.BytesIO(b)) 90 | 91 | def download_as_bytes(self): 92 | """Not a real boto3 method. Just convience for testing""" 93 | full_path = self.bucket.s3.file_per_key[(self.bucket.name, self.key)] 94 | 95 | with open(full_path, "rb") as f: 96 | return f.read() 97 | 98 | 99 | # 100 | 101 | 102 | class MockS3Client: 103 | def generate_presigned_url(self, ClientMethod, Params): 104 | return "https://mocks3/{}/{}?signed=Y".format(Params["Bucket"], Params["Key"]) 105 | 106 | 107 | def parse_presigned_url(url): 108 | g = re.match("https://mocks3/([^/]+)/([^?]+)\\?signed=Y", url) 109 | return g.group(1), g.group(2) 110 | 111 | 112 | # 113 | class MockSTS: 114 | def get_session_token(self, *args, **kwargs): 115 | expiration_seconds = kwargs.get("DurationSeconds", 900) 116 | datetime_expiration = datetime.datetime.now() + datetime.timedelta( 117 | 0, expiration_seconds 118 | ) 119 | dict_credentials = { 120 | "Credentials": { 121 | "AccessKeyId": "AccessKeyId", 122 | "Expiration": datetime_expiration.isoformat(), 123 | "SecretAccessKey": "SecretAccessKey", 124 | "SessionToken": "SessionToken", 125 | } 126 | } 127 | return dict_credentials 128 | 129 | 130 | # 131 | -------------------------------------------------------------------------------- /taiga2/tests/mock_s3_test.py: -------------------------------------------------------------------------------- 1 | from taiga2.tests.mock_s3 import MockS3 2 | import io 3 | 4 | 5 | def test_mock_s3_basics(tmpdir): 6 | s3 = MockS3(str(tmpdir)) 7 | 8 | path = str(tmpdir) + "/download" 9 | 10 | s3.Bucket("bucket").put_object("key", io.BytesIO(b"abc")) 11 | with open(path, "wb") as w: 12 | s3.Object("bucket", "key").download_fileobj(w) 13 | 14 | open(path, "rb").read() == b"abc" 15 | -------------------------------------------------------------------------------- /taiga2/tests/security_test.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/hello.txt: -------------------------------------------------------------------------------- 1 | Hello S3! 2 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/non-utf8-table.csv: -------------------------------------------------------------------------------- 1 | row,value,validutf8 2 | 1,®,N 3 | 2,R,Y 4 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/tall_matrix.csv: -------------------------------------------------------------------------------- 1 | ,a,b,c 2 | d,1.0,2.0,3.0 3 | e,4.0,5.0,6.0 4 | e1,4.0,5.0,6.0 5 | e2,4.0,5.0,6.0 6 | e3,4.0,5.0,6.0 7 | e4,4.0,5.0,6.0 8 | e5,4.0,5.0,6.0 9 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/tall_matrix.gct: -------------------------------------------------------------------------------- 1 | #1.2 2 | 7 2 3 | Name Description a b 4 | d D 1.0 2.0 5 | e E 4.0 5.0 6 | e1 E1 4.0 5.0 7 | e2 E2 4.0 5.0 8 | e3 E3 4.0 5.0 9 | e4 E4 4.0 5.0 10 | e5 E5 4.0 5.0 11 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/tall_table.csv: -------------------------------------------------------------------------------- 1 | a,b,c,d 2 | 1,2,Y,1.2 3 | 3,4,Y,2.0 4 | 5,6,N,4.0 5 | 1,2,Y,1.2 6 | 3,4,Y,2.0 7 | 5,6,N,4.0 8 | 1,2,Y,1.2 9 | 3,4,Y,2.0 10 | 5,6,N,4.0 11 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/tiny_matrix.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | d,1,2,3 3 | e,4,5,6 4 | -------------------------------------------------------------------------------- /taiga2/tests/test_files/tiny_table.csv: -------------------------------------------------------------------------------- 1 | a,b,c,d 2 | 1,2,Y,1.2 3 | 3,4,Y,2 4 | 5,,N,NA 5 | -------------------------------------------------------------------------------- /taiga2/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | def get_dict_from_response_jsonify(jsonified_response): 5 | """Utils function to get a dict from a Response built with flask.jsonify""" 6 | return json.loads(jsonified_response.get_data(as_text=True)) 7 | -------------------------------------------------------------------------------- /taiga2/third_party_clients/aws.py: -------------------------------------------------------------------------------- 1 | import flask 2 | import boto3 3 | import logging 4 | from flask import g 5 | import re 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class AWSClients: 11 | @property 12 | def s3(self): 13 | """ 14 | Get a s3 client using the credentials in the config. 15 | 16 | To mock in a test, set flask.g._s3_client in test setup 17 | """ 18 | config = flask.current_app.config 19 | if not hasattr(g, "_s3_resource"): 20 | aws_access_key_id = config["AWS_ACCESS_KEY_ID"] 21 | log.info("Getting S3 resource with access key %s", aws_access_key_id) 22 | g._s3_resource = boto3.resource( 23 | "s3", 24 | aws_access_key_id=aws_access_key_id, 25 | aws_secret_access_key=config["AWS_SECRET_ACCESS_KEY"], 26 | ) 27 | return g._s3_resource 28 | 29 | @property 30 | def s3_client(self): 31 | config = flask.current_app.config 32 | if not hasattr(g, "_s3_client"): 33 | aws_access_key_id = config["AWS_ACCESS_KEY_ID"] 34 | log.info("Getting S3 client with access key %s", aws_access_key_id) 35 | g._s3_client = boto3.client( 36 | "s3", 37 | # config=Config(signature_version='s3v4'), 38 | aws_access_key_id=aws_access_key_id, 39 | aws_secret_access_key=config["AWS_SECRET_ACCESS_KEY"], 40 | ) 41 | return g._s3_client 42 | 43 | @property 44 | def client_upload_sts(self): 45 | """ 46 | Get a STS client using credentials which are different the main credentials. These credentials should 47 | only have access to perform object puts to the appropriate bucket. 48 | 49 | To mock in a test, set flask.g._client_upload_sts_client in test setup 50 | """ 51 | config = flask.current_app.config 52 | if not hasattr(g, "_sts_client"): 53 | aws_access_key_id = config["CLIENT_UPLOAD_AWS_ACCESS_KEY_ID"] 54 | log.warn("Getting STS client with access key %s", aws_access_key_id) 55 | g._sts_client = boto3.client( 56 | "sts", 57 | aws_access_key_id=aws_access_key_id, 58 | aws_secret_access_key=config["CLIENT_UPLOAD_AWS_SECRET_ACCESS_KEY"], 59 | ) 60 | return g._sts_client 61 | 62 | 63 | aws = AWSClients() 64 | 65 | 66 | def parse_s3_url(url): 67 | g = re.match("s3://([^/]+)/(.*)", url) 68 | 69 | # TODO: update the urls in the db to always use the s3://bucket/key syntax 70 | if g is None: 71 | g = re.match("https?://([^.]+).s3.amazonaws.com/(.+)", url) 72 | 73 | assert g is not None, "Could not parse {} into bucket and key".format(repr(url)) 74 | 75 | return g.group(1), g.group(2) 76 | 77 | 78 | def create_s3_url(bucket, key): 79 | return "".join(["s3://", bucket, "/", key]) 80 | 81 | 82 | def create_signed_get_obj(bucket, key, filename): 83 | content_disposition = "attachment; filename={};".format(filename) 84 | # TODO: Set expiry on signing url 85 | signed_url = aws.s3_client.generate_presigned_url( 86 | ClientMethod="get_object", 87 | Params={ 88 | "Bucket": bucket, 89 | "Key": key, 90 | "ResponseContentDisposition": content_disposition, 91 | }, 92 | ) 93 | return signed_url 94 | -------------------------------------------------------------------------------- /taiga2/third_party_clients/gcs.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from typing import Optional, Tuple 3 | 4 | from google.cloud import storage, exceptions as gcs_exceptions 5 | 6 | 7 | def get_bucket(bucket_name: str) -> storage.Bucket: 8 | client = storage.Client() 9 | try: 10 | bucket = client.get_bucket(bucket_name) 11 | except gcs_exceptions.Forbidden as e: 12 | raise ValueError( 13 | "taiga-892@cds-logging.iam.gserviceaccount.com does not have storage.buckets.get access to bucket: {}".format( 14 | bucket_name 15 | ) 16 | ) 17 | except gcs_exceptions.NotFound as e: 18 | raise ValueError("No GCS bucket found: {}".format(bucket_name)) 19 | return bucket 20 | 21 | 22 | def create_signed_gcs_url(bucket_name, blob_name): 23 | client = storage.Client() 24 | bucket = client.get_bucket(bucket_name) 25 | blob = bucket.get_blob(blob_name) 26 | expiration_time = timedelta(minutes=1) 27 | 28 | url = blob.generate_signed_url(expiration=expiration_time, version="v4") 29 | return url 30 | 31 | 32 | def upload_from_file( 33 | file_obj, dest_bucket: str, dest_path: str, content_type: str, content_encoding: str 34 | ): 35 | bucket = get_bucket(dest_bucket) 36 | blob = storage.Blob(dest_path, bucket) 37 | blob.content_type = content_type 38 | blob.content_encoding = content_encoding 39 | try: 40 | blob.upload_from_file(file_obj) 41 | except gcs_exceptions.Forbidden as e: 42 | raise ValueError( 43 | "taiga-892@cds-logging.iam.gserviceaccount.com does not have storage.buckets.create access to bucket: {}".format( 44 | dest_bucket 45 | ) 46 | ) 47 | 48 | return blob 49 | 50 | 51 | def get_blob(bucket_name: str, object_name: str) -> Optional[storage.Blob]: 52 | bucket = get_bucket(bucket_name) 53 | blob = bucket.get_blob(object_name) 54 | 55 | return blob 56 | 57 | 58 | def parse_gcs_path(gcs_path: str) -> Tuple[str, str]: 59 | # remove prefix 60 | if gcs_path.startswith("gs://"): 61 | gcs_path = gcs_path.replace("gs://", "") 62 | 63 | if "/" not in gcs_path: 64 | raise ValueError( 65 | "Invalid GCS path. '{}' is not in the form 'bucket_name/object_name'".format( 66 | gcs_path 67 | ) 68 | ) 69 | 70 | bucket_name, object_name = gcs_path.split("/", 1) 71 | return bucket_name, object_name 72 | -------------------------------------------------------------------------------- /taiga2/ui.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import logging 3 | import flask 4 | from flask import Flask, render_template, send_from_directory, url_for, request, abort 5 | import os 6 | from taiga2.conf import load_config 7 | from taiga2.auth import init_front_auth 8 | from taiga2 import commands 9 | from .extensions import db as ext_db, migrate 10 | from taiga2 import models 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | def register_extensions(app): 16 | # Init the database with the app 17 | ext_db.init_app(app) 18 | migrate.init_app(app, ext_db) 19 | 20 | 21 | def register_commands(app): 22 | app.cli.add_command(commands.recreate_dev_db) 23 | app.cli.add_command(commands.run_worker) 24 | app.cli.add_command(commands.webpack) 25 | 26 | 27 | def create_app(settings_override=None, settings_file=None): 28 | 29 | app = Flask(__name__) 30 | 31 | settings_file = os.environ.get("TAIGA_SETTINGS_FILE", settings_file) 32 | load_config(app, settings_file=settings_file, settings_override=settings_override) 33 | 34 | register_extensions(app) 35 | register_commands(app) 36 | 37 | # Add hooks for managing the authentication before each request 38 | init_front_auth(app) 39 | 40 | # Register the routes 41 | app.add_url_rule("/", view_func=index) 42 | app.add_url_rule("/pseudostatic//", view_func=pseudostatic) 43 | app.add_url_rule("/", view_func=sendindex2) 44 | app.add_url_rule("/js/", view_func=static_f) 45 | 46 | @app.context_processor 47 | def inject_pseudostatic_url(): 48 | return dict(pseudostatic_url=pseudostatic_url) 49 | 50 | return app 51 | 52 | 53 | PSEUDOSTATIC_CACHE = {} 54 | 55 | 56 | def pseudostatic_url(name): 57 | static_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "static")) 58 | abs_filename = os.path.abspath(os.path.join(static_dir, name)) 59 | assert abs_filename.startswith(static_dir) 60 | 61 | hash = None 62 | if name in PSEUDOSTATIC_CACHE: 63 | hash, mtime = PSEUDOSTATIC_CACHE[abs_filename] 64 | if mtime != os.path.getmtime(abs_filename): 65 | hash = None 66 | 67 | if hash is None: 68 | mtime = os.path.getmtime(abs_filename) 69 | hash_md5 = hashlib.md5() 70 | with open(abs_filename, "rb") as fd: 71 | for chunk in iter(lambda: fd.read(40960), b""): 72 | hash_md5.update(chunk) 73 | hash = hash_md5.hexdigest() 74 | PSEUDOSTATIC_CACHE[abs_filename] = (hash, mtime) 75 | log.debug("Caching hash of %s as %s", abs_filename, hash) 76 | 77 | return flask.url_for("pseudostatic", hash=hash, filename=name) 78 | 79 | 80 | def render_index_html(): 81 | try: 82 | user_token = flask.g.current_user.token 83 | 84 | return render_template( 85 | "index.html", prefix=url_for("index"), user_token=user_token 86 | ) 87 | except AttributeError: 88 | abort(403) 89 | 90 | 91 | def index(): 92 | return render_index_html() 93 | 94 | 95 | def sendindex2(filename): 96 | return render_index_html() 97 | 98 | 99 | def static_f(filename): 100 | return send_from_directory(os.path.abspath("node_modules"), filename) 101 | 102 | 103 | def pseudostatic(hash, filename): 104 | static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") 105 | return flask.send_from_directory(static_dir, filename) 106 | -------------------------------------------------------------------------------- /taiga2/utils/exception_reporter.py: -------------------------------------------------------------------------------- 1 | from google.cloud import error_reporting 2 | from flask import request 3 | 4 | try: 5 | from flask import _app_ctx_stack as stack 6 | except ImportError: 7 | from flask import _request_ctx_stack as stack 8 | 9 | 10 | class ExceptionReporter: 11 | def __init__(self, service_name=None, app=None): 12 | self.service_name = service_name 13 | if app is not None: 14 | self.init_app(app) 15 | 16 | def init_app(self, app, service_name=None): 17 | if service_name is not None: 18 | self.service_name = service_name 19 | self.disabled = not app.config["REPORT_EXCEPTIONS"] 20 | # attempt to create a client, so we'll get an error on startup if there's a problem with credentials 21 | if not self.disabled: 22 | self._create_client() 23 | 24 | def report(self, with_request_context=True): 25 | if self.disabled: 26 | return 27 | 28 | client = self._get_client() 29 | if with_request_context: 30 | client.report_exception( 31 | http_context=error_reporting.build_flask_context(request) 32 | ) 33 | else: 34 | client.report_exception() 35 | 36 | def _create_client(self): 37 | return error_reporting.Client(service=self.service_name, project="cds-logging") 38 | 39 | def _get_client(self): 40 | ctx = stack.top 41 | if ctx is not None: 42 | if not hasattr(ctx, "stackdriver_client"): 43 | ctx.stackdriver_client = self._create_client() 44 | return ctx.stackdriver_client 45 | raise Exception("Missing context") 46 | -------------------------------------------------------------------------------- /taiga2/utils/fix_description_dataset_versions.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | import flask 4 | 5 | from urllib.parse import urlparse 6 | from sqlalchemy.orm.exc import NoResultFound 7 | 8 | from taiga2.api_app import create_app, create_db 9 | from taiga2.controllers import models_controller 10 | 11 | 12 | class DataFileInfo: 13 | def __init__(self, id, datafile, owner_email, version=None, creation_date=None): 14 | self.id = id 15 | self.datafile = datafile 16 | self.owner_email = owner_email 17 | self.version = version 18 | self.creation_date = creation_date 19 | 20 | 21 | class DatasetVersionFileInfo: 22 | def __init__(self): 23 | self.datafiles_info = [] 24 | 25 | 26 | def populate_db(dataset_version_with_datafile_csv_path): 27 | with open(dataset_version_with_datafile_csv_path) as dataset_version_file: 28 | print("Fixing the description and version of the dataset versions:") 29 | reader = csv.DictReader(dataset_version_file) 30 | 31 | for row in reader: 32 | dataset_permaname = row["permaname"] 33 | dataset_version_id = row["dataset_id"] 34 | dataset_version_version = row["version"] 35 | dataset_version_description = row["description"] 36 | 37 | try: 38 | dataset = models_controller.get_dataset_from_permaname( 39 | dataset_permaname 40 | ) 41 | except NoResultFound: 42 | print("Dataset {} was not found in db".format(dataset_permaname)) 43 | 44 | for from_dataset_dataset_versions in dataset.dataset_versions: 45 | if dataset_version_id == from_dataset_dataset_versions.id: 46 | if ( 47 | from_dataset_dataset_versions.description 48 | != dataset_version_description 49 | ): 50 | print( 51 | "Dataset {} => In Db: {}, in csv: '{}'".format( 52 | dataset.permaname, 53 | from_dataset_dataset_versions.description, 54 | dataset_version_description, 55 | ) 56 | ) 57 | models_controller.update_dataset_version_description( 58 | dataset_version_id=dataset_version_id, 59 | new_description=dataset_version_description, 60 | ) 61 | if ( 62 | int(dataset_version_version) 63 | != from_dataset_dataset_versions.version 64 | ): 65 | print( 66 | "Dataset version => In Db: {}, in csv: {}".format( 67 | from_dataset_dataset_versions.version, 68 | dataset_version_version, 69 | ) 70 | ) 71 | from_dataset_dataset_versions.version = int( 72 | dataset_version_version 73 | ) 74 | models_controller.db.session.add(from_dataset_dataset_versions) 75 | models_controller.db.session.commit() 76 | print("\n") 77 | 78 | 79 | if __name__ == "__main__": 80 | parser = argparse.ArgumentParser() 81 | 82 | parser.add_argument( 83 | "-s", 84 | "--settings", 85 | required=True, 86 | help="Settings used for the creation of the api/backends apps", 87 | ) 88 | parser.add_argument( 89 | "-dv", 90 | "--dataset_version", 91 | required=True, 92 | help="Dataset version with datafile csv file path", 93 | ) 94 | 95 | args = parser.parse_args() 96 | 97 | settings_path = args.settings 98 | dataset_version_with_datafile_csv_path = args.dataset_version 99 | 100 | print("Dataset version path: {}".format(dataset_version_with_datafile_csv_path)) 101 | 102 | api_app, backend_app = create_app(settings_file=settings_path) 103 | 104 | with backend_app.app_context(): 105 | # Use the next line only when you are sure you want to drop the db 106 | # models_controller.db.drop_all() 107 | # models_controller.db.create_all() 108 | populate_db(dataset_version_with_datafile_csv_path) 109 | -------------------------------------------------------------------------------- /travis/login_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -exv 3 | 4 | openssl aes-256-cbc -k "$super_secret_password" -in travis/travis-docker-push-account.json.enc -out travis/travis-docker-push-account.json -d 5 | cat travis/travis-docker-push-account.json | docker login -u _json_key --password-stdin https://us.gcr.io 6 | -------------------------------------------------------------------------------- /travis/push_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -exv 3 | 4 | REPO_URI=us.gcr.io/cds-docker-containers/taiga 5 | 6 | docker tag taiga:latest ${REPO_URI} 7 | docker push ${REPO_URI} -------------------------------------------------------------------------------- /travis/travis-docker-push-account.json.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/taiga/51a45d026cc487ef5c598677195b2839f1b551f9/travis/travis-docker-push-account.json.enc -------------------------------------------------------------------------------- /write_version.sh: -------------------------------------------------------------------------------- 1 | echo 'export const SHA = "'"$1"'"' > react_frontend/src/version.ts 2 | --------------------------------------------------------------------------------