├── .github └── workflows │ ├── auto_upload_to_vectorhub.yml │ ├── autodoc.yml │ ├── nightly-upload.yml │ ├── python-pytest-audio.yml │ ├── python-pytest-general.yml │ ├── python-pytest-image-text.yml │ ├── python-pytest-image.yml │ ├── python-pytest-text.yml │ └── update_badges.yml └── README.md /.github/workflows/auto_upload_to_vectorhub.yml: -------------------------------------------------------------------------------- 1 | name: Insert Documents Into Collection 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | run_test: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Pytest 3.7 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: '3.7' 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | pip install -e . 21 | python3 -m pip install vectorai-nightly 22 | python3 -m pip install torch torchvision 23 | python3 -m pip install transformers==3.5.1 24 | python3 -m pip install tensorflow-text 25 | python3 -m pip install sentencepiece 26 | - name: Creating models.json 27 | env: 28 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 29 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 30 | VH_COLLECTION_NAME: ${{secrets.VH_COLLECTION_NAME}} 31 | 32 | run: | 33 | python3 utils/upload_cards.py 34 | -------------------------------------------------------------------------------- /.github/workflows/autodoc.yml: -------------------------------------------------------------------------------- 1 | name: Run Autodoc 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | run_test: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Pytest 3.7 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: '3.7' 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | sudo apt-get install libsndfile1-dev 21 | pip install -e .['all'] 22 | - name: Updating docs 23 | run: | 24 | rm -rf docs 25 | rm -rf docsrc/_build 26 | rm -rf docsrc/modules.rst 27 | sphinx-apidoc -o docsrc vectorhub 28 | python -m sphinx docsrc/. docs 29 | touch docs/.nojekyll 30 | echo ${{ github.ref }} 31 | git add . 32 | git config --local user.email "action@github.com" 33 | git config --local user.name "GitHub Action" 34 | git commit -m "ci: Automated build push" -a | exit 0 35 | - name: Push changes 36 | if: github.ref == 'refs/heads/main' 37 | uses: ad-m/github-push-action@master 38 | with: 39 | github_token: ${{ secrets.SECRET_TOKEN }} 40 | branch: ${{ github.ref }} 41 | -------------------------------------------------------------------------------- /.github/workflows/nightly-upload.yml: -------------------------------------------------------------------------------- 1 | # name: Upload Vectorhub Nightly 2 | 3 | # on: 4 | # schedule: 5 | # - cron: "0 0 * * *" 6 | # # on: 7 | # # push: 8 | # # branches: 9 | # # - feature/add_vectorhub_nightly 10 | 11 | # jobs: 12 | # deploy: 13 | 14 | # runs-on: ubuntu-latest 15 | 16 | # steps: 17 | # - uses: actions/checkout@v2 18 | # - name: Set up Python 19 | # uses: actions/setup-python@v2 20 | # with: 21 | # python-version: '3.x' 22 | # - name: Install dependencies 23 | # run: | 24 | # python -m pip install --upgrade pip 25 | # pip install setuptools wheel twine 26 | # - name: Build and publish 27 | # env: 28 | # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 29 | # TWINE_PASSWORD: ${{ secrets.PYPI_PW }} 30 | # IS_VECTORHUB_NIGHTLY: True 31 | # run: | 32 | # python setup.py sdist bdist_wheel 33 | # twine upload dist/* 34 | -------------------------------------------------------------------------------- /.github/workflows/python-pytest-audio.yml: -------------------------------------------------------------------------------- 1 | name: Run Parallelized Pytest Audio 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'vectorhub/encoders/audio/**' 9 | - 'tests/encoders/audio/**' 10 | - 'tests/test_utils.py' 11 | - '.github/workflows/python-pytest-audio.yml' 12 | pull_request: 13 | branches: 14 | - main 15 | paths: 16 | - 'vectorhub/encoders/audio/**' 17 | - 'tests/encoders/audio/**' 18 | - 'tests/test_utils.py' 19 | - '.github/workflows/python-pytest-audio.yml' 20 | 21 | jobs: 22 | run_test: 23 | runs-on: ubuntu-latest 24 | strategy: 25 | max-parallel: 30 26 | matrix: 27 | python-version: [3.6, 3.7] 28 | test-path: 29 | - tests/encoders/audio/vectorai 30 | - tests/encoders/audio/tfhub/test_speech_embedding.py 31 | - tests/encoders/audio/tfhub/test_trill.py 32 | - tests/encoders/audio/tfhub/test_vggish.py 33 | - tests/encoders/audio/tfhub/test_yamnet.py 34 | - tests/encoders/audio/pytorch/test_fairseq.py 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: Set up Python ${{ matrix.python-version }} 38 | uses: actions/setup-python@v2 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | - name: Cache pip 42 | uses: actions/cache@v2 43 | with: 44 | path: ~/.cache/pip 45 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 46 | restore-keys: | 47 | ${{ runner.os }}-pip- 48 | ${{ runner.os }}- 49 | - name: Install base dependencies 50 | run: | 51 | sudo apt-get install libsndfile1 52 | python -m pip install --upgrade pip 53 | pip install -e . 54 | - name: Install dependencies for tests 55 | env: 56 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 57 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 58 | run: | 59 | pip install -r requirements.txt 60 | pip install -e ".[test]" 61 | pip install -e ".[encoders-audio-tfhub]" 62 | pip install -e ".[encoders-audio-pytorch]" 63 | - name: Run Tests 64 | env: 65 | VI_USERNAME: ${{ secrets.VI_USERNAME }} 66 | VI_API_KEY: ${{ secrets.VI_API_KEY }} 67 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 68 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 69 | run: python -m pytest ${{ matrix.test-path }} -------------------------------------------------------------------------------- /.github/workflows/python-pytest-general.yml: -------------------------------------------------------------------------------- 1 | name: Run Parallelized Pytest General 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'tests/test_import_utils.py' 9 | - 'tests/test_model_to_dict.py' 10 | - 'tests/test_utils.py' 11 | - 'tests/base/test_base.py' 12 | - '.github/workflows/python-pytest-general.yml' 13 | pull_request: 14 | branches: 15 | - main 16 | paths: 17 | - 'tests/test_import_utils.py' 18 | - 'tests/test_model_to_dict.py' 19 | - 'tests/test_utils.py' 20 | - 'tests/base/test_base.py' 21 | - '.github/workflows/python-pytest-general.yml' 22 | 23 | jobs: 24 | run_test: 25 | runs-on: ubuntu-latest 26 | strategy: 27 | max-parallel: 30 28 | matrix: 29 | python-version: [3.6, 3.7] 30 | test-path: 31 | - tests/test_import_utils.py 32 | - tests/test_model_to_dict.py 33 | - tests/test_utils.py 34 | - tests/base/test_base.py 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: Set up Python ${{ matrix.python-version }} 38 | uses: actions/setup-python@v2 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | - name: Cache pip 42 | uses: actions/cache@v2 43 | with: 44 | path: ~/.cache/pip 45 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 46 | restore-keys: | 47 | ${{ runner.os }}-pip- 48 | ${{ runner.os }}- 49 | - name: Install base dependencies 50 | run: | 51 | sudo apt-get install libsndfile1 52 | python -m pip install --upgrade pip 53 | pip install -e . 54 | - name: Install dependencies for tests 55 | env: 56 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 57 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 58 | run: | 59 | pip install -r requirements.txt 60 | pip install -e ".[test]" 61 | pip install -e ".[encoders-audio-tfhub]" 62 | - name: Run Tests 63 | env: 64 | VI_USERNAME: ${{ secrets.VI_USERNAME }} 65 | VI_API_KEY: ${{ secrets.VI_API_KEY }} 66 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 67 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 68 | run: python -m pytest ${{ matrix.test-path }} -------------------------------------------------------------------------------- /.github/workflows/python-pytest-image-text.yml: -------------------------------------------------------------------------------- 1 | name: Run Parallelized Pytest Image Text 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'vectorhub/bi_encoders/text_image/**' 9 | - 'tests/bi_encoders/text_image/**' 10 | - '.github/workflows/python-pytest-image-text.yml' 11 | pull_request: 12 | branches: 13 | - main 14 | paths: 15 | - 'vectorhub/bi_encoders/text_image/**' 16 | - 'tests/bi_encoders/text_image/**' 17 | - '.github/workflows/python-pytest-image-text.yml' 18 | 19 | jobs: 20 | run_test: 21 | runs-on: ubuntu-latest 22 | strategy: 23 | max-parallel: 30 24 | matrix: 25 | python-version: [3.6, 3.7] 26 | test-path: 27 | - tests/bi_encoders/text_image/torch 28 | steps: 29 | - uses: actions/checkout@v2 30 | - name: Set up Python ${{ matrix.python-version }} 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | - name: Cache pip 35 | uses: actions/cache@v2 36 | with: 37 | path: ~/.cache/pip 38 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 39 | restore-keys: | 40 | ${{ runner.os }}-pip- 41 | ${{ runner.os }}- 42 | - name: Install base dependencies 43 | run: | 44 | sudo apt-get install libsndfile1 45 | python -m pip install --upgrade pip 46 | pip install -e . 47 | - name: Install dependencies for tests 48 | env: 49 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 50 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 51 | run: | 52 | pip install -r requirements.txt 53 | pip install -e ".[test]" 54 | pip install -e ".[clip]" 55 | - name: Run Tests 56 | env: 57 | VI_USERNAME: ${{ secrets.VI_USERNAME }} 58 | VI_API_KEY: ${{ secrets.VI_API_KEY }} 59 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 60 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 61 | run: python -m pytest ${{ matrix.test-path }} -------------------------------------------------------------------------------- /.github/workflows/python-pytest-image.yml: -------------------------------------------------------------------------------- 1 | name: Run Parallelized Pytest Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'vectorhub/encoders/image/**' 9 | - 'tests/encoders/image/**' 10 | - '.github/workflows/python-pytest-image.yml' 11 | pull_request: 12 | branches: 13 | - main 14 | paths: 15 | - 'vectorhub/encoders/image/**' 16 | - 'tests/encoders/image/**' 17 | - '.github/workflows/python-pytest-image.yml' 18 | 19 | jobs: 20 | run_test: 21 | runs-on: ubuntu-latest 22 | strategy: 23 | max-parallel: 30 24 | matrix: 25 | python-version: [3.6, 3.7] 26 | test-path: 27 | - tests/encoders/image/vectorai 28 | - tests/encoders/image/tfhub/test_bit.py 29 | - tests/encoders/image/tfhub/test_inception_resnet.py 30 | - tests/encoders/image/tfhub/test_inception.py 31 | - tests/encoders/image/tfhub/test_mobilenet.py 32 | - tests/encoders/image/tfhub/test_resnet.py 33 | - tests/encoders/image/fastai 34 | - tests/encoders/face 35 | steps: 36 | - uses: actions/checkout@v2 37 | - name: Set up Python ${{ matrix.python-version }} 38 | uses: actions/setup-python@v2 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | - name: Cache pip 42 | uses: actions/cache@v2 43 | with: 44 | path: ~/.cache/pip 45 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 46 | restore-keys: | 47 | ${{ runner.os }}-pip- 48 | ${{ runner.os }}- 49 | - name: Install base dependencies 50 | run: | 51 | sudo apt-get install libsndfile1 52 | python -m pip install --upgrade pip 53 | pip install -e . 54 | - name: Install dependencies for tests 55 | run: | 56 | pip install -r requirements.txt 57 | pip install -e ".[test]" 58 | pip install -e ".[encoders-image-tfhub]" 59 | pip install -e ".[encoders-image-fastai]" 60 | pip install -e ".[encoders-image-tf-face-detection]" 61 | - name: Run Tests 62 | env: 63 | VI_USERNAME: ${{ secrets.VI_USERNAME }} 64 | VI_API_KEY: ${{ secrets.VI_API_KEY }} 65 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 66 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 67 | run: python -m pytest ${{ matrix.test-path }} -------------------------------------------------------------------------------- /.github/workflows/python-pytest-text.yml: -------------------------------------------------------------------------------- 1 | name: Run Parallelized Pytest Text 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'vectorhub/encoders/text/**' 9 | - 'tests/encoders/text/**' 10 | - '.github/workflows/python-pytest-text.yml' 11 | pull_request: 12 | branches: 13 | - main 14 | paths: 15 | - 'vectorhub/encoders/text/**' 16 | - 'tests/encoders/text/**' 17 | - '.github/workflows/python-pytest-text.yml' 18 | 19 | jobs: 20 | run_test: 21 | runs-on: ubuntu-latest 22 | strategy: 23 | max-parallel: 30 24 | matrix: 25 | python-version: [3.6, 3.7] 26 | test-path: 27 | - tests/encoders/text/vectorai 28 | - tests/encoders/text/tf_transformers 29 | - tests/encoders/text/torch_transformers 30 | - tests/encoders/text/tfhub/test_albert.py 31 | - tests/encoders/text/tfhub/test_bert.py 32 | - tests/encoders/text/tfhub/test_labse.py 33 | - tests/encoders/text/sentence_transformers 34 | - tests/encoders/text/tfhub/test_use.py 35 | - tests/encoders/text/tfhub/test_elmo.py 36 | - tests/encoders/text/tfhub/test_use_transformer.py 37 | - tests/encoders/text/tfhub/test_use_multi_transformer.py 38 | - tests/bi_encoders/qa/tfhub/test_lareqa_qa.py 39 | - tests/bi_encoders/qa/tfhub/test_use_qa.py 40 | steps: 41 | - uses: actions/checkout@v2 42 | - name: Set up Python ${{ matrix.python-version }} 43 | uses: actions/setup-python@v2 44 | with: 45 | python-version: ${{ matrix.python-version }} 46 | - name: Cache pip 47 | uses: actions/cache@v2 48 | with: 49 | path: ~/.cache/pip 50 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 51 | restore-keys: | 52 | ${{ runner.os }}-pip- 53 | ${{ runner.os }}- 54 | - name: Install base dependencies 55 | run: | 56 | sudo apt-get install libsndfile1 57 | python -m pip install --upgrade pip 58 | pip install -e . 59 | - name: Install dependencies for tests 60 | env: 61 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 62 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 63 | run: | 64 | pip install -r requirements.txt 65 | pip install -e ".[test]" 66 | pip install -e ".[encoders-text-tf-transformers]" 67 | pip install -e ".[encoders-text-torch-transformers]" 68 | pip install -e ".[encoders-text-sentence-transformers]" 69 | pip install -e ".[encoders-text-tfhub]" 70 | - name: Run Tests 71 | env: 72 | VI_USERNAME: ${{ secrets.VI_USERNAME }} 73 | VI_API_KEY: ${{ secrets.VI_API_KEY }} 74 | VH_USERNAME: ${{ secrets.VH_USERNAME }} 75 | VH_API_KEY: ${{ secrets.VH_API_KEY }} 76 | run: python -m pytest ${{ matrix.test-path }} 77 | -------------------------------------------------------------------------------- /.github/workflows/update_badges.yml: -------------------------------------------------------------------------------- 1 | # name: Update Badges 2 | 3 | # on: 4 | # schedule: 5 | # - cron: "0 0 * * *" 6 | 7 | # jobs: 8 | # run_test: 9 | # runs-on: ubuntu-latest 10 | # steps: 11 | # - uses: actions/checkout@v2 12 | # - name: Pytest 3.7 13 | # uses: actions/setup-python@v2 14 | # with: 15 | # python-version: '3.7' 16 | # - name: Download badges 17 | # run: | 18 | # python -m pip install requests 19 | # python utils/download_badges.py 20 | # echo ${{ github.ref }} 21 | # git add . 22 | # git config --local user.email "action@github.com" 23 | # git config --local user.name "GitHub Action" 24 | # git commit -m "ci: Automated build push" -a | exit 0 25 | # - name: Push changes 26 | # if: github.ref == 'refs/heads/main' 27 | # uses: ad-m/github-push-action@master 28 | # with: 29 | # github_token: ${{ secrets.SECRET_TOKEN }} 30 | # branch: ${{ github.ref }} 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | This repository is deprecated, and no longer maintained. If you are looking to learn about state-of-the-art techniques and models for turning data into vector embeddings, we recommend visiting our friends at VectorHub. 3 |

4 | 5 | --- 6 | Additional information: 7 | 8 | This repository was first set up by RelevanceAI - a platform to build and deploy AI apps and agents. 9 | 10 | VectorHub is a free and open-source educational platform to learn how to build information retrieval and feature engineering powered by vector embeddings. VectorHub is community-led and maintained by Superlinked - an open-source compute framework, focused on turning complex data into vector embeddings. 11 | --------------------------------------------------------------------------------