├── .github ├── github-teaser.png ├── vila-examples.png └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── README.md ├── datasets ├── README.md ├── download.sh ├── s2-vl-utils │ ├── README.md │ ├── cermine_loader.py │ ├── condense_dataset.py │ ├── config.json │ ├── download.py │ ├── requirements.txt │ ├── sources │ │ └── s2-vl-ver1 │ │ │ └── annotation_table.csv │ ├── vision_model_loader.py │ └── vision_postprocessor.py └── schema-token.json ├── examples ├── end2end-sci-pdf-parsing │ ├── Dockerfile │ ├── README.md │ ├── main.py │ ├── requirements.txt │ └── vila_pdf_parsing_service.py └── find-empty-unicode-chars │ ├── unicode-char-categories.csv │ ├── unicode-char-search.ipynb │ └── zero-length-unicode-chars.txt ├── requirements.txt ├── scripts ├── push_model_to_hub.sh ├── train_baseline.sh ├── train_hvila.sh └── train_ivila.sh ├── setup.py ├── src └── vila │ ├── __init__.py │ ├── automodel.py │ ├── constants.py │ ├── dataset │ ├── __init__.py │ └── preprocessors │ │ ├── __init__.py │ │ ├── base.py │ │ ├── config.py │ │ ├── grouping.py │ │ ├── hierarchical_modeling.py │ │ └── layout_indicator.py │ ├── models │ ├── __init__.py │ ├── configuration_hierarchical_model.py │ └── hierarchical_model.py │ ├── pdftools │ ├── __init__.py │ ├── base.py │ ├── datamodel.py │ ├── pdf_extractor.py │ └── pdfplumber_extractor.py │ ├── predictors.py │ └── utils.py ├── tests ├── fixtures │ ├── dummy_sample.json │ ├── large.pdf │ ├── regular.pdf │ └── vila-test.pdf ├── test_hierarchical_model.py ├── test_large_pdf.py ├── test_predictor.py ├── test_preprocessor.py ├── test_utils.py └── test_vila_run.py └── tools ├── arguments.py ├── create_hvila_model_base_weights.py ├── dataset_configs.py ├── generate-eval.py ├── push_model_to_hf_hub.py ├── train-baseline.py ├── train-hvila.py ├── train-ivila.py └── utils.py /.github/github-teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/.github/github-teaser.png -------------------------------------------------------------------------------- /.github/vila-examples.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/.github/vila-examples.png -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/README.md -------------------------------------------------------------------------------- /datasets/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/README.md -------------------------------------------------------------------------------- /datasets/download.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/download.sh -------------------------------------------------------------------------------- /datasets/s2-vl-utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/README.md -------------------------------------------------------------------------------- /datasets/s2-vl-utils/cermine_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/cermine_loader.py -------------------------------------------------------------------------------- /datasets/s2-vl-utils/condense_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/condense_dataset.py -------------------------------------------------------------------------------- /datasets/s2-vl-utils/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/config.json -------------------------------------------------------------------------------- /datasets/s2-vl-utils/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/download.py -------------------------------------------------------------------------------- /datasets/s2-vl-utils/requirements.txt: -------------------------------------------------------------------------------- 1 | bs4 2 | lxml 3 | PyPDF2 4 | tqdm 5 | requests -------------------------------------------------------------------------------- /datasets/s2-vl-utils/sources/s2-vl-ver1/annotation_table.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/sources/s2-vl-ver1/annotation_table.csv -------------------------------------------------------------------------------- /datasets/s2-vl-utils/vision_model_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/vision_model_loader.py -------------------------------------------------------------------------------- /datasets/s2-vl-utils/vision_postprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/s2-vl-utils/vision_postprocessor.py -------------------------------------------------------------------------------- /datasets/schema-token.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/datasets/schema-token.json -------------------------------------------------------------------------------- /examples/end2end-sci-pdf-parsing/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/end2end-sci-pdf-parsing/Dockerfile -------------------------------------------------------------------------------- /examples/end2end-sci-pdf-parsing/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/end2end-sci-pdf-parsing/README.md -------------------------------------------------------------------------------- /examples/end2end-sci-pdf-parsing/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/end2end-sci-pdf-parsing/main.py -------------------------------------------------------------------------------- /examples/end2end-sci-pdf-parsing/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/end2end-sci-pdf-parsing/requirements.txt -------------------------------------------------------------------------------- /examples/end2end-sci-pdf-parsing/vila_pdf_parsing_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/end2end-sci-pdf-parsing/vila_pdf_parsing_service.py -------------------------------------------------------------------------------- /examples/find-empty-unicode-chars/unicode-char-categories.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/find-empty-unicode-chars/unicode-char-categories.csv -------------------------------------------------------------------------------- /examples/find-empty-unicode-chars/unicode-char-search.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/find-empty-unicode-chars/unicode-char-search.ipynb -------------------------------------------------------------------------------- /examples/find-empty-unicode-chars/zero-length-unicode-chars.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/examples/find-empty-unicode-chars/zero-length-unicode-chars.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/push_model_to_hub.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/scripts/push_model_to_hub.sh -------------------------------------------------------------------------------- /scripts/train_baseline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/scripts/train_baseline.sh -------------------------------------------------------------------------------- /scripts/train_hvila.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/scripts/train_hvila.sh -------------------------------------------------------------------------------- /scripts/train_ivila.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/scripts/train_ivila.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/setup.py -------------------------------------------------------------------------------- /src/vila/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/__init__.py -------------------------------------------------------------------------------- /src/vila/automodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/automodel.py -------------------------------------------------------------------------------- /src/vila/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/constants.py -------------------------------------------------------------------------------- /src/vila/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/__init__.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/__init__.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/base.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/config.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/grouping.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/grouping.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/hierarchical_modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/hierarchical_modeling.py -------------------------------------------------------------------------------- /src/vila/dataset/preprocessors/layout_indicator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/dataset/preprocessors/layout_indicator.py -------------------------------------------------------------------------------- /src/vila/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/models/__init__.py -------------------------------------------------------------------------------- /src/vila/models/configuration_hierarchical_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/models/configuration_hierarchical_model.py -------------------------------------------------------------------------------- /src/vila/models/hierarchical_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/models/hierarchical_model.py -------------------------------------------------------------------------------- /src/vila/pdftools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/vila/pdftools/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/pdftools/base.py -------------------------------------------------------------------------------- /src/vila/pdftools/datamodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/pdftools/datamodel.py -------------------------------------------------------------------------------- /src/vila/pdftools/pdf_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/pdftools/pdf_extractor.py -------------------------------------------------------------------------------- /src/vila/pdftools/pdfplumber_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/pdftools/pdfplumber_extractor.py -------------------------------------------------------------------------------- /src/vila/predictors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/predictors.py -------------------------------------------------------------------------------- /src/vila/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/src/vila/utils.py -------------------------------------------------------------------------------- /tests/fixtures/dummy_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/fixtures/dummy_sample.json -------------------------------------------------------------------------------- /tests/fixtures/large.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/fixtures/large.pdf -------------------------------------------------------------------------------- /tests/fixtures/regular.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/fixtures/regular.pdf -------------------------------------------------------------------------------- /tests/fixtures/vila-test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/fixtures/vila-test.pdf -------------------------------------------------------------------------------- /tests/test_hierarchical_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_hierarchical_model.py -------------------------------------------------------------------------------- /tests/test_large_pdf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_large_pdf.py -------------------------------------------------------------------------------- /tests/test_predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_predictor.py -------------------------------------------------------------------------------- /tests/test_preprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_preprocessor.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /tests/test_vila_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tests/test_vila_run.py -------------------------------------------------------------------------------- /tools/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/arguments.py -------------------------------------------------------------------------------- /tools/create_hvila_model_base_weights.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/create_hvila_model_base_weights.py -------------------------------------------------------------------------------- /tools/dataset_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/dataset_configs.py -------------------------------------------------------------------------------- /tools/generate-eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/generate-eval.py -------------------------------------------------------------------------------- /tools/push_model_to_hf_hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/push_model_to_hf_hub.py -------------------------------------------------------------------------------- /tools/train-baseline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/train-baseline.py -------------------------------------------------------------------------------- /tools/train-hvila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/train-hvila.py -------------------------------------------------------------------------------- /tools/train-ivila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/train-ivila.py -------------------------------------------------------------------------------- /tools/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/vila/HEAD/tools/utils.py --------------------------------------------------------------------------------