├── examples ├── __init__.py ├── datasets │ ├── __init__.py │ ├── sign2mint.py │ ├── data_check.py │ └── dgs_types.py ├── load_all.py └── run-holistic.py ├── sign_language_datasets ├── __init__.py ├── utils │ ├── __init__.py │ ├── downloaders │ │ ├── __init__.py │ │ ├── youtube.py │ │ └── aslpro.py │ ├── signwriting │ │ ├── __init__.py │ │ └── ocr │ │ │ ├── __init__.py │ │ │ ├── assets │ │ │ ├── sign.png │ │ │ └── SuttonSignWritingOneD.ttf │ │ │ ├── ocr_test.py │ │ │ └── ocr.py │ ├── features │ │ └── __init__.py │ ├── get_pose_header.py │ └── torch_dataset.py └── datasets │ ├── asl_lex │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── asl_lex_test.py │ └── asl_lex.py │ ├── signtyp │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── signtyp_test.py │ └── signtyp.py │ ├── wmt_slt │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── openpose_135.poseheader │ ├── wmt_slt_test.py │ ├── README.md │ └── utils.py │ ├── dgs_corpus │ ├── checksums.tsv │ ├── splits │ │ ├── __init__.py │ │ └── create_document_split.py │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── openpose.poseheader │ ├── create_index.py │ ├── dgs_corpus_test.py │ └── dgs_utils.py │ ├── dgs_types │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── dgs_types_test.py │ └── make_poses.py │ ├── dicta_sign │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.header │ ├── dicta_sign_test.py │ └── dicta_sign.py │ ├── mediapi_skel │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.header │ ├── openpose.header │ ├── create_pose_headers.py │ ├── mediapi_skel_test.py │ ├── mediapi_utils.py │ └── mediapi_skel.py │ ├── sign_wordnet │ ├── checksums.tsv │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── sign_wordnet_test.py │ └── sign_wordnet.py │ ├── signsuisse │ ├── checksums.tsv │ ├── __init__.py │ ├── holistic.poseheader │ ├── dummy_data │ │ └── de.json │ ├── signsuisse_test.py │ ├── example.py │ └── download_holistic_gcs.py │ ├── autsl │ ├── dummy_data │ │ ├── labels.csv │ │ ├── videos │ │ │ ├── signer0_sample1000_color.mp4 │ │ │ └── signer0_sample1000_depth.mp4 │ │ └── poses │ │ │ └── holistic │ │ │ └── train │ │ │ └── signer0_sample1000.pose │ ├── __init__.py │ ├── holistic.poseheader │ ├── openpose_135.poseheader │ ├── autsl_test.py │ └── checksums.tsv │ ├── asl_signs │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── splits │ │ └── 1.0.0-uzh │ │ │ └── split.py │ └── asl_signs.py │ ├── how2sign │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── openpose.header │ ├── how2sign_test.py │ ├── checksums.tsv │ └── how2sign.py │ ├── ngt_corpus │ ├── __init__.py │ ├── ngt_corpus_test.py │ └── ngt_corpus_utils.py │ ├── pop_sign │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ └── pop_sign.py │ ├── sem_lex │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ └── holistic.poseheader │ ├── sign2mint │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── sign2mint_test.py │ └── sign2mint.py │ ├── signbank │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ └── signbank_test.py │ ├── wlasl │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── checksums.tsv │ ├── wlasl_test.py │ └── wlasl.py │ ├── asl_citizen │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ └── asl_citizen.py │ ├── bsl_corpus │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── checksums.tsv │ └── bsl_corpus_test.py │ ├── chicago_fs_wild │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── checksums.tsv │ └── chicago_fs_wild_test.py │ ├── spread_the_sign │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── holistic.poseheader │ ├── splits │ │ └── 1.0.0-uzh │ │ │ └── split.py │ └── spread_the_sign.py │ ├── swojs_glossario │ ├── dummy_data │ │ └── TODO-add_fake_data_in_this_directory.txt │ ├── __init__.py │ ├── swojs_glossario_test.py │ └── swojs_glossario.py │ ├── aslg_pc12 │ ├── __init__.py │ ├── checksums.tsv │ ├── dummy_data │ │ ├── sample-corpus-asl-en.asl │ │ └── sample-corpus-asl-en.en │ ├── aslg_pc12_test.py │ └── aslg_pc12.py │ ├── rwth_phoenix2014_t │ ├── __init__.py │ ├── pose.header │ ├── checksums.tsv │ ├── dummy_data │ │ ├── poses │ │ │ └── holistic │ │ │ │ ├── test │ │ │ │ └── 25October_2010_Monday_tagesschau-17.pose │ │ │ │ ├── train │ │ │ │ └── 25October_2010_Monday_tagesschau-17.pose │ │ │ │ └── validation │ │ │ │ └── 25October_2010_Monday_tagesschau-17.pose │ │ └── annotations │ │ │ └── PHOENIX-2014-T-release-v3 │ │ │ └── PHOENIX-2014-T │ │ │ ├── features │ │ │ └── fullFrame-210x260px │ │ │ │ ├── dev │ │ │ │ └── 25October_2010_Monday_tagesschau-17 │ │ │ │ │ ├── images0001.png │ │ │ │ │ └── images0002.png │ │ │ │ ├── test │ │ │ │ └── 25October_2010_Monday_tagesschau-17 │ │ │ │ │ ├── images0001.png │ │ │ │ │ └── images0002.png │ │ │ │ └── train │ │ │ │ └── 25October_2010_Monday_tagesschau-17 │ │ │ │ ├── images0001.png │ │ │ │ └── images0002.png │ │ │ └── annotations │ │ │ └── manual │ │ │ ├── PHOENIX-2014-T.dev.corpus.csv │ │ │ ├── PHOENIX-2014-T.test.corpus.csv │ │ │ └── PHOENIX-2014-T.train.corpus.csv │ ├── rwth_phoenix2014_t_test.py │ └── rwth_phoenix2014_t.py │ ├── warning.py │ ├── jw_sign │ └── README.md │ ├── __init__.py │ └── config.py ├── Makefile ├── .gitignore ├── MANIFEST.in ├── pyproject.toml └── .github └── workflows └── pypi-publish.yaml /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_lex/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signtyp/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/downloaders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign_wordnet/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/checksums.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/splits/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/dummy_data/labels.csv: -------------------------------------------------------------------------------- 1 | signer0_sample1000,164 -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_lex/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_signs/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/ngt_corpus/__init__.py: -------------------------------------------------------------------------------- 1 | from .ngt_corpus import NGTCorpus 2 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/pop_sign/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sem_lex/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign2mint/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signbank/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signtyp/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wlasl/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/features/__init__.py: -------------------------------------------------------------------------------- 1 | from .pose_feature import PoseFeature 2 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/ocr/__init__.py: -------------------------------------------------------------------------------- 1 | from .ocr import image_to_fsw 2 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_citizen/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/bsl_corpus/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign_wordnet/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/chicago_fs_wild/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/spread_the_sign/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/swojs_glossario/dummy_data/TODO-add_fake_data_in_this_directory.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/__init__.py: -------------------------------------------------------------------------------- 1 | """autsl dataset.""" 2 | 3 | from .autsl import AUTSL 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wlasl/__init__.py: -------------------------------------------------------------------------------- 1 | """wlasl dataset.""" 2 | 3 | from .wlasl import Wlasl 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_lex/__init__.py: -------------------------------------------------------------------------------- 1 | """asl_lex dataset.""" 2 | 3 | from .asl_lex import AslLex 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/pop_sign/__init__.py: -------------------------------------------------------------------------------- 1 | """Popsign dataset.""" 2 | 3 | from .pop_sign import PopSign 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sem_lex/__init__.py: -------------------------------------------------------------------------------- 1 | """asl-signs dataset.""" 2 | 3 | from .sem_lex import SemLex 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signtyp/__init__.py: -------------------------------------------------------------------------------- 1 | """SignTyp dataset.""" 2 | 3 | from .signtyp import SignTyp 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/__init__.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | from .wmt_slt import WMTSLT 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | style: 2 | black --line-length 140 --target-version py36 --skip-magic-trailing-comma sign_language_datasets 3 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_signs/__init__.py: -------------------------------------------------------------------------------- 1 | """asl-signs dataset.""" 2 | 3 | from .asl_signs import ASLSigns 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/__init__.py: -------------------------------------------------------------------------------- 1 | """aslg_pc12 dataset.""" 2 | 3 | from .aslg_pc12 import AslgPc12 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/__init__.py: -------------------------------------------------------------------------------- 1 | """dgs_types dataset.""" 2 | 3 | from .dgs_types import DgsTypes 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/__init__.py: -------------------------------------------------------------------------------- 1 | """how2sign dataset.""" 2 | 3 | from .how2sign import How2Sign 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signbank/__init__.py: -------------------------------------------------------------------------------- 1 | """SignBank dataset.""" 2 | 3 | from .signbank import SignBank 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/bsl_corpus/__init__.py: -------------------------------------------------------------------------------- 1 | """bsl_corpus dataset.""" 2 | 3 | from .bsl_corpus import BslCorpus 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/__init__.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | from .dicta_sign import DictaSign 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign2mint/__init__.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | from .sign2mint import Sign2MINT 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/__init__.py: -------------------------------------------------------------------------------- 1 | """signsuisse dataset.""" 2 | 3 | from .signsuisse import SignSuisse 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_citizen/__init__.py: -------------------------------------------------------------------------------- 1 | """asl-signs dataset.""" 2 | 3 | from .asl_citizen import ASLCitizen 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/__init__.py: -------------------------------------------------------------------------------- 1 | """mediapi_skel dataset.""" 2 | 3 | from .mediapi_skel import MediapiSkel 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign_wordnet/__init__.py: -------------------------------------------------------------------------------- 1 | """SignWordnet dataset.""" 2 | 3 | from .sign_wordnet import SignWordnet 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/spread_the_sign/__init__.py: -------------------------------------------------------------------------------- 1 | """asl-signs dataset.""" 2 | 3 | from .spread_the_sign import SpreadTheSign 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/chicago_fs_wild/__init__.py: -------------------------------------------------------------------------------- 1 | """ChicagoFSWild dataset.""" 2 | 3 | from .chicago_fs_wild import ChicagoFSWild 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/__init__.py: -------------------------------------------------------------------------------- 1 | """dgs_corpus dataset.""" 2 | 3 | from .dgs_corpus import DgsCorpus, DgsCorpusConfig 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/swojs_glossario/__init__.py: -------------------------------------------------------------------------------- 1 | """swojs_glossario dataset.""" 2 | 3 | from .swojs_glossario import SwojsGlossario 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/__init__.py: -------------------------------------------------------------------------------- 1 | """rwth_phoenix_2014_t dataset.""" 2 | 3 | from .rwth_phoenix2014_t import RWTHPhoenix2014T 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/autsl/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/openpose.header: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/how2sign/openpose.header -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/holistic.header: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/dicta_sign/holistic.header -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sem_lex/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/sem_lex/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/wmt_slt/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_signs/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/asl_signs/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/openpose_135.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/autsl/openpose_135.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/dgs_corpus/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/openpose.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/dgs_corpus/openpose.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/dgs_types/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/holistic.header: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/mediapi_skel/holistic.header -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/openpose.header: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/mediapi_skel/openpose.header -------------------------------------------------------------------------------- /sign_language_datasets/datasets/pop_sign/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/pop_sign/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/pose.header: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/pose.header -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/signsuisse/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/ocr/assets/sign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/utils/signwriting/ocr/assets/sign.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_citizen/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/asl_citizen/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/openpose_135.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/wmt_slt/openpose_135.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/chicago_fs_wild/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/chicago_fs_wild/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/datasets/spread_the_sign/holistic.poseheader: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/spread_the_sign/holistic.poseheader -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/ocr/assets/SuttonSignWritingOneD.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/utils/signwriting/ocr/assets/SuttonSignWritingOneD.ttf -------------------------------------------------------------------------------- /sign_language_datasets/datasets/bsl_corpus/checksums.tsv: -------------------------------------------------------------------------------- 1 | # TODO(bsl_corpus): If your dataset downloads files, then the checksums 2 | # will be automatically added here when running 3 | # `tfds build --register_checksums`. 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/dummy_data/videos/signer0_sample1000_color.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/autsl/dummy_data/videos/signer0_sample1000_color.mp4 -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/dummy_data/videos/signer0_sample1000_depth.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/autsl/dummy_data/videos/signer0_sample1000_depth.mp4 -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/dummy_data/poses/holistic/train/signer0_sample1000.pose: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/autsl/dummy_data/poses/holistic/train/signer0_sample1000.pose -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/checksums.tsv: -------------------------------------------------------------------------------- 1 | https://www-i6.informatik.rwth-aachen.de/ftp/pub/rwth-phoenix/2016/phoenix-2014-T.v3.tar.gz 43246233600 eb1e6b3d36611c43e6602ff436a4eb2c8c837c64b4db96016c205b81d549ae70 phoenix-2014-T.v3.tar.gz 2 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/test/25October_2010_Monday_tagesschau-17.pose: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/test/25October_2010_Monday_tagesschau-17.pose -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/train/25October_2010_Monday_tagesschau-17.pose: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/train/25October_2010_Monday_tagesschau-17.pose -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/validation/25October_2010_Monday_tagesschau-17.pose: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/poses/holistic/validation/25October_2010_Monday_tagesschau-17.pose -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | 3 | .idea/ 4 | old/ 5 | __pycache__ 6 | 7 | sign_language_datasets/datasets/msasl 8 | sign_language_datasets/datasets/ncslgr 9 | .coverage 10 | build/ 11 | dist/ 12 | sign_language_datasets.egg-info/ 13 | .DS_Store 14 | 15 | sign_language_datasets/datasets/spread_the_sign/splits/1.0.0-uzh/*.txt 16 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/chicago_fs_wild/checksums.tsv: -------------------------------------------------------------------------------- 1 | https://dl.ttic.edu/ChicagoFSWild.tgz 14270632063 82ac8146db14c854c6f201d36239a8b9ca05d171d7cad7556b710586db49645e ChicagoFSWild.tgz 2 | https://dl.ttic.edu/ChicagoFSWildPlus.tgz 87424483577 a949c426191f2cefb1ca8efbf80dfd90e3748fc1c45f2930fba3706ac8b688f6 ChicagoFSWildPlus.tgz 3 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/warning.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | 4 | def dataset_warning(dataset): 5 | warnings.warn( 6 | "This library provides access to data sets without claiming ownership over them or defining their licensing terms. " 7 | "Users who download data are responsible for checking the license of each individual data set." 8 | ) 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include sign_language_datasets *.header 2 | recursive-include sign_language_datasets *.poseheader 3 | recursive-include sign_language_datasets *.csv 4 | recursive-include sign_language_datasets *.tsv 5 | recursive-include sign_language_datasets *.png 6 | recursive-include sign_language_datasets *.ttf 7 | recursive-include sign_language_datasets *.json 8 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/checksums.tsv: -------------------------------------------------------------------------------- 1 | https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.asl 6457824 e550698336419edaaedfc51882541eb033db83257fa2167900ac0db9fae91633 sample-corpus-asl-en.asl 2 | https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.en 6315607 889732ced6ceaeab352efd647de65ac29e1ac7adb1206b2703d9d35432924f6a sample-corpus-asl-en.en 3 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/get_pose_header.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pose_format import Pose 4 | 5 | p = "/home/nlp/amit/sign-language/sign-language-datasets/old/autsl/holistic/train/" 6 | f_path = p + os.listdir(p)[0] 7 | 8 | f = open(f_path, "rb").read() 9 | pose = Pose.read(f) 10 | 11 | with open("../datasets/autsl/pose.poseheader", "wb") as fw: 12 | pose.header.write(fw) 13 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/dummy_data/de.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "uid": "111418", 4 | "name": "WEIHNACHTEN", 5 | "kategorie": "", 6 | "sprache": "de", 7 | "link": "https://signsuisse.sgb-fss.ch/index.php?id=9&L=0&tx_issignsuisselexikon_anzeige%5Bgebaerden%5D=111418&tx_issignsuisselexikon_anzeige%5Baction%5D=show&tx_issignsuisselexikon_anzeige%5Bcontroller%5D=Gebaerden" 8 | } 9 | ] -------------------------------------------------------------------------------- /sign_language_datasets/utils/downloaders/youtube.py: -------------------------------------------------------------------------------- 1 | def download_youtube(url, dst_path): 2 | import youtube_dl # Required for YouTube downloads 3 | 4 | ydl_opts = {"format": "bestvideo", "outtmpl": dst_path} 5 | with youtube_dl.YoutubeDL(ydl_opts) as ydl: 6 | try: 7 | ydl.download([url]) 8 | except youtube_dl.utils.DownloadError as e: 9 | print("Problem downloading youtube video", url) 10 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/jw_sign/README.md: -------------------------------------------------------------------------------- 1 | This is the automated loader for the dataset in the paper: 2 | **JWSign: A Highly Multilingual Corpus of Bible Translations for more Diversity in Sign Language Processing** 3 | 4 | **IMPORTANT NOTE: By using this dataset, you agree your usage of it is in agreement with [Jehovah’s Witnesses Terms of Use](https://www.jw.org/en/terms-of-use/).** 5 | 6 | We plan to release the dataset in November-December, 2023. 7 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/dev/25October_2010_Monday_tagesschau-17/images0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/dev/25October_2010_Monday_tagesschau-17/images0001.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/dev/25October_2010_Monday_tagesschau-17/images0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/dev/25October_2010_Monday_tagesschau-17/images0002.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/25October_2010_Monday_tagesschau-17/images0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/25October_2010_Monday_tagesschau-17/images0001.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/25October_2010_Monday_tagesschau-17/images0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/25October_2010_Monday_tagesschau-17/images0002.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/train/25October_2010_Monday_tagesschau-17/images0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/train/25October_2010_Monday_tagesschau-17/images0001.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/train/25October_2010_Monday_tagesschau-17/images0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/datasets/HEAD/sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/train/25October_2010_Monday_tagesschau-17/images0002.png -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/dummy_data/sample-corpus-asl-en.asl: -------------------------------------------------------------------------------- 1 | MEMBERSHIP PARLIAMENT SEE MINUTE 2 | APPROVAL MINUTE DESC-PREVIOUS SIT SEE MINUTE 3 | MEMBERSHIP PARLIAMENT SEE MINUTE 4 | VERIFICATION CREDENTIALS SEE MINUTE 5 | DOCUMENT RECEIVE SEE MINUTE 6 | WRITE STATEMENT AND DESC-ORAL QUESTION TABLE SEE MINUTE 7 | PETITION SEE MINUTE 8 | TEXT AGREEMENT DESC-FORWARD BY COUNCIL SEE MINUTE 9 | ACTION TAKE ON PARLIAMENT X-POSS RESOLUTION SEE MINUTE 10 | AGENDA FOR NEXT SIT SEE MINUTE -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/annotations/manual/PHOENIX-2014-T.dev.corpus.csv: -------------------------------------------------------------------------------- 1 | name|video|start|end|speaker|orth|translation 2 | 25October_2010_Monday_tagesschau-17|25October_2010_Monday_tagesschau-17/1/*.png|-1|-1|Signer01|REGEN SCHNEE REGION VERSCHWINDEN NORD REGEN KOENNEN REGION STERN KOENNEN SEHEN|regen und schnee lassen an den alpen in der nacht nach im norden und nordosten fallen hier und da schauer sonst ist das klar 3 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/annotations/manual/PHOENIX-2014-T.test.corpus.csv: -------------------------------------------------------------------------------- 1 | name|video|start|end|speaker|orth|translation 2 | 25October_2010_Monday_tagesschau-17|25October_2010_Monday_tagesschau-17/1/*.png|-1|-1|Signer01|REGEN SCHNEE REGION VERSCHWINDEN NORD REGEN KOENNEN REGION STERN KOENNEN SEHEN|regen und schnee lassen an den alpen in der nacht nach im norden und nordosten fallen hier und da schauer sonst ist das klar 3 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/dummy_data/annotations/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/annotations/manual/PHOENIX-2014-T.train.corpus.csv: -------------------------------------------------------------------------------- 1 | name|video|start|end|speaker|orth|translation 2 | 25October_2010_Monday_tagesschau-17|25October_2010_Monday_tagesschau-17/1/*.png|-1|-1|Signer01|REGEN SCHNEE REGION VERSCHWINDEN NORD REGEN KOENNEN REGION STERN KOENNEN SEHEN|regen und schnee lassen an den alpen in der nacht nach im norden und nordosten fallen hier und da schauer sonst ist das klar 3 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/dummy_data/sample-corpus-asl-en.en: -------------------------------------------------------------------------------- 1 | membership of parliament see minutes 2 | approval of minutes of previous sitting see minutes 3 | membership of parliament see minutes 4 | verification of credentials see minutes 5 | documents received see minutes 6 | written statements and oral questions tabling see minutes 7 | petitions see minutes 8 | texts of agreements forwarded by the council see minutes 9 | action taken on parliament's resolutions see minutes 10 | agenda for next sitting see minutes -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wlasl/checksums.tsv: -------------------------------------------------------------------------------- 1 | http://aslbricks.org/New/ASL-Videos/book.mp4 3012348 ec1016efc1f9879345bcc3babbb078a875b24ce7967ab2542cb8db030d5b5de9 book.mp4 2 | https://raw.githubusercontent.com/dxli94/WLASL/0ac8108282aba99226e29c066cb8eab847ef62da/start_kit/WLASL_v0.3.json 11932637 31ba5a5c421bdbff87494ee22ca6d4e276a8dfac44f43cc9a58b76ed1c4cd684 WLASL_v0.3.json 3 | https://www.robots.ox.ac.uk/~vgg/research/bsl1k/data/info/wlasl.tar 3970887680 4d5e53c05fa1bfc1dbe329a430b142592732fddde2639ef2239f6d2b8585959f wlasl.tar 4 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/aslg_pc12_test.py: -------------------------------------------------------------------------------- 1 | """aslg_pc12 dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | from . import aslg_pc12 6 | 7 | 8 | class AslgPc12Test(tfds.testing.DatasetBuilderTestCase): 9 | """Tests for aslg_pc12 dataset.""" 10 | 11 | DATASET_CLASS = aslg_pc12.AslgPc12 12 | SPLITS = {"train": 10} # Number of fake train example 13 | DL_DOWNLOAD_RESULT = ["sample-corpus-asl-en.asl", "sample-corpus-asl-en.en"] 14 | 15 | 16 | if __name__ == "__main__": 17 | tfds.testing.test_main() 18 | -------------------------------------------------------------------------------- /examples/load_all.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | from sign_language_datasets.datasets.config import SignDatasetConfig 4 | import tensorflow_datasets as tfds 5 | 6 | config = SignDatasetConfig( 7 | name="pose_holistic_paths2", 8 | version="3.0.0", 9 | include_video=False, 10 | include_pose="holistic", 11 | process_pose=False 12 | ) 13 | 14 | # Load the dgs_types dataset with the specified configuration 15 | dgs_types = tfds.load('dgs_types', builder_kwargs=dict(config=config)) 16 | 17 | for datum in dgs_types["train"].take(10): 18 | print(datum) -------------------------------------------------------------------------------- /sign_language_datasets/utils/downloaders/aslpro.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | from urllib.error import ContentTooShortError 3 | 4 | _ASLPRO_HEADERS = [ 5 | ("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7"), 6 | ("Referer", "http://www.aslpro.com/cgi-bin/aslpro/aslpro.cgi"), 7 | ] 8 | 9 | 10 | def download_aslpro(url: str, dst_path: str): 11 | # ASL Pro videos are in swf format 12 | opener = urllib.request.build_opener() 13 | opener.addheaders = _ASLPRO_HEADERS 14 | urllib.request.install_opener(opener) 15 | 16 | try: 17 | urllib.request.urlretrieve(url, dst_path) 18 | except ContentTooShortError: 19 | pass 20 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/create_pose_headers.py: -------------------------------------------------------------------------------- 1 | from pose_format import PoseHeader 2 | from pose_format.pose_header import PoseHeaderDimensions 3 | 4 | 5 | def get_pose_header(): 6 | from pose_format.utils.holistic import holistic_components 7 | 8 | pose_components = [c for c in holistic_components() if c.name in ["POSE_LANDMARKS", "LEFT_HAND_LANDMARKS", "RIGHT_HAND_LANDMARKS"]] 9 | dimensions = PoseHeaderDimensions(width=1, height=1, depth=1) 10 | return PoseHeader(version=0.1, dimensions=dimensions, components=pose_components) 11 | 12 | 13 | if __name__ == "__main__": 14 | pose_header = get_pose_header() 15 | with open("holistic.header", "wb") as f: 16 | pose_header.write(f) 17 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wlasl/wlasl_test.py: -------------------------------------------------------------------------------- 1 | """wlasl dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | from . import wlasl 6 | 7 | 8 | class WlaslTest(tfds.testing.DatasetBuilderTestCase): 9 | """Tests for wlasl dataset.""" 10 | 11 | # TODO(wlasl): 12 | DATASET_CLASS = wlasl.Wlasl 13 | SPLITS = {"train": 1} 14 | 15 | # If you are calling `download/download_and_extract` with a dict, like: 16 | # dl_manager.download({"some_key": "http://a.org/out.txt", ...}) 17 | # then the tests needs to provide the fake output paths relative to the 18 | # fake data directory 19 | # DL_EXTRACT_RESULT = {"some_key": "output_file1.txt", ...} 20 | 21 | 22 | if __name__ == "__main__": 23 | tfds.testing.test_main() 24 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/ocr/ocr_test.py: -------------------------------------------------------------------------------- 1 | """ocr tests""" 2 | 3 | import os 4 | from unittest import TestCase 5 | import cv2 6 | 7 | from sign_language_datasets.utils.signwriting.ocr import image_to_fsw 8 | 9 | 10 | class TestOCR(TestCase): 11 | def test_should_extract_fsw_from_image(self): 12 | dirname = os.path.dirname(__file__) 13 | img_path = os.path.join(dirname, "assets/sign.png") 14 | 15 | img_rgb = cv2.imread(img_path) 16 | symbols = ["S1f520", "S1f528", "S23c04", "S23c1c", "S2fb04", "S2ff00", "S33b10"] 17 | 18 | self.assertEqual( 19 | image_to_fsw(img_rgb, symbols), 20 | "M239x127S2ff00043x057S23c04071x118S23c1c028x118S1f520062x100S1f528035x100S2fb04054x181S33b10054x083", 21 | ) 22 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/autsl_test.py: -------------------------------------------------------------------------------- 1 | """autsl dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | from . import autsl 6 | 7 | 8 | class AutslTest(tfds.testing.DatasetBuilderTestCase): 9 | """Tests for autsl dataset.""" 10 | 11 | # TODO(autsl): 12 | DATASET_CLASS = autsl.AUTSL 13 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 14 | 15 | # If you are calling `download/download_and_extract` with a dict, like: 16 | # dl_manager.download({"some_key": "http://a.org/out.txt", ...}) 17 | # then the tests needs to provide the fake output paths relative to the 18 | # fake data directory 19 | # DL_EXTRACT_RESULT = {"some_key": "output_file1.txt", ...} 20 | 21 | 22 | if __name__ == "__main__": 23 | tfds.testing.test_main() 24 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signtyp/signtyp_test.py: -------------------------------------------------------------------------------- 1 | """SignTyp dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import signtyp 5 | 6 | 7 | class SignTypTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for SignTyp dataset.""" 9 | 10 | # TODO(SignTyp): 11 | DATASET_CLASS = signtyp.SignTyp 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_lex/asl_lex_test.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import asl_lex 5 | 6 | 7 | class AslLexTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for sign2mint dataset.""" 9 | 10 | # TODO(sign2mint): 11 | DATASET_CLASS = asl_lex.AslLex 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/wmt_slt_test.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import wmt_slt 5 | 6 | 7 | class WMTSLTTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for sign2mint dataset.""" 9 | 10 | # TODO(wmt_slt): 11 | DATASET_CLASS = wmt_slt.WMTSLT 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/how2sign_test.py: -------------------------------------------------------------------------------- 1 | """how2sign dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import how2sign 5 | 6 | 7 | class How2signTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for how2sign dataset.""" 9 | 10 | # TODO(how2sign): 11 | DATASET_CLASS = how2sign.How2Sign 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signbank/signbank_test.py: -------------------------------------------------------------------------------- 1 | """SignBank dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import signbank 5 | 6 | 7 | class SignbankTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for SignBank dataset.""" 9 | 10 | # TODO(SignBank): 11 | DATASET_CLASS = signbank.SignBank 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/dgs_types_test.py: -------------------------------------------------------------------------------- 1 | """dgs_types dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import dgs_types 5 | 6 | 7 | class DGSTypesTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for how2sign dataset.""" 9 | 10 | # TODO(how2sign): 11 | DATASET_CLASS = dgs_types.DgsTypes 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign2mint/sign2mint_test.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import sign2mint 5 | 6 | 7 | class Sign2MINTTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for sign2mint dataset.""" 9 | 10 | # TODO(sign2mint): 11 | DATASET_CLASS = sign2mint.Sign2MINT 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/bsl_corpus/bsl_corpus_test.py: -------------------------------------------------------------------------------- 1 | """bsl_corpus dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import bsl_corpus 5 | 6 | 7 | class BslCorpusTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for bsl_corpus dataset.""" 9 | 10 | # TODO(bsl_corpus): 11 | DATASET_CLASS = bsl_corpus.BslCorpus 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/dicta_sign_test.py: -------------------------------------------------------------------------------- 1 | """sign2mint dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import dicta_sign 5 | 6 | 7 | class DictaSignTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for sign2mint dataset.""" 9 | 10 | # TODO(sign2mint): 11 | DATASET_CLASS = dicta_sign.DictaSign 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign_wordnet/sign_wordnet_test.py: -------------------------------------------------------------------------------- 1 | """SignWordnet dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import SignWordnet 5 | 6 | 7 | class SignWordnetTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for SignWordnet dataset.""" 9 | 10 | # TODO(SignWordnet): 11 | DATASET_CLASS = SignWordnet 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/signsuisse_test.py: -------------------------------------------------------------------------------- 1 | """signsuisse dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import signsuisse 5 | 6 | 7 | class SignSuisseTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for signsuisse dataset.""" 9 | 10 | # TODO(signsuisse): 11 | DATASET_CLASS = signsuisse.SignSuisse 12 | SPLITS = {"train": 1} # Number of fake train example 13 | DL_EXTRACT_RESULT = ["de.json"] 14 | 15 | # If you are calling `download/download_and_extract` with a dict, like: 16 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 17 | # then the tests needs to provide the fake output paths relative to the 18 | # fake data directory 19 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 20 | 21 | 22 | if __name__ == "__main__": 23 | tfds.testing.test_main() 24 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/mediapi_skel_test.py: -------------------------------------------------------------------------------- 1 | """how2sign dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import mediapi_skel 5 | 6 | 7 | class MediapiSkelTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for how2sign dataset.""" 9 | 10 | # TODO(how2sign): 11 | DATASET_CLASS = mediapi_skel.MediapiSkel 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/chicago_fs_wild/chicago_fs_wild_test.py: -------------------------------------------------------------------------------- 1 | """how2sign dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | from . import chicagofswild 6 | 7 | 8 | class ChicagoFSWildTest(tfds.testing.DatasetBuilderTestCase): 9 | """Tests for how2sign dataset.""" 10 | 11 | # TODO(how2sign): 12 | DATASET_CLASS = chicagofswild.ChicagoFSWild 13 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 14 | 15 | # If you are calling `download/download_and_extract` with a dict, like: 16 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 17 | # then the tests needs to provide the fake output paths relative to the 18 | # fake data directory 19 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 20 | 21 | 22 | if __name__ == "__main__": 23 | tfds.testing.test_main() 24 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/swojs_glossario/swojs_glossario_test.py: -------------------------------------------------------------------------------- 1 | """swojs_glossario dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | from . import swojs_glossario 5 | 6 | 7 | class SwojsGlossarioTest(tfds.testing.DatasetBuilderTestCase): 8 | """Tests for swojs_glossario dataset.""" 9 | 10 | # TODO(swojs_glossario): 11 | DATASET_CLASS = swojs_glossario.SwojsGlossario 12 | SPLITS = {"train": 3, "test": 1} # Number of fake train example # Number of fake test example 13 | 14 | # If you are calling `download/download_and_extract` with a dict, like: 15 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 16 | # then the tests needs to provide the fake output paths relative to the 17 | # fake data directory 18 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 19 | 20 | 21 | if __name__ == "__main__": 22 | tfds.testing.test_main() 23 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_signs/splits/1.0.0-uzh/split.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | 4 | import pandas as pd 5 | 6 | 7 | seed = 3407 8 | csv_path = sys.argv[1] 9 | out_path = sys.argv[2] 10 | 11 | 12 | def write(filename, video_ids): 13 | with open(out_path + filename, "w") as f: 14 | for line in video_ids: 15 | f.write(f"{line}\n") 16 | 17 | 18 | df = pd.read_csv(csv_path) 19 | video_ids = df["sequence_id"].tolist() 20 | 21 | write("all.txt", video_ids) 22 | 23 | random.seed(seed) 24 | random.shuffle(video_ids) 25 | 26 | length = len(video_ids) 27 | val_ratio = 0.05 28 | val_idx = int(length * val_ratio) 29 | test_ratio = 0.05 30 | test_idx = val_idx + int(length * test_ratio) 31 | 32 | video_ids_val = video_ids[:val_idx] 33 | video_ids_test = video_ids[val_idx:test_idx] 34 | video_ids_train = video_ids[test_idx:] 35 | 36 | write("train.txt", video_ids_train) 37 | write("val.txt", video_ids_val) 38 | write("test.txt", video_ids_test) 39 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/spread_the_sign/splits/1.0.0-uzh/split.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | 4 | import pandas as pd 5 | 6 | 7 | seed = 3407 8 | csv_path = sys.argv[1] 9 | out_path = sys.argv[2] 10 | 11 | 12 | def write(filename, video_ids): 13 | with open(out_path + filename, "w") as f: 14 | for line in video_ids: 15 | f.write(f"{line}\n") 16 | 17 | 18 | df = pd.read_csv(csv_path) 19 | video_ids = df.index.values.tolist() 20 | 21 | write("all.txt", video_ids) 22 | 23 | random.seed(seed) 24 | random.shuffle(video_ids) 25 | 26 | length = len(video_ids) 27 | val_ratio = 0.01 28 | val_idx = int(length * val_ratio) 29 | test_ratio = 0.01 30 | test_idx = val_idx + int(length * test_ratio) 31 | 32 | video_ids_val = video_ids[:val_idx] 33 | video_ids_test = video_ids[val_idx:test_idx] 34 | video_ids_train = video_ids[test_idx:] 35 | 36 | write("train.txt", video_ids_train) 37 | write("val.txt", video_ids_val) 38 | write("test.txt", video_ids_test) 39 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .aslg_pc12 import AslgPc12 2 | from .asl_lex import AslLex 3 | from .autsl import AUTSL 4 | from .chicago_fs_wild import ChicagoFSWild 5 | from .config import SignDatasetConfig 6 | from .dgs_types import DgsTypes 7 | from .dgs_corpus import DgsCorpus 8 | from .dicta_sign import DictaSign 9 | from .how2sign import How2Sign 10 | from .mediapi_skel import MediapiSkel 11 | from .rwth_phoenix2014_t import RWTHPhoenix2014T 12 | from .sign2mint import Sign2MINT 13 | from .signbank import SignBank 14 | from .signtyp import SignTyp 15 | from .signsuisse import SignSuisse 16 | from .sign_wordnet import SignWordnet 17 | from .swojs_glossario import SwojsGlossario 18 | from .wlasl import Wlasl 19 | from .ngt_corpus import NGTCorpus 20 | from .bsl_corpus import BslCorpus 21 | from .wmt_slt import WMTSLT 22 | from .asl_signs import ASLSigns 23 | from .sem_lex import SemLex 24 | from .asl_citizen import ASLCitizen 25 | from .pop_sign import PopSign 26 | from .spread_the_sign import SpreadTheSign 27 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/rwth_phoenix2014_t_test.py: -------------------------------------------------------------------------------- 1 | """rwth_phoenix_2014_t dataset.""" 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | from . import rwth_phoenix2014_t 6 | 7 | 8 | class RWTHPhoenix2014TVideosTest(tfds.testing.DatasetBuilderTestCase): 9 | """Tests for rwth_phoenix_2014_t dataset.""" 10 | 11 | DATASET_CLASS = rwth_phoenix2014_t.RWTHPhoenix2014T 12 | SPLITS = {"train": 1, "validation": 1, "test": 1} 13 | OVERLAPPING_SPLITS = ["train", "validation", "test"] 14 | BUILDER_CONFIG_NAMES_TO_TEST = ["videos"] 15 | 16 | DL_EXTRACT_RESULT = ["annotations"] 17 | 18 | 19 | class RWTHPhoenix2014TPosesTest(tfds.testing.DatasetBuilderTestCase): 20 | """Tests for rwth_phoenix_2014_t dataset.""" 21 | 22 | DATASET_CLASS = rwth_phoenix2014_t.RWTHPhoenix2014T 23 | SPLITS = {"train": 1, "validation": 1, "test": 1} 24 | OVERLAPPING_SPLITS = ["train", "validation", "test"] 25 | BUILDER_CONFIG_NAMES_TO_TEST = ["poses"] 26 | 27 | DL_EXTRACT_RESULT = ["annotations", "poses"] 28 | 29 | 30 | if __name__ == "__main__": 31 | tfds.testing.test_main() 32 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/torch_dataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow_datasets as tfds 2 | import sign_language_datasets.datasets 3 | from sign_language_datasets.datasets.config import SignDatasetConfig 4 | 5 | import itertools 6 | import numpy as np 7 | 8 | import torch 9 | 10 | 11 | class TFDSTorchDataset(torch.utils.data.IterableDataset): 12 | def __init__(self, tf_dataset): 13 | self.dataset = tf_dataset 14 | 15 | def __len__(self): 16 | return len(self.dataset) 17 | 18 | def torch_obj(self, obj): 19 | if isinstance(obj, dict): 20 | return {k: self.torch_obj(v) for k, v in obj.items()} 21 | if isinstance(obj, np.ndarray): 22 | return torch.from_numpy(obj) 23 | return obj 24 | 25 | def __iter__(self): 26 | for datum in self.dataset: 27 | yield self.torch_obj(tfds.as_numpy(datum)) 28 | 29 | 30 | if __name__ == "__main__": 31 | # Fast download and load dataset using TFDS 32 | config = SignDatasetConfig(name="holistic-poses", version="1.0.0", include_video=False, include_pose="holistic") 33 | dicta_sign = tfds.load(name="dicta_sign", builder_kwargs={"config": config}) 34 | 35 | # Convert to torch dataset 36 | train_dataset = TFDSTorchDataset(dicta_sign["train"]) 37 | 38 | for datum in itertools.islice(train_dataset, 0, 10): 39 | print(datum) 40 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/ngt_corpus/ngt_corpus_test.py: -------------------------------------------------------------------------------- 1 | """ngt_corpus dataset tests.""" 2 | 3 | import itertools 4 | 5 | import tensorflow_datasets as tfds 6 | 7 | from unittest import TestCase 8 | 9 | from ... import datasets 10 | from ...datasets.config import SignDatasetConfig 11 | from ...datasets.ngt_corpus.ngt_corpus_utils import get_elan_sentences_ngt_corpus 12 | 13 | 14 | class TestNgtCorpus(TestCase): 15 | def test_ngt_corpus_loader(self): 16 | config = SignDatasetConfig(name="only-annotations", version="1.0.0", include_video=False) 17 | ngt = tfds.load(name="ngt_corpus", builder_kwargs={"config": config}) 18 | 19 | for datum in itertools.islice(ngt["train"], 0, 10): 20 | print(datum) 21 | 22 | 23 | class TestNgtCorpusUtils(TestCase): 24 | def test_ngt_get_elan_sentences(self): 25 | config = SignDatasetConfig(name="only-annotations", version="1.0.0", include_video=False) 26 | ngt = tfds.load(name="ngt_corpus", builder_kwargs={"config": config}) 27 | 28 | for datum in itertools.islice(ngt["train"], 0, 50): 29 | print(datum["id"].numpy().decode("utf-8")) 30 | elan_path = datum["paths"]["eaf"].numpy().decode("utf-8") 31 | 32 | sentences = get_elan_sentences_ngt_corpus(elan_path) 33 | 34 | for sentence in sentences: 35 | print(sentence) 36 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "sign-language-datasets" 7 | version = "v0.3.0" 8 | description = "TFDS Datasets for sign language" 9 | readme = "README.md" 10 | requires-python = ">=3.6" 11 | authors = [ 12 | { name = "Amit Moryossef", email = "amitmoryossef@gmail.com" } 13 | ] 14 | 15 | dependencies = [ 16 | "typing-extensions", 17 | "python-dotenv", 18 | "tqdm", 19 | "pose-format>=0.0.3", 20 | "tensorflow-datasets", 21 | "numpy", 22 | "Pillow", 23 | "requests", 24 | "etils", 25 | # For the BSL, DGS, NGT corpora 26 | "lxml", 27 | "pympi-ling", 28 | # Set the correct tensorflow version based on the platform 29 | 'tensorflow; platform_system != "Darwin"', 30 | 'tensorflow-macos; platform_system == "Darwin"', 31 | ] 32 | 33 | [project.optional-dependencies] 34 | video = [ 35 | "opencv-python" 36 | ] 37 | dev = [ 38 | "pylint", 39 | "pytest", 40 | "pytest-cov", 41 | "black", 42 | ] 43 | 44 | [tool.setuptools.packages] 45 | find = {} 46 | 47 | [tool.setuptools.package-data] 48 | sign_language_datasets = [ 49 | "*.header", 50 | "*.poseheader", 51 | "*.csv", 52 | "*.tsv", 53 | "*.png", 54 | "*.ttf", 55 | "*.json", 56 | ] 57 | 58 | [tool.pytest.ini_options] 59 | addopts = "-v --cov=sign_language_datasets" 60 | filterwarnings = [ 61 | "ignore::DeprecationWarning:flatbuffers", 62 | ] 63 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/checksums.tsv: -------------------------------------------------------------------------------- 1 | https://drive.google.com/uc?id=1dYey1F_SeHets-UO8F9cE3VMhRBO-6e0&export=download 33033658988 98e36aa4377dfb3b5e59de1ba5af94ade0a77e9565cacb0c7183267480b95cd3 train_rgb_front_clips.zip 2 | https://drive.google.com/uc?id=1PIYIIOxR2vnUDzSHdq3uyoUCoIJvsuNW&export=download 23273015704 dbcd63cf43bdfcb53534db7caec2b867ddd0fd652ed785faf21f8217d8446f6b train_rgb_side_clips.zip 3 | https://drive.google.com/uc?id=1lnsDN-LxcsroOmetdG5_sXYXZ7setlS4&export=download 22619897332 9dc65af07dd3dabdeb18300453c2394bbb188ea03dd73c966dc23880b8672470 train_2D_keypoints.tar.gz 4 | https://drive.google.com/uc?id=1oVZyTWhHShyqshC2kUrfWnBF8apIR7Z1&export=download 1778074855 6cb48170131fdb7fc64d40381a3c4be41112ba4454444170f5a31af2227b6d36 val_rgb_front_clips.zip 5 | https://drive.google.com/uc?id=1vJVV777_bmSeA2_k7iGdZu2izooeKUrq&export=download 1237300839 333db1b9f187c5ebd98f1d8daff1c0e8bd2b99c858f46456ca39b003e5e07c48 val_rgb_side_clips.zip 6 | https://drive.google.com/uc?id=1aOhRknNWj8APdxHmwJdQrMo5xuIGNXxM&export=download 1242181909 83f28594bf474ce642b284ebafa13467cd71cdd8a3a19f8a33c88d5fa4100957 val_2D_keypoints.tar.gz 7 | https://drive.google.com/uc?id=1d6GHqu0_8IGiKbu3sTZHtMb0DGhbHSMu&export=download 2413152782 cb63633542f1c3f703ad76ec5e1420bfe9bf028afdb120508ed7cded16ed1b55 test_rgb_front_clips.zip 8 | https://drive.google.com/uc?id=1gKV_TloCbMyMhOdYvr_a-6I-PTf0Sjyi&export=download 1694343193 ec5013b24ed561c46e65ec60ce3a1669c8ad218269c0496aea5d0fefb67a8e8e test_rgb_side_clips.zip 9 | https://drive.google.com/uc?id=1quj8Ipm56pH65KAKK3Pc-sqZ0ozw2gSe&export=download 1696569782 7b8afc7f8e25e127a7a246edad89b4deba2e7abc71f9a36148a3259564fcd37f test_2D_keypoints.tar.gz 10 | -------------------------------------------------------------------------------- /.github/workflows/pypi-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish Python Package 2 | 3 | on: 4 | release: 5 | types: [ created ] 6 | 7 | jobs: 8 | build-n-publish: 9 | name: Build and publish Python 🐍 distributions 📦 to PyPI 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v4 14 | with: 15 | python-version: '3.11' 16 | 17 | - name: Extract release version 18 | id: get_version 19 | run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV 20 | 21 | - name: Update version in pyproject.toml 22 | run: | 23 | sed -i 's/^version = .*/version = "${{ env.version }}"/' pyproject.toml 24 | 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | pip install build 29 | 30 | - name: Build a binary wheel dist 31 | run: | 32 | python -m build 33 | 34 | - name: Quick test of the built dist 35 | run: | 36 | python -c "import sign_language_datasets" 37 | 38 | - name: Commit updated pyproject.toml 39 | run: | 40 | git fetch origin master 41 | git checkout master 42 | git config user.name "GitHub Actions" 43 | git config user.email "actions@github.com" 44 | git add pyproject.toml 45 | git commit -m "Bump version to ${{ env.version }}" 46 | git push origin master 47 | 48 | - name: Publish distribution 📦 to PyPI 49 | uses: pypa/gh-action-pypi-publish@release/v1 50 | with: 51 | user: __token__ 52 | password: ${{ secrets.PYPI_API_TOKEN }} 53 | -------------------------------------------------------------------------------- /examples/datasets/sign2mint.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import tensorflow_datasets as tfds 4 | import hashlib 5 | 6 | # noinspection PyUnresolvedReferences 7 | import sign_language_datasets.datasets 8 | from sign_language_datasets.datasets.config import SignDatasetConfig 9 | 10 | def sign2mint(): 11 | config = SignDatasetConfig(name="firestore", version="1.0.0", include_video=False) 12 | dataset = tfds.load(name='sign2_mint', builder_kwargs={"config": config}) 13 | 14 | for datum in dataset["train"]: 15 | name = datum['fachbegriff'].numpy().decode('utf-8') 16 | swu = datum['swu'].numpy().decode('utf-8') 17 | url = datum['video'].numpy().decode('utf-8') 18 | uid_raw = "sign2mint" + datum['id'].numpy().decode('utf-8') 19 | uid = 's2m' + hashlib.md5(uid_raw.encode()).hexdigest() 20 | 21 | doc = { 22 | "uid": uid, 23 | "url": url, 24 | "meta": { 25 | "visibility": "unlisted", 26 | "name": "Sign2MINT: " + name, 27 | "language": "gsg", 28 | "userId": "1aGEJfRxDqdpIbOnjU5i5X0quYu2" 29 | }, 30 | "context": { 31 | "id": uid_raw, 32 | "wiktionary": datum['wortlink'].numpy().decode('utf-8'), 33 | "definition": datum['definition'].numpy().decode('utf-8') 34 | } 35 | } 36 | 37 | captions = [ 38 | {"language": "de", "transcription": name}, 39 | {"language": "Sgnw", "transcription": swu}, 40 | ] 41 | 42 | yield doc, captions, None 43 | 44 | 45 | if __name__ == '__main__': 46 | with open('sign2mint.jsonl', 'w') as f: 47 | for doc, captions, _ in sign2mint(): 48 | f.write(json.dumps({"doc": doc, "captions": captions})) 49 | f.write('\n') 50 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/create_index.py: -------------------------------------------------------------------------------- 1 | """Helper file to crawl The Public DGS Corpus and create an up-to-date index of the dataset""" 2 | 3 | import json 4 | import re 5 | import urllib.request 6 | 7 | from sign_language_datasets.datasets.config import cloud_bucket_file 8 | 9 | corpus_path = "https://www.sign-lang.uni-hamburg.de/meinedgs/" 10 | 11 | index_data = {} 12 | 13 | with urllib.request.urlopen(corpus_path + "ling/start-name_en.html") as response: 14 | html = response.read().decode("utf-8") 15 | 16 | trs = re.findall('([\s\S]*?)', html) 17 | for tr_id, tr in trs: 18 | tds = re.findall("(?:()|([\s\S]*?) 0 else None for links in td_links] 21 | assert len(links) == 13 22 | 23 | index_data[tr_id] = { 24 | "transcript": links[0], 25 | "format": links[2], 26 | "ilex": links[4], 27 | "eaf": links[5], 28 | "video_a": links[6], 29 | "video_b": links[7], 30 | "video_c": links[8], 31 | "srt": links[9], 32 | "cmdi": links[11], 33 | "openpose": links[12], 34 | } 35 | 36 | # Add holistic 37 | for c in ["a", "b"]: 38 | holistic_path = cloud_bucket_file(f"poses/holistic/dgs_corpus/{tr_id}_{c}.pose") 39 | index_data[tr_id]["holistic_" + c] = holistic_path if index_data[tr_id]["video_" + c] is not None else None 40 | 41 | # Make sure parsing worked 42 | if index_data[tr_id]["openpose"] is not None: 43 | assert index_data[tr_id]["openpose"].endswith(".json.gz") 44 | 45 | with open("dgs.json", "w") as f: 46 | json.dump(index_data, f) 47 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/example.py: -------------------------------------------------------------------------------- 1 | import tensorflow_datasets as tfds 2 | 3 | # noinspection PyUnresolvedReferences 4 | from pose_format import PoseHeader, Pose 5 | from pose_format.numpy import NumPyPoseBody 6 | from pose_format.utils.reader import BufferReader 7 | 8 | from sign_language_datasets.datasets.signsuisse.signsuisse import _POSE_HEADERS 9 | from sign_language_datasets.datasets.config import SignDatasetConfig 10 | 11 | 12 | def load_pose(pose_header, tf_pose): 13 | fps = int(tf_pose["fps"].numpy()) 14 | pose_body = NumPyPoseBody(fps, tf_pose["data"].numpy(), tf_pose["conf"].numpy()) 15 | return Pose(pose_header, pose_body) 16 | 17 | 18 | config = SignDatasetConfig( 19 | name="example", version="1.0.0", include_video=True, include_pose="holistic", process_video=False, sample_size=10 20 | ) 21 | dataset = tfds.load(name="sign_suisse", builder_kwargs={"config": config}) 22 | 23 | # pylint: disable=protected-access 24 | with open(_POSE_HEADERS[config.include_pose], "rb") as buffer: 25 | pose_header = PoseHeader.read(BufferReader(buffer.read())) 26 | 27 | for datum in dataset["train"]: 28 | print("Concept: ", datum["name"].numpy().decode("utf-8")) 29 | print("category: ", datum["category"].numpy().decode("utf-8")) 30 | print("Video: ", datum["video"].numpy().decode("utf-8")) 31 | print("Pose path: ", datum["pose"]["path"].numpy().decode("utf-8")) 32 | print("Pose: ", load_pose(pose_header, datum["pose"])) 33 | print("---") 34 | print("Example: ", datum["exampleText"].numpy().decode("utf-8")) 35 | print("Example Video: ", datum["exampleVideo"].numpy().decode("utf-8")) 36 | print("Example Pose path: ", datum["examplePose"]["path"].numpy().decode("utf-8")) 37 | print("Example Pose: ", load_pose(pose_header, datum["examplePose"])) 38 | print("---") 39 | print("\n\n") 40 | -------------------------------------------------------------------------------- /examples/run-holistic.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from os import path 3 | 4 | import tensorflow_datasets as tfds 5 | from datasets import tqdm 6 | from dotenv import load_dotenv 7 | 8 | # noinspection PyUnresolvedReferences 9 | import sign_language_datasets.datasets 10 | 11 | # noinspection PyUnresolvedReferences 12 | import sign_language_datasets.datasets.dgs_corpus 13 | from sign_language_datasets.datasets.config import SignDatasetConfig 14 | 15 | from pose_format.utils.holistic import load_holistic 16 | 17 | import cv2 18 | 19 | 20 | def load_video(cap: cv2.VideoCapture): 21 | while True: 22 | ret, frame = cap.read() 23 | if not ret: 24 | break 25 | yield cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 26 | 27 | 28 | config = SignDatasetConfig(name="full2", version="1.0.0", include_video=True, process_video=False, include_pose=None) 29 | dgs_types = tfds.load(name='dgs_types', builder_kwargs={"config": config}) 30 | 31 | decode_str = lambda s: s.numpy().decode('utf-8') 32 | for datum in tqdm(dgs_types["train"]): 33 | _id = decode_str(datum['id']) 34 | for view, video in zip(datum["views"]["name"], datum["views"]["video"]): 35 | view = decode_str(view) 36 | video = decode_str(video) 37 | 38 | pose_file_path = path.join("/home/nlp/amit/WWW/datasets/poses/holistic/dgs_types", f"{_id}_{view}.pose") 39 | if not path.exists(pose_file_path): 40 | cap = cv2.VideoCapture(video) 41 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 42 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 43 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 44 | all_frames = list(load_video(cap)) 45 | 46 | pose = load_holistic(all_frames, fps=fps, width=width, height=height, depth=width, progress=False) 47 | with open(pose_file_path, "wb") as f: 48 | pose.write(f) 49 | 50 | cap.release() 51 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/README.md: -------------------------------------------------------------------------------- 1 | # WMT-SLT 2 | 3 | Until [merged by `tfds`](https://github.com/tensorflow/datasets/pull/3988), please install our branch, supporting authentication in datasets: 4 | ```bash 5 | pip install git+ssh://git@github.com:AmitMY/datasets-1.git 6 | ``` 7 | 8 | Then install conditional dependencies: 9 | 10 | ```bash 11 | pip install srt mediapipe 12 | ``` 13 | 14 | ```py 15 | 16 | import tensorflow_datasets as tfds 17 | # noinspection PyUnresolvedReferences 18 | import sign_language_datasets.datasets 19 | from sign_language_datasets.datasets.config import SignDatasetConfig 20 | 21 | # Populate your access tokens 22 | TOKENS = { 23 | "zenodo_focusnews_token": "TODO", 24 | "zenodo_srf_videos_token": "TODO", 25 | "zenodo_srf_poses_token": "TODO" 26 | } 27 | 28 | # Load only the annotations, and include path to video files 29 | config = SignDatasetConfig(name="annotations", version="1.0.0", process_video=False) 30 | wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS}) 31 | 32 | # Load the annotations and openpose poses 33 | config = SignDatasetConfig(name="openpose", version="1.0.0", process_video=False, include_pose='openpose') 34 | wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS}) 35 | 36 | # Load the annotations and mediapipe holistic poses 37 | config = SignDatasetConfig(name="holistic", version="1.0.0", process_video=False, include_pose='holistic') 38 | wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS}) 39 | 40 | # Load the full video frames as a tensor 41 | config = SignDatasetConfig(name="videos", version="1.0.0", process_video=True) 42 | wmtslt = tfds.load(name='wmtslt', builder_kwargs={"config": config, **TOKENS}) 43 | 44 | decode_str = lambda s: s.numpy().decode('utf-8') 45 | for datum in itertools.islice(wmtslt["train"], 0, 10): 46 | subtitle = datum['subtitle'] 47 | print(decode_str(subtitle['start']), '-', decode_str(subtitle['end'])) 48 | print(decode_str(subtitle['text']), '\n') 49 | 50 | ``` -------------------------------------------------------------------------------- /examples/datasets/data_check.py: -------------------------------------------------------------------------------- 1 | # from pose_format import Pose 2 | # from sign_language_datasets.datasets.dgs_corpus.dgs_corpus import _POSE_HEADERS 3 | # from sign_language_datasets.utils.features import PoseFeature 4 | # 5 | # files = [ 6 | # "/home/nlp/amit/tensorflow_datasets/downloads/nlp.biu.ac.il_amit_data_pose_holi_dgs_coreCZFUT1DjfgqeTJH8Be7LXAdZjfIcl6MFu9-a0QsMiM.pose", 7 | # f"/home/nlp/amit/WWW/datasets/poses/holistic/dgs_corpus/1248625-15324720-15465943_a.pose", 8 | # "/home/nlp/amit/tensorflow_datasets/downloads/nlp.biu.ac.il_amit_data_pose_holi_dgs_corQoeprd_NyhZVoKS6jLUg0R4Yjart_u7A0LK2dpvd2f4.pose", 9 | # f"/home/nlp/amit/WWW/datasets/poses/holistic/dgs_corpus/1248625-15324720-15465943_b.pose", 10 | # ] 11 | # 12 | # for f_name in files: 13 | # with open(f_name, "rb") as f: 14 | # pose = Pose.read(f.read()) 15 | # 16 | # print("fps", pose.body.fps) 17 | # print("length", len(pose.body.data)) 18 | # 19 | # stride = 50 / 25 20 | # pose_feature = PoseFeature(shape=(None, 1, 543, 3), stride=stride, header_path=_POSE_HEADERS["holistic"]) 21 | # 22 | # encoded = pose_feature.encode_example(pose) 23 | # print("fps", encoded["fps"]) 24 | # print("shape", encoded["data"].shape) 25 | # print("") 26 | # # [00:32<04:31, 1.90it/s]{'id': '1247205_b', 'fps': 25, 'shape': (868, 1, 75, 3), 'length': 868, 'timestamps': tensor(34.6800)} 27 | # 28 | # # fps 50 29 | # # length 42623 30 | # # fps 25 31 | # # shape (21312, 1, 543, 3) 32 | # # 33 | # # fps 50 34 | # # length 42623 35 | # # fps 25 36 | # # shape (21312, 1, 543, 3) 37 | import json 38 | import os 39 | 40 | directory = "/home/nlp/amit/WWW/datasets/poses/holistic/dgs_corpus/" 41 | 42 | # Get list of files in the directory 43 | files = os.listdir(directory) 44 | 45 | # Create empty dictionary 46 | file_sizes = {} 47 | 48 | # Loop through each file in the directory 49 | for file in files: 50 | # Get full file path 51 | file_path = os.path.join(directory, file) 52 | 53 | # Get size of file in bytes 54 | file_size = os.path.getsize(file_path) 55 | 56 | # Add file name and size to dictionary 57 | file_sizes["https://nlp.biu.ac.il/~amit/datasets/poses/holistic/dgs_corpus/" + file] = file_size 58 | 59 | # Print the dictionary 60 | print(file_sizes) 61 | 62 | json.dump(file_sizes, open("holistic_pose_size.json", "w")) -------------------------------------------------------------------------------- /sign_language_datasets/datasets/aslg_pc12/aslg_pc12.py: -------------------------------------------------------------------------------- 1 | """ASLG-PC12: Synthetic English-ASL Gloss Parallel Corpus 2012""" 2 | 3 | import tensorflow_datasets as tfds 4 | from tensorflow.io.gfile import GFile 5 | 6 | from sign_language_datasets.datasets.warning import dataset_warning 7 | 8 | _DESCRIPTION = """ 9 | A large synthetic collection of parallel English and ASL-Gloss texts. 10 | There are two string features: text, and gloss. 11 | """ 12 | 13 | _CITATION = """ 14 | @inproceedings{othman2012english, 15 | title={English-asl gloss parallel corpus 2012: Aslg-pc12}, 16 | author={Othman, Achraf and Jemni, Mohamed}, 17 | booktitle={5th Workshop on the Representation and Processing of Sign Languages: Interactions between Corpus and Lexicon LREC}, 18 | year={2012} 19 | } 20 | """ 21 | 22 | _GLOSS_URL = "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.asl" 23 | _TEXT_URL = "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.en" 24 | 25 | 26 | class AslgPc12(tfds.core.GeneratorBasedBuilder): 27 | """DatasetBuilder for aslg_pc12 dataset.""" 28 | 29 | VERSION = tfds.core.Version("0.0.1") 30 | RELEASE_NOTES = {"0.0.1": "Sample of the full corpus"} 31 | 32 | def _info(self) -> tfds.core.DatasetInfo: 33 | """Returns the dataset metadata.""" 34 | 35 | return tfds.core.DatasetInfo( 36 | builder=self, 37 | description=_DESCRIPTION, 38 | features=tfds.features.FeaturesDict({"gloss": tfds.features.Text(), "text": tfds.features.Text()}), 39 | supervised_keys=("gloss", "text"), # Set to `None` to disable 40 | homepage="https://achrafothman.net/site/asl-smt/", 41 | citation=_CITATION, 42 | ) 43 | 44 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 45 | """Returns SplitGenerators.""" 46 | dataset_warning(self) 47 | 48 | gloss_path, text_path = dl_manager.download([_GLOSS_URL, _TEXT_URL]) 49 | 50 | return {"train": self._generate_examples(gloss_path, text_path)} 51 | 52 | def _generate_examples(self, gloss_path: str, text_path: str): 53 | """Yields examples.""" 54 | 55 | with GFile(gloss_path, "r") as gloss_f: 56 | with GFile(text_path, "r") as text_f: 57 | for i, (gloss, text) in enumerate(zip(gloss_f, text_f)): 58 | yield i, {"gloss": gloss, "text": text} 59 | -------------------------------------------------------------------------------- /examples/datasets/dgs_types.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | import tensorflow_datasets as tfds 5 | import hashlib 6 | 7 | # noinspection PyUnresolvedReferences 8 | import sign_language_datasets.datasets 9 | from sign_language_datasets.datasets.config import SignDatasetConfig 10 | 11 | 12 | def clear_gloss(gloss: str): 13 | text = re.search(r'^(.*?)(?=\d|$)', gloss) 14 | word = text.group(1) 15 | return word[0].upper() + word[1:].lower() 16 | 17 | 18 | def dgs_types(): 19 | config = SignDatasetConfig(name="firestore", version="1.0.0", include_video=False) 20 | dataset = tfds.load(name='dgs_types', builder_kwargs={"config": config}) 21 | 22 | for datum in dataset["train"]: 23 | uid_raw = "dgs_types_" + datum['id'].numpy().decode('utf-8') 24 | 25 | glosses = [clear_gloss(g.numpy().decode('utf-8')) for g in datum['glosses']] 26 | captions = [{"language": "de", "transcription": g} for g in glosses] 27 | captions.append({"language": "hns", "transcription": datum['hamnosys'].numpy().decode('utf-8')}) 28 | 29 | views = {view.numpy().decode('utf-8'): { 30 | "video": video.numpy().decode('utf-8'), 31 | "pose": None 32 | } for view, video in zip(datum["views"]["name"], datum["views"]["video"])} 33 | 34 | id_func = lambda view_name: 'dgstypes' + hashlib.md5((uid_raw + view_name).encode()).hexdigest() 35 | ids = [id_func(view_name) for view_name in views.keys()] 36 | 37 | for view_name, view_data in views.items(): 38 | this_id = id_func(view_name) 39 | doc = { 40 | "uid": this_id, 41 | "url": view_data["video"], 42 | "meta": { 43 | "visibility": "unlisted", 44 | "name": f"DGS Types: {' / '.join(glosses)} ({view_name})", 45 | "language": 'gsg', # German Sign Language 46 | "userId": "dxnkijRxuTP5JEQMaJu88Cxxlu52" # DGS Account 47 | }, 48 | "related": [_id for _id in ids if _id != this_id], 49 | "context": { 50 | "id": uid_raw 51 | } 52 | } 53 | 54 | yield doc, captions, None 55 | 56 | 57 | if __name__ == '__main__': 58 | with open('dgs_types.jsonl', 'w') as f: 59 | for doc, captions, _ in dgs_types(): 60 | f.write(json.dumps({"doc": doc, "captions": captions})) 61 | f.write('\n') 62 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_types/make_poses.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | 3 | import os 4 | import tensorflow_datasets as tfds 5 | from pose_format import Pose 6 | from tqdm import tqdm 7 | from dotenv import load_dotenv 8 | 9 | # noinspection PyUnresolvedReferences 10 | import sign_language_datasets.datasets 11 | 12 | # noinspection PyUnresolvedReferences 13 | import sign_language_datasets.datasets.dgs_corpus 14 | from sign_language_datasets.datasets.config import SignDatasetConfig 15 | 16 | from pose_format.utils.holistic import load_holistic 17 | 18 | import cv2 19 | 20 | 21 | def load_video(cap: cv2.VideoCapture): 22 | while True: 23 | ret, frame = cap.read() 24 | if not ret: 25 | break 26 | yield cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 27 | 28 | 29 | config = SignDatasetConfig(name="full3", version="1.0.0", include_video=True, process_video=False, include_pose=None) 30 | dgs_types = tfds.load(name="dgs_types", builder_kwargs={"config": config}) 31 | 32 | decode_str = lambda s: s.numpy().decode("utf-8") 33 | for datum in tqdm(dgs_types["train"]): 34 | _id = decode_str(datum["id"]) 35 | for view, video in zip(datum["views"]["name"], datum["views"]["video"]): 36 | view = decode_str(view) 37 | video = decode_str(video) 38 | 39 | pose_file_path = path.join("/home/nlp/amit/WWW/datasets/poses/holistic/dgs_types", f"{_id}_{view}.pose") 40 | if path.exists(pose_file_path): 41 | with open(pose_file_path, "rb") as f: 42 | pose = Pose.read(f.read()) 43 | if pose.body.data.shape[2] != 576: 44 | print(f"Skipping {_id}_{view} because it has {pose.body.shape[2]} landmarks instead of 576") 45 | os.remove(pose_file_path) 46 | 47 | if not path.exists(pose_file_path): 48 | print(f"Processing {_id}_{view}...") 49 | cap = cv2.VideoCapture(video) 50 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 51 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 52 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 53 | all_frames = list(load_video(cap)) 54 | 55 | pose = load_holistic(all_frames, fps=fps, width=width, height=height, depth=width, progress=False) 56 | with open(pose_file_path, "wb") as f: 57 | pose.write(f) 58 | 59 | cap.release() 60 | 61 | 62 | with open("holistic.poseheader", "wb") as f: 63 | pose.header.write(f) 64 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_lex/asl_lex.py: -------------------------------------------------------------------------------- 1 | """A database of lexical and phonological properties of American Sign Language signs. """ 2 | 3 | from os import path 4 | import tensorflow_datasets as tfds 5 | import csv 6 | 7 | from ..warning import dataset_warning 8 | from ...datasets.config import SignDatasetConfig 9 | 10 | _DESCRIPTION = """ 11 | ASL-LEX is a database of lexical and phonological properties of American Sign Language signs. 12 | It was first released in 2016 with nearly 1,000 signs. 13 | ASL-LEX was updated in Fall 2020 with greatly expanded information and an increased size of 2,723 signs. 14 | """ 15 | 16 | _CITATION = """ 17 | @article{caselli2017asl, 18 | title={ASL-LEX: A lexical database of American Sign Language}, 19 | author={Caselli, Naomi K and Sehyr, Zed Sevcikova and Cohen-Goldberg, Ariel M and Emmorey, Karen}, 20 | journal={Behavior research methods}, 21 | volume={49}, 22 | pages={784--801}, 23 | year={2017}, 24 | publisher={Springer} 25 | } 26 | """ 27 | 28 | header_file = path.join(path.dirname(path.realpath(__file__)), "data-key.csv") 29 | 30 | 31 | class AslLex(tfds.core.GeneratorBasedBuilder): 32 | """DatasetBuilder for asl-lex dataset.""" 33 | 34 | VERSION = tfds.core.Version("2.0.0") 35 | RELEASE_NOTES = { 36 | "1.0.0": "Initial release with nearly 1,000 signs.", 37 | "2.0.0": "greatly expanded information and an increased size of 2,723 signs", 38 | } 39 | 40 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=True), SignDatasetConfig(name="annotations", include_video=False)] 41 | 42 | def _info(self) -> tfds.core.DatasetInfo: 43 | """Returns the dataset metadata.""" 44 | 45 | # Open the CSV file 46 | with open(header_file, "r", encoding="utf-8") as csvfile: 47 | # Create a CSV reader object 48 | reader = csv.DictReader(csvfile) 49 | self.features = {row["VariableName(CSV&OSF)"]: tfds.features.Text() for row in reader} 50 | 51 | return tfds.core.DatasetInfo( 52 | builder=self, 53 | description=_DESCRIPTION, 54 | features=tfds.features.FeaturesDict(self.features), 55 | homepage="https://asl-lex.org/", 56 | supervised_keys=None, 57 | citation=_CITATION, 58 | ) 59 | 60 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 61 | dataset_warning(self) 62 | 63 | csv_path = dl_manager.download_and_extract("https://osf.io/download/9nygd/") 64 | 65 | return {"train": self._generate_examples(csv_path)} 66 | 67 | def _generate_examples(self, csv_path: str): 68 | """Yields examples.""" 69 | 70 | with open(csv_path, "r", encoding="ISO-8859-1") as csvfile: 71 | # Create a CSV reader object 72 | reader = csv.DictReader(csvfile) 73 | 74 | for i, row in enumerate(reader): 75 | new_row = {f: row[f] for f in self.features} 76 | yield str(i), new_row 77 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/ngt_corpus/ngt_corpus_utils.py: -------------------------------------------------------------------------------- 1 | import pympi 2 | 3 | """ 4 | Notes: 5 | 6 | Annotation guidelines of third release: https://archive.mpi.nl/tla/islandora/object/tla%3A1839_00_0000_0000_0021_6AC3_1 7 | 8 | Gloss annotations: 9 | - Glosses are annotated separately for each hand 10 | - the gloss tiers are called "GlossL" and "GlossR" 11 | - two-handed signs are annotated identically on the tiers of both hands (this also inflates the overall count of 12 | annotated glosses 13 | 14 | The translations into Dutch are in the tier called "TranslationNarrow". 15 | 16 | Both gloss and translation tiers exist for both speakers S1 and S2. Hence, the actual tier names of interest are: 17 | - GlossL S1 18 | - GlossL S2 19 | - GlossR S1 20 | - GlossR S2 21 | - TranslationNarrow S1 22 | - TranslationNarrow S2 23 | 24 | Not all ELAN files have gloss annotations or translations. The code needs to assume that any tier type could be missing. 25 | """ 26 | 27 | 28 | def get_elan_sentences_ngt_corpus(elan_path: str): 29 | eaf = pympi.Elan.Eaf(elan_path) # TODO add "suppress_version_warning=True" when pympi 1.7 is released 30 | 31 | timeslots = eaf.timeslots 32 | 33 | for participant in ["S1", "S2"]: 34 | dutch_tier_name = "TranslationNarrow " + participant 35 | if dutch_tier_name not in eaf.tiers: 36 | continue 37 | 38 | # tiers is defined as follows (http://dopefishh.github.io/pympi/Elan.html): 39 | # {tier_name -> (aligned_annotations, reference_annotations, attributes, ordinal)} 40 | # aligned_annotations: 41 | # [{id -> (begin_ts, end_ts, value, svg_ref)}] 42 | # reference_annotations: 43 | # [{id -> (reference, value, previous, svg_ref)}] 44 | # 45 | # - "ts" means timeslot, which references a time value in miliseconds 46 | # - "value" is the actual annotation content 47 | # - "svg_ref" is an optional reference to an SVG image that is always None in our files 48 | 49 | dutch_text = list(eaf.tiers[dutch_tier_name][0].values()) 50 | 51 | # collect all glosses in the entire ELAN file 52 | 53 | all_glosses = [] 54 | 55 | for hand in ["R", "L"]: 56 | hand_tier = "Gloss" + hand + " " + participant 57 | if hand_tier not in eaf.tiers: 58 | continue 59 | 60 | glosses = {} 61 | 62 | for gloss_id, (start, end, value, _) in eaf.tiers[hand_tier][0].items(): 63 | glosses[gloss_id] = {"start": timeslots[start], "end": timeslots[end], "gloss": value, "hand": hand} 64 | 65 | all_glosses += list(glosses.values()) 66 | 67 | for start, end, value, _ in dutch_text: 68 | sentence = {"participant": participant, "start": timeslots[start], "end": timeslots[end], "dutch": value} 69 | 70 | # Add glosses whose timestamps are within this sentence 71 | glosses_in_sentence = [item for item in all_glosses if item["start"] >= sentence["start"] and item["end"] <= sentence["end"]] 72 | 73 | sentence["glosses"] = list(sorted(glosses_in_sentence, key=lambda d: d["start"])) 74 | 75 | yield sentence 76 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/splits/create_document_split.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """dgs_document_split.ipynb 3 | 4 | Automatically generated by Colaboratory. 5 | 6 | Original file is located at 7 | https://colab.research.google.com/drive/19pHmLuIEAKFn4BqVr7cwNRaVxQHWKNI7 8 | """ 9 | 10 | # ! pip install sign-language-datasets==0.0.12 11 | 12 | import json 13 | 14 | import numpy as np 15 | 16 | import tensorflow_datasets as tfds 17 | import sign_language_datasets.datasets 18 | from sign_language_datasets.datasets.config import SignDatasetConfig 19 | from sign_language_datasets.datasets.dgs_corpus.dgs_utils import get_elan_sentences 20 | 21 | from typing import Optional, Tuple 22 | 23 | np.random.seed(1) 24 | 25 | # Videos 1177918 and 1432043 have 25 fps, start and end frame won't match 26 | 27 | INCORRECT_FRAMERATE = ["1432043", "1177918"] 28 | 29 | 30 | def get_split_indexes(total_size: int, dev_size: int, test_size) -> Tuple[np.array, np.array, np.array]: 31 | train_indexes = np.arange(total_size, dtype=np.int32) 32 | 33 | np.random.shuffle(train_indexes) 34 | 35 | # high inclusive 36 | 37 | dev_indexes = np.random.choice(train_indexes, size=(dev_size,), replace=False) 38 | 39 | remaining_train_indexes = np.asarray([i for i in train_indexes if i not in dev_indexes]) 40 | 41 | test_indexes = np.random.choice(remaining_train_indexes, size=(test_size,), replace=False) 42 | 43 | remaining_train_indexes = np.asarray([i for i in remaining_train_indexes if i not in test_indexes]) 44 | 45 | return remaining_train_indexes, dev_indexes, test_indexes 46 | 47 | 48 | config = SignDatasetConfig(name="only-annotations", version="1.0.0", include_video=False, include_pose=None) 49 | dgs_corpus = tfds.load("dgs_corpus", builder_kwargs=dict(config=config)) 50 | 51 | 52 | def get_split(dev_size: int, test_size: int): 53 | ids = np.array([datum["id"].numpy().decode("utf-8") for datum in dgs_corpus["train"] if datum["id"] not in INCORRECT_FRAMERATE]) 54 | 55 | train_indexes, dev_indexes, test_indexes = get_split_indexes(len(ids), dev_size=dev_size, test_size=test_size) 56 | 57 | print("Number of entire files in each split:") 58 | print(str({"train": len(train_indexes), "dev": len(dev_indexes), "test": len(test_indexes)})) 59 | 60 | return { 61 | "dgs_corpus_version": "3.0.0", 62 | "train": list(ids[train_indexes]), 63 | "dev": list(ids[dev_indexes]), 64 | "test": list(ids[test_indexes]), 65 | } 66 | 67 | 68 | split = get_split(dev_size=10, test_size=10) 69 | 70 | with open("split.json", "w") as outfile: 71 | json.dump(split, outfile, indent=4) 72 | 73 | # ! cat split.json 74 | 75 | """## compute sentence statistics for this split""" 76 | 77 | 78 | def get_split_name_from_id(_id: str) -> str: 79 | for key in split.keys(): 80 | if _id in split[key]: 81 | return key 82 | 83 | return "none" 84 | 85 | 86 | sentences_found = {"train": 0, "dev": 0, "test": 0, "none": 0} 87 | 88 | for datum in dgs_corpus["train"]: 89 | _id = datum["id"].numpy().decode("utf-8") 90 | 91 | split_name = get_split_name_from_id(_id) 92 | 93 | elan_path = datum["paths"]["eaf"].numpy().decode("utf-8") 94 | sentences = get_elan_sentences(elan_path) 95 | 96 | for sentence in sentences: 97 | gloss_sequence = " ".join([s["gloss"] for s in sentence["glosses"]]) 98 | german_sentence = sentence["german"] 99 | 100 | if gloss_sequence != "" and german_sentence != "": 101 | sentences_found[split_name] += 1 102 | 103 | print(sentences_found) 104 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/config.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Tuple 2 | 3 | import tensorflow_datasets as tfds 4 | 5 | 6 | def cloud_bucket_file(file_name: str) -> str: 7 | return f"https://sign-lanugage-datasets.sign-mt.cloud/{file_name}" 8 | 9 | 10 | class SignDatasetConfig(tfds.core.BuilderConfig): 11 | """General BuilderConfig for sign language datasets.""" 12 | 13 | def __init__( 14 | self, 15 | include_video: bool = True, 16 | process_video: bool = None, 17 | include_pose: Optional[str] = None, 18 | process_pose: bool = True, 19 | fps: Optional[float] = None, 20 | resolution: Optional[Tuple[int, int]] = None, 21 | sample_size: Optional[int] = None, 22 | extra: dict = {}, 23 | **kwargs, 24 | ): 25 | """Constructs a RWTHPhoenix2014TConfig. 26 | Args: 27 | include_video: bool, whether to include videos in the data. 28 | process_video: bool, whether to load the videos as tensors or paths 29 | include_pose: str, what pose data to include. 30 | fps: float, what pose data to include. 31 | resolution: (int, int), what resolution of videos to load. 32 | split: specify a known split identifier (optional) 33 | **kwargs: keyword arguments forwarded to super. 34 | """ 35 | super(SignDatasetConfig, self).__init__(**kwargs) 36 | self.include_video = include_video 37 | self.process_video = process_video if process_video is not None else include_video 38 | self.include_pose = include_pose.lower() if include_pose is not None else None 39 | self.process_pose = process_pose 40 | 41 | self.fps = fps 42 | self.resolution = resolution 43 | self.sample_size = sample_size 44 | self.extra = extra 45 | 46 | def ffmpeg_args(self): 47 | args: List[str] = [] 48 | 49 | if self.fps is not None: 50 | args += ["-r", str(self.fps)] 51 | 52 | if self.resolution is not None: 53 | w, h = self.resolution 54 | ratio = str(w) + ":" + str(h) 55 | scale = ratio + ":force_original_aspect_ratio=decrease" 56 | pad = ratio + ":(ow-iw)/2:(oh-ih)/2" 57 | args += ["-vf", "scale=" + scale + ",pad=" + pad + ",setsar=1"] 58 | 59 | return args 60 | 61 | def video_feature(self, default_resolution: Tuple[int, int], channels=3): 62 | w, h = self.resolution if self.resolution is not None else default_resolution 63 | 64 | ffmpeg_extra_args = self.ffmpeg_args() 65 | return VideoFeature(shape=(None, h, w, channels), ffmpeg_extra_args=ffmpeg_extra_args) 66 | 67 | 68 | class VideoFeature(tfds.features.Video): 69 | def encode_example(self, video_or_path_or_fobj): 70 | """TFDS does not scale images, so this class handles it""" 71 | 72 | # Load images if list of file paths 73 | if isinstance(video_or_path_or_fobj, list) and isinstance(video_or_path_or_fobj[0], str): 74 | import cv2 75 | _, h, w, _ = self.shape 76 | video_or_path_or_fobj = [cv2.resize(cv2.imread(f), (w, h)) for f in video_or_path_or_fobj] 77 | 78 | # In case where additional ffmpeg parameters are needed 79 | if isinstance(video_or_path_or_fobj, dict) and "video" in video_or_path_or_fobj and isinstance(video_or_path_or_fobj["video"], str): 80 | old_args = list(self._extra_ffmpeg_args) 81 | self._extra_ffmpeg_args += video_or_path_or_fobj["ffmpeg_args"] 82 | result = super(VideoFeature, self).encode_example(video_or_path_or_fobj["video"]) 83 | self._extra_ffmpeg_args = old_args 84 | return result 85 | 86 | return super(VideoFeature, self).encode_example(video_or_path_or_fobj) 87 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/autsl/checksums.tsv: -------------------------------------------------------------------------------- 1 | http://158.109.8.102/AuTSL/data/test/test_set_xsaft57.zip.001 1048576000 3578d6890bb9e0846d6f27c625db05a0ab94144f5c732c3b1ce35b94552d8e81 test_set_xsaft57.zip.001 2 | http://158.109.8.102/AuTSL/data/test/test_set_xsaft57.zip.002 1048576000 9b8ff4ffa53a38d889de011f4537d801812ee69c35fe27958d142256f3c14aa1 test_set_xsaft57.zip.002 3 | http://158.109.8.102/AuTSL/data/test/test_set_xsaft57.zip.003 596691542 9124e715ae33722d8a0acbae620f73bca9893712fa6633e42d6e2716612fc9ff test_set_xsaft57.zip.003 4 | http://158.109.8.102/AuTSL/data/train/train_labels.csv 653817 13ab78789655d5ccde3a056968bc3c3433002a459d9ae383822bab5223e7b86c train_labels.csv 5 | http://158.109.8.102/AuTSL/data/train/train_labels.csv 653817 13ab78789655d5ccde3a056968bc3c3433002a459d9ae383822bab5223e7b86c train_labels.csv 6 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.001 1048576000 6955ac2ae2b4226d51263411e09c0fd9d249d57803dca9ca6da2f94ad7ca4800 train_set_vfbha39.zip.001 7 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.002 1048576000 7cfc37cb53a4a771ba872c3fd4ad6faf20002a92120315f9938612a8b214ff71 train_set_vfbha39.zip.002 8 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.003 1048576000 b800a5b007494243f8030e743cf42e78018f7634c94cdbfa2608591182514fae train_set_vfbha39.zip.003 9 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.004 1048576000 bb34b4bbb72a1e9817cf8305e58052d1baa409970be81c579149dc1011adbba9 train_set_vfbha39.zip.004 10 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.005 1048576000 d74d82c6e69c8810a5f9d6d7796d4463422c84a08e0c1a7d10e4fb77081ba170 train_set_vfbha39.zip.005 11 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.006 1048576000 2254e0e40b3d7587ba74da7db17d97a9146057214496ef8feeea3d316c708270 train_set_vfbha39.zip.006 12 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.007 1048576000 e42410cd2243a7aacc8b98a24ba824aadc867ee04de014565f51700c23238f44 train_set_vfbha39.zip.007 13 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.008 1048576000 eb939ae4ad7f3669d6a2bfa2217a05c18229fc12c2d97c0fb0332cbda3ef3238 train_set_vfbha39.zip.008 14 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.009 1048576000 e97c8602f4acb5c30325bcad507d014457bd3e1cc733338f0498c11776e837e5 train_set_vfbha39.zip.009 15 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.010 1048576000 37d2ffe47b14c74a56830bbacfd712a9d0d391cfde95aaf4a7471aaa6779394c train_set_vfbha39.zip.010 16 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.011 1048576000 07533a380dbf3729417a39942c808cda6914d0a675acf1996c243353f93bf8ea train_set_vfbha39.zip.011 17 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.012 1048576000 1b7bbf8b8019d8a32b7df835a833acc60aff1a719b80e4ab6db9cb41a6f8e7a7 train_set_vfbha39.zip.012 18 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.013 1048576000 3001bbe8b1302740e64f779bd5af89480a544a15a3dbbf3415461e56b3270d05 train_set_vfbha39.zip.013 19 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.014 1048576000 a7df50a9b75cad2332a8b3208de2031b5e1895c8ec48cd377396965dd06ef03d train_set_vfbha39.zip.014 20 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.015 1048576000 5449633b9be06ed88b0cc8f0f07cfced3bcc3a9d37229747b0d602332fe43fc5 train_set_vfbha39.zip.015 21 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.016 1048576000 9f975f03e341d4cbc6c7f62ab3321e28cfd9662cc36dc01c03a079a7924edd6d train_set_vfbha39.zip.016 22 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.017 1048576000 901d23bc22e289b82c0a83eebd2c49d1e40d74a932d7a2ff19382d1dc1a34cdb train_set_vfbha39.zip.017 23 | http://158.109.8.102/AuTSL/data/train/train_set_vfbha39.zip.018 688988403 4ccf01c068f06293a937a078cefe4849269f3924cac6e33865acb8df7024da87 train_set_vfbha39.zip.018 24 | http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.001 1048576000 007a96f8e15055e370adaa3b7e8ca9e3b76abd269f677bc33808fc739401ce49 val_set_bjhfy68.zip.001 25 | http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.002 1048576000 c138b73f87f952008fe21b18466bc924d539532fd348c9a7e775761c27294591 val_set_bjhfy68.zip.002 26 | http://158.109.8.102/AuTSL/data/validation/val_set_bjhfy68.zip.003 1028644777 ed1549f54ddc5b4cf71d89743888daacde11f28616e52d9b5133bccf2f9fd0b1 val_set_bjhfy68.zip.003 27 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signsuisse/download_holistic_gcs.py: -------------------------------------------------------------------------------- 1 | import os 2 | from google.cloud import storage 3 | from pose_format import Pose 4 | import tarfile 5 | 6 | from tqdm import tqdm 7 | 8 | 9 | def fix_pose_file(file_path: str): 10 | with open(file_path, "rb") as buffer: 11 | pose = Pose.read(buffer.read()) 12 | if len(pose.header.components) == 4: 13 | return 14 | 15 | print(f"Fixing {file_path}") 16 | 17 | pose = pose.get_components(["POSE_LANDMARKS", "FACE_LANDMARKS", "LEFT_HAND_LANDMARKS", "RIGHT_HAND_LANDMARKS"]) 18 | 19 | with open(file_path, "wb") as f: 20 | pose.write(f) 21 | 22 | 23 | def download_blob(bucket_name, source_blob_name, destination_file_name): 24 | """Downloads a blob from the bucket.""" 25 | storage_client = storage.Client() 26 | bucket = storage_client.bucket(bucket_name) 27 | blob = bucket.blob(source_blob_name) 28 | 29 | blob.download_to_filename(destination_file_name) 30 | print(f"Blob {source_blob_name} downloaded to {destination_file_name}.") 31 | 32 | fix_pose_file(destination_file_name) 33 | 34 | 35 | def list_blobs_with_prefix(bucket_name, prefix): 36 | """Lists all the blobs in the bucket with the given prefix.""" 37 | storage_client = storage.Client() 38 | blobs = storage_client.list_blobs(bucket_name, prefix=prefix) 39 | 40 | return blobs 41 | 42 | 43 | def validate_pose_file(file_name: str, buffer: bytes): 44 | pose = Pose.read(buffer) 45 | data_frames, data_people, data_points, data_dimensions = pose.body.data.shape 46 | conf_frames, conf_people, conf_points = pose.body.confidence.shape 47 | assert data_frames == conf_frames, f"Pose {file_name} has different number of frames in data and confidence" 48 | assert data_people == conf_people, f"Pose {file_name} has different number of people in data and confidence" 49 | assert data_points == conf_points, f"Pose {file_name} has different number of points in data and confidence" 50 | assert data_points == 543, f"Pose {file_name} has different number of points in data ({data_points})" 51 | assert data_dimensions == 3, f"Pose {file_name} has different number of dimensions in data ({data_dimensions})" 52 | 53 | 54 | def validate_tar_files(tar_file_name): 55 | """Iterates over the files in a tar archive and validates each one.""" 56 | with tarfile.open(tar_file_name, "r") as tar: 57 | for member in tqdm(tar.getmembers(), desc="Validating files"): 58 | f = tar.extractfile(member) 59 | if f is not None: 60 | try: 61 | validate_pose_file(member.name, f.read()) 62 | except Exception as e: 63 | print(f"Error validating {member.name}") 64 | print(e) 65 | 66 | 67 | def list_files_in_tar_gz(file_path): 68 | with tarfile.open(file_path, "r") as tar: 69 | file_names = tar.getnames() 70 | return [os.path.basename(f) for f in file_names] 71 | 72 | 73 | def main(): 74 | bucket_name = "sign-mt-poses" 75 | prefix = "external/ss" 76 | blobs = list_blobs_with_prefix(bucket_name, prefix) 77 | 78 | destination_tar = "/home/nlp/amit/WWW/datasets/poses/holistic/signsuisse.tar" 79 | existing_files = set(list_files_in_tar_gz(destination_tar)) 80 | print(f"Found {len(existing_files)} valid files in {destination_tar}") 81 | print(list(existing_files)[:10]) 82 | 83 | with tarfile.open(destination_tar, "a") as tar: 84 | for blob in tqdm(blobs, desc="Reading files"): 85 | if not blob.name.endswith("/"): 86 | file_name = os.path.basename(blob.name) 87 | if file_name not in existing_files: 88 | download_blob(bucket_name, blob.name, file_name) 89 | tar.add(file_name) 90 | os.remove(file_name) 91 | 92 | validate_tar_files(destination_tar) 93 | 94 | 95 | if __name__ == "__main__": 96 | main() 97 | 98 | # Extract: tar xf signsuisse.tar --directory signsuisse 99 | # Compress: tar -cvf signsuisse.tar -C signsuisse . 100 | # import os 101 | # 102 | # from pose_format import Pose 103 | # 104 | # directory = '/home/nlp/amit/WWW/datasets/poses/holistic/signsuisse' 105 | # for filename in tqdm(os.listdir(directory)): 106 | # if filename.endswith(".pose"): 107 | # file_path = os.path.join(directory, filename) 108 | # 109 | # fix_pose_file(file_path) 110 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/spread_the_sign/spread_the_sign.py: -------------------------------------------------------------------------------- 1 | """Spreadthesign""" 2 | 3 | import csv 4 | from os import path 5 | 6 | 7 | import tensorflow as tf 8 | import tensorflow_datasets as tfds 9 | from tensorflow.io.gfile import GFile 10 | 11 | from pose_format import Pose 12 | 13 | from ..warning import dataset_warning 14 | from ...datasets.config import SignDatasetConfig 15 | 16 | _DESCRIPTION = """ 17 | SpreadTheSign2 is a notable multilingual dictio- nary containing around 23,000 words with up to 41 different spoken-sign language pairs and more than 600,000 videos in total. 18 | """ 19 | 20 | _CITATION = """ 21 | """ 22 | 23 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader")} 24 | 25 | _KNOWN_SPLITS = {"1.0.0-uzh": path.join(path.dirname(path.realpath(__file__)), "splits/1.0.0-uzh")} 26 | 27 | 28 | class SpreadTheSign(tfds.core.GeneratorBasedBuilder): 29 | """DatasetBuilder for Spreadthesign dataset.""" 30 | 31 | VERSION = tfds.core.Version("1.0.0") 32 | RELEASE_NOTES = {"1.0.0": "Initial release."} 33 | 34 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_pose="holistic")] 35 | 36 | def _info(self) -> tfds.core.DatasetInfo: 37 | """Returns the dataset metadata.""" 38 | 39 | features = { 40 | "id": tfds.features.Text(), 41 | "text": tfds.features.Text(), 42 | "sign_language": tfds.features.Text(), 43 | "spoken_language": tfds.features.Text(), 44 | "pose_path": tfds.features.Text(), 45 | "pose_length": tf.float32, 46 | } 47 | 48 | return tfds.core.DatasetInfo( 49 | builder=self, 50 | description=_DESCRIPTION, 51 | features=tfds.features.FeaturesDict(features), 52 | homepage="https://www.spreadthesign.com/", 53 | supervised_keys=None, 54 | citation=_CITATION, 55 | ) 56 | 57 | def _load_split_ids(self, split: str): 58 | split_dir = _KNOWN_SPLITS[self._builder_config.extra["split"]] 59 | 60 | with open(path.join(split_dir, f"{split}.txt")) as f: 61 | ids = [] 62 | for line in f: 63 | id = line.rstrip("\n") 64 | ids.append(id) 65 | 66 | return ids 67 | 68 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 69 | """Returns SplitGenerators.""" 70 | dataset_warning(self) 71 | 72 | pose_dir = self._builder_config.extra["pose_dir"] 73 | 74 | if "split" in self._builder_config.extra: 75 | train_args = {"pose_dir": pose_dir, "ids": self._load_split_ids("train")} 76 | val_args = {"pose_dir": pose_dir, "ids": self._load_split_ids("val")} 77 | test_args = {"pose_dir": pose_dir, "ids": self._load_split_ids("test")} 78 | 79 | return [ 80 | tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs=train_args), 81 | tfds.core.SplitGenerator(name=tfds.Split.VALIDATION, gen_kwargs=val_args), 82 | tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs=test_args), 83 | ] 84 | else: 85 | return [tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"pose_dir": pose_dir})] 86 | 87 | def _generate_examples(self, pose_dir: str, ids: list = []): 88 | """Yields examples.""" 89 | 90 | with GFile(self._builder_config.extra["csv_path"], "r") as csv_file: 91 | csv_data = csv.reader(csv_file, delimiter=",") 92 | next(csv_data) # Ignore the header 93 | 94 | for i, row in enumerate(csv_data): 95 | datum = {"id": str(i), "text": row[3], "sign_language": row[1], "spoken_language": row[2]} 96 | 97 | if len(ids) > 0 and (datum["id"] not in ids): 98 | continue 99 | 100 | if self.builder_config.include_pose is not None: 101 | if self.builder_config.include_pose == "holistic": 102 | mediapipe_path = path.join(pose_dir, row[0]) 103 | 104 | if path.exists(mediapipe_path): 105 | datum["pose_path"] = mediapipe_path 106 | with open(mediapipe_path, "rb") as f: 107 | pose = Pose.read(f.read()) 108 | datum["pose_length"] = pose.body.data.shape[0] 109 | 110 | yield datum["id"], datum 111 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/pop_sign/pop_sign.py: -------------------------------------------------------------------------------- 1 | """PopSign ASL v1.0: An Isolated American Sign Language Dataset Collected via Smartphones""" 2 | 3 | import csv 4 | import os 5 | from os import path 6 | 7 | import numpy as np 8 | 9 | import tensorflow as tf 10 | import tensorflow_datasets as tfds 11 | 12 | from pose_format import Pose 13 | from pose_format import Pose, PoseHeader 14 | from pose_format.numpy import NumPyPoseBody 15 | from pose_format.pose_header import PoseHeaderDimensions 16 | 17 | from sign_language_datasets.utils.features import PoseFeature 18 | 19 | from ..warning import dataset_warning 20 | from ...datasets.config import SignDatasetConfig, cloud_bucket_file 21 | 22 | _DESCRIPTION = """ 23 | PopSign ASL v1.0 dataset collects examples of 250 isolated American Sign Language (ASL) signs using Pixel 4A smartphone selfie cameras in a variety of environments. 24 | """ 25 | 26 | _CITATION = """ 27 | @article{starner2024popsign, 28 | title={PopSign ASL v1. 0: An Isolated American Sign Language Dataset Collected via Smartphones}, 29 | author={Starner, Thad and Forbes, Sean and So, Matthew and Martin, David and Sridhar, Rohit and Deshpande, Gururaj and Sepah, Sam and Shahryar, Sahir and Bhardwaj, Khushi and Kwok, Tyler and others}, 30 | journal={Advances in Neural Information Processing Systems}, 31 | volume={36}, 32 | year={2024} 33 | } 34 | """ 35 | 36 | _DOWNLOAD_URL = "https://signdata.cc.gatech.edu/" 37 | 38 | 39 | # _POSE_URLS = {"holistic": cloud_bucket_file("poses/holistic/ASLCitizen.zip")} 40 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader")} 41 | 42 | 43 | class PopSign(tfds.core.GeneratorBasedBuilder): 44 | """DatasetBuilder for Popsign dataset.""" 45 | 46 | VERSION = tfds.core.Version("1.0.0") 47 | RELEASE_NOTES = {"1.0.0": "Initial release."} 48 | 49 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_pose="holistic")] 50 | 51 | def _info(self) -> tfds.core.DatasetInfo: 52 | """Returns the dataset metadata.""" 53 | 54 | features = { 55 | "id": tfds.features.Text(), 56 | "text": tfds.features.Text(), 57 | # "signer_id": tfds.features.Text(), 58 | } 59 | 60 | # TODO: add videos 61 | 62 | if self._builder_config.include_pose == "holistic": 63 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 64 | stride = 1 if self._builder_config.fps is None else 30 / self._builder_config.fps 65 | features["pose"] = PoseFeature(shape=(None, 1, 576, 3), header_path=pose_header_path, stride=stride) 66 | 67 | return tfds.core.DatasetInfo( 68 | builder=self, 69 | description=_DESCRIPTION, 70 | features=tfds.features.FeaturesDict(features), 71 | homepage=_DOWNLOAD_URL, 72 | supervised_keys=None, 73 | citation=_CITATION, 74 | ) 75 | 76 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 77 | """Returns SplitGenerators.""" 78 | dataset_warning(self) 79 | 80 | # too expensive to host the poses at the moment, need to specify a local path 81 | # poses_dir = str(dl_manager.download_and_extract(_POSE_URLS["holistic"])) 82 | poses_dir = self._builder_config.extra["pose_dir"] 83 | 84 | return [ 85 | tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"poses_dir": path.join(poses_dir, 'train')}), 86 | tfds.core.SplitGenerator( 87 | name=tfds.Split.VALIDATION, gen_kwargs={"poses_dir": path.join(poses_dir, 'val')} 88 | ), 89 | tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs={"poses_dir": path.join(poses_dir, 'test')}), 90 | ] 91 | 92 | def _generate_examples(self, poses_dir: str): 93 | """Yields examples.""" 94 | 95 | for label in os.listdir(poses_dir): 96 | subdirectory_path = os.path.join(poses_dir, label) 97 | if os.path.isdir(subdirectory_path): 98 | for filename in os.listdir(subdirectory_path): 99 | if filename.endswith('.pose'): 100 | datum = {"id": filename.replace('.pose', ''), "text": label} 101 | mediapipe_path = path.join(subdirectory_path, filename) 102 | 103 | with open(mediapipe_path, "rb") as f: 104 | try: 105 | pose = Pose.read(f.read()) 106 | datum["pose"] = pose 107 | 108 | yield datum["id"], datum 109 | except Exception as e: 110 | print(e) 111 | 112 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_citizen/asl_citizen.py: -------------------------------------------------------------------------------- 1 | """ASL Citizen - A Community-Sourced Dataset for Advancing Isolated Sign Language Recognition""" 2 | 3 | import csv 4 | import tarfile 5 | from os import path 6 | 7 | import numpy as np 8 | 9 | import tensorflow as tf 10 | import tensorflow_datasets as tfds 11 | from tensorflow.io.gfile import GFile 12 | 13 | from pose_format import Pose 14 | from pose_format import Pose, PoseHeader 15 | from pose_format.numpy import NumPyPoseBody 16 | from pose_format.pose_header import PoseHeaderDimensions 17 | 18 | from sign_language_datasets.utils.features import PoseFeature 19 | 20 | from ..warning import dataset_warning 21 | from ...datasets.config import SignDatasetConfig, cloud_bucket_file 22 | 23 | _DESCRIPTION = """ 24 | The dataset contains about 84k video recordings of 2.7k isolated signs from American Sign Language (ASL). 25 | """ 26 | 27 | _CITATION = """ 28 | @article{desai2023asl, 29 | title={ASL Citizen: A Community-Sourced Dataset for Advancing Isolated Sign Language Recognition}, 30 | author={Desai, Aashaka and Berger, Lauren and Minakov, Fyodor O and Milan, Vanessa and Singh, Chinmay and Pumphrey, Kriston and Ladner, Richard E and Daum{\'e} III, Hal and Lu, Alex X and Caselli, Naomi and Bragg, Danielle}, 31 | journal={arXiv preprint arXiv:2304.05934}, 32 | year={2023} 33 | } 34 | """ 35 | 36 | _DOWNLOAD_URL = "https://download.microsoft.com/download/b/8/8/b88c0bae-e6c1-43e1-8726-98cf5af36ca4/ASL_Citizen.zip" 37 | 38 | 39 | _POSE_URLS = {"holistic": cloud_bucket_file("poses/holistic/ASLCitizen.zip")} 40 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader")} 41 | 42 | 43 | class ASLCitizen(tfds.core.GeneratorBasedBuilder): 44 | """DatasetBuilder for ASL Citizen dataset.""" 45 | 46 | VERSION = tfds.core.Version("1.0.0") 47 | RELEASE_NOTES = {"1.0.0": "Initial release."} 48 | 49 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_pose="holistic")] 50 | 51 | def _info(self) -> tfds.core.DatasetInfo: 52 | """Returns the dataset metadata.""" 53 | 54 | features = { 55 | "id": tfds.features.Text(), 56 | "text": tfds.features.Text(), 57 | "signer_id": tfds.features.Text(), 58 | "asl_lex_code": tfds.features.Text(), 59 | } 60 | 61 | # TODO: add videos 62 | 63 | if self._builder_config.include_pose == "holistic": 64 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 65 | stride = 1 if self._builder_config.fps is None else 30 / self._builder_config.fps 66 | features["pose"] = PoseFeature(shape=(None, 1, 576, 3), header_path=pose_header_path, stride=stride) 67 | 68 | return tfds.core.DatasetInfo( 69 | builder=self, 70 | description=_DESCRIPTION, 71 | features=tfds.features.FeaturesDict(features), 72 | homepage="https://www.microsoft.com/en-us/research/project/asl-citizen/", 73 | supervised_keys=None, 74 | citation=_CITATION, 75 | ) 76 | 77 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 78 | """Returns SplitGenerators.""" 79 | dataset_warning(self) 80 | 81 | archive = dl_manager.download_and_extract(_DOWNLOAD_URL) 82 | poses_dir = str(dl_manager.download_and_extract(_POSE_URLS["holistic"])) 83 | 84 | return [ 85 | tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"archive_path": archive, "split": "train", "poses_dir": poses_dir}), 86 | tfds.core.SplitGenerator( 87 | name=tfds.Split.VALIDATION, gen_kwargs={"archive_path": archive, "split": "val", "poses_dir": poses_dir} 88 | ), 89 | tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs={"archive_path": archive, "split": "test", "poses_dir": poses_dir}), 90 | ] 91 | 92 | def _generate_examples(self, archive_path: str, split: str, poses_dir: str): 93 | """Yields examples.""" 94 | 95 | with GFile(path.join(archive_path, "ASL_Citizen", "splits", f"{split}.csv"), "r") as csv_file: 96 | csv_data = csv.reader(csv_file, delimiter=",") 97 | next(csv_data) # Ignore the header 98 | 99 | for i, row in enumerate(csv_data): 100 | datum = {"id": str(i), "text": row[2], "signer_id": row[0], "asl_lex_code": row[3]} 101 | 102 | if self.builder_config.include_pose is not None: 103 | if self.builder_config.include_pose == "holistic": 104 | mediapipe_path = path.join(poses_dir, "poses", row[1].replace(".mp4", ".pose")) 105 | 106 | if path.exists(mediapipe_path): 107 | with open(mediapipe_path, "rb") as f: 108 | pose = Pose.read(f.read()) 109 | datum["pose"] = pose 110 | else: 111 | datum["pose"] = None 112 | 113 | yield datum["id"], datum 114 | -------------------------------------------------------------------------------- /sign_language_datasets/utils/signwriting/ocr/ocr.py: -------------------------------------------------------------------------------- 1 | """ 2 | This SignWriting OCR code is adapted from 3 | https://colab.research.google.com/drive/1x0OupzZNkQW1rCiDjEe1LX5V9k8_YF69#scrollTo=_1YX_pILnjFe 4 | """ 5 | 6 | import functools 7 | from typing import List 8 | 9 | import numpy as np 10 | from numpy.lib.stride_tricks import as_strided 11 | import os 12 | 13 | 14 | def strided_convolution(image, weight, stride=1): 15 | im_h, im_w = image.shape 16 | f_h, f_w = weight.shape 17 | 18 | out_shape = (1 + (im_h - f_h) // stride, 1 + (im_w - f_w) // stride, f_h, f_w) 19 | out_strides = (image.strides[0] * stride, image.strides[1] * stride, image.strides[0], image.strides[1]) 20 | windows = as_strided(image, shape=out_shape, strides=out_strides) 21 | 22 | return np.tensordot(windows, weight, axes=((2, 3), (0, 1))) 23 | 24 | 25 | def rgb2bin(img, neg=0.1): 26 | return np.where(img[:, :, 0] < 200, 1, neg) 27 | 28 | 29 | def key2swu(key): 30 | return chr(0x40001 + ((int(key[1:4], 16) - 256) * 96) + ((int(key[4:5], 16)) * 16) + int(key[5:6], 16)) 31 | 32 | 33 | def shape_pos(shape): 34 | x, y = shape[:2] 35 | return f"{x:03}x{y:03}" 36 | 37 | 38 | def crop_whitespace(img): 39 | import cv2 40 | 41 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 42 | gray = 255 * (gray < 128).astype(np.uint8) # To invert the text to white 43 | coords = cv2.findNonZero(gray) # Find all non-zero points (text) 44 | x, y, w, h = cv2.boundingRect(coords) # Find minimum spanning bounding box 45 | if y > 0: 46 | y -= 1 47 | h += 1 48 | if x > 0: 49 | x -= 1 50 | w += 1 51 | if y + h < img.shape[0]: 52 | h += 1 53 | if x + w < img.shape[1]: 54 | w += 1 55 | 56 | return cv2.copyMakeBorder(img[y : y + h, x : x + w], 2, 2, 2, 2, cv2.BORDER_CONSTANT, value=(255, 255, 255)) 57 | 58 | 59 | @functools.lru_cache() 60 | def get_font(): 61 | from PIL import ImageFont 62 | 63 | dirname = os.path.dirname(__file__) 64 | font_path = os.path.join(dirname, "assets/SuttonSignWritingOneD.ttf") 65 | 66 | return ImageFont.truetype(font_path, 30) 67 | 68 | 69 | def image_to_fsw(image: np.ndarray, symbols: List[str]) -> str: 70 | import cv2 71 | 72 | try: 73 | from PIL import Image, ImageDraw 74 | except ImportError: 75 | raise ImportError("Please install pillow with: pip install Pillow") 76 | 77 | font = get_font() 78 | 79 | # Adding border for conv calc to go over borders 80 | img_rgb = cv2.copyMakeBorder(image.copy(), 30, 30, 30, 30, cv2.BORDER_CONSTANT, value=(255, 255, 255)) 81 | img_bin = rgb2bin(img_rgb, -0.5) 82 | 83 | img_width, img_height, _ = img_rgb.shape 84 | final_sign = "M" + shape_pos([500, 500]) 85 | # final_sign = "A" + "".join(symbols) + "M" + shape_pos(img_rgb.shape) 86 | 87 | # Create all convolutions 88 | templates = [] 89 | convs = [] 90 | for symbol in symbols: 91 | swu = key2swu(symbol) 92 | (width, height), (offset_x, offset_y) = font.font.getsize(swu) 93 | img = Image.new("RGB", (width - offset_x, height - offset_y), (255, 255, 255)) 94 | d = ImageDraw.Draw(img) 95 | d.text((-offset_x, -offset_y), swu, font=font, fill=(0, 0, 0)) 96 | 97 | template = crop_whitespace(np.array(img)) 98 | templates.append(template) 99 | 100 | conv = strided_convolution(img_bin, rgb2bin(template, -0.5)) 101 | convs.append(conv) 102 | 103 | # Save video? 104 | # out = cv2.VideoWriter('project.avi',cv2.VideoWriter_fourcc(*'DIVX'), 15, img_rgb.shape[:2]) 105 | 106 | # Find max symbol 107 | while len(symbols) > 0: 108 | # 1. Find best match 109 | best_match = {"symbol": "", "point": (), "score": 0} 110 | for symbol, conv in zip(symbols, convs): 111 | point = np.unravel_index(np.argmax(conv, axis=None), conv.shape) 112 | score = conv[point] 113 | if score > best_match["score"]: 114 | best_match = {"symbol": symbol, "point": point, "score": score} 115 | 116 | idx = symbols.index(best_match["symbol"]) 117 | 118 | # 2. Select best match 119 | pt = best_match["point"][::-1] 120 | position = [int(500 - img_width / 2) + pt[0], int(500 - img_height / 2) + pt[1]] 121 | final_sign += best_match["symbol"] + shape_pos(position) 122 | 123 | w, h = templates[idx].shape[:-1] 124 | cv2.rectangle(img_rgb, pt, (pt[0] + h, pt[1] + w), (0, 0, 255), 1) 125 | 126 | # 3. Remove symbol from list 127 | # print(best_match) 128 | # cv2_imshow(templates[idx]) 129 | # cv2_imshow(convs[idx]) 130 | # out.write(img_rgb) 131 | symbols.pop(idx) 132 | convs.pop(idx) 133 | templates.pop(idx) 134 | 135 | # 4. Prevent collisions with this match 136 | for conv in convs: 137 | x, y = pt 138 | conv[y - 2 : y + 4, x - 2 : x + 4] = 0 139 | 140 | return final_sign 141 | 142 | 143 | if __name__ == "__main__": 144 | import cv2 145 | 146 | img_rgb = cv2.imread("assets/sign.png") 147 | symbols = ["S1f520", "S1f528", "S23c04", "S23c1c", "S2fb04", "S2ff00", "S33b10"] 148 | 149 | fsw = image_to_fsw(img_rgb, symbols) 150 | print("fsw", fsw) 151 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/swojs_glossario/swojs_glossario.py: -------------------------------------------------------------------------------- 1 | """A Brazilian Sign Language (Libras) lexicon.""" 2 | 3 | import json 4 | 5 | import tensorflow as tf 6 | import tensorflow_datasets as tfds 7 | 8 | from ..warning import dataset_warning 9 | from ...datasets import SignDatasetConfig 10 | 11 | _DESCRIPTION = """ 12 | A lexicon for Brazilian Sign Language (Libras), including videos, and SignWriting transcription. 13 | """ 14 | 15 | # TODO(swojs_glossario): BibTeX citation 16 | _CITATION = """ 17 | """ 18 | 19 | 20 | def get_media_id(datum): 21 | if "o:media" not in datum or len(datum["o:media"]) == 0: 22 | return None 23 | return datum["o:media"][0]["o:id"] 24 | 25 | 26 | def get_transcription(datum): 27 | vals = [] 28 | if "oc:transcription" in datum: 29 | vals = datum["oc:transcription"] 30 | elif "http://purl.org/linguistics/gold:writtenRealization" in datum: 31 | vals = datum["http://purl.org/linguistics/gold:writtenRealization"] 32 | 33 | return [t["@value"] for t in vals] 34 | 35 | 36 | class SwojsGlossario(tfds.core.GeneratorBasedBuilder): 37 | """DatasetBuilder for swojs_glossario dataset.""" 38 | 39 | VERSION = tfds.core.Version("1.0.0") 40 | RELEASE_NOTES = {"1.0.0": "Initial release."} 41 | 42 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=True), SignDatasetConfig(name="annotations", include_video=False)] 43 | 44 | def _info(self) -> tfds.core.DatasetInfo: 45 | """Returns the dataset metadata.""" 46 | 47 | features = { 48 | "id": tf.int32, 49 | "title": tfds.features.Text(), 50 | "created_date": tfds.features.Text(), 51 | "modified_date": tfds.features.Text(), 52 | "sign_writing": tfds.features.Sequence(tfds.features.Text()), 53 | "spoken_language": tfds.features.Sequence(tfds.features.Text()), 54 | "signed_language": tfds.features.Sequence(tfds.features.Text()), 55 | } 56 | 57 | if self._builder_config.include_video and self._builder_config.process_video: 58 | features["video"] = self._builder_config.video_feature((1920, 1080)) 59 | else: 60 | features["video"] = tfds.features.Text() 61 | 62 | return tfds.core.DatasetInfo( 63 | builder=self, 64 | description=_DESCRIPTION, 65 | features=tfds.features.FeaturesDict(features), 66 | homepage="http://swojs.ibict.br/portal/s/swojs/page/glossario", 67 | supervised_keys=None, 68 | citation=_CITATION, 69 | ) 70 | 71 | def get_media_dict(self, media_path): 72 | with open(media_path, "r", encoding="utf-8") as g: 73 | media = json.load(g) 74 | return {m["o:id"]: m["o:original_url"] for m in media} 75 | 76 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 77 | dataset_warning(self) 78 | 79 | annotations_path, media_path = dl_manager.download( 80 | ["http://swojs.ibict.br/portal/api/items?page", "http://swojs.ibict.br/portal/api/media?page"] 81 | ) 82 | 83 | media_dict = self.get_media_dict(media_path) 84 | 85 | local_videos = {} 86 | if self._builder_config.include_video and self._builder_config.process_video: 87 | with open(annotations_path, "r", encoding="utf-8") as f: 88 | annotations = json.load(f) 89 | 90 | medias = [_id for _id in [get_media_id(a) for a in annotations] if _id is not None] 91 | media_urls = [media_dict[_id] for _id in medias] 92 | 93 | video_paths = dl_manager.download(media_urls) 94 | local_videos = {k: v for k, v in zip(medias, video_paths)} 95 | 96 | return {"train": self._generate_examples(annotations_path, local_videos, media_dict)} 97 | 98 | def _generate_examples(self, annotations_path, local_videos, media_dict): 99 | """Yields examples.""" 100 | 101 | with open(annotations_path, "r", encoding="utf-8") as f: 102 | data = json.load(f) 103 | for datum in data: 104 | features = { 105 | "id": datum["o:id"], 106 | "title": datum["o:title"] if datum["o:title"] is not None else "", 107 | "created_date": datum["o:created"]["@value"], 108 | "modified_date": datum["o:modified"]["@value"], 109 | "sign_writing": get_transcription(datum), 110 | "spoken_language": [], 111 | "signed_language": [], 112 | } 113 | 114 | if "http://www.lingvoj.org/ontology:originalLanguage" in datum: 115 | features["spoken_language"] = [t["@value"] for t in datum["http://www.lingvoj.org/ontology:originalLanguage"]] 116 | 117 | if "http://www.lingvoj.org/ontology:targetLanguage" in datum: 118 | features["signed_language"] = [t["@value"] for t in datum["http://www.lingvoj.org/ontology:targetLanguage"]] 119 | 120 | media_id = get_media_id(datum) 121 | if media_id in local_videos: 122 | features["video"] = local_videos[media_id] 123 | elif media_id is not None: 124 | features["video"] = media_dict[media_id] 125 | else: 126 | features["video"] = "" 127 | 128 | yield str(features["id"]), features 129 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/signtyp/signtyp.py: -------------------------------------------------------------------------------- 1 | """SignTyp dataset.""" 2 | 3 | import re 4 | from datetime import datetime 5 | 6 | import requests 7 | import tensorflow as tf 8 | import tensorflow_datasets as tfds 9 | 10 | from ..warning import dataset_warning 11 | from ...datasets import SignDatasetConfig 12 | 13 | _DESCRIPTION = """ 14 | SignTyp is a work in progress. We currently have over 20,000 videos for signers from Brazil, China, Croatia, Denmark, Egypt, Estonia, Ethiopia, France, Germany, Haiti, Honduras, Iraq, Italy,Lithuania, Nepal, Nicaragua, the Philippines, Portugal, Russia, Rwanda, Sri Lanka, Turkey, Uganda, the United States and Venezuela. We have complete sets of approximately 1000 video recordings each for China, Denmark, Egypt, Ethiopia, Haiti, Honduras, Iraq, Lithuania, Portugal,the Phillipines, Russia (2 signers), Turkey, and the United States. Clipping has been completed for all sets except a second Russian signer. 15 | 16 | We currently have SignWriting transcriptions for approximately 9,000 signs, with more in progress. Each transcription includes detailed information on handshapes, orientation, actions, locations, as well as special sequencing and articulator relationship information. There is a downloadable file for all transcribed information for sign language researchers. 17 | 18 | SignWriting transcription can be seen in our SignPuddles - see link below. 19 | 20 | The project is sponsored by NSF (National Science Foundation) and is made possible by grant BCS-1049510 to the University of Connecticut and Harry van der Hulst. 21 | """ 22 | 23 | # TODO(SignTyp): BibTeX citation 24 | _CITATION = """ 25 | """ 26 | 27 | 28 | class SignTyp(tfds.core.GeneratorBasedBuilder): 29 | """DatasetBuilder for SignTyp dataset.""" 30 | 31 | VERSION = tfds.core.Version("1.0.0") 32 | RELEASE_NOTES = {"1.0.0": "Initial release."} 33 | 34 | BUILDER_CONFIGS = [ 35 | SignDatasetConfig(name="default", include_video=True, process_video=False, extra={"PHPSESSID": "hj9co07ct7f5noq529no9u09l4"}) 36 | ] 37 | 38 | def _info(self) -> tfds.core.DatasetInfo: 39 | """Returns the dataset metadata.""" 40 | 41 | features = { 42 | "id": tf.int32, 43 | "created_date": tfds.features.Text(), 44 | "modified_date": tfds.features.Text(), 45 | "sign_writing": tfds.features.Text(), 46 | "sign_language": tfds.features.Text(), 47 | "video": tfds.features.Text(), 48 | } 49 | 50 | return tfds.core.DatasetInfo( 51 | builder=self, 52 | description=_DESCRIPTION, 53 | features=tfds.features.FeaturesDict(features), 54 | homepage="https://signtyp.uconn.edu/", 55 | supervised_keys=None, 56 | citation=_CITATION, 57 | ) 58 | 59 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 60 | """Returns SplitGenerators.""" 61 | dataset_warning(self) 62 | 63 | if "PHPSESSID" not in self._builder_config.extra: 64 | raise Exception( 65 | "Missing PHPSESSID extra parameter. Go to https://signtyp.uconn.edu/signpuddle/ and copy your PHPSESSID from any network request." 66 | ) 67 | 68 | cookies = {"PHPSESSID": self._builder_config.extra["PHPSESSID"]} 69 | 70 | headers = { 71 | "Connection": "keep-alive", 72 | "Cache-Control": "max-age=0", 73 | "sec-ch-ua": '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"', 74 | "sec-ch-ua-mobile": "?0", 75 | "sec-ch-ua-platform": '"Windows"', 76 | "Upgrade-Insecure-Requests": "1", 77 | "Origin": "https://signtyp.uconn.edu", 78 | "Content-Type": "application/x-www-form-urlencoded", 79 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", 80 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 81 | "Sec-Fetch-Site": "same-origin", 82 | "Sec-Fetch-Mode": "navigate", 83 | "Sec-Fetch-User": "?1", 84 | "Sec-Fetch-Dest": "document", 85 | "Referer": "https://signtyp.uconn.edu/signpuddle/export.php?ui=1&sgn=9032", 86 | "Accept-Language": "en-US,en;q=0.9,he;q=0.8", 87 | } 88 | 89 | data = {"ex_source": "All", "action": "Download"} 90 | 91 | res = requests.post("https://signtyp.uconn.edu/signpuddle/export.php", data=data, headers=headers, cookies=cookies) 92 | 93 | spml = res.text 94 | 95 | if not spml.startswith(''): 96 | raise Exception("PHPSESSID might be expired.") 97 | 98 | return {"train": self._generate_examples(spml)} 99 | 100 | def _generate_examples(self, spml): 101 | regex = '(.*?)<.*?lxsg: (.*?)].*(http.*?)"' 102 | 103 | for row in spml.splitlines(): 104 | datum = re.search(regex, row) 105 | 106 | if datum is not None: 107 | _id = datum.group(1) 108 | 109 | yield _id, { 110 | "id": int(_id), 111 | "created_date": str(datetime.fromtimestamp(int(datum.group(2)))), 112 | "modified_date": str(datetime.fromtimestamp(int(datum.group(3)))), 113 | "sign_writing": datum.group(4), 114 | "sign_language": datum.group(5), 115 | "video": datum.group(6), 116 | } 117 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign2mint/sign2mint.py: -------------------------------------------------------------------------------- 1 | """A German Sign Language (DGS) lexicon for natural science subjects.""" 2 | 3 | import json 4 | import os 5 | 6 | import tensorflow as tf 7 | import tensorflow_datasets as tfds 8 | 9 | from ..warning import dataset_warning 10 | from ...datasets import SignDatasetConfig 11 | import urllib.request 12 | 13 | 14 | from ...utils.signwriting.ocr import image_to_fsw 15 | 16 | _DESCRIPTION = """ 17 | The specialist signs developed in the project break down barriers for deaf people 18 | and thus facilitate their access to education in natural science subjects. 19 | Improved communication enables deaf people in the MINT subjects to participate better in research and science, 20 | from school to university to independent work. With the MINT specialist sign language dictionary, 21 | you have a better chance of developing your scientific ideas as a researcher; 22 | it makes work in the laboratory easier and enriches your own career planning, as research results, 23 | for example, can be better communicated. For the first time, teachers, 24 | students and pupils have access to a uniform and constantly expanding MINT specialist sign language dictionary, 25 | """ 26 | 27 | # TODO(sign2mint): BibTeX citation 28 | _CITATION = """ 29 | """ 30 | 31 | OCR_CACHE_PATH = os.path.join(os.path.dirname(__file__), "ocr_cache.txt") 32 | 33 | 34 | class Sign2MINT(tfds.core.GeneratorBasedBuilder): 35 | """DatasetBuilder for sign2mint dataset.""" 36 | 37 | VERSION = tfds.core.Version("1.0.0") 38 | RELEASE_NOTES = {"1.0.0": "Initial release."} 39 | 40 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=True), SignDatasetConfig(name="annotations", include_video=False)] 41 | 42 | def _info(self) -> tfds.core.DatasetInfo: 43 | """Returns the dataset metadata.""" 44 | 45 | features = { 46 | "id": tfds.features.Text(), 47 | "fachbegriff": tfds.features.Text(), 48 | "fachgebiete": tfds.features.Sequence(tfds.features.Text()), 49 | "ursprung": tfds.features.Sequence(tfds.features.Text()), 50 | "verwendungskontext": tfds.features.Sequence(tfds.features.Text()), 51 | "definition": tfds.features.Text(), 52 | "bedeutungsnummern": tfds.features.Text(), 53 | "wortlink": tfds.features.Text(), 54 | "wikipedialink": tfds.features.Text(), 55 | "otherlink": tfds.features.Text(), 56 | "variants": tf.int32, 57 | "gebaerdenschrift": {"symbolIds": tfds.features.Sequence(tfds.features.Text())}, 58 | "swu": tfds.features.Text(), 59 | } 60 | 61 | if self._builder_config.include_video and self._builder_config.process_video: 62 | features["video"] = self._builder_config.video_feature((1920, 1080)) 63 | else: 64 | features["video"] = tfds.features.Text() 65 | 66 | return tfds.core.DatasetInfo( 67 | builder=self, 68 | description=_DESCRIPTION, 69 | features=tfds.features.FeaturesDict(features), 70 | homepage="https://sign2mint.de/", 71 | supervised_keys=None, 72 | citation=_CITATION, 73 | ) 74 | 75 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 76 | dataset_warning(self) 77 | 78 | annotations_path = dl_manager.download("https://sign2mint.de/api/entries/all/") 79 | 80 | local_videos = {} 81 | if self._builder_config.include_video and self._builder_config.process_video: 82 | with open(annotations_path, "r", encoding="utf-8") as f: 83 | annotations = json.load(f) 84 | videos = [a["videoLink"] for a in annotations] 85 | video_paths = dl_manager.download(videos) 86 | local_videos = {k: v for k, v in zip(videos, video_paths)} 87 | 88 | return {"train": self._generate_examples(annotations_path, local_videos)} 89 | 90 | ocr_cache = None 91 | 92 | def _ocr(self, datum): 93 | # sign2mint used to include only a photo of signwriting, which required OCR to get the FSW 94 | import cv2 95 | if self.ocr_cache is None: 96 | with open(OCR_CACHE_PATH, "r") as f: 97 | lines = [l.split(" ") for l in f.readlines()] 98 | self.ocr_cache = {l[0]: l[1] for l in lines} 99 | 100 | image_url = datum["gebaerdenschrift"]["url"].replace("&transparent=true", "") 101 | if image_url in self.ocr_cache: 102 | return self.ocr_cache[image_url] 103 | 104 | urllib.request.urlretrieve(image_url, "sign.png") 105 | 106 | img_rgb = cv2.imread("sign.png") 107 | 108 | symbols = datum["gebaerdenschrift"]["symbolIds"] 109 | 110 | fsw = image_to_fsw(img_rgb, symbols) 111 | 112 | with open(OCR_CACHE_PATH, "a") as f: 113 | f.write(image_url + " " + fsw + "\n") 114 | 115 | return fsw 116 | 117 | def _generate_examples(self, annotations_path, local_videos): 118 | """Yields examples.""" 119 | 120 | with open(annotations_path, "r", encoding="utf-8") as f: 121 | data = json.load(f) 122 | for datum in data: 123 | del datum["empfehlung"] # remove unused property 124 | 125 | video_link = datum["videoLink"] 126 | del datum["videoLink"] 127 | del datum["videoThumbnailLink"] 128 | 129 | datum["video"] = local_videos[video_link] if video_link in local_videos else video_link 130 | datum["gebaerdenschrift"]["symbolIds"] = [ 131 | s["symbolKey"] for s in datum["gebaerdenschrift"]["symbolIds"] if s["symbolKey"] != "" 132 | ] 133 | # datum["gebaerdenschrift"]["fsw"] = self._ocr(datum) 134 | 135 | yield datum["id"], datum 136 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/dgs_corpus_test.py: -------------------------------------------------------------------------------- 1 | """dgs_corpus dataset.""" 2 | 3 | import json 4 | import gzip 5 | import shutil 6 | import tempfile 7 | import numpy as np 8 | import tensorflow_datasets as tfds 9 | 10 | from unittest import TestCase 11 | from typing import List, Dict 12 | from contextlib import contextmanager 13 | 14 | from . import dgs_corpus 15 | from pose_format.pose import Pose 16 | 17 | 18 | OPENPOSE_TOTAL_KEYPOINTS = 137 19 | OPENPOSE_COMPONENTS_USED = ["pose_keypoints_2d", "face_keypoints_2d", "hand_left_keypoints_2d", "hand_right_keypoints_2d"] 20 | OPENPOSE_COMPONENTS_UNUSED = ["pose_keypoints_3d", "face_keypoints_3d", "hand_left_keypoints_3d", "hand_right_keypoints_3d"] 21 | OPENPOSE_NUM_POINTS_PER_COMPONENT = { 22 | "pose_keypoints_2d": 25, 23 | "face_keypoints_2d": 70, 24 | "hand_left_keypoints_2d": 21, 25 | "hand_right_keypoints_2d": 21, 26 | } 27 | 28 | 29 | def _generate_dgs_openpose_data(camera_names: List[str], num_frames: int = 10, num_people: int = 1) -> List[Dict]: 30 | data = [] # type: List[Dict] 31 | 32 | for camera_name in camera_names: 33 | camera_dict = {"camera": camera_name} 34 | camera_dict["id"] = np.random.randint(10000000) 35 | camera_dict["width"] = 1280 36 | camera_dict["height"] = 720 37 | 38 | camera_dict["frames"] = {} 39 | 40 | for frame_id in range(num_frames): 41 | frame_dict = {"version": 1.4, "people": []} 42 | 43 | for _ in range(num_people): 44 | keypoints_dict = {} 45 | for component in OPENPOSE_COMPONENTS_USED: 46 | num_keypoints = OPENPOSE_NUM_POINTS_PER_COMPONENT[component] 47 | 48 | # for each keypoint: one value for each dimension, plus confidence 49 | num_pose_values = 3 * num_keypoints 50 | pose_data = np.random.random_sample((num_pose_values,)).tolist() 51 | keypoints_dict[component] = pose_data 52 | 53 | for component in OPENPOSE_COMPONENTS_UNUSED: 54 | keypoints_dict[component] = [] 55 | 56 | frame_dict["people"].append(keypoints_dict) 57 | 58 | camera_dict["frames"][str(frame_id)] = frame_dict 59 | 60 | data.append(camera_dict) 61 | 62 | return data 63 | 64 | 65 | def _generate_dgs_openpose_file( 66 | filehandle: tempfile.NamedTemporaryFile, camera_names: List[str], num_frames: int = 10, num_people: int = 1 67 | ) -> None: 68 | data = _generate_dgs_openpose_data(camera_names, num_frames, num_people) 69 | json.dump(data, filehandle) 70 | filehandle.flush() 71 | 72 | 73 | def _gzip_file(filepath_in: str, filepath_out: str) -> None: 74 | with open(filepath_in, "rb") as filehandle_in: 75 | with gzip.open(filepath_out, "wb") as filehandle_out: 76 | shutil.copyfileobj(filehandle_in, filehandle_out) 77 | 78 | 79 | @contextmanager 80 | def _create_tmp_dgs_openpose_file(camera_names: List[str], num_frames: int = 10, num_people: int = 1) -> str: 81 | with tempfile.NamedTemporaryFile(mode="w+") as filehandle: 82 | _generate_dgs_openpose_file(filehandle, camera_names, num_frames, num_people) 83 | 84 | filepath_zipped = filehandle.name + ".gz" 85 | _gzip_file(filehandle.name, filepath_zipped) 86 | 87 | yield filepath_zipped 88 | 89 | 90 | class TestDgsCorpusAuxiliaryFunctions(TestCase): 91 | def test_convert_dgs_dict_to_openpose_frames(self): 92 | input_dict = {"7": {"people": [1, 2, 3]}, "8": {"people": [4, 5, 6]}} 93 | 94 | expected_output = {7: {"people": [1, 2, 3], "frame_id": 7}, 8: {"people": [4, 5, 6], "frame_id": 8}} 95 | 96 | actual_output = dgs_corpus.convert_dgs_dict_to_openpose_frames(input_dict) 97 | 98 | self.assertDictEqual(actual_output, expected_output) 99 | 100 | def test_get_poses_return_type(self): 101 | camera_names_in_mock_data = ["a", "b", "c"] 102 | num_frames_in_mock_data = 10 103 | num_people_in_mock_data = 1 104 | 105 | people_to_extract = {"a", "b"} 106 | 107 | with _create_tmp_dgs_openpose_file( 108 | camera_names=camera_names_in_mock_data, num_frames=num_frames_in_mock_data, num_people=num_people_in_mock_data 109 | ) as filepath: 110 | poses = dgs_corpus.get_openpose(openpose_path=filepath, fps=50, people=people_to_extract) 111 | 112 | for pose in poses.values(): 113 | self.assertIsInstance(pose, Pose) 114 | 115 | def test_get_poses_subset_of_camera_names(self): 116 | camera_names_in_mock_data = ["a2", "b1", "c5"] 117 | num_frames_in_mock_data = 10 118 | num_people_in_mock_data = 1 119 | 120 | people_to_extract = {"a", "b"} 121 | 122 | with _create_tmp_dgs_openpose_file( 123 | camera_names=camera_names_in_mock_data, num_frames=num_frames_in_mock_data, num_people=num_people_in_mock_data 124 | ) as filepath: 125 | poses = dgs_corpus.get_openpose(openpose_path=filepath, fps=50, people=people_to_extract) 126 | 127 | for person in poses.keys(): 128 | self.assertTrue(person[0] in people_to_extract) 129 | 130 | 131 | ''' 132 | class DgsCorpusTest(tfds.testing.DatasetBuilderTestCase): 133 | """Tests for dgs_corpus dataset.""" 134 | 135 | # TODO(dgs_corpus): 136 | DATASET_CLASS = dgs_corpus.DgsCorpus 137 | SPLITS = { 138 | "train": 3, # Number of fake train example 139 | "test": 1, # Number of fake test example 140 | } 141 | 142 | # If you are calling `download/download_and_extract` with a dict, like: 143 | # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) 144 | # then the tests needs to provide the fake output paths relative to the 145 | # fake data directory 146 | # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} 147 | ''' 148 | 149 | if __name__ == "__main__": 150 | tfds.testing.test_main() 151 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/mediapi_utils.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import io 3 | import zipfile 4 | 5 | import numpy as np 6 | from pose_format.numpy import NumPyPoseBody 7 | from tqdm import tqdm 8 | 9 | 10 | def convert_time(vtt_time): 11 | hhmmss, fraction = vtt_time.split(".") 12 | h, m, s = hhmmss.split(":") 13 | hhmmss = int(h) * 3600 + int(m) * 60 + int(s) 14 | return hhmmss + float(int(fraction) / 1000) 15 | 16 | 17 | def read_pose_tsv_file(pose_tsv: bytes, num_keypoints: int = 33): 18 | # TSV file where the first cell is the frame ID 19 | # and the rest of the cells are the pose data (x, y, z, confidence) for every landmark 20 | rows = pose_tsv.decode("utf-8").strip().split("\n") 21 | rows = [row.strip().split("\t") for row in rows] 22 | 23 | num_frames = max(int(row[0]) for row in rows) 24 | tensor = np.zeros((num_frames + 1, num_keypoints, 4), dtype=np.float32) 25 | 26 | for row in rows: 27 | frame_id = int(row[0]) 28 | pose_data = row[1:] 29 | if len(pose_data) > 0: 30 | vec = np.array(pose_data).reshape(-1, 4) 31 | if len(vec) != num_keypoints: 32 | print(f"Warning: pose data has wrong number of keypoints ({len(vec)} instead of {num_keypoints})") 33 | if num_keypoints == 21: # hands 34 | vec[:, 3] = 1 35 | tensor[frame_id] = vec 36 | 37 | return tensor 38 | 39 | 40 | def load_pose_body_from_zip(zip_file, fps: float) -> NumPyPoseBody: 41 | face, left_hand, n, body, right_hand = sorted(zip_file.namelist()) 42 | left_hand_tensor = read_pose_tsv_file(zip_file.read(left_hand), 21) 43 | right_hand_tensor = read_pose_tsv_file(zip_file.read(right_hand), 21) 44 | body_tensor = read_pose_tsv_file(zip_file.read(body), 33) 45 | # TODO: we can add the face if needed 46 | pose_tensor = np.expand_dims(np.concatenate([body_tensor, left_hand_tensor, right_hand_tensor], axis=1), axis=1) 47 | 48 | data = pose_tensor[:, :, :, :3] 49 | confidence = np.round(pose_tensor[:, :, :, 3]) 50 | return NumPyPoseBody(fps=fps, data=data, confidence=confidence) 51 | 52 | 53 | def read_mediapi_set(mediapi_path: str, pose_path: str = None, split="test", pose_type="holistic"): 54 | try: 55 | import webvtt 56 | except ImportError: 57 | raise ImportError("Please install webvtt with: pip install webvtt-py") 58 | 59 | pose_zip = "mediapipe_zips.zip" if pose_type == "holistic" else "openpose_zips.zip" 60 | mediapipe_zips = f"mediapi-skel/7/data/{pose_zip}" 61 | subtitle_zips = "mediapi-skel/7/data/subtitles.zip" 62 | information = "mediapi-skel/7/information/video_information.csv" 63 | 64 | with zipfile.ZipFile(mediapi_path, "r") as root_zip: 65 | # Open the information csv using DictReader 66 | information_info = root_zip.getinfo(information) 67 | with root_zip.open(information_info, "r") as information_file: 68 | text = io.TextIOWrapper(information_file) 69 | reader = csv.DictReader(text) 70 | split_data = [row for row in reader if row["train/dev/test"] == split] 71 | 72 | print("Sample test data:", split_data[0]) 73 | 74 | name_number = lambda name: int(name.split("/")[-1].split(".")[0]) if not name.endswith("/") else -1 75 | 76 | # Open the subtitles zip and extract the subtitles for every test datum 77 | subtitle_info = root_zip.getinfo(subtitle_zips) 78 | with root_zip.open(subtitle_info, "r") as subtitle_file: 79 | nested_zip = zipfile.ZipFile(subtitle_file, "r") 80 | number_name = {name_number(name): name for name in nested_zip.namelist()} 81 | 82 | for datum in split_data: 83 | webvtt_text = nested_zip.read(number_name[int(datum["video"])]) 84 | buffer = io.StringIO(webvtt_text.decode("utf-8")) 85 | datum["subtitles"] = list(webvtt.read_buffer(buffer)) 86 | datum["subtitles"] = [ 87 | {"start_time": convert_time(c.start), "end_time": convert_time(c.end), "text": c.text} for c in datum["subtitles"] 88 | ] 89 | 90 | print("Sample subtitle:", split_data[0]["subtitles"]) 91 | 92 | # Open the mediapipe zips and extract the mediapipe data for every test datum 93 | if pose_path is None: 94 | print( 95 | "No pose path given. This means the extraction of mediapipe data will be slow. " 96 | "Consider extracting the poses zip using the following command:" 97 | "unzip -j mediapi-skel.zip mediapi-skel/7/data/mediapipe_zips.zip -d ." 98 | "And then pass MEDIAPI_POSE_PATH=mediapipe_zips.zip to this script." 99 | ) 100 | mediapipe_info = root_zip.getinfo(mediapipe_zips) 101 | mediapipe_file = root_zip.open(mediapipe_info, "r") 102 | mediapipe_zip = zipfile.ZipFile(mediapipe_file, "r") 103 | else: 104 | mediapipe_zip = zipfile.ZipFile(pose_path, "r") 105 | 106 | print("mediapipe files", mediapipe_zip.namelist()) 107 | number_name = {name_number(name): name for name in mediapipe_zip.namelist() if "/00001/" not in name} 108 | for datum in split_data: 109 | with zipfile.ZipFile(mediapipe_zip.open(number_name[int(datum["video"])]), "r") as nested_zip: 110 | datum["id"] = str(datum["video"]) 111 | datum["metadata"] = { 112 | "fps": float(datum["fps"].replace(",", ".")), 113 | "duration": float(datum["fps"].replace(",", ".")), 114 | "frames": int(datum["frames"]), 115 | "height": int(datum["height"]), 116 | "width": int(datum["width"]), 117 | } 118 | if pose_type is not None: 119 | try: 120 | datum["pose"] = load_pose_body_from_zip(nested_zip, fps=datum["metadata"]["fps"]) 121 | except Exception as e: 122 | print("Failed to load pose for", datum["id"], e) 123 | print(datum) 124 | continue 125 | yield datum 126 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wlasl/wlasl.py: -------------------------------------------------------------------------------- 1 | """WLASL: A large-scale dataset for Word-Level American Sign Language""" 2 | 3 | import json 4 | 5 | import tensorflow as tf 6 | import tensorflow_datasets as tfds 7 | from tensorflow.io.gfile import GFile 8 | from tqdm import tqdm 9 | 10 | from ..warning import dataset_warning 11 | from ...datasets.config import SignDatasetConfig 12 | from ...utils.downloaders.aslpro import download_aslpro 13 | from ...utils.downloaders.youtube import download_youtube 14 | 15 | _DESCRIPTION = """ 16 | A large-scale dataset for Word-Level American Sign Language 17 | """ 18 | 19 | _CITATION = """ 20 | @inproceedings{dataset:li2020word, 21 | title={Word-level Deep Sign Language Recognition from Video: A New Large-scale Dataset and Methods Comparison}, 22 | author={Li, Dongxu and Rodriguez, Cristian and Yu, Xin and Li, Hongdong}, 23 | booktitle={The IEEE Winter Conference on Applications of Computer Vision}, 24 | pages={1459--1469}, 25 | year={2020} 26 | } 27 | """ 28 | 29 | _INDEX_URL = "https://raw.githubusercontent.com/dxli94/WLASL/0ac8108282aba99226e29c066cb8eab847ef62da/start_kit/WLASL_v0.3.json" 30 | 31 | _POSE_URLS = {"openpose": "https://www.robots.ox.ac.uk/~vgg/research/bsl1k/data/info/wlasl.tar"} 32 | 33 | 34 | class Wlasl(tfds.core.GeneratorBasedBuilder): 35 | """DatasetBuilder for wlasl dataset.""" 36 | 37 | VERSION = tfds.core.Version("0.3.0") 38 | RELEASE_NOTES = {"0.3.0": "fix deafasl links"} 39 | 40 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=False, include_pose="openpose")] 41 | 42 | def _info(self) -> tfds.core.DatasetInfo: 43 | """Returns the dataset metadata.""" 44 | 45 | features = { 46 | "id": tfds.features.Text(), 47 | "signer": tfds.features.Text(), 48 | "gloss": tfds.features.Text(), 49 | "gloss_variation": tf.int32, 50 | "video_file": {"start": tf.int32, "end": tf.int32, "remote": tfds.features.Text()}, 51 | "fps": tf.int32, 52 | "bbox": tfds.features.BBoxFeature(), 53 | } 54 | 55 | if self._builder_config.include_video: 56 | features["video"] = self._builder_config.video_feature((None, None)) 57 | features["video_file"]["local"] = tfds.features.Text() 58 | 59 | if self._builder_config.include_pose == "openpose": 60 | features["pose"] = { 61 | "data": tfds.features.Tensor(shape=(None, 1, 137, 2), dtype=tf.float32), 62 | "conf": tfds.features.Tensor(shape=(None, 1, 137), dtype=tf.float32), 63 | } 64 | 65 | return tfds.core.DatasetInfo( 66 | builder=self, 67 | description=_DESCRIPTION, 68 | features=tfds.features.FeaturesDict(features), 69 | supervised_keys=("video", "gloss"), 70 | homepage="https://dxli94.github.io/WLASL/", 71 | citation=_CITATION, 72 | ) 73 | 74 | def _download_video(self, url: str, dl_manager): 75 | print("url", url) 76 | 77 | if "aslpro" in url: 78 | return dl_manager.download_custom(url, download_aslpro) 79 | elif "youtube" in url or "youtu.be" in url: 80 | return dl_manager.download_custom(url, download_youtube) 81 | else: 82 | return dl_manager.download(url) 83 | 84 | def _download_videos(self, data, dl_manager): 85 | videos = {} 86 | for datum in data: 87 | for instance in datum["instances"]: 88 | videos[instance["video_id"]] = instance["url"] 89 | 90 | paths = {} 91 | for video_id, video in tqdm(videos.items(), total=len(videos)): 92 | print(video_id, video) 93 | try: 94 | paths[video_id] = self._download_video(video, dl_manager) 95 | except (FileNotFoundError, ConnectionError) as err: 96 | print("Failed to download", video, str(err)) 97 | 98 | return paths 99 | 100 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 101 | """Returns SplitGenerators.""" 102 | dataset_warning(self) 103 | 104 | data_index_path = dl_manager.download(_INDEX_URL) 105 | 106 | # Download videos and update paths 107 | with GFile(data_index_path, "r") as f: 108 | data = json.load(f) 109 | 110 | if self._builder_config.include_video: 111 | paths = self._download_videos(data, dl_manager) 112 | for datum in data: 113 | for instance in datum["instances"]: 114 | instance["video"] = paths[instance["video_id"]] if instance["video_id"] in paths else None 115 | 116 | if self._builder_config.include_pose == "openpose": 117 | pose_path = dl_manager.download_and_extract(_POSE_URLS[self._builder_config.include_pose]) 118 | else: 119 | pose_path = None 120 | 121 | return { 122 | "train": self._generate_examples(data, pose_path, "train"), 123 | "validation": self._generate_examples(data, pose_path, "val"), 124 | "test": self._generate_examples(data, pose_path, "test"), 125 | } 126 | 127 | def _generate_examples(self, data, pose_path, split): 128 | """Yields examples.""" 129 | 130 | counter = 0 131 | for datum in data: 132 | counter += len(datum["instances"]) 133 | print("counter", counter) 134 | 135 | # print(data) 136 | # print(pose_path) 137 | # print(len(data)) 138 | 139 | raise Exception("die") 140 | # counter = 0 141 | # for datum in data: 142 | # for instance in datum["instances"]: 143 | # if instance["split"] == split and instance["video"] is not None: 144 | # yield counter, { 145 | # "video": instance["video"], 146 | # "url": instance["url"], 147 | # "start": instance["frame_start"], 148 | # "end": instance["frame_end"], 149 | # "fps": instance["fps"], 150 | # "signer_id": instance["signer_id"], 151 | # "bbox": instance["bbox"], 152 | # "gloss": datum["gloss"], 153 | # "gloss_variation": instance["variation_id"], 154 | # } 155 | # counter += 1 156 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dgs_corpus/dgs_utils.py: -------------------------------------------------------------------------------- 1 | import pympi 2 | 3 | from lxml import etree 4 | from typing import Dict, List 5 | 6 | 7 | def get_elan_sentences(elan_path: str): 8 | eaf = pympi.Elan.Eaf(elan_path) # TODO add "suppress_version_warning=True" when pympi 1.7 is released 9 | 10 | timeslots = eaf.timeslots 11 | 12 | for participant in ["A", "B"]: 13 | german_tier_name = "Deutsche_Übersetzung_" + participant 14 | if german_tier_name not in eaf.tiers: 15 | continue 16 | 17 | german_text = eaf.tiers[german_tier_name][0] 18 | 19 | english_tier_name = "Translation_into_English_" + participant 20 | english_text = list(eaf.tiers[english_tier_name][0].values()) if english_tier_name in eaf.tiers else [] 21 | 22 | all_glosses = [] 23 | for hand in ["r", "l"]: 24 | hand_tier = "Lexem_Gebärde_" + hand + "_" + participant 25 | if hand_tier not in eaf.tiers: 26 | continue 27 | 28 | gloss = { 29 | _id: {"start": timeslots[s], "end": timeslots[e], "gloss": val, "hand": hand} 30 | for _id, (s, e, val, _) in eaf.tiers[hand_tier][0].items() 31 | } 32 | for tier in ["Lexeme_Sign", "Gebärde", "Sign"]: 33 | items = eaf.tiers[tier + "_" + hand + "_" + participant][1] 34 | for ref, val, _1, _2 in items.values(): 35 | if ref in gloss: # 2 files have a missing reference 36 | gloss[ref][tier] = val 37 | 38 | all_glosses += list(gloss.values()) 39 | 40 | all_mouthings = [] 41 | 42 | tier_name = "Mundbild_Mundgestik_" + participant 43 | items = eaf.tiers[tier_name][0] 44 | 45 | # structure of entries: 46 | # {'a2768296': ('ts42', 'ts43', 'tochter', None), ... } 47 | 48 | for s, e, val, _ in items.values(): 49 | mouthing_entry = {"start": timeslots[s], "end": timeslots[e], "mouthing": val} 50 | all_mouthings.append(mouthing_entry) 51 | 52 | for _id, (s, e, val, _) in german_text.items(): 53 | sentence = {"id": _id, "participant": participant, "start": timeslots[s], "end": timeslots[e], "german": val} 54 | 55 | # Add English sentence 56 | english_sentence = [val2 for (s2, e2, val2, _) in english_text if s == s2 and e == e2] 57 | sentence["english"] = english_sentence[0] if len(english_sentence) > 0 else None 58 | 59 | # Add glosses 60 | sentence["glosses"] = list( 61 | sorted( 62 | [item for item in all_glosses if item["start"] >= sentence["start"] and item["end"] <= sentence["end"]], 63 | key=lambda d: d["start"], 64 | ) 65 | ) 66 | 67 | # add mouthings 68 | sentence["mouthings"] = list( 69 | sorted( 70 | [item for item in all_mouthings if item["start"] >= sentence["start"] and item["end"] <= sentence["end"]], 71 | key=lambda d: d["start"], 72 | ) 73 | ) 74 | 75 | yield sentence 76 | 77 | 78 | def get_child_elements(root: etree.ElementTree, element_name: str, attributes_to_extract: List[str]) -> Dict[str, Dict[str, str]]: 79 | """ 80 | 81 | :param root: 82 | :param element_name: 83 | :param attributes_to_extract: 84 | :return: 85 | """ 86 | 87 | elements = root.xpath("/ilex-data/" + element_name) # type: List[etree.Element] 88 | 89 | by_id = {} 90 | 91 | for element in elements: 92 | id_ = element.get("id") 93 | by_id[id_] = {} 94 | for attribute_name in attributes_to_extract: 95 | value = element.get(attribute_name) 96 | by_id[id_][attribute_name] = value 97 | 98 | return by_id 99 | 100 | 101 | def get_signer_ids_from_ilex(ilex_path: str) -> Dict[str, List[str]]: 102 | """ 103 | 104 | File structure: 105 | 106 | 107 | 109 | 111 | 113 | 115 | 117 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | :param ilex_path: 129 | :return: 130 | """ 131 | 132 | root = etree.parse(ilex_path) 133 | 134 | informant_dict = get_child_elements(root=root, element_name="informant", attributes_to_extract=["name"]) 135 | 136 | camera_perspective_dict = get_child_elements( 137 | root=root, element_name="camera_perspective", attributes_to_extract=["visible_persons", "code"] 138 | ) 139 | 140 | signer_identities_by_perspective = {} # type: Dict[str, List[str]] 141 | 142 | for camera_perspective in camera_perspective_dict.values(): 143 | # extract A, B or C without trailing numbers 144 | 145 | clean_code = camera_perspective["code"][0].lower() 146 | 147 | # remove enclosing "{" and "}" for list of informant ids 148 | 149 | ids_of_visible_persons = camera_perspective["visible_persons"][1:-1].split(",") 150 | 151 | names_of_visible_persons = [informant_dict[id_]["name"] for id_ in ids_of_visible_persons] 152 | 153 | signer_identities_by_perspective[clean_code] = names_of_visible_persons 154 | 155 | return signer_identities_by_perspective 156 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/rwth_phoenix2014_t/rwth_phoenix2014_t.py: -------------------------------------------------------------------------------- 1 | """RWTH-PHOENIX 2014 T: Parallel Corpus of Sign Language Video, Gloss and Translation""" 2 | 3 | import csv 4 | from os import path 5 | 6 | import tensorflow as tf 7 | import tensorflow_datasets as tfds 8 | from tensorflow.io.gfile import GFile 9 | 10 | from ..warning import dataset_warning 11 | from ...datasets.config import SignDatasetConfig, cloud_bucket_file 12 | from ...utils.features import PoseFeature 13 | 14 | _DESCRIPTION = """ 15 | Parallel Corpus of German Sign Language of the weather, including video, gloss and translation. 16 | Additional poses extracted by MediaPipe Holistic are also available. 17 | """ 18 | 19 | _CITATION = """ 20 | @inproceedings{cihan2018neural, 21 | title={Neural sign language translation}, 22 | author={Cihan Camgoz, Necati and Hadfield, Simon and Koller, Oscar and Ney, Hermann and Bowden, Richard}, 23 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 24 | pages={7784--7793}, 25 | year={2018} 26 | } 27 | @article{koller2015continuous, 28 | title={Continuous sign language recognition: 29 | Towards large vocabulary statistical recognition systems handling multiple signers}, 30 | author={Koller, Oscar and Forster, Jens and Ney, Hermann}, 31 | journal={Computer Vision and Image Understanding}, 32 | volume={141}, 33 | pages={108--125}, 34 | year={2015}, 35 | publisher={Elsevier} 36 | } 37 | """ 38 | 39 | _VIDEO_ANNOTATIONS_URL = "https://www-i6.informatik.rwth-aachen.de/ftp/pub/rwth-phoenix/2016/phoenix-2014-T.v3.tar.gz" 40 | _ANNOTATIONS_URL = cloud_bucket_file("public/phoenix-annotations.tar.gz") 41 | 42 | _POSE_URLS = {"holistic": cloud_bucket_file("poses/holistic/phoenix.tar.gz")} 43 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "pose.header")} 44 | 45 | 46 | class RWTHPhoenix2014T(tfds.core.GeneratorBasedBuilder): 47 | """DatasetBuilder for rwth_phoenix_2014_t dataset.""" 48 | 49 | VERSION = tfds.core.Version("3.0.0") 50 | RELEASE_NOTES = {"3.0.0": "Initial release."} 51 | 52 | BUILDER_CONFIGS = [ 53 | SignDatasetConfig(name="default", include_video=True, include_pose="holistic"), 54 | SignDatasetConfig(name="videos", include_video=True, include_pose=None), 55 | SignDatasetConfig(name="poses", include_video=False, include_pose="holistic"), 56 | SignDatasetConfig(name="annotations", include_video=False, include_pose=None), 57 | ] 58 | 59 | def _info(self) -> tfds.core.DatasetInfo: 60 | """Returns the dataset metadata.""" 61 | 62 | features = {"id": tfds.features.Text(), "signer": tfds.features.Text(), "gloss": tfds.features.Text(), "text": tfds.features.Text()} 63 | 64 | if self._builder_config.include_video: 65 | features["fps"] = tf.int32 66 | features["video"] = self._builder_config.video_feature((210, 260)) 67 | 68 | if self._builder_config.include_pose == "holistic": 69 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 70 | stride = 1 if self._builder_config.fps is None else 25 / self._builder_config.fps 71 | features["pose"] = PoseFeature(shape=(None, 1, 543, 3), header_path=pose_header_path, stride=stride) 72 | 73 | return tfds.core.DatasetInfo( 74 | builder=self, 75 | description=_DESCRIPTION, 76 | features=tfds.features.FeaturesDict(features), 77 | homepage="https://www-i6.informatik.rwth-aachen.de/~koller/RWTH-PHOENIX-2014-T/", 78 | supervised_keys=None, 79 | citation=_CITATION, 80 | ) 81 | 82 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 83 | """Returns SplitGenerators.""" 84 | dataset_warning(self) 85 | 86 | urls = [_VIDEO_ANNOTATIONS_URL if self._builder_config.include_video else _ANNOTATIONS_URL] 87 | 88 | if self._builder_config.include_pose is not None: 89 | urls.append(_POSE_URLS[self._builder_config.include_pose]) 90 | 91 | dl_manager._verify_ssl = False # Due to https://github.com/sign-language-processing/datasets/issues/51 92 | downloads = dl_manager.download_and_extract(urls) 93 | annotations_path = path.join(downloads[0], "PHOENIX-2014-T-release-v3", "PHOENIX-2014-T") 94 | 95 | if self._builder_config.include_pose == "holistic": 96 | pose_path = path.join(downloads[1], "holistic") 97 | else: 98 | pose_path = None 99 | 100 | return [ 101 | tfds.core.SplitGenerator( 102 | name=tfds.Split.VALIDATION, gen_kwargs={"annotations_path": annotations_path, "pose_path": pose_path, "split": "dev"} 103 | ), 104 | tfds.core.SplitGenerator( 105 | name=tfds.Split.TEST, gen_kwargs={"annotations_path": annotations_path, "pose_path": pose_path, "split": "test"} 106 | ), 107 | tfds.core.SplitGenerator( 108 | name=tfds.Split.TRAIN, gen_kwargs={"annotations_path": annotations_path, "pose_path": pose_path, "split": "train"} 109 | ), 110 | ] 111 | 112 | def _generate_examples(self, annotations_path: str, pose_path: str, split: str): 113 | """Yields examples.""" 114 | 115 | filepath = path.join(annotations_path, "annotations", "manual", "PHOENIX-2014-T." + split + ".corpus.csv") 116 | images_path = path.join(annotations_path, "features", "fullFrame-210x260px", split) 117 | poses_path = path.join(pose_path, split) if pose_path is not None else None 118 | 119 | with GFile(filepath, "r") as f: 120 | data = csv.DictReader(f, delimiter="|", quoting=csv.QUOTE_NONE) 121 | for row in data: 122 | datum = {"id": row["name"], "signer": row["speaker"], "gloss": row["orth"], "text": row["translation"]} 123 | 124 | if self._builder_config.include_video: 125 | frames_base = path.join(images_path, row["video"])[:-7] 126 | datum["video"] = [ 127 | path.join(frames_base, name) 128 | for name in sorted(tf.io.gfile.listdir(frames_base)) 129 | if name != "createDnnTrainingLabels-profile.py.lprof" 130 | ] 131 | datum["fps"] = self._builder_config.fps if self._builder_config.fps is not None else 25 132 | 133 | if poses_path is not None: 134 | datum["pose"] = path.join(poses_path, datum["id"] + ".pose") 135 | 136 | yield datum["id"], datum 137 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/sign_wordnet/sign_wordnet.py: -------------------------------------------------------------------------------- 1 | """The Multilingual Sign Language Wordnet""" 2 | 3 | import csv 4 | 5 | import tensorflow_datasets as tfds 6 | 7 | from ..warning import dataset_warning 8 | from ...datasets import SignDatasetConfig 9 | 10 | _DESCRIPTION = """ 11 | This is an ongoing project to add several European sign languages to the Open Multilingual Wordnet. It is part of the ongoing EASIER project, whose goal is to create an intelligent translation framework for sign languages. This Wordnet aims at connecting signs with a shared meaning accross languages. 12 | 13 | The current available version covers German Sign Language, Greek Sign Language, British Sign Language, Sign Language of the Netherlands, and French Sign Language. More than 9 000 signs have been successfully linked to one or more synsets, with 15 000 more still being processed. It is a work in progress, and the coverage is limited. 14 | """ 15 | 16 | _CITATION = """ 17 | @inproceedings{Bigeard2022IntroducingSL, 18 | title={Introducing Sign Languages to a Multilingual Wordnet: Bootstrapping Corpora and Lexical Resources of Greek Sign Language and German Sign Language}, 19 | author={Sam Bigeard and Marc Schulder and Maria Kopf and Thomas Hanke and Kyriaki Vasilaki and Anna Vacalopoulou and Theodoros Goulas and Lida Dimou and Evita F. Fotinea and Eleni Efthimiou}, 20 | booktitle={SIGNLANG}, 21 | year={2022}, 22 | url={https://api.semanticscholar.org/CorpusID:250086794} 23 | } 24 | """ 25 | 26 | # Map between Sign Wornet language codes and IANA language codes 27 | IANA_MAP = {"bsl": "bfi", "dgs": "gsg", "gsl": "gss", "lsf": "fsl", "ngt": "dse"} 28 | 29 | 30 | def no_space(d: dict): 31 | for k, v in d.items(): 32 | if isinstance(v, str): 33 | d[k] = v.strip() 34 | return d 35 | 36 | 37 | class SignWordnet(tfds.core.GeneratorBasedBuilder): 38 | """DatasetBuilder for SignWordnet dataset.""" 39 | 40 | VERSION = tfds.core.Version("0.2.0") 41 | RELEASE_NOTES = {"0.2.0": "Initial release."} 42 | 43 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=True)] 44 | 45 | def _info(self) -> tfds.core.DatasetInfo: 46 | """Returns the dataset metadata.""" 47 | 48 | features = { 49 | "id": tfds.features.Text(), 50 | "sign_language": tfds.features.Text(), 51 | "sign_language_name": tfds.features.Text(), 52 | "translations": tfds.features.Sequence({"language": tfds.features.Text(), "text": tfds.features.Text()}), 53 | "links": tfds.features.Text(), 54 | "gloss": tfds.features.Text(), 55 | "video": tfds.features.Text(), 56 | "synset_id": tfds.features.Text(), 57 | "confidence": tfds.features.Text(), 58 | } 59 | 60 | return tfds.core.DatasetInfo( 61 | builder=self, 62 | description=_DESCRIPTION, 63 | features=tfds.features.FeaturesDict(features), 64 | homepage="https://www.sign-lang.uni-hamburg.de/easier/sign-wordnet/index.html", 65 | supervised_keys=None, 66 | citation=_CITATION, 67 | ) 68 | 69 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 70 | """Returns SplitGenerators.""" 71 | dataset_warning(self) 72 | 73 | try: 74 | import nltk 75 | except ImportError: 76 | raise ImportError("Please install nltk with: pip install nltk") 77 | 78 | nltk.download("wordnet") 79 | nltk.download("omw-1.4") 80 | nltk.download("extended_omw") 81 | 82 | signs_data_url = "https://www.sign-lang.uni-hamburg.de/easier/sign-wordnet/static/csv/sign_data.csv" 83 | accepted_synsets_url = "https://www.sign-lang.uni-hamburg.de/easier/sign-wordnet/static/csv/synset_links_accepted.csv" 84 | custom_synsets_url = "https://www.sign-lang.uni-hamburg.de/easier/sign-wordnet/static/csv/custom_synsets.csv" 85 | 86 | signs_data, accepted_synsets, custom_synsets = dl_manager.download([signs_data_url, accepted_synsets_url, custom_synsets_url]) 87 | 88 | return {"train": self._generate_examples(signs_data, accepted_synsets, custom_synsets)} 89 | 90 | def _generate_examples(self, signs_data: str, accepted_synsets: str, custom_synsets: str): 91 | from nltk.corpus import wordnet as wn 92 | 93 | # List all available languages in OMW 94 | languages = wn.langs() 95 | print("Translation Languages", languages) 96 | 97 | with open(custom_synsets, "r", encoding="utf-8") as csvfile: 98 | custom_synsets_dict = {row["synset_id"]: no_space(row) for row in csv.DictReader(csvfile)} 99 | 100 | with open(signs_data, "r", encoding="utf-8") as csvfile: 101 | signs_data_dict = {row["sign_id_wordnet"]: no_space(row) for row in csv.DictReader(csvfile)} 102 | 103 | with open(accepted_synsets, "r", encoding="utf-8") as csvfile: 104 | accepted_synsets_dict = {row["sign_id"]: no_space(row) for row in csv.DictReader(csvfile)} 105 | 106 | for _id, row in signs_data_dict.items(): 107 | accepted_synset = accepted_synsets_dict.get(_id, {}) 108 | synset_id = accepted_synset.get("synset_id", "") # Looks like omw.00672433-v 109 | translations = [] 110 | 111 | if synset_id != "": # Looks like omw.00672433-v 112 | if synset_id in custom_synsets_dict: 113 | translations.append({"language": "en", "text": custom_synsets_dict[synset_id]["words_english"]}) 114 | else: 115 | # TODO: track open issue https://github.com/omwn/omw-data/issues/35 116 | _, omw_id = synset_id.split(".") 117 | omw_offset, omw_pos = omw_id.split("-") 118 | synset = wn.synset_from_pos_and_offset(omw_pos, int(omw_offset)) 119 | 120 | # Get translations in all languages 121 | for lang in languages: 122 | lemmas = synset.lemmas(lang=lang) 123 | if lemmas: # If there are lemmas for this language 124 | for lemma in lemmas: 125 | translations.append({"language": lang, "text": lemma.name()}) 126 | 127 | yield _id, { 128 | "id": _id, 129 | "sign_language": IANA_MAP[row["language"]], 130 | "sign_language_name": row["language"], 131 | "translations": translations, 132 | "links": row["links"], 133 | "gloss": row["gloss"], 134 | "video": row["video"], 135 | "synset_id": accepted_synset.get("synset_id", ""), 136 | "confidence": accepted_synset.get("confidence", ""), 137 | } 138 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/dicta_sign/dicta_sign.py: -------------------------------------------------------------------------------- 1 | """A list of 1000+ concepts with a sign language equivalent in each of the four project languages.""" 2 | 3 | from os import path 4 | import tensorflow_datasets as tfds 5 | import re 6 | 7 | from ..config import cloud_bucket_file 8 | from ..warning import dataset_warning 9 | from ...datasets import SignDatasetConfig 10 | from ...utils.features import PoseFeature 11 | 12 | _DESCRIPTION = """ 13 | A list of 1000+ concepts with a sign language equivalent in each of the four project languages 14 | """ 15 | 16 | _CITATION = """ 17 | @inproceedings{efthimiou2010dicta, 18 | title={Dicta-sign--sign language recognition, generation and modelling: a research effort with applications in deaf communication}, 19 | author={Efthimiou, Eleni and Fontinea, Stavroula-Evita and Hanke, Thomas and Glauert, John and Bowden, Rihard and Braffort, Annelies and Collet, Christophe and Maragos, Petros and Goudenove, Fran{\c{c}}ois}, 20 | booktitle={Proceedings of the 4th Workshop on the Representation and Processing of Sign Languages: Corpora and Sign Language Technologies}, 21 | pages={80--83}, 22 | year={2010} 23 | } 24 | """ 25 | 26 | SPOKEN_LANGUAGES = {"BSL": "en", "DGS": "de", "LSF": "fr", "GSL": "el"} 27 | 28 | _POSE_URLS = {"holistic": cloud_bucket_file("poses/holistic/dicta_sign.tar.gz")} 29 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.header")} 30 | 31 | 32 | class DictaSign(tfds.core.GeneratorBasedBuilder): 33 | """DatasetBuilder for sign2mint dataset.""" 34 | 35 | VERSION = tfds.core.Version("1.0.0") 36 | RELEASE_NOTES = {"1.0.0": "Initial release."} 37 | 38 | BUILDER_CONFIGS = [ 39 | SignDatasetConfig(name="default", include_video=True, include_pose="holistic"), 40 | SignDatasetConfig(name="poses", include_video=False, include_pose="holistic"), 41 | SignDatasetConfig(name="annotations", include_video=False), 42 | ] 43 | 44 | def _info(self) -> tfds.core.DatasetInfo: 45 | """Returns the dataset metadata.""" 46 | 47 | features = { 48 | "id": tfds.features.Text(), 49 | "signed_language": tfds.features.Text(), 50 | "spoken_language": tfds.features.Text(), 51 | "text": tfds.features.Text(), 52 | "text_en": tfds.features.Text(), 53 | "gloss": tfds.features.Text(), 54 | "hamnosys": tfds.features.Text(), 55 | } 56 | 57 | resolution = (320, 240) 58 | if self._builder_config.resolution is None: 59 | # Required in order to pad all videos equally 60 | self._builder_config.resolution = resolution 61 | 62 | if self._builder_config.include_video and self._builder_config.process_video: 63 | features["video"] = self._builder_config.video_feature(resolution) 64 | else: 65 | features["video"] = tfds.features.Text() 66 | 67 | if self._builder_config.include_pose == "holistic": 68 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 69 | stride = 1 if self._builder_config.fps is None else 25 / self._builder_config.fps 70 | features["pose"] = PoseFeature(shape=(None, 1, 543, 3), header_path=pose_header_path, stride=stride) 71 | 72 | return tfds.core.DatasetInfo( 73 | builder=self, 74 | description=_DESCRIPTION, 75 | features=tfds.features.FeaturesDict(features), 76 | homepage="https://www.sign-lang.uni-hamburg.de/dicta-sign/portal/", 77 | supervised_keys=None, 78 | citation=_CITATION, 79 | ) 80 | 81 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 82 | dataset_warning(self) 83 | 84 | concepts_path = dl_manager.download("https://www.sign-lang.uni-hamburg.de/dicta-sign/portal/concepts/concepts_eng.html") 85 | 86 | if self._builder_config.include_pose is not None: 87 | poses_path = dl_manager.download_and_extract(_POSE_URLS[self._builder_config.include_pose]) 88 | else: 89 | poses_path = None 90 | 91 | regex = r"(.*?)<[\s\S]*?title-l1\">(.*?)&[\s\S]*?(.*?)<\/div[\s\S]*?\"hns\">(.*?)<" 103 | concepts = [] 104 | for concept_path in concept_paths: 105 | with open(concept_path, "r", encoding="utf-8") as concept_f: 106 | matches = re.findall(regex, concept_f.read(), re.MULTILINE) 107 | 108 | assert len(matches) == 4, "Concept does not include 4 matches {}".format(concept_path) 109 | 110 | for match in matches: 111 | concept = { 112 | "id": match[2].split("/")[1].split(".")[0] + "_" + match[1], 113 | "signed_language": match[1], 114 | "spoken_language": SPOKEN_LANGUAGES[match[1]], 115 | "text": match[0], 116 | "text_en": matches[0][0], 117 | "gloss": match[3], 118 | "hamnosys": match[4], 119 | "video": "https://www.sign-lang.uni-hamburg.de/dicta-sign/portal/concepts/" + match[2], 120 | } 121 | if not any(map(lambda t: concept["video"].endswith(t), filter_videos)): 122 | concepts.append(concept) 123 | 124 | if self._builder_config.include_video: 125 | video_urls = [c["video"] for c in concepts] 126 | video_paths = dl_manager.download(video_urls) 127 | 128 | for video_path, concept in zip(video_paths, concepts): 129 | concept["video"] = video_path if self._builder_config.process_video else str(video_path) 130 | 131 | return {"train": self._generate_examples(concepts, poses_path)} 132 | 133 | def _generate_examples(self, concepts, poses_path: str): 134 | """Yields examples.""" 135 | 136 | for concept in concepts: 137 | if poses_path is not None: 138 | concept["pose"] = path.join(poses_path, "dicta_sign", concept["id"] + ".pose") 139 | 140 | yield concept["id"], concept 141 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/how2sign/how2sign.py: -------------------------------------------------------------------------------- 1 | """How2Sign: A multimodal and multiview continuous American Sign Language (ASL) dataset""" 2 | 3 | import os 4 | from itertools import chain 5 | from os import path 6 | 7 | import tensorflow as tf 8 | import tensorflow_datasets as tfds 9 | from pose_format.utils.openpose import load_openpose_directory 10 | 11 | from ..warning import dataset_warning 12 | from ...datasets.config import SignDatasetConfig 13 | from ...utils.features import PoseFeature 14 | 15 | _DESCRIPTION = """ 16 | A multimodal and multiview continuous American Sign Language (ASL) dataset, 17 | consisting of a parallel corpus of more than 80 hours of sign language videos and a set of corresponding modalities 18 | including speech, English transcripts, and depth. 19 | """ 20 | 21 | _CITATION = """ 22 | @inproceedings{Duarte_CVPR2021, 23 | title={{How2Sign: A Large-scale Multimodal Dataset for Continuous American Sign Language}}, 24 | author={Duarte, Amanda and Palaskar, Shruti and Ventura, Lucas and Ghadiyaram, Deepti and DeHaan, Kenneth and 25 | Metze, Florian and Torres, Jordi and Giro-i-Nieto, Xavier}, 26 | booktitle={Conference on Computer Vision and Pattern Recognition (CVPR)}, 27 | year={2021} 28 | } 29 | """ 30 | 31 | _SPLITS = { 32 | tfds.Split.TRAIN: { 33 | "rgb_clips_front": "https://docs.google.com/uc?export=download&id=1VX7n0jjW0pW3GEdgOks3z8nqE6iI6EnW", 34 | "rgb_clips_side": "https://docs.google.com/uc?export=download&id=1oiw861NGp4CKKFO3iuHGSCgTyQ-DXHW7", 35 | "bfh_2d_front": "https://drive.usercontent.google.com/download?id=1TBX7hLraMiiLucknM1mhblNVomO9-Y0r&export=download&authuser=0", 36 | "bfh_2d_side": None, 37 | "translation": None, 38 | }, 39 | tfds.Split.VALIDATION: { 40 | "rgb_clips_front": "https://docs.google.com/uc?export=download&id=1DhLH8tIBn9HsTzUJUfsEOGcP4l9EvOiO", 41 | "rgb_clips_side": "https://docs.google.com/uc?export=download&id=1mxL7kJPNUzJ6zoaqJyxF1Krnjo4F-eQG", 42 | "bfh_2d_front": "https://drive.usercontent.google.com/download?id=1JmEsU0GYUD5iVdefMOZpeWa_iYnmK_7w&export=download&authuser=0", 43 | "bfh_2d_side": None, 44 | "translation": None, 45 | }, 46 | tfds.Split.TEST: { 47 | "rgb_clips_front": "https://docs.google.com/uc?export=download&id=1qTIXFsu8M55HrCiaGv7vZ7GkdB3ubjaG", 48 | "rgb_clips_side": "https://docs.google.com/uc?export=download&id=1j9v9P7UdMJ0_FVWg8H95cqx4DMSsrdbH", 49 | "bfh_2d_front": "https://drive.usercontent.google.com/download?id=1g8tzzW5BNPzHXlamuMQOvdwlHRa-29Vp&export=download&authuser=0", 50 | "bfh_2d_side": None, 51 | "translation": None, 52 | }, 53 | } 54 | 55 | _POSE_HEADERS = {"openpose": path.join(path.dirname(path.realpath(__file__)), "openpose.header")} 56 | 57 | 58 | class How2Sign(tfds.core.GeneratorBasedBuilder): 59 | """DatasetBuilder for how2sign dataset.""" 60 | 61 | VERSION = tfds.core.Version("1.0.0") 62 | RELEASE_NOTES = {"1.0.0": "Initial release."} 63 | 64 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_video=True, include_pose="openpose")] 65 | 66 | def _info(self) -> tfds.core.DatasetInfo: 67 | """Returns the dataset metadata.""" 68 | 69 | features = {"id": tfds.features.Text(), "fps": tf.int32} 70 | 71 | if self._builder_config.include_video: 72 | features["video"] = { 73 | "front": self._builder_config.video_feature((1280, 720)), 74 | "side": self._builder_config.video_feature((1280, 720)), 75 | } 76 | 77 | if self._builder_config.include_pose == "openpose": 78 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 79 | stride = 1 if self._builder_config.fps is None else 24 / self._builder_config.fps 80 | features["pose"] = { 81 | "front": PoseFeature(shape=(None, 1, 137, 2), header_path=pose_header_path, stride=stride), 82 | # "side": PoseFeature(shape=(None, 1, 137, 2), header_path=pose_header_path, stride=stride), 83 | } 84 | 85 | return tfds.core.DatasetInfo( 86 | builder=self, 87 | description=_DESCRIPTION, 88 | features=tfds.features.FeaturesDict(features), 89 | homepage="https://how2sign.github.io/", 90 | supervised_keys=None, 91 | citation=_CITATION, 92 | ) 93 | 94 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 95 | """Returns SplitGenerators.""" 96 | dataset_warning(self) 97 | 98 | # Define what files are required to download 99 | download_keys = [] 100 | if self._builder_config.include_video is not None: 101 | download_keys += ["rgb_clips_front", "rgb_clips_side"] 102 | if self._builder_config.include_pose is not None: 103 | download_keys += ["bfh_2d_front", "bfh_2d_side"] 104 | 105 | urls = chain.from_iterable([[split[k] for k in download_keys] for split in _SPLITS.values()]) 106 | urls = [url for url in urls if url is not None] 107 | 108 | downloads = dl_manager.download_and_extract(urls) 109 | url_map = {u: d for u, d in zip(urls, downloads)} # Map local paths 110 | 111 | return [ 112 | tfds.core.SplitGenerator(name=name, gen_kwargs={k: url_map[v] if v is not None else None for k, v in split.items()}) 113 | for name, split in _SPLITS.items() 114 | ] 115 | 116 | def _generate_examples(self, rgb_clips_front: str, rgb_clips_side: str, bfh_2d_front: str, bfh_2d_side: str, translation: str): 117 | """Yields examples.""" 118 | 119 | # TODO get ids from translation file 120 | ids = [] 121 | ids = [p[: -len("-rgb_front.mp4")] for p in os.listdir(path.join(rgb_clips_front, "raw_videos"))] 122 | ids = ids[:10] 123 | 124 | for _id in ids: 125 | datum = {"id": _id, "fps": 24} 126 | 127 | if self.builder_config.include_video: 128 | datum["video"] = { 129 | "front": path.join(rgb_clips_front, "raw_videos", _id + "-rgb_front.mp4"), 130 | "side": path.join(rgb_clips_side, "raw_videos", _id + "-rgb_side.mp4"), 131 | } 132 | 133 | if self._builder_config.include_pose == "openpose": 134 | front_path = path.join(bfh_2d_front, "openpose_output", "json", _id + "-rgb_front") 135 | front_pose = load_openpose_directory(front_path, fps=24, width=1280, height=720) 136 | 137 | # TODO add side pose when available 138 | # side_path = path.join(bfh_2d_side, 'openpose_output', 'json', _id + '-rgb_side') 139 | # side_pose = load_openpose_directory(side_path, fps=24, width=1280, height=720) 140 | 141 | datum["pose"] = {"front": front_pose} 142 | 143 | yield _id, datum 144 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/asl_signs/asl_signs.py: -------------------------------------------------------------------------------- 1 | """asl-signs dataset for Google - Isolated Sign Language Recognition (kaggle)""" 2 | 3 | import csv 4 | import functools 5 | from os import path 6 | 7 | import numpy as np 8 | import pyarrow.parquet as pq 9 | import tensorflow_datasets as tfds 10 | from pose_format import Pose, PoseHeader 11 | from pose_format.numpy import NumPyPoseBody 12 | from pose_format.pose_header import PoseHeaderDimensions 13 | from tensorflow.io.gfile import GFile 14 | 15 | from sign_language_datasets.utils.features import PoseFeature 16 | from ..warning import dataset_warning 17 | from ...datasets.config import SignDatasetConfig 18 | 19 | _DESCRIPTION = """ 20 | The Isolated Sign Language Recognition corpus (version 1.0.0) is a collection of hand and facial landmarks generated by Mediapipe version 0.9.0.1 on ~100k videos of isolated signs performed by 21 Deaf signers from a 250-sign vocabulary. 21 | """ 22 | 23 | _CITATION = """ 24 | @misc{asl-signs, 25 | author = {Ashley Chow, Glenn Cameron, Mark Sherwood, Phil Culliton, Sam Sepah, Sohier Dane, Thad Starner}, 26 | title = {Google - Isolated Sign Language Recognition}, 27 | publisher = {Kaggle}, 28 | year = {2023}, 29 | url = {https://kaggle.com/competitions/asl-signs} 30 | } 31 | """ 32 | 33 | _POSE_HEADERS = {"holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.poseheader")} 34 | 35 | _KNOWN_SPLITS = {"1.0.0-uzh": path.join(path.dirname(path.realpath(__file__)), "splits/1.0.0-uzh")} 36 | 37 | 38 | @functools.lru_cache() 39 | def get_pose_header(): 40 | from pose_format.utils.holistic import holistic_components 41 | 42 | # create a new Pose instance 43 | # FIXME: raw video resolution and fps as well as estimated keypoint confidence are unknown 44 | # resolution and fps are hardcoded to the default values for a Pixel 4a smartphone at the moment 45 | width = 1080 46 | height = 720 47 | dimensions = PoseHeaderDimensions(width=width, height=height, depth=1000) 48 | return PoseHeader(version=0.1, dimensions=dimensions, components=holistic_components("XYZC")[:-1]) # no world landmarks 49 | 50 | 51 | class ASLSigns(tfds.core.GeneratorBasedBuilder): 52 | """DatasetBuilder for asl-signs dataset.""" 53 | 54 | VERSION = tfds.core.Version("1.0.0") 55 | RELEASE_NOTES = {"1.0.0": "Initial release."} 56 | 57 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_pose="holistic")] 58 | 59 | def _info(self) -> tfds.core.DatasetInfo: 60 | """Returns the dataset metadata.""" 61 | 62 | features = {"id": tfds.features.Text(), "text": tfds.features.Text(), "signer_id": tfds.features.Text()} 63 | 64 | if self._builder_config.include_pose == "holistic": 65 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 66 | stride = 1 if self._builder_config.fps is None else 30 / self._builder_config.fps 67 | features["pose"] = PoseFeature(shape=(None, 1, 543, 3), header_path=pose_header_path, stride=stride) 68 | 69 | return tfds.core.DatasetInfo( 70 | builder=self, 71 | description=_DESCRIPTION, 72 | features=tfds.features.FeaturesDict(features), 73 | homepage="https://www.kaggle.com/competitions/asl-signs/overview", 74 | supervised_keys=None, 75 | citation=_CITATION, 76 | ) 77 | 78 | def _load_split_ids(self, split: str): 79 | split_dir = _KNOWN_SPLITS[self._builder_config.extra["split"]] 80 | 81 | with open(path.join(split_dir, f"{split}.txt")) as f: 82 | ids = [] 83 | for line in f: 84 | id = line.rstrip("\n") 85 | ids.append(id) 86 | 87 | return ids 88 | 89 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 90 | """Returns SplitGenerators.""" 91 | dataset_warning(self) 92 | 93 | # Note: This function requires the Kaggle CLI tool. Read the installation guide at https://www.kaggle.com/docs/api 94 | archive = dl_manager.download_kaggle_data("asl-signs") 95 | 96 | if "split" in self._builder_config.extra: 97 | train_args = {"archive_path": archive, "ids": self._load_split_ids("train")} 98 | val_args = {"archive_path": archive, "ids": self._load_split_ids("val")} 99 | test_args = {"archive_path": archive, "ids": self._load_split_ids("test")} 100 | 101 | return [ 102 | tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs=train_args), 103 | tfds.core.SplitGenerator(name=tfds.Split.VALIDATION, gen_kwargs=val_args), 104 | tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs=test_args), 105 | ] 106 | else: 107 | return [tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={"archive_path": archive})] 108 | 109 | def _generate_examples(self, archive_path: str, ids: list = []): 110 | """Yields examples.""" 111 | 112 | with GFile(path.join(archive_path, "train.csv"), "r") as csv_file: 113 | csv_data = csv.reader(csv_file, delimiter=",") 114 | next(csv_data) # Ignore the header 115 | 116 | for i, row in enumerate(csv_data): 117 | datum = {"id": row[2], "text": row[3], "signer_id": row[1]} 118 | 119 | if len(ids) > 0 and (datum["id"] not in ids): 120 | continue 121 | 122 | if self.builder_config.include_pose is not None: 123 | if self.builder_config.include_pose == "holistic": 124 | # get data from parquet file 125 | parquet_path = path.join(archive_path, row[0]) 126 | pose_df = pq.read_table(parquet_path).to_pandas() 127 | num_frames = len(pose_df["frame"].drop_duplicates()) 128 | dimensions = ["x", "y", "z"] 129 | 130 | # reorder keypoints 131 | FACE = np.arange(0, 468).tolist() 132 | LHAND = np.arange(468, 489).tolist() 133 | POSE = np.arange(489, 522).tolist() 134 | RHAND = np.arange(522, 543).tolist() 135 | # rearrage the order of keypoints 136 | points = POSE + FACE + LHAND + RHAND 137 | pose_data = pose_df[dimensions].to_numpy().reshape(num_frames, -1, len(dimensions)) 138 | pose_data = pose_data[:, points, :] 139 | 140 | header = get_pose_header() 141 | 142 | # add the person dimension 143 | pose_data = np.expand_dims(pose_data, 1) 144 | # TODO: revert the normalization done in the original dataset 145 | # pose_data = pose_data * np.array([width, height, 1.0]) 146 | body = NumPyPoseBody( 147 | fps=30, data=pose_data, confidence=np.ones(shape=(pose_data.shape[0], 1, header.total_points())) 148 | ) 149 | pose = Pose(header, body) 150 | datum["pose"] = pose 151 | 152 | yield datum["id"], datum 153 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/mediapi_skel/mediapi_skel.py: -------------------------------------------------------------------------------- 1 | """MEDIAPI-SKEL is a bilingual corpus of French Sign Language (LSF) in the form of pose data (coordinates of skeleton points) and French in the form of subtitles. It can be used for academic research.""" 2 | 3 | import os 4 | from os import path 5 | 6 | import tensorflow as tf 7 | import tensorflow_datasets as tfds 8 | 9 | from .mediapi_utils import read_mediapi_set 10 | from ..warning import dataset_warning 11 | from ...datasets.config import SignDatasetConfig 12 | from ...utils.features import PoseFeature 13 | 14 | _DESCRIPTION = """ 15 | MEDIAPI-SKEL is a bilingual corpus of French Sign Language (LSF) in the form of pose data (coordinates of skeleton points) and French in the form of subtitles. It can be used for academic research. 16 | 17 | To constitute this corpus, we used 368 subtitled videos totaling 27 hours of LSF footage and written French produced by Média'Pi!, a media company producing bilingual content with LSF and written French . 18 | 19 | We cannot provide the original videos, which can be accessed via a subscription to Média'Pi!. We provide the French subtitles an WebVTT format, aligned with two types of data: openpose and mediapipe holistic (in version 2 of the repository), calculated from each of the videos. 20 | 21 | Data des cription: 22 | X corresponds to the number of the corresponding video, from 00001 to 00368. 23 | - Subtitles: For each X video, we provide a X.fr.vtt file containing the corresponding French subtitles. 24 | - OpenPose: For each X video, we supply a X.zip archive containing as many json files as there are frames in the video, named X_Y_keypoints.json, where Y is the frame number. Each json file contains the coordinates of 137 points (25 for the body, 2x21 for the hands and 70 for the face). 25 | - Mediapipe Holistic: For each X video, we supply a X.zip archive containing 5 csv files. X_N contains the image size (width and height), X_left_hand, X_right_hand, X_face and X_body contain the data for the hands, face and body respectively, with 1 line per frame. The 1st column shows the frame number, then for each point on the skeleton (21 for each hand, 468 for the face and 33 for the body), there is the following information: x (in pixels), y (in pixels), z (between -1 and 0), visibility (0 or 1 depending on whether the point is visible or not). 26 | 27 | Directories and files: 28 | Data directory 29 | - openpose_zips.zip 30 | - mediapipe_zips.zip 31 | - subtitles.zip 32 | Information directory 33 | - video_information.csv: contains information about the resolution, frame rate, duration and number of frames of each video, as well as a breakdown of videos in train/dev/test sets. 34 | - example: directory containing a short extract 35 | MEDIAPI-SKEL is a 2D-skeleton video corpus of French Sign Language (LSF) with French subtitles. It can be used by members of public university and research structures. 36 | """ 37 | 38 | _CITATION = """ 39 | @inproceedings{bull-etal-2020-mediapi, 40 | title = "{MEDIAPI}-{SKEL} - A 2{D}-Skeleton Video Database of {F}rench {S}ign {L}anguage With Aligned {F}rench Subtitles", 41 | author = "Bull, Hannah and 42 | Braffort, Annelies and 43 | Gouiff{\`e}s, Mich{\`e}le", 44 | booktitle = "Proceedings of the Twelfth Language Resources and Evaluation Conference", 45 | month = may, 46 | year = "2020", 47 | address = "Marseille, France", 48 | publisher = "European Language Resources Association", 49 | url = "https://aclanthology.org/2020.lrec-1.743", 50 | pages = "6063--6068", 51 | language = "English", 52 | ISBN = "979-10-95546-34-4", 53 | } 54 | """ 55 | 56 | _POSE_HEADERS = { 57 | "holistic": path.join(path.dirname(path.realpath(__file__)), "holistic.header"), 58 | "openpose": path.join(path.dirname(path.realpath(__file__)), "openpose.header"), 59 | } 60 | 61 | _SPLITS = {tfds.Split.TRAIN: "train", tfds.Split.VALIDATION: "dev", tfds.Split.TEST: "test"} 62 | 63 | 64 | class MediapiSkel(tfds.core.GeneratorBasedBuilder): 65 | """DatasetBuilder for mediapi_skel dataset.""" 66 | 67 | VERSION = tfds.core.Version("1.0.0") 68 | RELEASE_NOTES = {"1.0.0": "Initial release."} 69 | 70 | BUILDER_CONFIGS = [SignDatasetConfig(name="default", include_pose="openpose")] 71 | 72 | def _info(self) -> tfds.core.DatasetInfo: 73 | """Returns the dataset metadata.""" 74 | 75 | features = { 76 | "id": tfds.features.Text(), 77 | "metadata": {"fps": tf.int32, "height": tf.int32, "width": tf.int32, "duration": tf.float32, "frames": tf.int32}, 78 | "subtitles": tfds.features.Sequence({"start_time": tf.float32, "end_time": tf.float32, "text": tfds.features.Text()}), 79 | } 80 | 81 | if self._builder_config.include_pose is not None: 82 | pose_header_path = _POSE_HEADERS[self._builder_config.include_pose] 83 | if self._builder_config.fps is not None: 84 | print("Pose FPS is not implemented for mediapi_skel dataset (since the original fps is not consistent)") 85 | 86 | if self._builder_config.include_pose == "openpose": 87 | pose_shape = (None, 1, 137, 2) 88 | raise NotImplementedError("Openpose is available, but not yet implemented for mediapi_skel dataset.") 89 | 90 | if self._builder_config.include_pose == "holistic": 91 | pose_shape = (None, 1, 75, 3) 92 | 93 | features["pose"] = PoseFeature(shape=pose_shape, stride=1, header_path=pose_header_path) 94 | 95 | return tfds.core.DatasetInfo( 96 | builder=self, 97 | description=_DESCRIPTION, 98 | features=tfds.features.FeaturesDict(features), 99 | homepage="https://www.ortolang.fr/market/corpora/mediapi-skel/v2", 100 | supervised_keys=None, 101 | citation=_CITATION, 102 | ) 103 | 104 | def _split_generators(self, dl_manager: tfds.download.DownloadManager): 105 | """Returns SplitGenerators.""" 106 | dataset_warning(self) 107 | 108 | ZIP_PATH = os.environ.get("MEDIAPI_PATH", None) 109 | if ZIP_PATH is None: 110 | raise ValueError("MEDIAPI_PATH environment variable not set.") 111 | if not os.path.exists(ZIP_PATH): 112 | raise ValueError(f"MEDIAPI_PATH environment variable points to non-existent path: {ZIP_PATH}") 113 | 114 | if self._builder_config.include_video: 115 | raise NotImplementedError("Video does not exist for mediapi_skel dataset.") 116 | 117 | POSE_ZIP_PATH = os.environ.get("MEDIAPI_POSE_PATH", None) 118 | if POSE_ZIP_PATH is not None and not os.path.exists(POSE_ZIP_PATH): 119 | raise ValueError(f"MEDIAPI_POSE_PATH environment variable points to non-existent path: {POSE_ZIP_PATH}") 120 | 121 | return [ 122 | tfds.core.SplitGenerator( 123 | name=name, 124 | gen_kwargs={ 125 | "data": read_mediapi_set( 126 | mediapi_path=ZIP_PATH, pose_path=POSE_ZIP_PATH, split=split, pose_type=self._builder_config.include_pose 127 | ) 128 | }, 129 | ) 130 | for name, split in _SPLITS.items() 131 | ] 132 | 133 | def _generate_examples(self, data: iter): 134 | """Yields examples.""" 135 | 136 | for datum in data: 137 | yield datum["id"], {"id": datum["id"], "metadata": datum["metadata"], "subtitles": datum["subtitles"], "pose": datum["pose"]} 138 | -------------------------------------------------------------------------------- /sign_language_datasets/datasets/wmt_slt/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utils file contains a subset of utils from: 3 | https://github.com/bricksdont/sign-sockeye-baselines/blob/f056c50765bf5f088419591b505b9f967605d483/scripts/preprocessing/convert_and_split_data.py 4 | """ 5 | 6 | import os 7 | import datetime 8 | from typing import Dict 9 | 10 | import tarfile 11 | import tempfile 12 | 13 | import numpy as np 14 | 15 | from pose_format import Pose, PoseHeader 16 | from pose_format.numpy import NumPyPoseBody 17 | from pose_format.pose_header import PoseHeaderDimensions 18 | from pose_format.utils.openpose_135 import load_openpose_135_directory 19 | from pose_format.utils.openpose import load_frames_directory_dict 20 | 21 | 22 | def extract_tar_xz_file(filepath: str, target_dir: str): 23 | """ 24 | 25 | :param filepath: 26 | :param target_dir: 27 | :return: 28 | """ 29 | with tarfile.open(filepath) as tar_handle: 30 | 31 | def is_within_directory(directory, target): 32 | abs_directory = os.path.abspath(directory) 33 | abs_target = os.path.abspath(target) 34 | 35 | prefix = os.path.commonprefix([abs_directory, abs_target]) 36 | 37 | return prefix == abs_directory 38 | 39 | def safe_extract(tar, path=".", members=None, *, numeric_owner=False): 40 | for member in tar.getmembers(): 41 | member_path = os.path.join(path, member.name) 42 | if not is_within_directory(path, member_path): 43 | raise Exception("Attempted Path Traversal in Tar File") 44 | 45 | tar.extractall(path, members, numeric_owner=numeric_owner) 46 | 47 | safe_extract(tar_handle, path=target_dir) 48 | 49 | 50 | def read_openpose_surrey_format(filepath: str, fps: int, width: int, height: int) -> Pose: 51 | """ 52 | Read files of the form "focusnews.071.openpose.tar.xz". 53 | Assumes a 135 keypoint Openpose model. 54 | 55 | :param filepath: 56 | :param fps: 57 | :return: 58 | """ 59 | with tempfile.TemporaryDirectory(prefix="extract_pose_file") as tmpdir_name: 60 | # extract tar.xz 61 | extract_tar_xz_file(filepath=filepath, target_dir=tmpdir_name) 62 | 63 | openpose_dir = os.path.join(tmpdir_name, "openpose") 64 | 65 | # load directory 66 | poses = load_openpose_135_directory(directory=openpose_dir, fps=fps, width=width, height=height) 67 | 68 | return poses 69 | 70 | 71 | def formatted_holistic_pose(width=1000, height=1000): 72 | try: 73 | import mediapipe as mp 74 | from pose_format.utils.holistic import holistic_components 75 | 76 | except ImportError: 77 | raise ImportError("Please install mediapipe with: pip install mediapipe") 78 | 79 | mp_holistic = mp.solutions.holistic 80 | FACEMESH_CONTOURS_POINTS = [str(p) for p in sorted(set([p for p_tup in list(mp_holistic.FACEMESH_CONTOURS) for p in p_tup]))] 81 | 82 | dimensions = PoseHeaderDimensions(width=width, height=height, depth=width) 83 | header = PoseHeader(version=0.1, dimensions=dimensions, components=holistic_components("XYZC", 10)) 84 | body = NumPyPoseBody( 85 | fps=10, data=np.zeros(shape=(1, 1, header.total_points(), 3)), confidence=np.zeros(shape=(1, 1, header.total_points())) 86 | ) 87 | pose = Pose(header, body) 88 | return pose.get_components( 89 | ["POSE_LANDMARKS", "FACE_LANDMARKS", "LEFT_HAND_LANDMARKS", "RIGHT_HAND_LANDMARKS"], {"FACE_LANDMARKS": FACEMESH_CONTOURS_POINTS} 90 | ) 91 | 92 | 93 | def load_mediapipe_directory(directory: str, fps: int, width: int, height: int) -> Pose: 94 | """ 95 | 96 | :param directory: 97 | :param fps: 98 | :return: 99 | """ 100 | 101 | frames = load_frames_directory_dict(directory=directory, pattern="(?:^|\D)?(\d+).*?.json") 102 | 103 | def load_mediapipe_frame(frame): 104 | def load_landmarks(name, num_points: int): 105 | points = [[float(p) for p in r.split(",")] for r in frame[name]["landmarks"]] 106 | points = [(ps + [1.0])[:4] for ps in points] # Add visibility to all points 107 | if len(points) == 0: 108 | points = [[0, 0, 0, 0] for _ in range(num_points)] 109 | return np.array([[x, y, z] for x, y, z, c in points]), np.array([c for x, y, z, c in points]) 110 | 111 | face_data, face_confidence = load_landmarks("face_landmarks", 128) 112 | body_data, body_confidence = load_landmarks("pose_landmarks", 33) 113 | lh_data, lh_confidence = load_landmarks("left_hand_landmarks", 21) 114 | rh_data, rh_confidence = load_landmarks("right_hand_landmarks", 21) 115 | data = np.concatenate([body_data, face_data, lh_data, rh_data]) 116 | conf = np.concatenate([body_confidence, face_confidence, lh_confidence, rh_confidence]) 117 | return data, conf 118 | 119 | def load_mediapipe_frames(): 120 | max_frames = int(max(frames.keys())) + 1 121 | pose_body_data = np.zeros(shape=(max_frames, 1, 21 + 21 + 33 + 128, 3), dtype=np.float) 122 | pose_body_conf = np.zeros(shape=(max_frames, 1, 21 + 21 + 33 + 128), dtype=np.float) 123 | for frame_id, frame in frames.items(): 124 | data, conf = load_mediapipe_frame(frame) 125 | pose_body_data[frame_id][0] = data 126 | pose_body_conf[frame_id][0] = conf 127 | return NumPyPoseBody(data=pose_body_data, confidence=pose_body_conf, fps=fps) 128 | 129 | pose = formatted_holistic_pose(width=width, height=height) 130 | 131 | pose.body = load_mediapipe_frames() 132 | 133 | return pose 134 | 135 | 136 | def read_mediapipe_surrey_format(filepath: str, fps: int, width: int, height: int) -> Pose: 137 | """ 138 | Read files of the form "focusnews.103.mediapipe.tar.xz" 139 | """ 140 | with tempfile.TemporaryDirectory(prefix="extract_pose_file") as tmpdir_name: 141 | # extract tar.xz 142 | extract_tar_xz_file(filepath=filepath, target_dir=tmpdir_name) 143 | poses_dir = os.path.join(tmpdir_name, "poses") 144 | # load directory 145 | pose = load_mediapipe_directory(directory=poses_dir, fps=fps, width=width, height=height) 146 | return pose 147 | 148 | 149 | def get_video_metadata(filename: str) -> Dict[str, int]: 150 | import cv2 151 | 152 | """ 153 | Get metadata from mp4 video. 154 | 155 | :param filename: 156 | :return: 157 | """ 158 | cap = cv2.VideoCapture(filename) 159 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 160 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 161 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 162 | cap.release() 163 | 164 | return {"fps": fps, "width": width, "height": height} 165 | 166 | 167 | def milliseconds_to_frame_index(milliseconds: int, fps: int) -> int: 168 | """ 169 | :param miliseconds: 170 | :param fps: 171 | :return: 172 | """ 173 | return int(fps * (milliseconds / 1000)) 174 | 175 | 176 | def convert_srt_time_to_frame(srt_time: datetime.timedelta, fps: int) -> int: 177 | """ 178 | datetime.timedelta(seconds=4, microseconds=71000) 179 | 180 | :param srt_time: 181 | :param fps: 182 | :return: 183 | """ 184 | seconds, microseconds = srt_time.seconds, srt_time.microseconds 185 | 186 | milliseconds = int((seconds * 1000) + (microseconds / 1000)) 187 | 188 | return milliseconds_to_frame_index(milliseconds=milliseconds, fps=fps) 189 | 190 | 191 | def reduce_pose_people(pose: Pose): 192 | pose.body.data = pose.body.data[:, 0:1, :, :] 193 | pose.body.confidence = pose.body.confidence[:, 0:1, :] 194 | --------------------------------------------------------------------------------