├── .gitignore ├── BBFN ├── LICENSE ├── README.md ├── environment.yml ├── img │ ├── model2.png │ ├── results2.png │ └── singlelayer.png ├── requirements.txt ├── scripts │ └── run.sh └── src │ ├── __pycache__ │ ├── config.cpython-37.pyc │ ├── create_dataset.cpython-37.pyc │ ├── data_loader.cpython-37.pyc │ ├── models.cpython-37.pyc │ └── solver.cpython-37.pyc │ ├── config.py │ ├── create_dataset.py │ ├── data_loader.py │ ├── main.py │ ├── models.py │ ├── modules │ ├── __pycache__ │ │ ├── encoders.cpython-37.pyc │ │ ├── multihead_attention.cpython-37.pyc │ │ ├── position_embedding.cpython-37.pyc │ │ └── transformer.cpython-37.pyc │ ├── encoders.py │ ├── multihead_attention.py │ ├── position_embedding.py │ └── transformer.py │ ├── run.sh │ ├── solver.py │ ├── test.py │ └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── eval_metrics.cpython-37.pyc │ ├── functions.cpython-37.pyc │ └── tools.cpython-37.pyc │ ├── eval_metrics.py │ ├── functions.py │ └── tools.py ├── LICENSE ├── MISA ├── LICENSE ├── README.md ├── environment.yml ├── misa-pic.png ├── requirments.txt └── src │ ├── __pycache__ │ ├── config.cpython-37.pyc │ ├── config.cpython-38.pyc │ ├── create_dataset.cpython-37.pyc │ ├── data_loader.cpython-37.pyc │ ├── models.cpython-37.pyc │ └── solver.cpython-37.pyc │ ├── config.py │ ├── create_dataset.py │ ├── data_loader.py │ ├── models.py │ ├── output │ ├── run.sh │ ├── solver.py │ ├── test.log │ ├── test.py │ ├── train.log │ ├── train.py │ └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── convert.cpython-37.pyc │ ├── functions.cpython-37.pyc │ └── time_track.cpython-37.pyc │ ├── convert.py │ ├── functions.py │ └── time_track.py ├── Modality_Analysis_NAACL2022_FINAL.pdf ├── Mult_BERT ├── LICENSE ├── README.md ├── __pycache__ │ ├── config.cpython-37.pyc │ ├── create_dataset.cpython-37.pyc │ ├── data_loader.cpython-37.pyc │ └── encoders.cpython-37.pyc ├── config.py ├── create_dataset.py ├── data_loader.py ├── encoders.py ├── imgs │ ├── architecture.png │ └── cm.png ├── main.py ├── modules │ ├── __pycache__ │ │ ├── multihead_attention.cpython-37.pyc │ │ ├── position_embedding.cpython-37.pyc │ │ └── transformer.cpython-37.pyc │ ├── multihead_attention.py │ ├── position_embedding.py │ └── transformer.py ├── requiorements.txt ├── run.sh ├── src │ ├── README.md │ ├── __pycache__ │ │ ├── config.cpython-37.pyc │ │ ├── ctc.cpython-37.pyc │ │ ├── data_loader.cpython-37.pyc │ │ ├── dataset.cpython-37.pyc │ │ ├── eval_metrics.cpython-37.pyc │ │ ├── models.cpython-37.pyc │ │ ├── train.cpython-37.pyc │ │ └── utils.cpython-37.pyc │ ├── ctc.py │ ├── dataset.py │ ├── eval_metrics.py │ ├── models.py │ ├── train.py │ └── utils.py └── warp-ctc │ ├── .travis.yml │ ├── CMakeLists.txt │ ├── LICENSE │ ├── README.md │ ├── build │ ├── CMakeCache.txt │ ├── CMakeFiles │ │ ├── 3.10.2 │ │ │ ├── CMakeCCompiler.cmake │ │ │ ├── CMakeCXXCompiler.cmake │ │ │ ├── CMakeDetermineCompilerABI_C.bin │ │ │ ├── CMakeDetermineCompilerABI_CXX.bin │ │ │ ├── CMakeSystem.cmake │ │ │ ├── CompilerIdC │ │ │ │ ├── CMakeCCompilerId.c │ │ │ │ └── a.out │ │ │ └── CompilerIdCXX │ │ │ │ ├── CMakeCXXCompilerId.cpp │ │ │ │ └── a.out │ │ ├── CMakeDirectoryInformation.cmake │ │ ├── CMakeOutput.log │ │ ├── Makefile.cmake │ │ ├── Makefile2 │ │ ├── TargetDirectories.txt │ │ ├── cmake.check_cache │ │ ├── feature_tests.bin │ │ ├── feature_tests.c │ │ ├── feature_tests.cxx │ │ ├── progress.marks │ │ ├── test_cpu.dir │ │ │ ├── CXX.includecache │ │ │ ├── DependInfo.cmake │ │ │ ├── build.make │ │ │ ├── cmake_clean.cmake │ │ │ ├── depend.internal │ │ │ ├── depend.make │ │ │ ├── flags.make │ │ │ ├── link.txt │ │ │ ├── progress.make │ │ │ └── tests │ │ │ │ ├── random.cpp.o │ │ │ │ └── test_cpu.cpp.o │ │ └── warpctc.dir │ │ │ ├── CXX.includecache │ │ │ ├── DependInfo.cmake │ │ │ ├── build.make │ │ │ ├── cmake_clean.cmake │ │ │ ├── depend.internal │ │ │ ├── depend.make │ │ │ ├── flags.make │ │ │ ├── link.txt │ │ │ ├── progress.make │ │ │ └── src │ │ │ └── ctc_entrypoint.cpp.o │ ├── Makefile │ ├── cmake_install.cmake │ ├── libwarpctc.so │ └── test_cpu │ ├── doc │ ├── baidu-research-logo-small.png │ └── deep-speech-ctc-small.png │ ├── include │ ├── contrib │ │ └── moderngpu │ │ │ ├── LICENSE │ │ │ └── include │ │ │ ├── device │ │ │ ├── ctaloadbalance.cuh │ │ │ ├── ctamerge.cuh │ │ │ ├── ctascan.cuh │ │ │ ├── ctasearch.cuh │ │ │ ├── ctasegreduce.cuh │ │ │ ├── ctasegscan.cuh │ │ │ ├── ctasegsort.cuh │ │ │ ├── ctasortedsearch.cuh │ │ │ ├── devicetypes.cuh │ │ │ ├── deviceutil.cuh │ │ │ ├── intrinsics.cuh │ │ │ ├── loadstore.cuh │ │ │ ├── serialsets.cuh │ │ │ └── sortnetwork.cuh │ │ │ ├── mgpudevice.cuh │ │ │ ├── mgpuenums.h │ │ │ └── util │ │ │ └── static.h │ ├── ctc.h │ └── detail │ │ ├── cpu_ctc.h │ │ ├── ctc_helper.h │ │ ├── gpu_ctc.h │ │ ├── gpu_ctc_kernels.h │ │ ├── hostdevice.h │ │ └── reduce.h │ ├── pytorch_binding │ ├── .gitignore │ ├── setup.cfg │ ├── setup.py │ ├── src │ │ ├── binding.cpp │ │ ├── cpu_binding.h │ │ └── gpu_binding.h │ ├── tests │ │ ├── test_cpu.py │ │ └── test_gpu.py │ └── warpctc_pytorch │ │ └── __init__.py │ ├── src │ ├── ctc_entrypoint.cpp │ ├── ctc_entrypoint.cu │ └── reduce.cu │ └── tests │ ├── random.cpp │ ├── test.h │ ├── test_cpu.cpp │ └── test_gpu.cu ├── Multimodal-Infomax ├── LICENSE ├── README.md ├── environment.yml ├── img │ └── ModelFigSingle.png └── src │ ├── __pycache__ │ ├── config.cpython-37.pyc │ ├── create_dataset.cpython-37.pyc │ ├── data_loader.cpython-37.pyc │ ├── model.cpython-37.pyc │ └── solver.cpython-37.pyc │ ├── config.py │ ├── create_dataset.py │ ├── data_loader.py │ ├── main.py │ ├── model.py │ ├── output │ ├── output_bert_large.log │ ├── run.sh │ ├── solver.py │ ├── test.py │ └── test.sh ├── README.md └── Self-MM ├── LICENSE ├── README.md ├── Recent_work.md ├── assets └── MainModel.png ├── config ├── __pycache__ │ ├── config_regression.cpython-37.pyc │ └── config_tune.cpython-37.pyc ├── config_regression.py └── config_tune.py ├── data ├── DataPre.py ├── TextPre.py ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ └── load_data.cpython-37.pyc ├── getFeature.py ├── getLengths.py └── load_data.py ├── logs ├── self_mm-mosei.log └── self_mm-mosi.log ├── models ├── AMIO.py ├── __init__.py ├── __pycache__ │ ├── AMIO.cpython-37.pyc │ └── __init__.cpython-37.pyc ├── multiTask │ ├── SELF_MM.py │ ├── __init__.py │ └── __pycache__ │ │ ├── SELF_MM.cpython-37.pyc │ │ └── __init__.cpython-37.pyc └── subNets │ ├── AlignNets.py │ ├── BertTextEncoder.py │ ├── FeatureNets.py │ ├── __init__.py │ ├── __pycache__ │ ├── AlignNets.cpython-37.pyc │ ├── BertTextEncoder.cpython-37.pyc │ └── __init__.cpython-37.pyc │ └── transformers_encoder │ ├── __init__.py │ ├── multihead_attention.py │ ├── position_embedding.py │ └── transformer.py ├── plt.py ├── requirements.txt ├── run.py ├── run.sh ├── test.py ├── trains ├── ATIO.py ├── __init__.py ├── __pycache__ │ ├── ATIO.cpython-37.pyc │ └── __init__.cpython-37.pyc └── multiTask │ ├── SELF_MM.py │ ├── __init__.py │ └── __pycache__ │ ├── SELF_MM.cpython-37.pyc │ └── __init__.cpython-37.pyc └── utils ├── __init__.py ├── __pycache__ ├── __init__.cpython-37.pyc ├── functions.cpython-37.pyc └── metricsTop.cpython-37.pyc ├── functions.py └── metricsTop.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /BBFN/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Deep Cognition and Language Research (DeCLaRe) Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /BBFN/README.md: -------------------------------------------------------------------------------- 1 | # Bi-Bimodal Modality Fusion for Correlation-Controlled Multimodal Sentiment Analysis 2 | 3 | 🔥🔥 BBFN has won the best paper award honourable mention at ICMI 2021! 4 | 5 | This repository contains official implementation of the paper: [Bi-Bimodal Modality Fusion for Correlation-Controlled Multimodal Sentiment Analysis (ICMI 2021)](https://arxiv.org/abs/2107.13669) 6 | 7 | 8 | 💎 If you would be interested in other multimodal works in our DeCLaRe Lab, please visit the [clustered repository](https://github.com/declare-lab/multimodal-deep-learning) 9 | 10 | ## Model Architecture 11 | 12 | Overview of our Bi-Bimodal Fusion Network (BBFN). It learns two text-related pairs of representations, text-acoustic and text-visual by enforcing each pair of modalities to complement mutually. Finally, the four (two pairs) head representations are concatenated 13 | to generate the final prediction. 14 | 15 | ![Alt text](img/model2.png?raw=true "Model") 16 | 17 | A single complementation layer: two identical pipelines (left and right) propagate the main modality and fuse that 18 | with complementary modality with regularization and gated control. 19 | 20 | ![Alt text](img/singlelayer.png?raw=true "Model") 21 | 22 | ## Results 23 | 24 | Results on the test set of CMU-MOSI and CMU-MOSEI dataset. Notation: △ indicates results in the corresponding line are excerpted from previous papers; † means the results are reproduced with publicly visible source code and applicable hyperparameter setting; ‡ shows the results have experienced paired t-test with 𝑝 < 0.05 and demonstrate significant improvement over MISA, the state-of-the-art model. 25 | 26 | ![Alt text](img/results2.png?raw=true "Model") 27 | 28 | ## Usage 29 | 1. Set up conda environemnt 30 | ``` 31 | conda env create -f environment.yml 32 | conda activate BBFN 33 | ``` 34 | 35 | 2. Install [CMU Multimodal SDK](https://github.com/A2Zadeh/CMU-MultimodalSDK) 36 | 37 | 3. Set `sdk_dir` in `src/config.py` to the path of CMU-MultimodalSDK 38 | 39 | 4. Train the model 40 | ``` 41 | cd src 42 | python main.py --dataset --data_path 43 | ``` 44 | We provide a script `scripts/run.sh` for your reference. 45 | 46 | ## Citation 47 | Please cite our paper if you find our work useful for your research: 48 | ```bibtex 49 | @article{han2021bi, 50 | title={Bi-Bimodal Modality Fusion for Correlation-Controlled Multimodal Sentiment Analysis}, 51 | author={Han, Wei and Chen, Hui and Gelbukh, Alexander and Zadeh, Amir and Morency, Louis-philippe and Poria, Soujanya}, 52 | journal={ICMI 2021}, 53 | year={2021} 54 | } 55 | ``` 56 | 57 | ## Contact 58 | Should you have any question, feel free to contact me through [henryhan88888@gmail.com](henryhan88888@gmail.com) 59 | 60 | -------------------------------------------------------------------------------- /BBFN/img/model2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/img/model2.png -------------------------------------------------------------------------------- /BBFN/img/results2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/img/results2.png -------------------------------------------------------------------------------- /BBFN/img/singlelayer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/img/singlelayer.png -------------------------------------------------------------------------------- /BBFN/requirements.txt: -------------------------------------------------------------------------------- 1 | conda create --name bbfn python==3.7.5 2 | conda activate bbfn 3 | 4 | pip install torch==1.7.0 5 | pip install scikit-learn==0.21.3 6 | pip install transformers==4.0.0 7 | pip install ipywidgets 8 | 9 | git clone https://github.com/A2Zadeh/CMU-MultimodalSDK.git 10 | export PYTHONPATH="/path/to/cloned/directory/CMU-MultimodalSDK:$PYTHONPATH" 11 | pip install h5py validators tqdm numpy argparse requests colorama 12 | 13 | 14 | -------------------------------------------------------------------------------- /BBFN/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GPU_NO=$1 4 | 5 | if [ -z $GPU_NO ];then 6 | GPU_NO=0 7 | fi 8 | 9 | CUDA_VISIBLE_DEVICES=$GPU_NO python main.py --dataset mosi --data_path ./datasets/MOSI -------------------------------------------------------------------------------- /BBFN/src/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/__pycache__/create_dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/__pycache__/create_dataset.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/__pycache__/solver.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/__pycache__/solver.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from datetime import datetime 4 | from collections import defaultdict 5 | from datetime import datetime 6 | from pathlib import Path 7 | import pprint 8 | from torch import optim 9 | import torch.nn as nn 10 | 11 | # path to a pretrained word embedding file 12 | word_emb_path = '/home/yingting/Glove/glove.840B.300d.txt' 13 | assert(word_emb_path is not None) 14 | 15 | 16 | username = Path.home().name 17 | project_dir = Path(__file__).resolve().parent.parent 18 | sdk_dir = project_dir.joinpath('CMU-MultimodalSDK') 19 | data_dir = project_dir.joinpath('datasets') 20 | data_dict = {'mosi': data_dir.joinpath('MOSI'), 'mosei': data_dir.joinpath( 21 | 'MOSEI'), 'ur_funny': data_dir.joinpath('UR_FUNNY')} 22 | optimizer_dict = {'RMSprop': optim.RMSprop, 'Adam': optim.Adam} 23 | activation_dict = {'elu': nn.ELU, "hardshrink": nn.Hardshrink, "hardtanh": nn.Hardtanh, 24 | "leakyrelu": nn.LeakyReLU, "prelu": nn.PReLU, "relu": nn.ReLU, "rrelu": nn.RReLU, 25 | "tanh": nn.Tanh} 26 | 27 | 28 | def str2bool(v): 29 | """string to boolean""" 30 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 31 | return True 32 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 33 | return False 34 | else: 35 | raise argparse.ArgumentTypeError('Boolean value expected.') 36 | 37 | 38 | class Config(object): 39 | def __init__(self, data, mode='train'): 40 | """Configuration Class: set kwargs as class attributes with setattr""" 41 | # if kwargs is not None: 42 | # for key, value in kwargs.items(): 43 | # if key == 'optimizer': 44 | # value = optimizer_dict[value] 45 | # if key == 'activation': 46 | # value = activation_dict[value] 47 | # setattr(self, key, value) 48 | 49 | # Dataset directory: ex) ./datasets/cornell/ 50 | self.dataset_dir = data_dict[data.lower()] 51 | self.sdk_dir = sdk_dir 52 | self.mode = mode 53 | # Glove path 54 | self.word_emb_path = word_emb_path 55 | 56 | # Data Split ex) 'train', 'valid', 'test' 57 | # self.data_dir = self.dataset_dir.joinpath(self.mode) 58 | self.data_dir = self.dataset_dir 59 | 60 | def __str__(self): 61 | """Pretty-print configurations in alphabetical order""" 62 | config_str = 'Configurations\n' 63 | config_str += pprint.pformat(self.__dict__) 64 | return config_str 65 | 66 | 67 | def get_config(dataset='mosi', mode='train', batch_size=32, use_bert=False): 68 | config = Config(data=dataset, mode=mode) 69 | 70 | config.dataset = dataset 71 | config.batch_size = batch_size 72 | config.use_bert = use_bert 73 | 74 | # if dataset == "mosi": 75 | # config.num_classes = 1 76 | # elif dataset == "mosei": 77 | # config.num_classes = 1 78 | # elif dataset == "ur_funny": 79 | # config.num_classes = 2 80 | # else: 81 | # print("No dataset mentioned") 82 | # exit() 83 | 84 | # Namespace => Dictionary 85 | # kwargs = vars(kwargs) 86 | # kwargs.update(optional_kwargs) 87 | 88 | return config 89 | -------------------------------------------------------------------------------- /BBFN/src/data_loader.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | from tqdm import tqdm_notebook 4 | from collections import defaultdict 5 | 6 | import torch 7 | import torch.nn as nn 8 | from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence 9 | from torch.utils.data import DataLoader, Dataset 10 | from transformers import * 11 | 12 | from create_dataset import MOSI, MOSEI, UR_FUNNY, PAD, UNK 13 | 14 | bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') 15 | 16 | 17 | class MSADataset(Dataset): 18 | def __init__(self, config): 19 | self.config = config 20 | 21 | ## Fetch dataset 22 | if "mosi" in str(config.data_dir).lower(): 23 | dataset = MOSI(config) 24 | elif "mosei" in str(config.data_dir).lower(): 25 | dataset = MOSEI(config) 26 | elif "ur_funny" in str(config.data_dir).lower(): 27 | dataset = UR_FUNNY(config) 28 | else: 29 | print("Dataset not defined correctly") 30 | exit() 31 | 32 | self.data, self.word2id, self.pretrained_emb = dataset.get_data(config.mode) 33 | self.len = len(self.data) 34 | 35 | config.visual_size = self.data[0][0][1].shape[1] 36 | config.acoustic_size = self.data[0][0][2].shape[1] 37 | 38 | config.word2id = self.word2id 39 | config.pretrained_emb = self.pretrained_emb 40 | 41 | @property 42 | def lav_dim(self): 43 | l_dim = 768 if self.config.use_bert else 300 44 | return l_dim, self.data[0][0][2].shape[1], self.data[0][0][1].shape[1] 45 | 46 | @property 47 | def lav_len(self): 48 | # keep this part dummy for code compatibility. lengths are only used in CTC module 49 | return 0, 0, 0 50 | 51 | def __getitem__(self, index): 52 | return self.data[index] 53 | 54 | def __len__(self): 55 | return self.len 56 | 57 | 58 | def get_loader(hp, config, shuffle=True): 59 | """Load DataLoader of given DialogDataset""" 60 | 61 | dataset = MSADataset(config) 62 | 63 | print(config.mode) 64 | config.data_len = len(dataset) 65 | 66 | config.lav_dim = dataset.lav_dim 67 | config.lav_len = dataset.lav_len 68 | 69 | if config.mode == 'train': 70 | hp.n_train = len(dataset) 71 | elif config.mode == 'valid': 72 | hp.n_valid = len(dataset) 73 | elif config.mode == 'test': 74 | hp.n_test = len(dataset) 75 | 76 | def collate_fn(batch): 77 | ''' 78 | Collate functions assume batch = [Dataset[i] for i in index_set] 79 | ''' 80 | # for later use we sort the batch in descending order of length 81 | batch = sorted(batch, key=lambda x: x[0][0].shape[0], reverse=True) 82 | 83 | # get the data out of the batch - use pad sequence util functions from PyTorch to pad things 84 | 85 | 86 | labels = torch.cat([torch.from_numpy(sample[1]) for sample in batch], dim=0) 87 | 88 | # MOSEI sentiment labels locate in the first column of sentiment matrix 89 | if labels.size(1) == 7: 90 | labels = labels[:,0][:,None] 91 | 92 | sentences = pad_sequence([torch.LongTensor(sample[0][0]) for sample in batch], padding_value=PAD) 93 | visual = pad_sequence([torch.FloatTensor(sample[0][1]) for sample in batch]) 94 | acoustic = pad_sequence([torch.FloatTensor(sample[0][2]) for sample in batch]) 95 | 96 | 97 | ## BERT-based features input prep 98 | 99 | SENT_LEN = sentences.size(0) 100 | # Create bert indices using tokenizer 101 | 102 | bert_details = [] 103 | for sample in batch: 104 | text = " ".join(sample[0][3]) 105 | encoded_bert_sent = bert_tokenizer.encode_plus( 106 | text, max_length=SENT_LEN+2, add_special_tokens=True, pad_to_max_length=True) 107 | bert_details.append(encoded_bert_sent) 108 | 109 | # Bert things are batch_first 110 | bert_sentences = torch.LongTensor([sample["input_ids"] for sample in bert_details]) 111 | bert_sentence_types = torch.LongTensor([sample["token_type_ids"] for sample in bert_details]) 112 | bert_sentence_att_mask = torch.LongTensor([sample["attention_mask"] for sample in bert_details]) 113 | 114 | # lengths are useful later in using RNNs 115 | lengths = torch.LongTensor([sample[0][0].shape[0] for sample in batch]) 116 | 117 | return sentences, visual, acoustic, labels, lengths, bert_sentences, bert_sentence_types, bert_sentence_att_mask 118 | 119 | 120 | data_loader = DataLoader( 121 | dataset=dataset, 122 | batch_size=config.batch_size, 123 | shuffle=shuffle, 124 | collate_fn=collate_fn) 125 | 126 | return data_loader -------------------------------------------------------------------------------- /BBFN/src/modules/__pycache__/encoders.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/modules/__pycache__/encoders.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/modules/__pycache__/multihead_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/modules/__pycache__/multihead_attention.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/modules/__pycache__/position_embedding.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/modules/__pycache__/position_embedding.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/modules/__pycache__/transformer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/modules/__pycache__/transformer.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/modules/position_embedding.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | # Code adapted from the fairseq repo. 7 | 8 | def make_positions(tensor, padding_idx, left_pad): 9 | """Replace non-padding symbols with their position numbers. 10 | Position numbers begin at padding_idx+1. 11 | Padding symbols are ignored, but it is necessary to specify whether padding 12 | is added on the left side (left_pad=True) or right side (left_pad=False). 13 | """ 14 | max_pos = padding_idx + 1 + tensor.size(1) 15 | device = tensor.get_device() 16 | buf_name = f'range_buf_{device}' 17 | if not hasattr(make_positions, buf_name): 18 | setattr(make_positions, buf_name, tensor.new()) 19 | setattr(make_positions, buf_name, getattr(make_positions, buf_name).type_as(tensor)) 20 | if getattr(make_positions, buf_name).numel() < max_pos: 21 | torch.arange(padding_idx + 1, max_pos, out=getattr(make_positions, buf_name)) 22 | mask = tensor.ne(padding_idx) 23 | positions = getattr(make_positions, buf_name)[:tensor.size(1)].expand_as(tensor) 24 | if left_pad: 25 | positions = positions - mask.size(1) + mask.long().sum(dim=1).unsqueeze(1) 26 | new_tensor = tensor.clone() 27 | return new_tensor.masked_scatter_(mask, positions[mask]).long() 28 | 29 | class SinusoidalPositionalEmbedding(nn.Module): 30 | """This module produces sinusoidal positional embeddings of any length. 31 | Padding symbols are ignored, but it is necessary to specify whether padding 32 | is added on the left side (left_pad=True) or right side (left_pad=False). 33 | """ 34 | 35 | def __init__(self, embedding_dim, padding_idx=0, left_pad=0, init_size=128): 36 | super().__init__() 37 | self.embedding_dim = embedding_dim 38 | self.padding_idx = padding_idx 39 | self.left_pad = left_pad 40 | self.weights = dict() # device --> actual weight; due to nn.DataParallel :-( 41 | self.register_buffer('_float_tensor', torch.FloatTensor(1)) 42 | 43 | @staticmethod 44 | def get_embedding(num_embeddings, embedding_dim, padding_idx=None): 45 | """Build sinusoidal embeddings. 46 | This matches the implementation in tensor2tensor, but differs slightly 47 | from the description in Section 3.5 of "Attention Is All You Need". 48 | """ 49 | half_dim = embedding_dim // 2 50 | emb_c1 = math.log(10000) / (half_dim - 1) 51 | 52 | emb_c2 = torch.arange(embedding_dim, dtype=torch.int32) 53 | 54 | emb = torch.exp((emb_c2 // 2).to(torch.float) * -emb_c1) # (embedding_dim,) 55 | emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) # (num_emb, embedding_dim) 56 | 57 | # assign sinusoidal positional embedding to correct positions 58 | emb[:,emb_c2 % 2 == 0] = torch.sin(emb[:,emb_c2 % 2 == 0]) 59 | emb[:,emb_c2 % 2 == 1] = torch.cos(emb[:,emb_c2 % 2 == 1]) 60 | 61 | # emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1) # (num_emb, half_dim*2) 62 | 63 | if embedding_dim % 2 == 1: 64 | # zero pad 65 | emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) 66 | if padding_idx is not None: 67 | emb[padding_idx, :] = 0 68 | 69 | return emb 70 | 71 | def forward(self, input): 72 | """Input is expected to be of size [bsz x seqlen].""" 73 | bsz, seq_len = input.size() 74 | max_pos = self.padding_idx + 1 + seq_len 75 | device = input.get_device() 76 | if device not in self.weights or max_pos > self.weights[device].size(0): 77 | # recompute/expand embeddings if needed 78 | self.weights[device] = SinusoidalPositionalEmbedding.get_embedding( 79 | max_pos, 80 | self.embedding_dim, 81 | self.padding_idx, 82 | ) 83 | self.weights[device] = self.weights[device].type_as(self._float_tensor) 84 | positions = make_positions(input, self.padding_idx, self.left_pad) 85 | return self.weights[device].index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach() 86 | 87 | def max_positions(self): 88 | """Maximum number of supported positions.""" 89 | return int(1e5) # an arbitrary large number -------------------------------------------------------------------------------- /BBFN/src/run.sh: -------------------------------------------------------------------------------- 1 | #TRAIN 2 | CUDA_VISIBLE_DEVICES=1 python main.py \ 3 | --dataset mosi \ 4 | --data_path ./datasets/MOSI \ 5 | --use_bert \ 6 | --lr 1e-4 \ 7 | --train_method missing \ 8 | --train_changed_modal language \ 9 | --train_changed_pct 0.3 10 | 11 | #TEST 12 | # CUDA_VISIBLE_DEVICES=1 python test.py \ 13 | # --dataset mosi \ 14 | # --data_path ./datasets/MOSI \ 15 | # --use_bert \ 16 | # --lr 1e-4 \ 17 | # --train_method missing \ 18 | # --train_changed_modal language \ 19 | # --train_changed_pct 0.3 \ 20 | # --test_method missing \ 21 | # --test_changed_modal language \ 22 | # --test_changed_pct 0 \ 23 | # --is_test -------------------------------------------------------------------------------- /BBFN/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .functions import * 2 | from .eval_metrics import * -------------------------------------------------------------------------------- /BBFN/src/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/utils/__pycache__/eval_metrics.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/utils/__pycache__/eval_metrics.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/utils/__pycache__/functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/utils/__pycache__/functions.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/utils/__pycache__/tools.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/BBFN/src/utils/__pycache__/tools.cpython-37.pyc -------------------------------------------------------------------------------- /BBFN/src/utils/functions.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Function 2 | import torch.nn as nn 3 | import torch 4 | import torch.autograd.functional as F 5 | 6 | """ 7 | Adapted from https://github.com/fungtion/DSN/blob/master/functions.py 8 | """ 9 | 10 | class ReverseLayerF(Function): 11 | 12 | @staticmethod 13 | def forward(ctx, x, p): 14 | ctx.p = p 15 | 16 | return x.view_as(x) 17 | 18 | @staticmethod 19 | def backward(ctx, grad_output): 20 | output = grad_output.neg() * ctx.p 21 | 22 | return output, None 23 | 24 | 25 | class MSE(nn.Module): 26 | def __init__(self): 27 | super(MSE, self).__init__() 28 | 29 | def forward(self, pred, real): 30 | diffs = torch.add(real, -pred) 31 | n = torch.numel(diffs.data) 32 | mse = torch.sum(diffs.pow(2)) / n 33 | 34 | return mse 35 | 36 | 37 | class SIMSE(nn.Module): 38 | 39 | def __init__(self): 40 | super(SIMSE, self).__init__() 41 | 42 | def forward(self, pred, real): 43 | diffs = torch.add(real, - pred) 44 | n = torch.numel(diffs.data) 45 | simse = torch.sum(diffs).pow(2) / (n ** 2) 46 | 47 | return simse 48 | 49 | 50 | class DiffLoss(nn.Module): 51 | 52 | def __init__(self): 53 | super(DiffLoss, self).__init__() 54 | 55 | def forward(self, input1, input2): 56 | 57 | batch_size = input1.size(0) 58 | input1 = input1.view(batch_size, -1) 59 | input2 = input2.view(batch_size, -1) 60 | 61 | # Zero mean 62 | input1_mean = torch.mean(input1, dim=0, keepdims=True) 63 | input2_mean = torch.mean(input2, dim=0, keepdims=True) 64 | input1 = input1 - input1_mean 65 | input2 = input2 - input2_mean 66 | 67 | input1_l2_norm = torch.norm(input1, p=2, dim=1, keepdim=True).detach() 68 | input1_l2 = input1.div(input1_l2_norm.expand_as(input1) + 1e-6) 69 | 70 | 71 | input2_l2_norm = torch.norm(input2, p=2, dim=1, keepdim=True).detach() 72 | input2_l2 = input2.div(input2_l2_norm.expand_as(input2) + 1e-6) 73 | 74 | diff_loss = torch.mean((input1_l2.t().mm(input2_l2)).pow(2)) 75 | 76 | return diff_loss 77 | 78 | class CMD(nn.Module): 79 | """ 80 | Adapted from https://github.com/wzell/cmd/blob/master/models/domain_regularizer.py 81 | """ 82 | 83 | def __init__(self): 84 | super(CMD, self).__init__() 85 | 86 | def forward(self, x1, x2, n_moments): 87 | mx1 = torch.mean(x1, 0) 88 | mx2 = torch.mean(x2, 0) 89 | sx1 = x1-mx1 90 | sx2 = x2-mx2 91 | dm = self.matchnorm(mx1, mx2) 92 | scms = dm 93 | for i in range(n_moments - 1): 94 | scms += self.scm(sx1, sx2, i + 2) 95 | return scms 96 | 97 | def matchnorm(self, x1, x2): 98 | power = torch.pow(x1-x2,2) 99 | summed = torch.sum(power) 100 | sqrt = summed**(0.5) 101 | return sqrt 102 | # return ((x1-x2)**2).sum().sqrt() 103 | 104 | def scm(self, sx1, sx2, k): 105 | ss1 = torch.mean(torch.pow(sx1, k), 0) 106 | ss2 = torch.mean(torch.pow(sx2, k), 0) 107 | return self.matchnorm(ss1, ss2) 108 | 109 | class AttSoftmax(torch.autograd.Function): 110 | 111 | @staticmethod 112 | def forward(ctx, input, dim): 113 | attn_weights = F.softmax(input, dim=dim) 114 | attn_weights_mask = attn_weights.isnan() 115 | 116 | ctx.attn_weights_mask = attn_weights_mask 117 | # save_for_backward only for saving inputs and outputs 118 | ctx.dim = dim 119 | output = torch.masked_fill(attn_weights, attn_weights_mask, 0.0) 120 | ctx.save_for_backward(output) 121 | 122 | return output 123 | 124 | @staticmethod 125 | def backward(ctx, grad_output): 126 | output, = ctx.saved_tensors # read from tuple 127 | 128 | attn_weights_mask = ctx.attn_weights_mask 129 | dim = ctx.attn_weights_mask 130 | 131 | grad_output = grad_output * output 132 | 133 | # grad_non_mask = grad_output - grad_output.sum(dim=dim, keepdim=True) 134 | grad_non_mask = grad_output - grad_output*grad_output.sum(dim=-1).unsqueeze(-1) 135 | out_grad=torch.masked_fill(grad_non_mask, attn_weights_mask, 0.0) 136 | return out_grad, None 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /BBFN/src/utils/tools.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | 5 | def save_load_name(args, name=''): 6 | if args.aligned: 7 | name = name if len(name) > 0 else 'aligned_model' 8 | elif not args.aligned: 9 | name = name if len(name) > 0 else 'nonaligned_model' 10 | 11 | # return name + '_' + args.model 12 | return name 13 | 14 | 15 | def save_model(args, model, name=''): 16 | name = save_load_name(args, name) 17 | # if not os.path.exists('pre_trained_models'): 18 | # os.mkdir('pre_trained_models') 19 | # torch.save(model, f'pre_trained_models/{name}.pt') 20 | 21 | # save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={0 if args.train_method=="missing" else "N"}/' 22 | # if not os.path.exists(save_dir): 23 | # os.makedirs(save_dir) 24 | 25 | if args.train_method=="missing": 26 | save_mode = f'0' 27 | elif args.train_method=="g_noise": 28 | save_mode = f'N' 29 | elif args.train_method=="hybird": 30 | save_mode = f'H' 31 | else: 32 | raise 33 | save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={save_mode}/' 34 | if not os.path.exists(save_dir): 35 | os.makedirs(save_dir) 36 | 37 | print("-------------------------save_dir") 38 | print(save_dir) 39 | 40 | torch.save(model, f'{save_dir}/{name}.pt') 41 | 42 | 43 | def load_model(args, name=''): 44 | name = save_load_name(args, name) 45 | # model = torch.load(f'pre_trained_models/{name}.pt') 46 | 47 | # save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={0 if args.train_method=="missing" else "N"}/' 48 | # if not os.path.exists(save_dir): 49 | # os.makedirs(save_dir) 50 | 51 | if args.train_method=="missing": 52 | save_mode = f'0' 53 | elif args.train_method=="g_noise": 54 | save_mode = f'N' 55 | elif args.train_method=="hybird": 56 | save_mode = f'H' 57 | else: 58 | raise 59 | save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={save_mode}/' 60 | print("-------------------------save_dir") 61 | print(save_dir) 62 | if not os.path.exists(save_dir): 63 | os.makedirs(save_dir) 64 | 65 | model = torch.load(f'{save_dir}/{name}.pt') 66 | 67 | return model 68 | 69 | def to_gpu(x, on_cpu=False, gpu_id=None): 70 | """Tensor => Variable""" 71 | if torch.cuda.is_available() and not on_cpu: 72 | x = x.cuda(gpu_id) 73 | return x 74 | 75 | def to_cpu(x): 76 | """Variable => Tensor""" 77 | if torch.cuda.is_available(): 78 | x = x.cpu() 79 | return x.data 80 | -------------------------------------------------------------------------------- /MISA/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Deep Cognition and Language Research (DeCLaRe) Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MISA/README.md: -------------------------------------------------------------------------------- 1 | # MISA: Modality-Invariant and -Specific Representations for Multimodal Sentiment Analysis 2 | Code for the [ACM MM 2020](https://2020.acmmm.org) paper [MISA: Modality-Invariant and -Specific Representations for Multimodal Sentiment Analysis](https://arxiv.org/pdf/2005.03545.pdf) 3 | 4 | 5 |

6 | 7 |

8 | 9 | 10 | 11 | ### Setup the environment 12 | 13 | We work with a conda environment. 14 | 15 | ``` 16 | conda env create -f environment.yml 17 | conda activate misa-code 18 | ``` 19 | 20 | ### Data Download 21 | 22 | - Install [CMU Multimodal SDK](https://github.com/A2Zadeh/CMU-MultimodalSDK). Ensure, you can perform ```from mmsdk import mmdatasdk```. 23 | - Option 1: Download [pre-computed splits](https://drive.google.com/drive/folders/1IBwWNH0XjPnZWaAlP1U2tIJH6Rb3noMI?usp=sharing) and place the contents inside ```datasets``` folder. 24 | - Option 2: Re-create splits by downloading data from MMSDK. For this, simply run the code as detailed next. 25 | 26 | ### Running the code 27 | 28 | 1. ```cd src``` 29 | 2. Set ```word_emb_path``` in ```config.py``` to [glove file](http://nlp.stanford.edu/data/glove.840B.300d.zip). 30 | 3. Set ```sdk_dir``` to the path of CMU-MultimodalSDK. 31 | 2. ```python train.py --data mosi```. Replace ```mosi``` with ```mosei``` or ```ur_funny``` for other datasets. 32 | 33 | ### Citation 34 | 35 | If this paper is useful for your research, please cite us at: 36 | 37 | ``` 38 | @article{hazarika2020misa, 39 | title={MISA: Modality-Invariant and-Specific Representations for Multimodal Sentiment Analysis}, 40 | author={Hazarika, Devamanyu and Zimmermann, Roger and Poria, Soujanya}, 41 | journal={arXiv preprint arXiv:2005.03545}, 42 | year={2020} 43 | } 44 | ``` 45 | 46 | ### Contact 47 | 48 | For any questions, please email at [hazarika@comp.nus.edu.sg](mailto:hazarika@comp.nus.edu.sg) 49 | -------------------------------------------------------------------------------- /MISA/misa-pic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/misa-pic.png -------------------------------------------------------------------------------- /MISA/requirments.txt: -------------------------------------------------------------------------------- 1 | 2 | conda create --name misa python==3.7.5 3 | conda activate misa 4 | 5 | pip install torch==1.3.1 6 | pip install transformers==2.5.1 gensim==3.8.1 7 | pip install -U scikit-learn scipy matplotlib 8 | pip install ipywidgets 9 | 10 | git clone https://github.com/A2Zadeh/CMU-MultimodalSDK.git 11 | export PYTHONPATH="/path/to/cloned/directory/CMU-MultimodalSDK:$PYTHONPATH" 12 | pip install h5py validators tqdm numpy argparse requests colorama 13 | 14 | 15 | -------------------------------------------------------------------------------- /MISA/src/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/__pycache__/config.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/config.cpython-38.pyc -------------------------------------------------------------------------------- /MISA/src/__pycache__/create_dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/create_dataset.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/__pycache__/solver.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/__pycache__/solver.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/data_loader.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | from tqdm import tqdm_notebook 4 | from collections import defaultdict 5 | 6 | import torch 7 | import torch.nn as nn 8 | from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence 9 | from torch.utils.data import DataLoader, Dataset 10 | from transformers import * 11 | 12 | from create_dataset import MOSI, MOSEI, UR_FUNNY, PAD, UNK 13 | 14 | 15 | bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') 16 | 17 | 18 | class MSADataset(Dataset): 19 | def __init__(self, config): 20 | 21 | ## Fetch dataset 22 | if "mosi" in str(config.data_dir).lower(): 23 | dataset = MOSI(config) 24 | elif "mosei" in str(config.data_dir).lower(): 25 | dataset = MOSEI(config) 26 | elif "ur_funny" in str(config.data_dir).lower(): 27 | dataset = UR_FUNNY(config) 28 | else: 29 | print("Dataset not defined correctly") 30 | exit() 31 | 32 | self.data, self.word2id, self.pretrained_emb = dataset.get_data(config.mode) 33 | self.len = len(self.data) 34 | 35 | config.visual_size = self.data[0][0][1].shape[1] 36 | config.acoustic_size = self.data[0][0][2].shape[1] 37 | 38 | config.word2id = self.word2id 39 | config.pretrained_emb = self.pretrained_emb 40 | 41 | 42 | def __getitem__(self, index): 43 | return self.data[index] 44 | 45 | def __len__(self): 46 | return self.len 47 | 48 | 49 | 50 | def get_loader(config, shuffle=True): 51 | """Load DataLoader of given DialogDataset""" 52 | 53 | dataset = MSADataset(config) 54 | 55 | print(config.mode) 56 | config.data_len = len(dataset) 57 | 58 | 59 | def collate_fn(batch): 60 | ''' 61 | Collate functions assume batch = [Dataset[i] for i in index_set] 62 | ''' 63 | # for later use we sort the batch in descending order of length 64 | batch = sorted(batch, key=lambda x: x[0][0].shape[0], reverse=True) 65 | 66 | # get the data out of the batch - use pad sequence util functions from PyTorch to pad things 67 | 68 | 69 | labels = torch.cat([torch.from_numpy(sample[1]) for sample in batch], dim=0) 70 | sentences = pad_sequence([torch.LongTensor(sample[0][0]) for sample in batch], padding_value=PAD) 71 | visual = pad_sequence([torch.FloatTensor(sample[0][1]) for sample in batch]) 72 | acoustic = pad_sequence([torch.FloatTensor(sample[0][2]) for sample in batch]) 73 | 74 | ## BERT-based features input prep 75 | 76 | SENT_LEN = sentences.size(0) 77 | # Create bert indices using tokenizer 78 | 79 | bert_details = [] 80 | for sample in batch: 81 | text = " ".join(sample[0][3]) 82 | encoded_bert_sent = bert_tokenizer.encode_plus( 83 | text, max_length=SENT_LEN+2, add_special_tokens=True, pad_to_max_length=True) 84 | bert_details.append(encoded_bert_sent) 85 | 86 | 87 | # Bert things are batch_first 88 | bert_sentences = torch.LongTensor([sample["input_ids"] for sample in bert_details]) 89 | bert_sentence_types = torch.LongTensor([sample["token_type_ids"] for sample in bert_details]) 90 | bert_sentence_att_mask = torch.LongTensor([sample["attention_mask"] for sample in bert_details]) 91 | 92 | 93 | # lengths are useful later in using RNNs 94 | lengths = torch.LongTensor([sample[0][0].shape[0] for sample in batch]) 95 | 96 | return sentences, visual, acoustic, labels, lengths, bert_sentences, bert_sentence_types, bert_sentence_att_mask 97 | 98 | 99 | data_loader = DataLoader( 100 | dataset=dataset, 101 | batch_size=config.batch_size, 102 | shuffle=shuffle, 103 | collate_fn=collate_fn) 104 | 105 | return data_loader 106 | -------------------------------------------------------------------------------- /MISA/src/run.sh: -------------------------------------------------------------------------------- 1 | ####TRAIN 2 | CUDA_VISIBLE_DEVICES=3 python train.py \ 3 | --data mosi \ 4 | --learning_rate 1e-5 \ 5 | --optimizer RMSprop \ 6 | --activation hardtanh \ 7 | --train_method missing \ 8 | --train_changed_modal language \ 9 | --train_changed_pct 0.3 10 | 11 | ###TEST 12 | # CUDA_VISIBLE_DEVICES=3 python test.py \ 13 | # --data mosi \ 14 | # --learning_rate 1e-5 \ 15 | # --optimizer RMSprop \ 16 | # --activation hardtanh \ 17 | # --train_method missing \ 18 | # --train_changed_modal language \ 19 | # --train_changed_pct 0.3 \ 20 | # --test_method missing \ 21 | # --test_changed_modal language \ 22 | # --test_changed_pct 0 \ 23 | # --is_test 24 | 25 | -------------------------------------------------------------------------------- /MISA/src/test.log: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | Truncation was not explicitely activated but `max_length` is provided a specific value, please use `truncation=True` to explicitely truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`. 3 | /home/yingting/anaconda3/envs/misa/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:1944: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert). 4 | FutureWarning, 5 | mosi 6 | test 7 | mae: 0.8594561 8 | corr: 0.727530167007793 9 | mult_acc: 0.39504373177842567 10 | Classification Report (pos/neg) : 11 | precision recall f1-score support 12 | 13 | False 0.85260 0.77836 0.81379 379 14 | True 0.72903 0.81588 0.77002 277 15 | 16 | accuracy 0.79421 656 17 | macro avg 0.79082 0.79712 0.79191 656 18 | weighted avg 0.80042 0.79421 0.79531 656 19 | 20 | Accuracy (pos/neg) 0.7942073170731707 21 | Classification Report (non-neg/neg) : 22 | precision recall f1-score support 23 | 24 | False 0.81044 0.77836 0.79408 379 25 | True 0.73913 0.77524 0.75676 307 26 | 27 | accuracy 0.77697 686 28 | macro avg 0.77478 0.77680 0.77542 686 29 | weighted avg 0.77853 0.77697 0.77738 686 30 | 31 | Accuracy (non-neg/neg) 0.7769679300291545 32 | -------------------------------------------------------------------------------- /MISA/src/test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | from random import random 5 | from sklearn.metrics import f1_score, classification_report, accuracy_score 6 | 7 | from config import get_config, activation_dict 8 | from data_loader import get_loader 9 | from solver import Solver 10 | from utils import to_gpu 11 | import models 12 | 13 | import warnings 14 | 15 | warnings.filterwarnings('ignore') 16 | 17 | if __name__ == '__main__': 18 | # Setting random seed 19 | # random_name = str(random()) 20 | # random_seed = 336 21 | # torch.manual_seed(random_seed) 22 | # torch.cuda.manual_seed_all(random_seed) 23 | # torch.backends.cudnn.deterministic = True 24 | # torch.backends.cudnn.benchmark = False 25 | # np.random.seed(random_seed) 26 | 27 | # Setting the config for each stage 28 | train_config = get_config(mode='train') 29 | dev_config = get_config(mode='dev') 30 | test_config = get_config(mode='test') 31 | 32 | # print(test_config) 33 | 34 | # Creating pytorch dataloaders 35 | train_data_loader = get_loader(train_config, shuffle = True) 36 | dev_data_loader = get_loader(dev_config, shuffle = False) 37 | test_data_loader = get_loader(test_config, shuffle = False) 38 | 39 | # load model 40 | model = getattr(models, test_config.model)(test_config) 41 | # if test_config.data == 'mosi': 42 | # path = f'checkpoints/best_model_mosi.std' 43 | 44 | # model.load_state_dict(torch.load( 45 | # f'checkpoints/model_2021-11-14_00:02:34.std')) # mosi 46 | # model.load_state_dict(torch.load(f'checkpoints/best_model_mosi.std')) 47 | # elif test_config.data == 'mosei': 48 | # path = f'checkpoints/best_model_mosei.std' 49 | # path = f'checkpoints/mosei/best_30%L=0_train/best_model_mosei.std' 50 | # path = f'checkpoints/mosei/best_30%L=N_train/best_model_mosei.std' 51 | 52 | # model.load_state_dict(torch.load( 53 | # f'checkpoints/model_2021-11-14_14:14:10.std')) # mosei 54 | # model.load_state_dict(torch.load(f'checkpoints/best_model_mosei.std')) 55 | 56 | 57 | # Solver is a wrapper for model traiing and testing 58 | solver = Solver 59 | solver = solver(train_config, dev_config, test_config, train_data_loader, dev_data_loader, test_data_loader, is_train=False, model=model) 60 | 61 | # Build the model 62 | solver.build() 63 | solver.criterion = nn.MSELoss(reduction="mean") 64 | 65 | # test the model (test scores will be returned based on dev performance) 66 | solver.eval(mode="test", to_print=True) 67 | 68 | -------------------------------------------------------------------------------- /MISA/src/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | from random import random 5 | 6 | from config import get_config, activation_dict 7 | from data_loader import get_loader 8 | from solver import Solver 9 | 10 | import torch 11 | import torch.nn as nn 12 | from torch.nn import functional as F 13 | 14 | import warnings 15 | import models 16 | warnings.filterwarnings('ignore') 17 | 18 | if __name__ == '__main__': 19 | 20 | # Setting random seed 21 | random_name = str(random()) 22 | random_seed = 337 23 | torch.manual_seed(random_seed) 24 | torch.cuda.manual_seed_all(random_seed) 25 | torch.backends.cudnn.deterministic = True 26 | torch.backends.cudnn.benchmark = False 27 | np.random.seed(random_seed) 28 | 29 | # Setting the config for each stage 30 | train_config = get_config(mode='train') 31 | dev_config = get_config(mode='dev') 32 | test_config = get_config(mode='test') 33 | 34 | print(train_config) 35 | 36 | # Creating pytorch dataloaders 37 | train_data_loader = get_loader(train_config, shuffle = True) 38 | dev_data_loader = get_loader(dev_config, shuffle = False) 39 | test_data_loader = get_loader(test_config, shuffle = False) 40 | 41 | 42 | # Solver is a wrapper for model traiing and testing 43 | solver = Solver 44 | solver = solver(train_config, dev_config, test_config, train_data_loader, dev_data_loader, test_data_loader, is_train=True) 45 | 46 | # Build the model 47 | solver.build() 48 | 49 | # Train the model (test scores will be returned based on dev performance) 50 | solver.train() -------------------------------------------------------------------------------- /MISA/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .convert import * 2 | from .time_track import time_desc_decorator 3 | from .functions import * -------------------------------------------------------------------------------- /MISA/src/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/utils/__pycache__/convert.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/utils/__pycache__/convert.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/utils/__pycache__/functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/utils/__pycache__/functions.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/utils/__pycache__/time_track.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/MISA/src/utils/__pycache__/time_track.cpython-37.pyc -------------------------------------------------------------------------------- /MISA/src/utils/convert.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def to_gpu(x, on_cpu=False, gpu_id=None): 4 | """Tensor => Variable""" 5 | if torch.cuda.is_available() and not on_cpu: 6 | x = x.cuda(gpu_id) 7 | return x 8 | 9 | def to_cpu(x): 10 | """Variable => Tensor""" 11 | if torch.cuda.is_available(): 12 | x = x.cpu() 13 | return x.data -------------------------------------------------------------------------------- /MISA/src/utils/functions.py: -------------------------------------------------------------------------------- 1 | from torch.autograd import Function 2 | import torch.nn as nn 3 | import torch 4 | 5 | """ 6 | Adapted from https://github.com/fungtion/DSN/blob/master/functions.py 7 | """ 8 | 9 | class ReverseLayerF(Function): 10 | 11 | @staticmethod 12 | def forward(ctx, x, p): 13 | ctx.p = p 14 | 15 | return x.view_as(x) 16 | 17 | @staticmethod 18 | def backward(ctx, grad_output): 19 | output = grad_output.neg() * ctx.p 20 | 21 | return output, None 22 | 23 | 24 | class MSE(nn.Module): 25 | def __init__(self): 26 | super(MSE, self).__init__() 27 | 28 | def forward(self, pred, real): 29 | diffs = torch.add(real, -pred) 30 | n = torch.numel(diffs.data) 31 | mse = torch.sum(diffs.pow(2)) / n 32 | 33 | return mse 34 | 35 | 36 | class SIMSE(nn.Module): 37 | 38 | def __init__(self): 39 | super(SIMSE, self).__init__() 40 | 41 | def forward(self, pred, real): 42 | diffs = torch.add(real, - pred) 43 | n = torch.numel(diffs.data) 44 | simse = torch.sum(diffs).pow(2) / (n ** 2) 45 | 46 | return simse 47 | 48 | 49 | class DiffLoss(nn.Module): 50 | 51 | def __init__(self): 52 | super(DiffLoss, self).__init__() 53 | 54 | def forward(self, input1, input2): 55 | 56 | batch_size = input1.size(0) 57 | input1 = input1.view(batch_size, -1) 58 | input2 = input2.view(batch_size, -1) 59 | 60 | # Zero mean 61 | input1_mean = torch.mean(input1, dim=0, keepdims=True) 62 | input2_mean = torch.mean(input2, dim=0, keepdims=True) 63 | input1 = input1 - input1_mean 64 | input2 = input2 - input2_mean 65 | 66 | input1_l2_norm = torch.norm(input1, p=2, dim=1, keepdim=True).detach() 67 | input1_l2 = input1.div(input1_l2_norm.expand_as(input1) + 1e-6) 68 | 69 | 70 | input2_l2_norm = torch.norm(input2, p=2, dim=1, keepdim=True).detach() 71 | input2_l2 = input2.div(input2_l2_norm.expand_as(input2) + 1e-6) 72 | 73 | diff_loss = torch.mean((input1_l2.t().mm(input2_l2)).pow(2)) 74 | 75 | return diff_loss 76 | 77 | class CMD(nn.Module): 78 | """ 79 | Adapted from https://github.com/wzell/cmd/blob/master/models/domain_regularizer.py 80 | """ 81 | 82 | def __init__(self): 83 | super(CMD, self).__init__() 84 | 85 | def forward(self, x1, x2, n_moments): 86 | mx1 = torch.mean(x1, 0) 87 | mx2 = torch.mean(x2, 0) 88 | sx1 = x1-mx1 89 | sx2 = x2-mx2 90 | dm = self.matchnorm(mx1, mx2) 91 | scms = dm 92 | for i in range(n_moments - 1): 93 | scms += self.scm(sx1, sx2, i + 2) 94 | return scms 95 | 96 | def matchnorm(self, x1, x2): 97 | power = torch.pow(x1-x2,2) 98 | summed = torch.sum(power) 99 | sqrt = summed**(0.5) 100 | return sqrt 101 | # return ((x1-x2)**2).sum().sqrt() 102 | 103 | def scm(self, sx1, sx2, k): 104 | ss1 = torch.mean(torch.pow(sx1, k), 0) 105 | ss2 = torch.mean(torch.pow(sx2, k), 0) 106 | return self.matchnorm(ss1, ss2) 107 | -------------------------------------------------------------------------------- /MISA/src/utils/time_track.py: -------------------------------------------------------------------------------- 1 | import time 2 | from functools import partial 3 | 4 | 5 | def base_time_desc_decorator(method, desc='test_description'): 6 | def timed(*args, **kwargs): 7 | 8 | # Print Description 9 | # print('#' * 50) 10 | print(desc) 11 | # print('#' * 50 + '\n') 12 | 13 | # Calculation Runtime 14 | start = time.time() 15 | 16 | # Run Method 17 | try: 18 | result = method(*args, **kwargs) 19 | except TypeError: 20 | result = method(**kwargs) 21 | 22 | # Print Runtime 23 | print('Done! It took {:.2} secs\n'.format(time.time() - start)) 24 | 25 | if result is not None: 26 | return result 27 | 28 | return timed 29 | 30 | 31 | def time_desc_decorator(desc): return partial(base_time_desc_decorator, desc=desc) 32 | 33 | 34 | @time_desc_decorator('this is description') 35 | def time_test(arg, kwarg='this is kwarg'): 36 | time.sleep(3) 37 | print('Inside of time_test') 38 | print('printing arg: ', arg) 39 | print('printing kwarg: ', kwarg) 40 | 41 | 42 | @time_desc_decorator('this is second description') 43 | def no_arg_method(): 44 | print('this method has no argument') 45 | 46 | 47 | if __name__ == '__main__': 48 | time_test('hello', kwarg=3) 49 | time_test(3) 50 | no_arg_method() 51 | -------------------------------------------------------------------------------- /Modality_Analysis_NAACL2022_FINAL.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Modality_Analysis_NAACL2022_FINAL.pdf -------------------------------------------------------------------------------- /Mult_BERT/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yao-Hung Hubert Tsai and Shaojie Bai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Mult_BERT/README.md: -------------------------------------------------------------------------------- 1 | ![Python 3.6](https://img.shields.io/badge/python-3.6-green.svg) 2 | 3 | # Multimodal Transformer for Unaligned Multimodal Language Sequences 4 | 5 | > Pytorch implementation for learning Multimodal Transformer for unaligned multimodal language sequences. 6 | 7 | Correspondence to: 8 | - Yao-Hung Hubert Tsai (yaohungt@cs.cmu.edu) 9 | - Shaojie Bai (shaojieb@andrew.cmu.edu) 10 | 11 | ## Paper 12 | [**Multimodal Transformer for Unaligned Multimodal Language Sequences**](https://arxiv.org/pdf/1906.00295.pdf)
13 | [Yao-Hung Hubert Tsai](https://yaohungt.github.io) *, [Shaojie Bai](https://jerrybai1995.github.io) *, [Paul Pu Liang](http://www.cs.cmu.edu/~pliang/), [J. Zico Kolter](http://zicokolter.com), [Louis-Philippe Morency](https://www.cs.cmu.edu/~morency/), and [Ruslan Salakhutdinov](https://www.cs.cmu.edu/~rsalakhu/)
14 | Association for Computational Linguistics (ACL), 2019. (*equal contribution) 15 | 16 | Please cite our paper if you find our work useful for your research: 17 | 18 | ```tex 19 | @inproceedings{tsai2019MULT, 20 | title={Multimodal Transformer for Unaligned Multimodal Language Sequences}, 21 | author={Tsai, Yao-Hung Hubert and Bai, Shaojie and Liang, Paul Pu and Kolter, J. Zico and Morency, Louis-Philippe and Salakhutdinov, Ruslan}, 22 | booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 23 | month = {7}, 24 | year={2019}, 25 | address = {Florence, Italy}, 26 | publisher = {Association for Computational Linguistics}, 27 | } 28 | ``` 29 | 30 | ## Overview 31 | 32 | ### Overall Architecture for Multimodal Transformer 33 |

34 | 35 | 36 | Multimodal Transformer (MulT) merges multimodal time-series via a feed-forward fusion process from multiple directional pairwise crossmodal transformers. Specifically, each crossmodal transformer serves to repeatedly reinforce a *target modality* with the low-level features from another *source modality* by learning the attention across the two modalities' features. A MulT architecture hence models all pairs of modalities with such crossmodal transformers, followed by sequence models (e.g., self-attention transformer) that predicts using the fused features. 37 | 38 | 39 | ### Crossmodal Attention for Two Sequences from Distinct Modalities 40 |

41 | 42 | 43 | The core of our proposed model are crossmodal transformer and crossmodal attention module. 44 | 45 | ## Usage 46 | 47 | ### Prerequisites 48 | - Python 3.6/3.7 49 | - [Pytorch (>=1.0.0) and torchvision](https://pytorch.org/) 50 | - CUDA 10.0 or above 51 | 52 | ### Datasets 53 | 54 | Data files (containing processed MOSI, MOSEI and IEMOCAP datasets) can be downloaded from [here](https://www.dropbox.com/sh/hyzpgx1hp9nj37s/AAB7FhBqJOFDw2hEyvv2ZXHxa?dl=0). 55 | 56 | To retrieve the meta information and the raw data, please refer to the [SDK for these datasets](https://github.com/A2Zadeh/CMU-MultimodalSDK). 57 | 58 | ### Run the Code 59 | 60 | 1. Create (empty) folders for data and pre-trained models: 61 | ~~~~ 62 | mkdir data pre_trained_models 63 | ~~~~ 64 | 65 | and put the downloaded data in 'data/'. 66 | 67 | 2. Command as follows 68 | ~~~~ 69 | python main.py [--FLAGS] 70 | ~~~~ 71 | 72 | Note that the defualt arguments are for unaligned version of MOSEI. For other datasets, please refer to Supplmentary. 73 | 74 | ### If Using CTC 75 | 76 | Transformer requires no CTC module. However, as we describe in the paper, CTC module offers an alternative to applying other kinds of sequence models (e.g., recurrent architectures) to unaligned multimodal streams. 77 | 78 | If you want to use the CTC module, plesase install warp-ctc from [here](https://github.com/baidu-research/warp-ctc). 79 | 80 | The quick version: 81 | ~~~~ 82 | git clone https://github.com/SeanNaren/warp-ctc.git 83 | cd warp-ctc 84 | mkdir build; cd build 85 | cmake .. 86 | make 87 | cd ../pytorch_binding 88 | python setup.py install 89 | export WARP_CTC_PATH=/home/xxx/warp-ctc/build 90 | ~~~~ 91 | 92 | ### Acknowledgement 93 | Some portion of the code were adapted from the [fairseq](https://github.com/pytorch/fairseq) repo. 94 | 95 | 96 | -------------------------------------------------------------------------------- /Mult_BERT/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/__pycache__/create_dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/__pycache__/create_dataset.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/__pycache__/encoders.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/__pycache__/encoders.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from datetime import datetime 4 | from collections import defaultdict 5 | from datetime import datetime 6 | from pathlib import Path 7 | import pprint 8 | from torch import optim 9 | import torch.nn as nn 10 | 11 | # path to a pretrained word embedding file 12 | word_emb_path = '/home/yingting/Glove/glove.840B.300d.txt' 13 | assert(word_emb_path is not None) 14 | 15 | 16 | username = Path.home().name 17 | project_dir = Path(__file__).resolve().parent 18 | sdk_dir = project_dir.joinpath('CMU-MultimodalSDK') 19 | data_dir = project_dir.joinpath('datasets') 20 | data_dict = {'mosi': data_dir.joinpath('MOSI'), 'mosei': data_dir.joinpath( 21 | 'MOSEI'), 'ur_funny': data_dir.joinpath('UR_FUNNY')} 22 | optimizer_dict = {'RMSprop': optim.RMSprop, 'Adam': optim.Adam} 23 | activation_dict = {'elu': nn.ELU, "hardshrink": nn.Hardshrink, "hardtanh": nn.Hardtanh, 24 | "leakyrelu": nn.LeakyReLU, "prelu": nn.PReLU, "relu": nn.ReLU, "rrelu": nn.RReLU, 25 | "tanh": nn.Tanh} 26 | 27 | 28 | def str2bool(v): 29 | """string to boolean""" 30 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 31 | return True 32 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 33 | return False 34 | else: 35 | raise argparse.ArgumentTypeError('Boolean value expected.') 36 | 37 | 38 | class Config(object): 39 | def __init__(self, data, mode='train'): 40 | """Configuration Class: set kwargs as class attributes with setattr""" 41 | # if kwargs is not None: 42 | # for key, value in kwargs.items(): 43 | # if key == 'optimizer': 44 | # value = optimizer_dict[value] 45 | # if key == 'activation': 46 | # value = activation_dict[value] 47 | # setattr(self, key, value) 48 | 49 | # Dataset directory: ex) ./datasets/cornell/ 50 | self.dataset_dir = data_dict[data.lower()] 51 | self.sdk_dir = sdk_dir 52 | self.mode = mode 53 | # Glove path 54 | self.word_emb_path = word_emb_path 55 | 56 | # Data Split ex) 'train', 'valid', 'test' 57 | # self.data_dir = self.dataset_dir.joinpath(self.mode) 58 | self.data_dir = self.dataset_dir 59 | 60 | def __str__(self): 61 | """Pretty-print configurations in alphabetical order""" 62 | config_str = 'Configurations\n' 63 | config_str += pprint.pformat(self.__dict__) 64 | return config_str 65 | 66 | 67 | def get_config(dataset='mosi', mode='train', batch_size=32, use_bert=False): 68 | config = Config(data=dataset, mode=mode) 69 | 70 | config.dataset = dataset 71 | config.batch_size = batch_size 72 | config.use_bert = use_bert 73 | 74 | # if dataset == "mosi": 75 | # config.num_classes = 1 76 | # elif dataset == "mosei": 77 | # config.num_classes = 1 78 | # elif dataset == "ur_funny": 79 | # config.num_classes = 2 80 | # else: 81 | # print("No dataset mentioned") 82 | # exit() 83 | 84 | # Namespace => Dictionary 85 | # kwargs = vars(kwargs) 86 | # kwargs.update(optional_kwargs) 87 | 88 | return config 89 | -------------------------------------------------------------------------------- /Mult_BERT/data_loader.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | from tqdm import tqdm_notebook 4 | from collections import defaultdict 5 | 6 | import torch 7 | import torch.nn as nn 8 | from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence 9 | from torch.utils.data import DataLoader, Dataset 10 | # from transformers import * 11 | from transformers import BertTokenizer 12 | 13 | from create_dataset import MOSI, MOSEI, UR_FUNNY, PAD, UNK 14 | 15 | bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') 16 | 17 | 18 | class MSADataset(Dataset): 19 | def __init__(self, config): 20 | self.config = config 21 | 22 | ## Fetch dataset 23 | if "mosi" in str(config.data_dir).lower(): 24 | dataset = MOSI(config) 25 | elif "mosei" in str(config.data_dir).lower(): 26 | dataset = MOSEI(config) 27 | elif "ur_funny" in str(config.data_dir).lower(): 28 | dataset = UR_FUNNY(config) 29 | else: 30 | print("Dataset not defined correctly") 31 | exit() 32 | 33 | self.data, self.word2id, self.pretrained_emb = dataset.get_data(config.mode) 34 | self.len = len(self.data) 35 | 36 | config.visual_size = self.data[0][0][1].shape[1] 37 | config.acoustic_size = self.data[0][0][2].shape[1] 38 | 39 | config.word2id = self.word2id 40 | config.pretrained_emb = self.pretrained_emb 41 | 42 | @property 43 | def lav_dim(self): 44 | l_dim = 768 if self.config.use_bert else 300 45 | return l_dim, self.data[0][0][2].shape[1], self.data[0][0][1].shape[1] 46 | 47 | @property 48 | def lav_len(self): 49 | # keep this part dummy for code compatibility. lengths are only used in CTC module 50 | return 0, 0, 0 51 | 52 | def __getitem__(self, index): 53 | return self.data[index] 54 | 55 | def __len__(self): 56 | return self.len 57 | 58 | 59 | def get_loader(hp, config, shuffle=True): 60 | """Load DataLoader of given DialogDataset""" 61 | 62 | dataset = MSADataset(config) 63 | 64 | print(config.mode) 65 | config.data_len = len(dataset) 66 | 67 | config.lav_dim = dataset.lav_dim 68 | config.lav_len = dataset.lav_len 69 | 70 | if config.mode == 'train': 71 | hp.n_train = len(dataset) 72 | elif config.mode == 'valid': 73 | hp.n_valid = len(dataset) 74 | elif config.mode == 'test': 75 | hp.n_test = len(dataset) 76 | 77 | def collate_fn(batch): 78 | ''' 79 | Collate functions assume batch = [Dataset[i] for i in index_set] 80 | ''' 81 | # for later use we sort the batch in descending order of length 82 | batch = sorted(batch, key=lambda x: x[0][0].shape[0], reverse=True) 83 | 84 | # get the data out of the batch - use pad sequence util functions from PyTorch to pad things 85 | 86 | 87 | labels = torch.cat([torch.from_numpy(sample[1]) for sample in batch], dim=0) 88 | 89 | # MOSEI sentiment labels locate in the first column of sentiment matrix 90 | if labels.size(1) == 7: 91 | labels = labels[:,0][:,None] 92 | 93 | sentences = pad_sequence([torch.LongTensor(sample[0][0]) for sample in batch], padding_value=PAD) 94 | visual = pad_sequence([torch.FloatTensor(sample[0][1]) for sample in batch]) 95 | acoustic = pad_sequence([torch.FloatTensor(sample[0][2]) for sample in batch]) 96 | 97 | 98 | ## BERT-based features input prep 99 | 100 | SENT_LEN = sentences.size(0) 101 | # Create bert indices using tokenizer 102 | 103 | bert_details = [] 104 | for sample in batch: 105 | text = " ".join(sample[0][3]) 106 | encoded_bert_sent = bert_tokenizer.encode_plus( 107 | text, max_length=SENT_LEN+2, add_special_tokens=True, pad_to_max_length=True) 108 | bert_details.append(encoded_bert_sent) 109 | 110 | # Bert things are batch_first 111 | bert_sentences = torch.LongTensor([sample["input_ids"] for sample in bert_details]) 112 | bert_sentence_types = torch.LongTensor([sample["token_type_ids"] for sample in bert_details]) 113 | bert_sentence_att_mask = torch.LongTensor([sample["attention_mask"] for sample in bert_details]) 114 | 115 | # lengths are useful later in using RNNs 116 | lengths = torch.LongTensor([sample[0][0].shape[0] for sample in batch]) 117 | 118 | return sentences, visual, acoustic, labels, lengths, bert_sentences, bert_sentence_types, bert_sentence_att_mask 119 | 120 | 121 | data_loader = DataLoader( 122 | dataset=dataset, 123 | batch_size=config.batch_size, 124 | shuffle=shuffle, 125 | collate_fn=collate_fn) 126 | 127 | return data_loader -------------------------------------------------------------------------------- /Mult_BERT/imgs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/imgs/architecture.png -------------------------------------------------------------------------------- /Mult_BERT/imgs/cm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/imgs/cm.png -------------------------------------------------------------------------------- /Mult_BERT/modules/__pycache__/multihead_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/modules/__pycache__/multihead_attention.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/modules/__pycache__/position_embedding.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/modules/__pycache__/position_embedding.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/modules/__pycache__/transformer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/modules/__pycache__/transformer.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/modules/position_embedding.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | # Code adapted from the fairseq repo. 7 | 8 | def make_positions(tensor, padding_idx, left_pad): 9 | """Replace non-padding symbols with their position numbers. 10 | Position numbers begin at padding_idx+1. 11 | Padding symbols are ignored, but it is necessary to specify whether padding 12 | is added on the left side (left_pad=True) or right side (left_pad=False). 13 | """ 14 | max_pos = padding_idx + 1 + tensor.size(1) 15 | device = tensor.get_device() 16 | buf_name = f'range_buf_{device}' 17 | if not hasattr(make_positions, buf_name): 18 | setattr(make_positions, buf_name, tensor.new()) 19 | setattr(make_positions, buf_name, getattr(make_positions, buf_name).type_as(tensor)) 20 | if getattr(make_positions, buf_name).numel() < max_pos: 21 | torch.arange(padding_idx + 1, max_pos, out=getattr(make_positions, buf_name)) 22 | mask = tensor.ne(padding_idx) 23 | positions = getattr(make_positions, buf_name)[:tensor.size(1)].expand_as(tensor) 24 | if left_pad: 25 | positions = positions - mask.size(1) + mask.long().sum(dim=1).unsqueeze(1) 26 | new_tensor = tensor.clone() 27 | return new_tensor.masked_scatter_(mask, positions[mask]).long() 28 | 29 | 30 | class SinusoidalPositionalEmbedding(nn.Module): 31 | """This module produces sinusoidal positional embeddings of any length. 32 | Padding symbols are ignored, but it is necessary to specify whether padding 33 | is added on the left side (left_pad=True) or right side (left_pad=False). 34 | """ 35 | 36 | def __init__(self, embedding_dim, padding_idx=0, left_pad=0, init_size=128): 37 | super().__init__() 38 | self.embedding_dim = embedding_dim 39 | self.padding_idx = padding_idx 40 | self.left_pad = left_pad 41 | self.weights = dict() # device --> actual weight; due to nn.DataParallel :-( 42 | self.register_buffer('_float_tensor', torch.FloatTensor(1)) 43 | 44 | @staticmethod 45 | def get_embedding(num_embeddings, embedding_dim, padding_idx=None): 46 | """Build sinusoidal embeddings. 47 | This matches the implementation in tensor2tensor, but differs slightly 48 | from the description in Section 3.5 of "Attention Is All You Need". 49 | """ 50 | half_dim = embedding_dim // 2 51 | emb = math.log(10000) / (half_dim - 1) 52 | emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb) 53 | emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) 54 | emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1) 55 | if embedding_dim % 2 == 1: 56 | # zero pad 57 | emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) 58 | if padding_idx is not None: 59 | emb[padding_idx, :] = 0 60 | return emb 61 | 62 | def forward(self, input): 63 | """Input is expected to be of size [bsz x seqlen].""" 64 | bsz, seq_len = input.size() 65 | max_pos = self.padding_idx + 1 + seq_len 66 | device = input.get_device() 67 | if device not in self.weights or max_pos > self.weights[device].size(0): 68 | # recompute/expand embeddings if needed 69 | self.weights[device] = SinusoidalPositionalEmbedding.get_embedding( 70 | max_pos, 71 | self.embedding_dim, 72 | self.padding_idx, 73 | ) 74 | self.weights[device] = self.weights[device].type_as(self._float_tensor) 75 | positions = make_positions(input, self.padding_idx, self.left_pad) 76 | return self.weights[device].index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach() 77 | 78 | def max_positions(self): 79 | """Maximum number of supported positions.""" 80 | return int(1e5) # an arbitrary large number -------------------------------------------------------------------------------- /Mult_BERT/requiorements.txt: -------------------------------------------------------------------------------- 1 | conda create --name mult python==3.7 2 | conda activate mult 3 | 4 | pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html 5 | pip install torch==1.8.0 scipy scikit-learn 6 | pip install transformers==4.12.5 7 | pip install h5py validators tqdm numpy argparse requests colorama ipywidgets 8 | 9 | -------------------------------------------------------------------------------- /Mult_BERT/run.sh: -------------------------------------------------------------------------------- 1 | ##TRAIN 2 | CUDA_VISIBLE_DEVICES=0,1 python main.py \ 3 | --dataset mosi \ 4 | --batch_size 128 \ 5 | --lr 1e-4 \ 6 | --optim Adam \ 7 | --num_heads 10 \ 8 | --embed_dropout 0.3 \ 9 | --attn_dropout 0.2 \ 10 | --out_dropout 0.1 \ 11 | --clip 0.8 \ 12 | --num_epochs 100 \ 13 | --aligned \ 14 | --nlevels 4 \ 15 | --use_bert \ 16 | --train_method missing \ 17 | --train_changed_modal language \ 18 | --train_changed_pct 0.3 19 | ##TEST 20 | # CUDA_VISIBLE_DEVICES=0,1 python main.py \ 21 | # --dataset mosi \ 22 | # --batch_size 128 \ 23 | # --lr 1e-4 \ 24 | # --optim Adam \ 25 | # --num_heads 10 \ 26 | # --embed_dropout 0.3 \ 27 | # --attn_dropout 0.2 \ 28 | # --out_dropout 0.1 \ 29 | # --clip 0.8 \ 30 | # --num_epochs 100 \ 31 | # --aligned \ 32 | # --nlevels 4 \ 33 | # --use_bert \ 34 | # --train_method missing \ 35 | # --train_changed_modal language \ 36 | # --train_changed_pct 0.3 \ 37 | # --test_method missing \ 38 | # --test_changed_modal language \ 39 | # --test_changed_pct 0 --is_test 40 | -------------------------------------------------------------------------------- /Mult_BERT/src/README.md: -------------------------------------------------------------------------------- 1 | # MulT model and Multimodal Sentiment Analysis Benchmarks 2 | 3 | This directory contains the model architecture for Multimodal Transformer (MulT) as well as the three major multimodal sentiment analysis benchmarks we used in the paper. All datasets should be put in the `../data` folder (if it does not exist, use `mkdir` to create one). Depending on the dataset, we may have different sets of evaluation metrics (see `eval_metrics.py`). 4 | -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/ctc.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/ctc.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/dataset.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/eval_metrics.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/eval_metrics.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/src/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /Mult_BERT/src/ctc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | class CTCModule(nn.Module): 6 | def __init__(self, in_dim, out_seq_len): 7 | ''' 8 | This module is performing alignment from A (e.g., audio) to B (e.g., text). 9 | :param in_dim: Dimension for input modality A 10 | :param out_seq_len: Sequence length for output modality B 11 | ''' 12 | super(CTCModule, self).__init__() 13 | # Use LSTM for predicting the position from A to B 14 | self.pred_output_position_inclu_blank = nn.LSTM(in_dim, out_seq_len+1, num_layers=2, batch_first=True) # 1 denoting blank 15 | 16 | self.out_seq_len = out_seq_len 17 | 18 | self.softmax = nn.Softmax(dim=2) 19 | def forward(self, x): 20 | ''' 21 | :input x: Input with shape [batch_size x in_seq_len x in_dim] 22 | ''' 23 | # NOTE that the index 0 refers to blank. 24 | pred_output_position_inclu_blank, _ = self.pred_output_position_inclu_blank(x) 25 | 26 | prob_pred_output_position_inclu_blank = self.softmax(pred_output_position_inclu_blank) # batch_size x in_seq_len x out_seq_len+1 27 | prob_pred_output_position = prob_pred_output_position_inclu_blank[:, :, 1:] # batch_size x in_seq_len x out_seq_len 28 | prob_pred_output_position = prob_pred_output_position.transpose(1,2) # batch_size x out_seq_len x in_seq_len 29 | pseudo_aligned_out = torch.bmm(prob_pred_output_position, x) # batch_size x out_seq_len x in_dim 30 | 31 | # pseudo_aligned_out is regarded as the aligned A (w.r.t B) 32 | return pseudo_aligned_out, (pred_output_position_inclu_blank) 33 | 34 | 35 | -------------------------------------------------------------------------------- /Mult_BERT/src/dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torch.utils.data.dataset import Dataset 3 | import pickle 4 | import os 5 | from scipy import signal 6 | import torch 7 | 8 | if torch.cuda.is_available(): 9 | torch.set_default_tensor_type('torch.cuda.FloatTensor') 10 | else: 11 | torch.set_default_tensor_type('torch.FloatTensor') 12 | 13 | ############################################################################################ 14 | # This file provides basic processing script for the multimodal datasets we use. For other 15 | # datasets, small modifications may be needed (depending on the type of the data, etc.) 16 | ############################################################################################ 17 | 18 | 19 | class Multimodal_Datasets(Dataset): 20 | def __init__(self, dataset_path, data='mosei_senti', split_type='train', if_align=False): 21 | super(Multimodal_Datasets, self).__init__() 22 | dataset_path = os.path.join(dataset_path, data+'_data.pkl' if if_align else data+'_data_noalign.pkl' ) 23 | dataset = pickle.load(open(dataset_path, 'rb')) 24 | 25 | # These are torch tensors 26 | self.vision = torch.tensor(dataset[split_type]['vision'].astype(np.float32)).cpu().detach() 27 | self.text = torch.tensor(dataset[split_type]['text'].astype(np.float32)).cpu().detach() 28 | self.audio = dataset[split_type]['audio'].astype(np.float32) 29 | self.audio[self.audio == -np.inf] = 0 30 | self.audio = torch.tensor(self.audio).cpu().detach() 31 | self.labels = torch.tensor(dataset[split_type]['labels'].astype(np.float32)).cpu().detach() 32 | 33 | # Note: this is STILL an numpy array 34 | self.meta = dataset[split_type]['id'] if 'id' in dataset[split_type].keys() else None 35 | 36 | self.data = data 37 | 38 | self.n_modalities = 3 # vision/ text/ audio 39 | 40 | def get_n_modalities(self): 41 | return self.n_modalities 42 | def get_seq_len(self): 43 | return self.text.shape[1], self.audio.shape[1], self.vision.shape[1] 44 | def get_dim(self): 45 | return self.text.shape[2], self.audio.shape[2], self.vision.shape[2] 46 | def get_lbl_info(self): 47 | # return number_of_labels, label_dim 48 | return self.labels.shape[1], self.labels.shape[2] 49 | def __len__(self): 50 | return len(self.labels) 51 | def __getitem__(self, index): 52 | X = (index, self.text[index], self.audio[index], self.vision[index]) 53 | Y = self.labels[index] 54 | META = (0,0,0) if self.meta is None else (self.meta[index][0], self.meta[index][1], self.meta[index][2]) 55 | if self.data == 'mosi': 56 | META = (self.meta[index][0].decode('UTF-8'), self.meta[index][1].decode('UTF-8'), self.meta[index][2].decode('UTF-8')) 57 | if self.data == 'iemocap': 58 | Y = torch.argmax(Y, dim=-1) 59 | return X, Y, META 60 | 61 | -------------------------------------------------------------------------------- /Mult_BERT/src/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | from src.dataset import Multimodal_Datasets 4 | 5 | from torch.utils.data import DataLoader 6 | import numpy as np 7 | 8 | def get_data(args, dataset, split='train'): 9 | alignment = 'a' if args.aligned else 'na' 10 | data_path = os.path.join(args.data_path, dataset) + f'_{split}_{alignment}.dt' 11 | if not os.path.exists(data_path): 12 | print(f" - Creating new {split} data") 13 | data = Multimodal_Datasets(args.data_path, dataset, split, args.aligned) 14 | torch.save(data, data_path) 15 | else: 16 | print(f" - Found cached {split} data") 17 | data = torch.load(data_path) 18 | return data 19 | 20 | 21 | def save_load_name(args, name=''): 22 | if args.aligned: 23 | name = name if len(name) > 0 else 'aligned_model' 24 | elif not args.aligned: 25 | name = name if len(name) > 0 else 'nonaligned_model' 26 | 27 | return name + '_' + args.dataset 28 | 29 | 30 | def save_model(args, model, name=''): 31 | name = save_load_name(args, name) 32 | 33 | if args.train_method=="missing": 34 | save_mode = f'0' 35 | elif args.train_method=="g_noise": 36 | save_mode = f'N' 37 | elif args.train_method=="hybird": 38 | save_mode = f'H' 39 | else: 40 | raise 41 | save_dir = f'checkpoints/{args.dataset}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={save_mode}' 42 | if not os.path.exists(save_dir): 43 | os.makedirs(save_dir) 44 | 45 | print("-------------------------save_dir") 46 | print(save_dir) 47 | torch.save(model, f'{save_dir}/{name}.pt') 48 | 49 | 50 | def load_model(args, name=''): 51 | name = save_load_name(args, name) 52 | 53 | if args.train_method=="missing": 54 | save_mode = f'0' 55 | elif args.train_method=="g_noise": 56 | save_mode = f'N' 57 | elif args.train_method=="hybird": 58 | save_mode = f'H' 59 | else: 60 | raise 61 | save_dir = f'checkpoints/{args.dataset}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={save_mode}' 62 | if not os.path.exists(save_dir): 63 | os.makedirs(save_dir) 64 | 65 | print("-------------------------save_dir") 66 | print(save_dir) 67 | model = torch.load(f'{save_dir}/{name}.pt') 68 | return model 69 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: xenial 3 | language: python 4 | python: 5 | - "3.6" 6 | install: 7 | ## Start conda installation 8 | # https://conda.io/docs/user-guide/tasks/use-conda-with-travis-ci.html 9 | - sudo apt-get update 10 | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 11 | - bash miniconda.sh -b -p $HOME/miniconda 12 | - export PATH="$HOME/miniconda/bin:$PATH" 13 | - hash -r 14 | - conda config --set always_yes yes --set changeps1 no 15 | - conda update -q conda 16 | # Useful for debugging any issues with conda 17 | - conda info -a 18 | ## End conda installation 19 | 20 | - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pytest-flakes 21 | - source activate test-environment 22 | - conda install pytorch torchvision -c pytorch 23 | - mkdir build && cd build && cmake .. && make && sudo make install && cd .. && sudo ldconfig 24 | - cd pytorch_binding && python setup.py install && cd .. && sudo ldconfig 25 | 26 | script: 27 | - cd pytorch_binding && py.test tests && cd .. 28 | - cd pytorch_binding && py.test --flakes --ignore=warpctc_pytorch/_warp_ctc && cd .. 29 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | IF (APPLE) 2 | cmake_minimum_required(VERSION 3.4) 3 | ELSE() 4 | cmake_minimum_required(VERSION 2.8) 5 | ENDIF() 6 | 7 | project(ctc_release) 8 | 9 | IF (NOT APPLE) 10 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2") 11 | ENDIF() 12 | 13 | IF (APPLE) 14 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -O2") 15 | add_definitions(-DAPPLE) 16 | ENDIF() 17 | 18 | include_directories(include) 19 | 20 | FIND_PACKAGE(CUDA 6.5) 21 | MESSAGE(STATUS "cuda found ${CUDA_FOUND}") 22 | 23 | option(WITH_GPU "compile warp-ctc with cuda." ${CUDA_FOUND}) 24 | option(WITH_OMP "compile warp-ctc with openmp." ON) 25 | 26 | if(NOT WITH_OMP) 27 | add_definitions(-DCTC_DISABLE_OMP) 28 | endif() 29 | if (WITH_OMP) 30 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") 31 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fopenmp") 32 | endif() 33 | 34 | # need to be at least 30 or __shfl_down in reduce wont compile 35 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_30,code=sm_30 -O2") 36 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_35,code=sm_35") 37 | 38 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_50,code=sm_50") 39 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_52,code=sm_52") 40 | IF(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5) 41 | SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -D_MWAITXINTRIN_H_INCLUDED -D_FORCE_INLINES") 42 | ENDIF() 43 | 44 | IF (CUDA_VERSION GREATER 7.6) 45 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_60,code=sm_60") 46 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_61,code=sm_61") 47 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_62,code=sm_62") 48 | ENDIF() 49 | 50 | IF (CUDA_VERSION GREATER 8.9) 51 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -gencode arch=compute_70,code=sm_70") 52 | ENDIF() 53 | 54 | if (NOT APPLE) 55 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --std=c++14") 56 | set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}") 57 | ENDIF() 58 | 59 | IF (APPLE) 60 | EXEC_PROGRAM(uname ARGS -v OUTPUT_VARIABLE DARWIN_VERSION) 61 | STRING(REGEX MATCH "[0-9]+" DARWIN_VERSION ${DARWIN_VERSION}) 62 | MESSAGE(STATUS "DARWIN_VERSION=${DARWIN_VERSION}") 63 | 64 | #for el capitain have to use rpath 65 | 66 | IF (DARWIN_VERSION LESS 15) 67 | set(CMAKE_SKIP_RPATH TRUE) 68 | ENDIF () 69 | 70 | ELSE() 71 | #always skip for linux 72 | set(CMAKE_SKIP_RPATH TRUE) 73 | ENDIF() 74 | 75 | 76 | IF (WITH_GPU) 77 | 78 | MESSAGE(STATUS "Building shared library with GPU support") 79 | 80 | CUDA_ADD_LIBRARY(warpctc SHARED src/ctc_entrypoint.cu src/reduce.cu) 81 | IF (!Torch_FOUND) 82 | TARGET_LINK_LIBRARIES(warpctc ${CUDA_curand_LIBRARY}) 83 | ENDIF() 84 | 85 | add_executable(test_cpu tests/test_cpu.cpp tests/random.cpp ) 86 | TARGET_LINK_LIBRARIES(test_cpu warpctc) 87 | SET_TARGET_PROPERTIES(test_cpu PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14") 88 | 89 | cuda_add_executable(test_gpu tests/test_gpu.cu tests/random.cpp ) 90 | TARGET_LINK_LIBRARIES(test_gpu warpctc ${CUDA_curand_LIBRARY}) 91 | SET_TARGET_PROPERTIES(test_gpu PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14") 92 | 93 | INSTALL(TARGETS warpctc 94 | RUNTIME DESTINATION "bin" 95 | LIBRARY DESTINATION "lib" 96 | ARCHIVE DESTINATION "lib") 97 | 98 | INSTALL(FILES include/ctc.h DESTINATION "include") 99 | ELSE() 100 | MESSAGE(STATUS "Building shared library with no GPU support") 101 | 102 | if (NOT APPLE) 103 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -O2") 104 | ENDIF() 105 | 106 | ADD_LIBRARY(warpctc SHARED src/ctc_entrypoint.cpp) 107 | 108 | add_executable(test_cpu tests/test_cpu.cpp tests/random.cpp ) 109 | TARGET_LINK_LIBRARIES(test_cpu warpctc) 110 | SET_TARGET_PROPERTIES(test_cpu PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14") 111 | 112 | INSTALL(TARGETS warpctc 113 | RUNTIME DESTINATION "bin" 114 | LIBRARY DESTINATION "lib" 115 | ARCHIVE DESTINATION "lib") 116 | 117 | INSTALL(FILES include/ctc.h DESTINATION "include") 118 | ENDIF() 119 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/README.md: -------------------------------------------------------------------------------- 1 | # PyTorch bindings for Warp-ctc 2 | 3 | [![Build Status](https://travis-ci.org/SeanNaren/warp-ctc.svg?branch=pytorch_bindings)](https://travis-ci.org/SeanNaren/warp-ctc) 4 | 5 | This is an extension onto the original repo found [here](https://github.com/baidu-research/warp-ctc). 6 | 7 | ## Installation 8 | 9 | Install [PyTorch](https://github.com/pytorch/pytorch#installation) v0.4. 10 | 11 | `WARP_CTC_PATH` should be set to the location of a built WarpCTC 12 | (i.e. `libwarpctc.so`). This defaults to `../build`, so from within a 13 | new warp-ctc clone you could build WarpCTC like this: 14 | 15 | ```bash 16 | git clone https://github.com/SeanNaren/warp-ctc.git 17 | cd warp-ctc 18 | mkdir build; cd build 19 | cmake .. 20 | make 21 | ``` 22 | 23 | Now install the bindings: 24 | ```bash 25 | cd pytorch_binding 26 | python setup.py install 27 | ``` 28 | 29 | If you try the above and get a dlopen error on OSX with anaconda3 (as recommended by pytorch): 30 | ```bash 31 | cd ../pytorch_binding 32 | python setup.py install 33 | cd ../build 34 | cp libwarpctc.dylib /Users/$WHOAMI/anaconda3/lib 35 | ``` 36 | This will resolve the library not loaded error. This can be easily modified to work with other python installs if needed. 37 | 38 | Example to use the bindings below. 39 | 40 | ```python 41 | import torch 42 | from warpctc_pytorch import CTCLoss 43 | ctc_loss = CTCLoss() 44 | # expected shape of seqLength x batchSize x alphabet_size 45 | probs = torch.FloatTensor([[[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]]]).transpose(0, 1).contiguous() 46 | labels = torch.IntTensor([1, 2]) 47 | label_sizes = torch.IntTensor([2]) 48 | probs_sizes = torch.IntTensor([2]) 49 | probs.requires_grad_(True) # tells autograd to compute gradients for probs 50 | cost = ctc_loss(probs, labels, probs_sizes, label_sizes) 51 | cost.backward() 52 | ``` 53 | 54 | ## Documentation 55 | 56 | ``` 57 | CTCLoss(size_average=False, length_average=False) 58 | # size_average (bool): normalize the loss by the batch size (default: False) 59 | # length_average (bool): normalize the loss by the total number of frames in the batch. If True, supersedes size_average (default: False) 60 | 61 | forward(acts, labels, act_lens, label_lens) 62 | # acts: Tensor of (seqLength x batch x outputDim) containing output activations from network (before softmax) 63 | # labels: 1 dimensional Tensor containing all the targets of the batch in one large sequence 64 | # act_lens: Tensor of size (batch) containing size of each output sequence from the network 65 | # label_lens: Tensor of (batch) containing label length of each example 66 | ``` -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeCCompiler.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_C_COMPILER "/usr/bin/cc") 2 | set(CMAKE_C_COMPILER_ARG1 "") 3 | set(CMAKE_C_COMPILER_ID "GNU") 4 | set(CMAKE_C_COMPILER_VERSION "7.5.0") 5 | set(CMAKE_C_COMPILER_VERSION_INTERNAL "") 6 | set(CMAKE_C_COMPILER_WRAPPER "") 7 | set(CMAKE_C_STANDARD_COMPUTED_DEFAULT "11") 8 | set(CMAKE_C_COMPILE_FEATURES "c_std_90;c_function_prototypes;c_std_99;c_restrict;c_variadic_macros;c_std_11;c_static_assert") 9 | set(CMAKE_C90_COMPILE_FEATURES "c_std_90;c_function_prototypes") 10 | set(CMAKE_C99_COMPILE_FEATURES "c_std_99;c_restrict;c_variadic_macros") 11 | set(CMAKE_C11_COMPILE_FEATURES "c_std_11;c_static_assert") 12 | 13 | set(CMAKE_C_PLATFORM_ID "Linux") 14 | set(CMAKE_C_SIMULATE_ID "") 15 | set(CMAKE_C_SIMULATE_VERSION "") 16 | 17 | 18 | 19 | set(CMAKE_AR "/usr/bin/ar") 20 | set(CMAKE_C_COMPILER_AR "/usr/bin/gcc-ar-7") 21 | set(CMAKE_RANLIB "/usr/bin/ranlib") 22 | set(CMAKE_C_COMPILER_RANLIB "/usr/bin/gcc-ranlib-7") 23 | set(CMAKE_LINKER "/usr/bin/ld") 24 | set(CMAKE_COMPILER_IS_GNUCC 1) 25 | set(CMAKE_C_COMPILER_LOADED 1) 26 | set(CMAKE_C_COMPILER_WORKS TRUE) 27 | set(CMAKE_C_ABI_COMPILED TRUE) 28 | set(CMAKE_COMPILER_IS_MINGW ) 29 | set(CMAKE_COMPILER_IS_CYGWIN ) 30 | if(CMAKE_COMPILER_IS_CYGWIN) 31 | set(CYGWIN 1) 32 | set(UNIX 1) 33 | endif() 34 | 35 | set(CMAKE_C_COMPILER_ENV_VAR "CC") 36 | 37 | if(CMAKE_COMPILER_IS_MINGW) 38 | set(MINGW 1) 39 | endif() 40 | set(CMAKE_C_COMPILER_ID_RUN 1) 41 | set(CMAKE_C_SOURCE_FILE_EXTENSIONS c;m) 42 | set(CMAKE_C_IGNORE_EXTENSIONS h;H;o;O;obj;OBJ;def;DEF;rc;RC) 43 | set(CMAKE_C_LINKER_PREFERENCE 10) 44 | 45 | # Save compiler ABI information. 46 | set(CMAKE_C_SIZEOF_DATA_PTR "8") 47 | set(CMAKE_C_COMPILER_ABI "ELF") 48 | set(CMAKE_C_LIBRARY_ARCHITECTURE "x86_64-linux-gnu") 49 | 50 | if(CMAKE_C_SIZEOF_DATA_PTR) 51 | set(CMAKE_SIZEOF_VOID_P "${CMAKE_C_SIZEOF_DATA_PTR}") 52 | endif() 53 | 54 | if(CMAKE_C_COMPILER_ABI) 55 | set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_C_COMPILER_ABI}") 56 | endif() 57 | 58 | if(CMAKE_C_LIBRARY_ARCHITECTURE) 59 | set(CMAKE_LIBRARY_ARCHITECTURE "x86_64-linux-gnu") 60 | endif() 61 | 62 | set(CMAKE_C_CL_SHOWINCLUDES_PREFIX "") 63 | if(CMAKE_C_CL_SHOWINCLUDES_PREFIX) 64 | set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_C_CL_SHOWINCLUDES_PREFIX}") 65 | endif() 66 | 67 | 68 | 69 | 70 | 71 | set(CMAKE_C_IMPLICIT_LINK_LIBRARIES "gcc;gcc_s;c;gcc;gcc_s") 72 | set(CMAKE_C_IMPLICIT_LINK_DIRECTORIES "/usr/lib/gcc/x86_64-linux-gnu/7;/usr/lib/x86_64-linux-gnu;/usr/lib;/lib/x86_64-linux-gnu;/lib") 73 | set(CMAKE_C_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "") 74 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeCXXCompiler.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_COMPILER "/usr/bin/c++") 2 | set(CMAKE_CXX_COMPILER_ARG1 "") 3 | set(CMAKE_CXX_COMPILER_ID "GNU") 4 | set(CMAKE_CXX_COMPILER_VERSION "7.5.0") 5 | set(CMAKE_CXX_COMPILER_VERSION_INTERNAL "") 6 | set(CMAKE_CXX_COMPILER_WRAPPER "") 7 | set(CMAKE_CXX_STANDARD_COMPUTED_DEFAULT "14") 8 | set(CMAKE_CXX_COMPILE_FEATURES "cxx_std_98;cxx_template_template_parameters;cxx_std_11;cxx_alias_templates;cxx_alignas;cxx_alignof;cxx_attributes;cxx_auto_type;cxx_constexpr;cxx_decltype;cxx_decltype_incomplete_return_types;cxx_default_function_template_args;cxx_defaulted_functions;cxx_defaulted_move_initializers;cxx_delegating_constructors;cxx_deleted_functions;cxx_enum_forward_declarations;cxx_explicit_conversions;cxx_extended_friend_declarations;cxx_extern_templates;cxx_final;cxx_func_identifier;cxx_generalized_initializers;cxx_inheriting_constructors;cxx_inline_namespaces;cxx_lambdas;cxx_local_type_template_args;cxx_long_long_type;cxx_noexcept;cxx_nonstatic_member_init;cxx_nullptr;cxx_override;cxx_range_for;cxx_raw_string_literals;cxx_reference_qualified_functions;cxx_right_angle_brackets;cxx_rvalue_references;cxx_sizeof_member;cxx_static_assert;cxx_strong_enums;cxx_thread_local;cxx_trailing_return_types;cxx_unicode_literals;cxx_uniform_initialization;cxx_unrestricted_unions;cxx_user_literals;cxx_variadic_macros;cxx_variadic_templates;cxx_std_14;cxx_aggregate_default_initializers;cxx_attribute_deprecated;cxx_binary_literals;cxx_contextual_conversions;cxx_decltype_auto;cxx_digit_separators;cxx_generic_lambdas;cxx_lambda_init_captures;cxx_relaxed_constexpr;cxx_return_type_deduction;cxx_variable_templates;cxx_std_17") 9 | set(CMAKE_CXX98_COMPILE_FEATURES "cxx_std_98;cxx_template_template_parameters") 10 | set(CMAKE_CXX11_COMPILE_FEATURES "cxx_std_11;cxx_alias_templates;cxx_alignas;cxx_alignof;cxx_attributes;cxx_auto_type;cxx_constexpr;cxx_decltype;cxx_decltype_incomplete_return_types;cxx_default_function_template_args;cxx_defaulted_functions;cxx_defaulted_move_initializers;cxx_delegating_constructors;cxx_deleted_functions;cxx_enum_forward_declarations;cxx_explicit_conversions;cxx_extended_friend_declarations;cxx_extern_templates;cxx_final;cxx_func_identifier;cxx_generalized_initializers;cxx_inheriting_constructors;cxx_inline_namespaces;cxx_lambdas;cxx_local_type_template_args;cxx_long_long_type;cxx_noexcept;cxx_nonstatic_member_init;cxx_nullptr;cxx_override;cxx_range_for;cxx_raw_string_literals;cxx_reference_qualified_functions;cxx_right_angle_brackets;cxx_rvalue_references;cxx_sizeof_member;cxx_static_assert;cxx_strong_enums;cxx_thread_local;cxx_trailing_return_types;cxx_unicode_literals;cxx_uniform_initialization;cxx_unrestricted_unions;cxx_user_literals;cxx_variadic_macros;cxx_variadic_templates") 11 | set(CMAKE_CXX14_COMPILE_FEATURES "cxx_std_14;cxx_aggregate_default_initializers;cxx_attribute_deprecated;cxx_binary_literals;cxx_contextual_conversions;cxx_decltype_auto;cxx_digit_separators;cxx_generic_lambdas;cxx_lambda_init_captures;cxx_relaxed_constexpr;cxx_return_type_deduction;cxx_variable_templates") 12 | set(CMAKE_CXX17_COMPILE_FEATURES "cxx_std_17") 13 | 14 | set(CMAKE_CXX_PLATFORM_ID "Linux") 15 | set(CMAKE_CXX_SIMULATE_ID "") 16 | set(CMAKE_CXX_SIMULATE_VERSION "") 17 | 18 | 19 | 20 | set(CMAKE_AR "/usr/bin/ar") 21 | set(CMAKE_CXX_COMPILER_AR "/usr/bin/gcc-ar-7") 22 | set(CMAKE_RANLIB "/usr/bin/ranlib") 23 | set(CMAKE_CXX_COMPILER_RANLIB "/usr/bin/gcc-ranlib-7") 24 | set(CMAKE_LINKER "/usr/bin/ld") 25 | set(CMAKE_COMPILER_IS_GNUCXX 1) 26 | set(CMAKE_CXX_COMPILER_LOADED 1) 27 | set(CMAKE_CXX_COMPILER_WORKS TRUE) 28 | set(CMAKE_CXX_ABI_COMPILED TRUE) 29 | set(CMAKE_COMPILER_IS_MINGW ) 30 | set(CMAKE_COMPILER_IS_CYGWIN ) 31 | if(CMAKE_COMPILER_IS_CYGWIN) 32 | set(CYGWIN 1) 33 | set(UNIX 1) 34 | endif() 35 | 36 | set(CMAKE_CXX_COMPILER_ENV_VAR "CXX") 37 | 38 | if(CMAKE_COMPILER_IS_MINGW) 39 | set(MINGW 1) 40 | endif() 41 | set(CMAKE_CXX_COMPILER_ID_RUN 1) 42 | set(CMAKE_CXX_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC) 43 | set(CMAKE_CXX_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;mm;CPP) 44 | set(CMAKE_CXX_LINKER_PREFERENCE 30) 45 | set(CMAKE_CXX_LINKER_PREFERENCE_PROPAGATES 1) 46 | 47 | # Save compiler ABI information. 48 | set(CMAKE_CXX_SIZEOF_DATA_PTR "8") 49 | set(CMAKE_CXX_COMPILER_ABI "ELF") 50 | set(CMAKE_CXX_LIBRARY_ARCHITECTURE "x86_64-linux-gnu") 51 | 52 | if(CMAKE_CXX_SIZEOF_DATA_PTR) 53 | set(CMAKE_SIZEOF_VOID_P "${CMAKE_CXX_SIZEOF_DATA_PTR}") 54 | endif() 55 | 56 | if(CMAKE_CXX_COMPILER_ABI) 57 | set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_CXX_COMPILER_ABI}") 58 | endif() 59 | 60 | if(CMAKE_CXX_LIBRARY_ARCHITECTURE) 61 | set(CMAKE_LIBRARY_ARCHITECTURE "x86_64-linux-gnu") 62 | endif() 63 | 64 | set(CMAKE_CXX_CL_SHOWINCLUDES_PREFIX "") 65 | if(CMAKE_CXX_CL_SHOWINCLUDES_PREFIX) 66 | set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_CXX_CL_SHOWINCLUDES_PREFIX}") 67 | endif() 68 | 69 | 70 | 71 | 72 | 73 | set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "stdc++;m;gcc_s;gcc;c;gcc_s;gcc") 74 | set(CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "/usr/lib/gcc/x86_64-linux-gnu/7;/usr/lib/x86_64-linux-gnu;/usr/lib;/lib/x86_64-linux-gnu;/lib") 75 | set(CMAKE_CXX_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "") 76 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeDetermineCompilerABI_C.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeDetermineCompilerABI_C.bin -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeDetermineCompilerABI_CXX.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeDetermineCompilerABI_CXX.bin -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CMakeSystem.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_HOST_SYSTEM "Linux-5.4.0-42-generic") 2 | set(CMAKE_HOST_SYSTEM_NAME "Linux") 3 | set(CMAKE_HOST_SYSTEM_VERSION "5.4.0-42-generic") 4 | set(CMAKE_HOST_SYSTEM_PROCESSOR "x86_64") 5 | 6 | 7 | 8 | set(CMAKE_SYSTEM "Linux-5.4.0-42-generic") 9 | set(CMAKE_SYSTEM_NAME "Linux") 10 | set(CMAKE_SYSTEM_VERSION "5.4.0-42-generic") 11 | set(CMAKE_SYSTEM_PROCESSOR "x86_64") 12 | 13 | set(CMAKE_CROSSCOMPILING "FALSE") 14 | 15 | set(CMAKE_SYSTEM_LOADED 1) 16 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CompilerIdC/a.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CompilerIdC/a.out -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CompilerIdCXX/a.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/3.10.2/CompilerIdCXX/a.out -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/CMakeDirectoryInformation.cmake: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | # Relative path conversion top directories. 5 | set(CMAKE_RELATIVE_PATH_TOP_SOURCE "/home/yingting/Multimodal-Transformer/warp-ctc") 6 | set(CMAKE_RELATIVE_PATH_TOP_BINARY "/home/yingting/Multimodal-Transformer/warp-ctc/build") 7 | 8 | # Force unix paths in dependencies. 9 | set(CMAKE_FORCE_UNIX_PATHS 1) 10 | 11 | 12 | # The C and CXX include file regular expressions for this directory. 13 | set(CMAKE_C_INCLUDE_REGEX_SCAN "^.*$") 14 | set(CMAKE_C_INCLUDE_REGEX_COMPLAIN "^$") 15 | set(CMAKE_CXX_INCLUDE_REGEX_SCAN ${CMAKE_C_INCLUDE_REGEX_SCAN}) 16 | set(CMAKE_CXX_INCLUDE_REGEX_COMPLAIN ${CMAKE_C_INCLUDE_REGEX_COMPLAIN}) 17 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/Makefile2: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | # Default target executed when no arguments are given to make. 5 | default_target: all 6 | 7 | .PHONY : default_target 8 | 9 | # The main recursive all target 10 | all: 11 | 12 | .PHONY : all 13 | 14 | # The main recursive preinstall target 15 | preinstall: 16 | 17 | .PHONY : preinstall 18 | 19 | #============================================================================= 20 | # Special targets provided by cmake. 21 | 22 | # Disable implicit rules so canonical targets will work. 23 | .SUFFIXES: 24 | 25 | 26 | # Remove some rules from gmake that .SUFFIXES does not remove. 27 | SUFFIXES = 28 | 29 | .SUFFIXES: .hpux_make_needs_suffix_list 30 | 31 | 32 | # Suppress display of executed commands. 33 | $(VERBOSE).SILENT: 34 | 35 | 36 | # A target that is always out of date. 37 | cmake_force: 38 | 39 | .PHONY : cmake_force 40 | 41 | #============================================================================= 42 | # Set environment variables for the build. 43 | 44 | # The shell in which to execute make rules. 45 | SHELL = /bin/sh 46 | 47 | # The CMake executable. 48 | CMAKE_COMMAND = /usr/bin/cmake 49 | 50 | # The command to remove a file. 51 | RM = /usr/bin/cmake -E remove -f 52 | 53 | # Escaping for special characters. 54 | EQUALS = = 55 | 56 | # The top-level source directory on which CMake was run. 57 | CMAKE_SOURCE_DIR = /home/yingting/Multimodal-Transformer/warp-ctc 58 | 59 | # The top-level build directory on which CMake was run. 60 | CMAKE_BINARY_DIR = /home/yingting/Multimodal-Transformer/warp-ctc/build 61 | 62 | #============================================================================= 63 | # Target rules for target CMakeFiles/test_cpu.dir 64 | 65 | # All Build rule for target. 66 | CMakeFiles/test_cpu.dir/all: CMakeFiles/warpctc.dir/all 67 | $(MAKE) -f CMakeFiles/test_cpu.dir/build.make CMakeFiles/test_cpu.dir/depend 68 | $(MAKE) -f CMakeFiles/test_cpu.dir/build.make CMakeFiles/test_cpu.dir/build 69 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles --progress-num=1,2,3 "Built target test_cpu" 70 | .PHONY : CMakeFiles/test_cpu.dir/all 71 | 72 | # Include target in all. 73 | all: CMakeFiles/test_cpu.dir/all 74 | 75 | .PHONY : all 76 | 77 | # Build rule for subdir invocation for target. 78 | CMakeFiles/test_cpu.dir/rule: cmake_check_build_system 79 | $(CMAKE_COMMAND) -E cmake_progress_start /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles 5 80 | $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/test_cpu.dir/all 81 | $(CMAKE_COMMAND) -E cmake_progress_start /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles 0 82 | .PHONY : CMakeFiles/test_cpu.dir/rule 83 | 84 | # Convenience name for target. 85 | test_cpu: CMakeFiles/test_cpu.dir/rule 86 | 87 | .PHONY : test_cpu 88 | 89 | # clean rule for target. 90 | CMakeFiles/test_cpu.dir/clean: 91 | $(MAKE) -f CMakeFiles/test_cpu.dir/build.make CMakeFiles/test_cpu.dir/clean 92 | .PHONY : CMakeFiles/test_cpu.dir/clean 93 | 94 | # clean rule for target. 95 | clean: CMakeFiles/test_cpu.dir/clean 96 | 97 | .PHONY : clean 98 | 99 | #============================================================================= 100 | # Target rules for target CMakeFiles/warpctc.dir 101 | 102 | # All Build rule for target. 103 | CMakeFiles/warpctc.dir/all: 104 | $(MAKE) -f CMakeFiles/warpctc.dir/build.make CMakeFiles/warpctc.dir/depend 105 | $(MAKE) -f CMakeFiles/warpctc.dir/build.make CMakeFiles/warpctc.dir/build 106 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles --progress-num=4,5 "Built target warpctc" 107 | .PHONY : CMakeFiles/warpctc.dir/all 108 | 109 | # Include target in all. 110 | all: CMakeFiles/warpctc.dir/all 111 | 112 | .PHONY : all 113 | 114 | # Build rule for subdir invocation for target. 115 | CMakeFiles/warpctc.dir/rule: cmake_check_build_system 116 | $(CMAKE_COMMAND) -E cmake_progress_start /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles 2 117 | $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/warpctc.dir/all 118 | $(CMAKE_COMMAND) -E cmake_progress_start /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles 0 119 | .PHONY : CMakeFiles/warpctc.dir/rule 120 | 121 | # Convenience name for target. 122 | warpctc: CMakeFiles/warpctc.dir/rule 123 | 124 | .PHONY : warpctc 125 | 126 | # clean rule for target. 127 | CMakeFiles/warpctc.dir/clean: 128 | $(MAKE) -f CMakeFiles/warpctc.dir/build.make CMakeFiles/warpctc.dir/clean 129 | .PHONY : CMakeFiles/warpctc.dir/clean 130 | 131 | # clean rule for target. 132 | clean: CMakeFiles/warpctc.dir/clean 133 | 134 | .PHONY : clean 135 | 136 | #============================================================================= 137 | # Special targets to cleanup operation of make. 138 | 139 | # Special rule to run CMake to check the build system integrity. 140 | # No rule that depends on this can have commands that come from listfiles 141 | # because they might be regenerated. 142 | cmake_check_build_system: 143 | $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 144 | .PHONY : cmake_check_build_system 145 | 146 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/TargetDirectories.txt: -------------------------------------------------------------------------------- 1 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/install/strip.dir 2 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/install/local.dir 3 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/test_cpu.dir 4 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/warpctc.dir 5 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/install.dir 6 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/edit_cache.dir 7 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/rebuild_cache.dir 8 | /home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/list_install_components.dir 9 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/cmake.check_cache: -------------------------------------------------------------------------------- 1 | # This file is generated by cmake for dependency checking of the CMakeCache.txt file 2 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/feature_tests.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/feature_tests.bin -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/feature_tests.c: -------------------------------------------------------------------------------- 1 | 2 | const char features[] = {"\n" 3 | "C_FEATURE:" 4 | #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 304 5 | "1" 6 | #else 7 | "0" 8 | #endif 9 | "c_function_prototypes\n" 10 | "C_FEATURE:" 11 | #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 304 && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L 12 | "1" 13 | #else 14 | "0" 15 | #endif 16 | "c_restrict\n" 17 | "C_FEATURE:" 18 | #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 406 && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201000L 19 | "1" 20 | #else 21 | "0" 22 | #endif 23 | "c_static_assert\n" 24 | "C_FEATURE:" 25 | #if (__GNUC__ * 100 + __GNUC_MINOR__) >= 304 && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L 26 | "1" 27 | #else 28 | "0" 29 | #endif 30 | "c_variadic_macros\n" 31 | 32 | }; 33 | 34 | int main(int argc, char** argv) { (void)argv; return features[argc]; } 35 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/progress.marks: -------------------------------------------------------------------------------- 1 | 5 2 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/CXX.includecache: -------------------------------------------------------------------------------- 1 | #IncludeRegexLine: ^[ ]*[#%][ ]*(include|import)[ ]*[<"]([^">]+)([">]) 2 | 3 | #IncludeRegexScan: ^.*$ 4 | 5 | #IncludeRegexComplain: ^$ 6 | 7 | #IncludeRegexTransform: 8 | 9 | ../include/ctc.h 10 | cstddef 11 | - 12 | 13 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/random.cpp 14 | vector 15 | - 16 | random 17 | - 18 | 19 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/test.h 20 | stdexcept 21 | - 22 | vector 23 | - 24 | limits 25 | - 26 | numeric 27 | - 28 | ctc.h 29 | - 30 | thrust/system_error.h 31 | - 32 | thrust/system/cuda/error.h 33 | - 34 | 35 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/test_cpu.cpp 36 | cmath 37 | - 38 | random 39 | - 40 | tuple 41 | - 42 | vector 43 | - 44 | iostream 45 | - 46 | ctc.h 47 | - 48 | test.h 49 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/test.h 50 | 51 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/DependInfo.cmake: -------------------------------------------------------------------------------- 1 | # The set of languages for which implicit dependencies are needed: 2 | set(CMAKE_DEPENDS_LANGUAGES 3 | "CXX" 4 | ) 5 | # The set of files for implicit dependencies of each language: 6 | set(CMAKE_DEPENDS_CHECK_CXX 7 | "/home/yingting/Multimodal-Transformer/warp-ctc/tests/random.cpp" "/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/random.cpp.o" 8 | "/home/yingting/Multimodal-Transformer/warp-ctc/tests/test_cpu.cpp" "/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o" 9 | ) 10 | set(CMAKE_CXX_COMPILER_ID "GNU") 11 | 12 | # The include file search paths: 13 | set(CMAKE_CXX_TARGET_INCLUDE_PATH 14 | "../include" 15 | ) 16 | 17 | # Targets to which this target links. 18 | set(CMAKE_TARGET_LINKED_INFO_FILES 19 | "/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/warpctc.dir/DependInfo.cmake" 20 | ) 21 | 22 | # Fortran module output directory. 23 | set(CMAKE_Fortran_TARGET_MODULE_DIR "") 24 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/cmake_clean.cmake: -------------------------------------------------------------------------------- 1 | file(REMOVE_RECURSE 2 | "CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o" 3 | "CMakeFiles/test_cpu.dir/tests/random.cpp.o" 4 | "test_cpu.pdb" 5 | "test_cpu" 6 | ) 7 | 8 | # Per-language clean rules from dependency scanning. 9 | foreach(lang CXX) 10 | include(CMakeFiles/test_cpu.dir/cmake_clean_${lang}.cmake OPTIONAL) 11 | endforeach() 12 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/depend.internal: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | CMakeFiles/test_cpu.dir/tests/random.cpp.o 5 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/random.cpp 6 | CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o 7 | ../include/ctc.h 8 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/test.h 9 | /home/yingting/Multimodal-Transformer/warp-ctc/tests/test_cpu.cpp 10 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/depend.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | CMakeFiles/test_cpu.dir/tests/random.cpp.o: ../tests/random.cpp 5 | 6 | CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o: ../include/ctc.h 7 | CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o: ../tests/test.h 8 | CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o: ../tests/test_cpu.cpp 9 | 10 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/flags.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | # compile CXX with /usr/bin/c++ 5 | CXX_FLAGS = -O2 -fopenmp -std=c++14 -O2 -O2 -fopenmp -std=c++14 -O2 --std=c++14 6 | 7 | CXX_DEFINES = 8 | 9 | CXX_INCLUDES = -I/home/yingting/Multimodal-Transformer/warp-ctc/include 10 | 11 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/link.txt: -------------------------------------------------------------------------------- 1 | /usr/bin/c++ -O2 -fopenmp -std=c++14 -O2 -rdynamic CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o CMakeFiles/test_cpu.dir/tests/random.cpp.o -o test_cpu libwarpctc.so 2 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/progress.make: -------------------------------------------------------------------------------- 1 | CMAKE_PROGRESS_1 = 1 2 | CMAKE_PROGRESS_2 = 2 3 | CMAKE_PROGRESS_3 = 3 4 | 5 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/random.cpp.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/random.cpp.o -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/test_cpu.dir/tests/test_cpu.cpp.o -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/CXX.includecache: -------------------------------------------------------------------------------- 1 | #IncludeRegexLine: ^[ ]*[#%][ ]*(include|import)[ ]*[<"]([^">]+)([">]) 2 | 3 | #IncludeRegexScan: ^.*$ 4 | 5 | #IncludeRegexComplain: ^$ 6 | 7 | #IncludeRegexTransform: 8 | 9 | ../include/contrib/moderngpu/include/device/ctamerge.cuh 10 | ctasearch.cuh 11 | ../include/contrib/moderngpu/include/device/ctasearch.cuh 12 | loadstore.cuh 13 | ../include/contrib/moderngpu/include/device/loadstore.cuh 14 | sortnetwork.cuh 15 | ../include/contrib/moderngpu/include/device/sortnetwork.cuh 16 | 17 | ../include/contrib/moderngpu/include/device/ctascan.cuh 18 | ../mgpuenums.h 19 | ../include/contrib/moderngpu/include/mgpuenums.h 20 | deviceutil.cuh 21 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 22 | intrinsics.cuh 23 | ../include/contrib/moderngpu/include/device/intrinsics.cuh 24 | 25 | ../include/contrib/moderngpu/include/device/ctasearch.cuh 26 | deviceutil.cuh 27 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 28 | ../mgpudevice.cuh 29 | ../include/contrib/moderngpu/include/mgpudevice.cuh 30 | 31 | ../include/contrib/moderngpu/include/device/ctasegscan.cuh 32 | ctascan.cuh 33 | ../include/contrib/moderngpu/include/device/ctascan.cuh 34 | 35 | ../include/contrib/moderngpu/include/device/devicetypes.cuh 36 | climits 37 | - 38 | ../util/static.h 39 | ../include/contrib/moderngpu/include/util/static.h 40 | 41 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 42 | intrinsics.cuh 43 | ../include/contrib/moderngpu/include/device/intrinsics.cuh 44 | 45 | ../include/contrib/moderngpu/include/device/intrinsics.cuh 46 | devicetypes.cuh 47 | ../include/contrib/moderngpu/include/device/devicetypes.cuh 48 | 49 | ../include/contrib/moderngpu/include/device/loadstore.cuh 50 | ../mgpudevice.cuh 51 | ../include/contrib/moderngpu/include/mgpudevice.cuh 52 | deviceutil.cuh 53 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 54 | intrinsics.cuh 55 | ../include/contrib/moderngpu/include/device/intrinsics.cuh 56 | 57 | ../include/contrib/moderngpu/include/device/sortnetwork.cuh 58 | deviceutil.cuh 59 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 60 | 61 | ../include/contrib/moderngpu/include/mgpudevice.cuh 62 | mgpuenums.h 63 | ../include/contrib/moderngpu/include/mgpuenums.h 64 | device/deviceutil.cuh 65 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 66 | device/loadstore.cuh 67 | ../include/contrib/moderngpu/include/device/loadstore.cuh 68 | device/ctasegscan.cuh 69 | ../include/contrib/moderngpu/include/device/ctasegscan.cuh 70 | 71 | ../include/contrib/moderngpu/include/mgpuenums.h 72 | 73 | ../include/contrib/moderngpu/include/util/static.h 74 | functional 75 | - 76 | iterator 77 | - 78 | cfloat 79 | - 80 | typeinfo 81 | - 82 | vector 83 | - 84 | list 85 | - 86 | map 87 | - 88 | algorithm 89 | - 90 | cassert 91 | - 92 | memory 93 | - 94 | cmath 95 | - 96 | cstdio 97 | - 98 | cstdlib 99 | - 100 | 101 | ../include/ctc.h 102 | cstddef 103 | - 104 | 105 | ../include/detail/cpu_ctc.h 106 | tuple 107 | - 108 | cmath 109 | - 110 | limits 111 | - 112 | algorithm 113 | - 114 | numeric 115 | - 116 | omp.h 117 | - 118 | ctc_helper.h 119 | ../include/detail/ctc_helper.h 120 | 121 | ../include/detail/ctc_helper.h 122 | limits 123 | - 124 | algorithm 125 | - 126 | cmath 127 | - 128 | hostdevice.h 129 | ../include/detail/hostdevice.h 130 | 131 | ../include/detail/gpu_ctc.h 132 | ctc_helper.h 133 | ../include/detail/ctc_helper.h 134 | gpu_ctc_kernels.h 135 | ../include/detail/gpu_ctc_kernels.h 136 | reduce.h 137 | ../include/detail/reduce.h 138 | 139 | ../include/detail/gpu_ctc_kernels.h 140 | contrib/moderngpu/include/device/ctascan.cuh 141 | - 142 | contrib/moderngpu/include/device/ctamerge.cuh 143 | - 144 | ctc_helper.h 145 | ../include/detail/ctc_helper.h 146 | 147 | ../include/detail/hostdevice.h 148 | 149 | ../include/detail/reduce.h 150 | 151 | /home/yingting/Multimodal-Transformer/warp-ctc/src/ctc_entrypoint.cpp 152 | cstddef 153 | - 154 | iostream 155 | - 156 | algorithm 157 | - 158 | ctc.h 159 | - 160 | detail/cpu_ctc.h 161 | /home/yingting/Multimodal-Transformer/warp-ctc/src/detail/cpu_ctc.h 162 | detail/gpu_ctc.h 163 | /home/yingting/Multimodal-Transformer/warp-ctc/src/detail/gpu_ctc.h 164 | 165 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/DependInfo.cmake: -------------------------------------------------------------------------------- 1 | # The set of languages for which implicit dependencies are needed: 2 | set(CMAKE_DEPENDS_LANGUAGES 3 | "CXX" 4 | ) 5 | # The set of files for implicit dependencies of each language: 6 | set(CMAKE_DEPENDS_CHECK_CXX 7 | "/home/yingting/Multimodal-Transformer/warp-ctc/src/ctc_entrypoint.cpp" "/home/yingting/Multimodal-Transformer/warp-ctc/build/CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o" 8 | ) 9 | set(CMAKE_CXX_COMPILER_ID "GNU") 10 | 11 | # The include file search paths: 12 | set(CMAKE_CXX_TARGET_INCLUDE_PATH 13 | "../include" 14 | ) 15 | 16 | # Targets to which this target links. 17 | set(CMAKE_TARGET_LINKED_INFO_FILES 18 | ) 19 | 20 | # Fortran module output directory. 21 | set(CMAKE_Fortran_TARGET_MODULE_DIR "") 22 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/cmake_clean.cmake: -------------------------------------------------------------------------------- 1 | file(REMOVE_RECURSE 2 | "CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o" 3 | "libwarpctc.pdb" 4 | "libwarpctc.so" 5 | ) 6 | 7 | # Per-language clean rules from dependency scanning. 8 | foreach(lang CXX) 9 | include(CMakeFiles/warpctc.dir/cmake_clean_${lang}.cmake OPTIONAL) 10 | endforeach() 11 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/depend.internal: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o 5 | ../include/contrib/moderngpu/include/device/ctamerge.cuh 6 | ../include/contrib/moderngpu/include/device/ctascan.cuh 7 | ../include/contrib/moderngpu/include/device/ctasearch.cuh 8 | ../include/contrib/moderngpu/include/device/ctasegscan.cuh 9 | ../include/contrib/moderngpu/include/device/devicetypes.cuh 10 | ../include/contrib/moderngpu/include/device/deviceutil.cuh 11 | ../include/contrib/moderngpu/include/device/intrinsics.cuh 12 | ../include/contrib/moderngpu/include/device/loadstore.cuh 13 | ../include/contrib/moderngpu/include/device/sortnetwork.cuh 14 | ../include/contrib/moderngpu/include/mgpudevice.cuh 15 | ../include/contrib/moderngpu/include/mgpuenums.h 16 | ../include/contrib/moderngpu/include/util/static.h 17 | ../include/ctc.h 18 | ../include/detail/cpu_ctc.h 19 | ../include/detail/ctc_helper.h 20 | ../include/detail/gpu_ctc.h 21 | ../include/detail/gpu_ctc_kernels.h 22 | ../include/detail/hostdevice.h 23 | ../include/detail/reduce.h 24 | /home/yingting/Multimodal-Transformer/warp-ctc/src/ctc_entrypoint.cpp 25 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/depend.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/ctamerge.cuh 5 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/ctascan.cuh 6 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/ctasearch.cuh 7 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/ctasegscan.cuh 8 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/devicetypes.cuh 9 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/deviceutil.cuh 10 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/intrinsics.cuh 11 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/loadstore.cuh 12 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/device/sortnetwork.cuh 13 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/mgpudevice.cuh 14 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/mgpuenums.h 15 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/contrib/moderngpu/include/util/static.h 16 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/ctc.h 17 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/cpu_ctc.h 18 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/ctc_helper.h 19 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/gpu_ctc.h 20 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/gpu_ctc_kernels.h 21 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/hostdevice.h 22 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../include/detail/reduce.h 23 | CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: ../src/ctc_entrypoint.cpp 24 | 25 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/flags.make: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.10 3 | 4 | # compile CXX with /usr/bin/c++ 5 | CXX_FLAGS = -O2 -fopenmp -std=c++14 -O2 -fPIC 6 | 7 | CXX_DEFINES = -Dwarpctc_EXPORTS 8 | 9 | CXX_INCLUDES = -I/home/yingting/Multimodal-Transformer/warp-ctc/include 10 | 11 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/link.txt: -------------------------------------------------------------------------------- 1 | /usr/bin/c++ -fPIC -O2 -fopenmp -std=c++14 -O2 -shared -Wl,-soname,libwarpctc.so -o libwarpctc.so CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o 2 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/progress.make: -------------------------------------------------------------------------------- 1 | CMAKE_PROGRESS_1 = 4 2 | CMAKE_PROGRESS_2 = 5 3 | 4 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/CMakeFiles/warpctc.dir/src/ctc_entrypoint.cpp.o -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/cmake_install.cmake: -------------------------------------------------------------------------------- 1 | # Install script for directory: /home/yingting/Multimodal-Transformer/warp-ctc 2 | 3 | # Set the install prefix 4 | if(NOT DEFINED CMAKE_INSTALL_PREFIX) 5 | set(CMAKE_INSTALL_PREFIX "/usr/local") 6 | endif() 7 | string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") 8 | 9 | # Set the install configuration name. 10 | if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME) 11 | if(BUILD_TYPE) 12 | string(REGEX REPLACE "^[^A-Za-z0-9_]+" "" 13 | CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}") 14 | else() 15 | set(CMAKE_INSTALL_CONFIG_NAME "") 16 | endif() 17 | message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"") 18 | endif() 19 | 20 | # Set the component getting installed. 21 | if(NOT CMAKE_INSTALL_COMPONENT) 22 | if(COMPONENT) 23 | message(STATUS "Install component: \"${COMPONENT}\"") 24 | set(CMAKE_INSTALL_COMPONENT "${COMPONENT}") 25 | else() 26 | set(CMAKE_INSTALL_COMPONENT) 27 | endif() 28 | endif() 29 | 30 | # Install shared libraries without execute permission? 31 | if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE) 32 | set(CMAKE_INSTALL_SO_NO_EXE "1") 33 | endif() 34 | 35 | # Is this installation the result of a crosscompile? 36 | if(NOT DEFINED CMAKE_CROSSCOMPILING) 37 | set(CMAKE_CROSSCOMPILING "FALSE") 38 | endif() 39 | 40 | if("x${CMAKE_INSTALL_COMPONENT}x" STREQUAL "xUnspecifiedx" OR NOT CMAKE_INSTALL_COMPONENT) 41 | file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/lib" TYPE SHARED_LIBRARY FILES "/home/yingting/Multimodal-Transformer/warp-ctc/build/libwarpctc.so") 42 | if(EXISTS "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libwarpctc.so" AND 43 | NOT IS_SYMLINK "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libwarpctc.so") 44 | if(CMAKE_INSTALL_DO_STRIP) 45 | execute_process(COMMAND "/usr/bin/strip" "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libwarpctc.so") 46 | endif() 47 | endif() 48 | endif() 49 | 50 | if("x${CMAKE_INSTALL_COMPONENT}x" STREQUAL "xUnspecifiedx" OR NOT CMAKE_INSTALL_COMPONENT) 51 | file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/include" TYPE FILE FILES "/home/yingting/Multimodal-Transformer/warp-ctc/include/ctc.h") 52 | endif() 53 | 54 | if(CMAKE_INSTALL_COMPONENT) 55 | set(CMAKE_INSTALL_MANIFEST "install_manifest_${CMAKE_INSTALL_COMPONENT}.txt") 56 | else() 57 | set(CMAKE_INSTALL_MANIFEST "install_manifest.txt") 58 | endif() 59 | 60 | string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT 61 | "${CMAKE_INSTALL_MANIFEST_FILES}") 62 | file(WRITE "/home/yingting/Multimodal-Transformer/warp-ctc/build/${CMAKE_INSTALL_MANIFEST}" 63 | "${CMAKE_INSTALL_MANIFEST_CONTENT}") 64 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/libwarpctc.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/libwarpctc.so -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/build/test_cpu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/build/test_cpu -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/doc/baidu-research-logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/doc/baidu-research-logo-small.png -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/doc/deep-speech-ctc-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Mult_BERT/warp-ctc/doc/deep-speech-ctc-small.png -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/include/contrib/moderngpu/LICENSE: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2013, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * * Redistributions of source code must retain the above copyright 7 | * notice, this list of conditions and the following disclaimer. 8 | * * Redistributions in binary form must reproduce the above copyright 9 | * notice, this list of conditions and the following disclaimer in the 10 | * documentation and/or other materials provided with the distribution. 11 | * * Neither the name of the NVIDIA CORPORATION nor the 12 | * names of its contributors may be used to endorse or promote products 13 | * derived from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY 19 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | * 26 | ******************************************************************************/ 27 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/include/contrib/moderngpu/include/mgpuenums.h: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2013, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are met: 6 | * * Redistributions of source code must retain the above copyright 7 | * notice, this list of conditions and the following disclaimer. 8 | * * Redistributions in binary form must reproduce the above copyright 9 | * notice, this list of conditions and the following disclaimer in the 10 | * documentation and/or other materials provided with the distribution. 11 | * * Neither the name of the NVIDIA CORPORATION nor the 12 | * names of its contributors may be used to endorse or promote products 13 | * derived from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 | * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY 19 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | * 26 | ******************************************************************************/ 27 | 28 | /****************************************************************************** 29 | * 30 | * Code and text by Sean Baxter, NVIDIA Research 31 | * See http://nvlabs.github.io/moderngpu for repository and documentation. 32 | * 33 | ******************************************************************************/ 34 | 35 | #pragma once 36 | 37 | namespace mgpu { 38 | 39 | enum MgpuBounds { 40 | MgpuBoundsLower, 41 | MgpuBoundsUpper 42 | }; 43 | 44 | enum MgpuScanType { 45 | MgpuScanTypeExc, 46 | MgpuScanTypeInc 47 | }; 48 | 49 | enum MgpuSearchType { 50 | MgpuSearchTypeNone, 51 | MgpuSearchTypeIndex, 52 | MgpuSearchTypeMatch, 53 | MgpuSearchTypeIndexMatch 54 | }; 55 | 56 | enum MgpuJoinKind { 57 | MgpuJoinKindInner, 58 | MgpuJoinKindLeft, 59 | MgpuJoinKindRight, 60 | MgpuJoinKindOuter 61 | }; 62 | 63 | enum MgpuSetOp { 64 | MgpuSetOpIntersection, 65 | MgpuSetOpUnion, 66 | MgpuSetOpDiff, 67 | MgpuSetOpSymDiff 68 | }; 69 | 70 | } // namespace mgpu 71 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/include/detail/ctc_helper.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "hostdevice.h" 8 | 9 | namespace ctc_helper { 10 | 11 | static const float threshold = 1e-1; 12 | 13 | template 14 | HOSTDEVICE 15 | T neg_inf() { return -T(INFINITY); } 16 | 17 | inline int div_up(int x, int y) { 18 | return (x + y - 1) / y; 19 | } 20 | 21 | template struct maximum { 22 | HOSTDEVICE 23 | Res operator()(const Arg& x, const Arg& y) const { 24 | return x < y ? y : x; 25 | } 26 | }; 27 | 28 | template struct add { 29 | HOSTDEVICE 30 | Res operator()(const Arg& x, const Arg& y) const { 31 | return x + y; 32 | } 33 | }; 34 | 35 | template struct identity { 36 | HOSTDEVICE Res operator()(const Arg& x) const {return Res(x);} 37 | }; 38 | 39 | template struct negate { 40 | HOSTDEVICE Res operator()(const Arg& x) const {return Res(-x);} 41 | }; 42 | 43 | template struct exponential { 44 | HOSTDEVICE Res operator()(const Arg& x) const {return std::exp(x);} 45 | }; 46 | 47 | template 48 | struct log_plus { 49 | typedef Res result_type; 50 | HOSTDEVICE 51 | Res operator()(const Arg1& p1, const Arg2& p2) { 52 | if (p1 == neg_inf()) 53 | return p2; 54 | if (p2 == neg_inf()) 55 | return p1; 56 | Res result = log1p(exp(-fabs(p1 - p2))) + maximum()(p1, p2); 57 | return result; 58 | } 59 | }; 60 | 61 | } 62 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/include/detail/hostdevice.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef __CUDACC__ 4 | #define HOSTDEVICE __host__ __device__ 5 | #else 6 | #define HOSTDEVICE 7 | #endif 8 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/include/detail/reduce.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | ctcStatus_t reduce_negate(const float* input, float* output, int rows, int cols, bool axis, cudaStream_t stream); 4 | ctcStatus_t reduce_exp(const float* input, float* output, int rows, int cols, bool axis, cudaStream_t stream); 5 | ctcStatus_t reduce_max(const float* input, float* output, int rows, int cols, bool axis, cudaStream_t stream); 6 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore compiled FFI location 2 | warpctc_pytorch/_warp_ctc 3 | 4 | # Created by https://www.gitignore.io/api/python 5 | 6 | ### Python ### 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule.* 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | 110 | # End of https://www.gitignore.io/api/python 111 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | import sys 4 | from setuptools import setup, find_packages 5 | 6 | from torch.utils.cpp_extension import BuildExtension, CppExtension 7 | import torch 8 | 9 | extra_compile_args = ['-std=c++14', '-fPIC'] 10 | warp_ctc_path = "../build" 11 | 12 | if platform.system() == 'Darwin': 13 | lib_ext = ".dylib" 14 | else: 15 | lib_ext = ".so" 16 | 17 | if "WARP_CTC_PATH" in os.environ: 18 | warp_ctc_path = os.environ["WARP_CTC_PATH"] 19 | if not os.path.exists(os.path.join(warp_ctc_path, "libwarpctc" + lib_ext)): 20 | print(("Could not find libwarpctc.so in {}.\n" 21 | "Build warp-ctc and set WARP_CTC_PATH to the location of" 22 | " libwarpctc.so (default is '../build')").format(warp_ctc_path)) 23 | sys.exit(1) 24 | 25 | include_dirs = [os.path.realpath('../include')] 26 | 27 | if torch.cuda.is_available() or "CUDA_HOME" in os.environ: 28 | enable_gpu = True 29 | else: 30 | print("Torch was not built with CUDA support, not building warp-ctc GPU extensions.") 31 | enable_gpu = False 32 | 33 | if enable_gpu: 34 | from torch.utils.cpp_extension import CUDAExtension 35 | 36 | build_extension = CUDAExtension 37 | extra_compile_args += ['-DWARPCTC_ENABLE_GPU'] 38 | else: 39 | build_extension = CppExtension 40 | 41 | ext_modules = [ 42 | build_extension( 43 | name='warpctc_pytorch._warp_ctc', 44 | language='c++', 45 | sources=['src/binding.cpp'], 46 | include_dirs=include_dirs, 47 | library_dirs=[os.path.realpath(warp_ctc_path)], 48 | libraries=['warpctc'], 49 | extra_link_args=['-Wl,-rpath,' + os.path.realpath(warp_ctc_path)], 50 | extra_compile_args=extra_compile_args 51 | ) 52 | ] 53 | 54 | setup( 55 | name="warpctc_pytorch", 56 | version="0.1", 57 | description="PyTorch wrapper for warp-ctc", 58 | url="https://github.com/baidu-research/warp-ctc", 59 | author="Jared Casper, Sean Naren", 60 | author_email="jared.casper@baidu.com, sean.narenthiran@digitalreasoning.com", 61 | license="Apache", 62 | packages=find_packages(), 63 | ext_modules=ext_modules, 64 | cmdclass={'build_ext': BuildExtension} 65 | ) 66 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/src/binding.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include 7 | 8 | #ifdef WARPCTC_ENABLE_GPU 9 | #include "ATen/cuda/CUDAContext.h" 10 | #include 11 | #include "ATen/cuda/CUDAEvent.h" 12 | 13 | #include "THC.h" 14 | extern THCState* state; 15 | #endif 16 | 17 | #include "ctc.h" 18 | 19 | int cpu_ctc(torch::Tensor probs, 20 | torch::Tensor grads, 21 | torch::Tensor labels, 22 | torch::Tensor label_sizes, 23 | torch::Tensor sizes, 24 | int minibatch_size, 25 | torch::Tensor costs, 26 | int blank_label) 27 | { 28 | float* probs_ptr = (float*)probs.data_ptr(); 29 | float* grads_ptr = grads.storage() ? (float*)grads.data_ptr() : NULL; 30 | int* sizes_ptr = (int*)sizes.data_ptr(); 31 | int* labels_ptr = (int*)labels.data_ptr(); 32 | int* label_sizes_ptr = (int*)label_sizes.data_ptr(); 33 | float* costs_ptr = (float*)costs.data_ptr(); 34 | 35 | const int probs_size = probs.size(2); 36 | 37 | ctcOptions options; 38 | memset(&options, 0, sizeof(options)); 39 | options.loc = CTC_CPU; 40 | options.num_threads = 0; // will use default number of threads 41 | options.blank_label = blank_label; 42 | 43 | #if defined(CTC_DISABLE_OMP) || defined(APPLE) 44 | // have to use at least one 45 | options.num_threads = std::max(options.num_threads, (unsigned int) 1); 46 | #endif 47 | 48 | size_t cpu_size_bytes; 49 | get_workspace_size(label_sizes_ptr, sizes_ptr, 50 | probs_size, minibatch_size, 51 | options, &cpu_size_bytes); 52 | 53 | float* cpu_workspace = new float[cpu_size_bytes / sizeof(float)]; 54 | 55 | compute_ctc_loss(probs_ptr, grads_ptr, 56 | labels_ptr, label_sizes_ptr, 57 | sizes_ptr, probs_size, 58 | minibatch_size, costs_ptr, 59 | cpu_workspace, options); 60 | 61 | delete[] cpu_workspace; 62 | return 1; 63 | } 64 | 65 | #ifdef WARPCTC_ENABLE_GPU 66 | int gpu_ctc(torch::Tensor probs, 67 | torch::Tensor grads, 68 | torch::Tensor labels, 69 | torch::Tensor label_sizes, 70 | torch::Tensor sizes, 71 | int minibatch_size, 72 | torch::Tensor costs, 73 | int blank_label) 74 | { 75 | float* probs_ptr = (float*)probs.data_ptr(); 76 | float* grads_ptr = grads.storage() ? (float*)grads.data_ptr() : NULL; 77 | int* sizes_ptr = (int*)sizes.data_ptr(); 78 | int* labels_ptr = (int*)labels.data_ptr(); 79 | int* label_sizes_ptr = (int*)label_sizes.data_ptr(); 80 | float* costs_ptr = (float*)costs.data_ptr(); 81 | 82 | const int probs_size = probs.size(2); 83 | 84 | ctcOptions options; 85 | memset(&options, 0, sizeof(options)); 86 | options.loc = CTC_GPU; 87 | options.blank_label = blank_label; 88 | options.stream = at::cuda::getCurrentCUDAStream(); 89 | 90 | size_t gpu_size_bytes; 91 | get_workspace_size(label_sizes_ptr, sizes_ptr, 92 | probs_size, minibatch_size, 93 | options, &gpu_size_bytes); 94 | 95 | void* gpu_workspace = THCudaMalloc(state, gpu_size_bytes); 96 | 97 | compute_ctc_loss(probs_ptr, grads_ptr, 98 | labels_ptr, label_sizes_ptr, 99 | sizes_ptr, probs_size, 100 | minibatch_size, costs_ptr, 101 | gpu_workspace, options); 102 | 103 | THCudaFree(state, (void *) gpu_workspace); 104 | return 1; 105 | } 106 | #endif 107 | 108 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 109 | m.def("cpu_ctc", &cpu_ctc, "CTC Loss function with cpu"); 110 | #ifdef WARPCTC_ENABLE_GPU 111 | m.def("gpu_ctc", &gpu_ctc, "CTC Loss function with gpu"); 112 | #endif 113 | } 114 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/src/cpu_binding.h: -------------------------------------------------------------------------------- 1 | /* 2 | int cpu_ctc(THFloatTensor *probs, 3 | THFloatTensor *grads, 4 | THIntTensor *labels_ptr, 5 | THIntTensor *label_sizes_ptr, 6 | THIntTensor *sizes, 7 | int minibatch_size, 8 | THFloatTensor *costs, 9 | int blank_label); 10 | */ 11 | 12 | int cpu_ctc(torch::Tensor probs, 13 | torch::Tensor grads, 14 | torch::Tensor labels, 15 | torch::Tensor label_sizes, 16 | torch::Tensor sizes, 17 | int minibatch_size, 18 | torch::Tensor costs, 19 | int blank_label); 20 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/src/gpu_binding.h: -------------------------------------------------------------------------------- 1 | /* 2 | int gpu_ctc(THCudaTensor *probs, 3 | THCudaTensor *grads, 4 | THIntTensor *labels_ptr, 5 | THIntTensor *label_sizes_ptr, 6 | THIntTensor *sizes, 7 | int minibatch_size, 8 | THFloatTensor *costs, 9 | int blank_label); 10 | */ 11 | 12 | int gpu_ctc(torch::Tensor probs, 13 | torch::Tensor grads, 14 | torch::Tensor labels, 15 | torch::Tensor label_sizes, 16 | torch::Tensor sizes, 17 | int minibatch_size, 18 | torch::Tensor costs, 19 | int blank_label); 20 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/tests/test_cpu.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import warpctc_pytorch as warp_ctc 3 | import pytest 4 | 5 | 6 | def test_simple(): 7 | probs = torch.FloatTensor([[[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]]]).transpose(0, 1).contiguous() 8 | grads = torch.zeros(probs.size()) 9 | labels = torch.IntTensor([1, 2]) 10 | label_sizes = torch.IntTensor([2]) 11 | sizes = torch.IntTensor(probs.size(1)).fill_(probs.size(0)) 12 | minibatch_size = probs.size(1) 13 | costs = torch.zeros(minibatch_size) 14 | warp_ctc.cpu_ctc(probs, 15 | grads, 16 | labels, 17 | label_sizes, 18 | sizes, 19 | minibatch_size, 20 | costs, 21 | 0) 22 | print('CPU_cost: %f' % costs.sum()) 23 | 24 | 25 | @pytest.mark.parametrize("multiplier", [1.0, 200.0]) 26 | def test_medium(multiplier): 27 | probs = torch.FloatTensor([ 28 | [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], 29 | [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]] 30 | ]).contiguous() * multiplier 31 | 32 | grads = torch.zeros(probs.size()) 33 | labels = torch.IntTensor([1, 2, 1, 2]) 34 | label_sizes = torch.IntTensor([2, 2]) 35 | sizes = torch.IntTensor([2, 2]) 36 | minibatch_size = probs.size(1) 37 | costs = torch.zeros(minibatch_size) 38 | warp_ctc.cpu_ctc(probs, 39 | grads, 40 | labels, 41 | label_sizes, 42 | sizes, 43 | minibatch_size, 44 | costs, 45 | 0) 46 | print('CPU_cost: %f' % costs.sum()) 47 | 48 | 49 | def test_empty_label(): 50 | probs = torch.FloatTensor([ 51 | [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], 52 | [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]] 53 | ]).contiguous() 54 | 55 | grads = torch.zeros(probs.size()) 56 | labels = torch.IntTensor([1, 2]) 57 | label_sizes = torch.IntTensor([2, 0]) 58 | sizes = torch.IntTensor([2, 2]) 59 | minibatch_size = probs.size(1) 60 | costs = torch.zeros(minibatch_size) 61 | warp_ctc.cpu_ctc(probs, 62 | grads, 63 | labels, 64 | label_sizes, 65 | sizes, 66 | minibatch_size, 67 | costs, 68 | 0) 69 | print('CPU_cost: %f' % costs.sum()) 70 | 71 | 72 | def test_CTCLoss(): 73 | probs = torch.FloatTensor([[ 74 | [0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1] 75 | ]]).transpose(0, 1).contiguous() 76 | labels = torch.IntTensor([1, 2]) 77 | label_sizes = torch.IntTensor([2]) 78 | probs_sizes = torch.IntTensor([2]) 79 | probs.requires_grad_(True) 80 | 81 | ctc_loss = warp_ctc.CTCLoss() 82 | cost = ctc_loss(probs, labels, probs_sizes, label_sizes) 83 | cost.backward() 84 | 85 | 86 | if __name__ == '__main__': 87 | pytest.main([__file__]) 88 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/tests/test_gpu.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import warpctc_pytorch as warp_ctc 3 | import pytest 4 | 5 | 6 | @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires GPU") 7 | def test_simple(): 8 | probs = torch.FloatTensor([[[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]]]).transpose(0, 1).contiguous() 9 | grads = torch.zeros(probs.size()) 10 | labels = torch.IntTensor([1, 2]) 11 | label_sizes = torch.IntTensor([2]) 12 | sizes = torch.IntTensor(probs.size(1)).fill_(probs.size(0)) 13 | minibatch_size = probs.size(1) 14 | costs = torch.zeros(minibatch_size) 15 | warp_ctc.cpu_ctc(probs, 16 | grads, 17 | labels, 18 | label_sizes, 19 | sizes, 20 | minibatch_size, 21 | costs, 22 | 0) 23 | print('CPU_cost: %f' % costs.sum()) 24 | probs = probs.clone().cuda() 25 | grads = torch.zeros(probs.size()).cuda() 26 | costs = torch.zeros(minibatch_size) 27 | warp_ctc.gpu_ctc(probs, 28 | grads, 29 | labels, 30 | label_sizes, 31 | sizes, 32 | minibatch_size, 33 | costs, 34 | 0) 35 | print('GPU_cost: %f' % costs.sum()) 36 | print(grads.view(grads.size(0) * grads.size(1), grads.size(2))) 37 | 38 | 39 | @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires GPU") 40 | @pytest.mark.parametrize("multiplier", [1.0, 200.0]) 41 | def test_medium(multiplier): 42 | probs = torch.FloatTensor([ 43 | [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], 44 | [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]] 45 | ]).contiguous() * multiplier 46 | 47 | grads = torch.zeros(probs.size()) 48 | labels = torch.IntTensor([1, 2, 1, 2]) 49 | label_sizes = torch.IntTensor([2, 2]) 50 | sizes = torch.IntTensor([2, 2]) 51 | minibatch_size = probs.size(1) 52 | costs = torch.zeros(minibatch_size) 53 | warp_ctc.cpu_ctc(probs, 54 | grads, 55 | labels, 56 | label_sizes, 57 | sizes, 58 | minibatch_size, 59 | costs, 60 | 0) 61 | print('CPU_cost: %f' % costs.sum()) 62 | probs = probs.clone().cuda() 63 | grads = torch.zeros(probs.size()).cuda() 64 | costs = torch.zeros(minibatch_size) 65 | warp_ctc.gpu_ctc(probs, 66 | grads, 67 | labels, 68 | label_sizes, 69 | sizes, 70 | minibatch_size, 71 | costs, 72 | 0) 73 | print('GPU_cost: %f' % costs.sum()) 74 | print(grads.view(grads.size(0) * grads.size(1), grads.size(2))) 75 | 76 | 77 | @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires GPU") 78 | def test_empty_label(): 79 | probs = torch.FloatTensor([ 80 | [[0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1]], 81 | [[0.6, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.5, 0.2, 0.1]] 82 | ]).contiguous() 83 | 84 | grads = torch.zeros(probs.size()) 85 | labels = torch.IntTensor([1, 2]) 86 | label_sizes = torch.IntTensor([2, 0]) 87 | sizes = torch.IntTensor([2, 2]) 88 | minibatch_size = probs.size(1) 89 | costs = torch.zeros(minibatch_size) 90 | warp_ctc.cpu_ctc(probs, 91 | grads, 92 | labels, 93 | label_sizes, 94 | sizes, 95 | minibatch_size, 96 | costs, 97 | 0) 98 | print('CPU_cost: %f' % costs.sum()) 99 | probs = probs.clone().cuda() 100 | grads = torch.zeros(probs.size()).cuda() 101 | costs = torch.zeros(minibatch_size) 102 | warp_ctc.gpu_ctc(probs, 103 | grads, 104 | labels, 105 | label_sizes, 106 | sizes, 107 | minibatch_size, 108 | costs, 109 | 0) 110 | print('GPU_cost: %f' % costs.sum()) 111 | print(grads.view(grads.size(0) * grads.size(1), grads.size(2))) 112 | 113 | 114 | @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires GPU") 115 | def test_CTCLoss(): 116 | probs = torch.FloatTensor([[ 117 | [0.1, 0.6, 0.1, 0.1, 0.1], [0.1, 0.1, 0.6, 0.1, 0.1] 118 | ]]).transpose(0, 1).contiguous().cuda() 119 | labels = torch.IntTensor([1, 2]) 120 | label_sizes = torch.IntTensor([2]) 121 | probs_sizes = torch.IntTensor([2]) 122 | probs.requires_grad_(True) 123 | 124 | ctc_loss = warp_ctc.CTCLoss() 125 | cost = ctc_loss(probs, labels, probs_sizes, label_sizes) 126 | cost.backward() 127 | 128 | 129 | if __name__ == '__main__': 130 | pytest.main([__file__]) 131 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/pytorch_binding/warpctc_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import warpctc_pytorch as warp_ctc 3 | from torch.autograd import Function 4 | from torch.nn import Module 5 | 6 | from ._warp_ctc import * # noqa 7 | 8 | 9 | def _assert_no_grad(tensor): 10 | assert not tensor.requires_grad, \ 11 | "gradients only computed for acts - please " \ 12 | "mark other tensors as not requiring gradients" 13 | 14 | 15 | class _CTC(Function): 16 | @staticmethod 17 | def forward(ctx, acts, labels, act_lens, label_lens, size_average=False, 18 | length_average=False, blank=0): 19 | is_cuda = True if acts.is_cuda else False 20 | acts = acts.contiguous() 21 | loss_func = warp_ctc.gpu_ctc if is_cuda else warp_ctc.cpu_ctc 22 | grads = torch.zeros(acts.size()).type_as(acts) 23 | minibatch_size = acts.size(1) 24 | costs = torch.zeros(minibatch_size).cpu() 25 | loss_func(acts, 26 | grads, 27 | labels, 28 | label_lens, 29 | act_lens, 30 | minibatch_size, 31 | costs, 32 | blank) 33 | 34 | costs = torch.FloatTensor([costs.sum()]) 35 | 36 | if length_average: 37 | # Compute the avg. log-probability per batch sample and frame. 38 | total_length = torch.sum(act_lens).item() 39 | grads = grads / total_length 40 | costs = costs / total_length 41 | elif size_average: 42 | # Compute the avg. log-probability per batch sample. 43 | grads = grads / minibatch_size 44 | costs = costs / minibatch_size 45 | 46 | ctx.grads = grads 47 | return costs 48 | 49 | @staticmethod 50 | def backward(ctx, grad_output): 51 | _grad_output = grad_output.to(ctx.grads.device) 52 | return ctx.grads.mul_(_grad_output), None, None, None, None, None, None 53 | 54 | class CTCLoss(Module): 55 | """ 56 | Parameters: 57 | size_average (bool): normalize the loss by the batch size 58 | (default: `False`) 59 | length_average (bool): normalize the loss by the total number of frames 60 | in the batch. If `True`, supersedes `size_average` 61 | (default: `False`) 62 | """ 63 | def __init__(self, blank=0, size_average=False, length_average=False): 64 | super(CTCLoss, self).__init__() 65 | self.ctc = _CTC.apply 66 | self.blank = blank 67 | self.size_average = size_average 68 | self.length_average = length_average 69 | 70 | def forward(self, acts, labels, act_lens, label_lens): 71 | """ 72 | acts: Tensor of (seqLength x batch x outputDim) containing output from network 73 | labels: 1 dimensional Tensor containing all the targets of the batch in one sequence 74 | act_lens: Tensor of size (batch) containing size of each output sequence from the network 75 | label_lens: Tensor of (batch) containing label length of each example 76 | """ 77 | assert len(labels.size()) == 1 # labels must be 1 dimensional 78 | _assert_no_grad(labels) 79 | _assert_no_grad(act_lens) 80 | _assert_no_grad(label_lens) 81 | return self.ctc(acts, labels, act_lens, label_lens, self.size_average, 82 | self.length_average, self.blank) 83 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/src/ctc_entrypoint.cu: -------------------------------------------------------------------------------- 1 | ctc_entrypoint.cpp -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/tests/random.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | 5 | std::vector 6 | genActs(int size) { 7 | std::vector arr(size); 8 | std::mt19937 gen(0); 9 | std::uniform_real_distribution<> dis(0, 1); 10 | for(int i = 0; i < size; ++i) 11 | arr[i] = dis(gen); 12 | return arr; 13 | } 14 | 15 | std::vector 16 | genLabels(int alphabet_size, int L) { 17 | std::vector label(L); 18 | 19 | std::mt19937 gen(1); 20 | std::uniform_int_distribution<> dis(1, alphabet_size - 1); 21 | 22 | for(int i = 0; i < L; ++i) { 23 | label[i] = dis(gen); 24 | } 25 | // guarantee repeats for testing 26 | if (L >= 3) { 27 | label[L / 2] = label[L / 2 + 1]; 28 | label[L / 2 - 1] = label[L / 2]; 29 | } 30 | return label; 31 | } 32 | 33 | -------------------------------------------------------------------------------- /Mult_BERT/warp-ctc/tests/test.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | #include 10 | 11 | inline void throw_on_error(ctcStatus_t status, const char* message) { 12 | if (status != CTC_STATUS_SUCCESS) { 13 | throw std::runtime_error(message + (", stat = " + 14 | std::string(ctcGetStatusString(status)))); 15 | } 16 | } 17 | 18 | #ifdef __CUDACC__ 19 | #include 20 | #include 21 | 22 | inline void throw_on_error(cudaError_t error, const char* message) { 23 | if (error) { 24 | throw thrust::system_error(error, thrust::cuda_category(), message); 25 | } 26 | } 27 | 28 | #endif 29 | 30 | std::vector genActs(int size); 31 | std::vector genLabels(int alphabet_size, int L); 32 | 33 | float rel_diff(const std::vector& grad, 34 | const std::vector& num_grad) { 35 | float diff = 0.; 36 | float tot = 0.; 37 | for(size_t idx = 0; idx < grad.size(); ++idx) { 38 | diff += (grad[idx] - num_grad[idx]) * (grad[idx] - num_grad[idx]); 39 | tot += grad[idx] * grad[idx]; 40 | } 41 | 42 | return diff / tot; 43 | } 44 | 45 | // Numerically stable softmax for a minibatch of 1 46 | void softmax(const float* const acts, 47 | int alphabet_size, int T, 48 | float *probs) { 49 | 50 | for (int t = 0; t < T; ++t) { 51 | 52 | float max_activation = 53 | -std::numeric_limits::infinity(); 54 | 55 | for (int a = 0; a < alphabet_size; ++a) 56 | max_activation = 57 | std::max(max_activation, acts[t*alphabet_size + a]); 58 | 59 | float denom = 0; 60 | for (int a = 0; a < alphabet_size; ++a) 61 | denom += std::exp(acts[t*alphabet_size + a] - max_activation); 62 | 63 | for (int a = 0; a < alphabet_size; ++a) 64 | probs[t*alphabet_size + a] = 65 | std::exp(acts[t*alphabet_size + a] - max_activation) / denom; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Multimodal-Infomax/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Deep Cognition and Language Research (DeCLaRe) Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Multimodal-Infomax/README.md: -------------------------------------------------------------------------------- 1 | # MultiModal-InfoMax 2 | 3 | This repository contains the official implementation code of the paper [Improving Multimodal Fusion with Hierarchical Mutual Information Maximization for Multimodal Sentiment Analysis](https://arxiv.org/pdf/2109.00412.pdf), accepted at **EMNLP 2021**. 4 | 5 | :fire: If you would be interested in other multimodal works in our DeCLaRe Lab, welcome to visit the [clustered repository](https://github.com/declare-lab/multimodal-deep-learning) 6 | 7 | ## Introduction 8 | Multimodal-informax (MMIM) synthesizes fusion results from multi-modality input through a two-level mutual information (MI) maximization. We use BA (Barber-Agakov) lower bound and contrastive predictive coding as the target function to be maximized. To facilitate the computation, we design an entropy estimation module with associated history data memory to facilitate the computation of BA lower bound and the training process. 9 | 10 | ![Alt text](img/ModelFigSingle.png?raw=true "Model") 11 | 12 | ## Usage 13 | 1. Download the CMU-MOSI and CMU-MOSEI dataset from [Google Drive](https://drive.google.com/drive/folders/1djN_EkrwoRLUt7Vq_QfNZgCl_24wBiIK?usp=sharing) or [Baidu Disk](https://pan.baidu.com/s/1Wxo4Bim9JhNmg8265p3ttQ) (extraction code: g3m2). Place them under the folder `Multimodal-Infomax/datasets` 14 | 15 | 2. Set up the environment (need conda prerequisite) 16 | ``` 17 | conda env create -f environment.yml 18 | conda activate MMIM 19 | ``` 20 | 21 | 3. Start training 22 | ``` 23 | python main.py --dataset mosi --contrast 24 | ``` 25 | 26 | ## Citation 27 | Please cite our paper if you find our work useful for your research: 28 | ```bibtex 29 | @article{han2021improving, 30 | title={Improving Multimodal Fusion with Hierarchical Mutual Information Maximization for Multimodal Sentiment Analysis}, 31 | author={Han, Wei and Chen, Hui and Poria, Soujanya}, 32 | journal={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, 33 | year={2021} 34 | } 35 | ``` 36 | 37 | ## Contact 38 | Should you have any question, feel free to contact me through [henryhan88888@gmail.com](henryhan88888@gmail.com) 39 | -------------------------------------------------------------------------------- /Multimodal-Infomax/img/ModelFigSingle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/img/ModelFigSingle.png -------------------------------------------------------------------------------- /Multimodal-Infomax/src/__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/src/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /Multimodal-Infomax/src/__pycache__/create_dataset.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/src/__pycache__/create_dataset.cpython-37.pyc -------------------------------------------------------------------------------- /Multimodal-Infomax/src/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/src/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /Multimodal-Infomax/src/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/src/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /Multimodal-Infomax/src/__pycache__/solver.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Multimodal-Infomax/src/__pycache__/solver.cpython-37.pyc -------------------------------------------------------------------------------- /Multimodal-Infomax/src/main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import argparse 3 | import numpy as np 4 | 5 | from utils import * 6 | from torch.utils.data import DataLoader 7 | from solver import Solver 8 | from config import get_args, get_config, output_dim_dict, criterion_dict 9 | from data_loader import get_loader 10 | 11 | 12 | def set_seed(seed): 13 | torch.set_default_tensor_type('torch.FloatTensor') 14 | torch.manual_seed(seed) 15 | if torch.cuda.is_available(): 16 | torch.cuda.manual_seed_all(seed) 17 | torch.set_default_tensor_type('torch.cuda.FloatTensor') 18 | 19 | torch.backends.cudnn.deterministic = True 20 | torch.backends.cudnn.benchmark = False 21 | use_cuda = True 22 | 23 | if __name__ == '__main__': 24 | args = get_args() 25 | dataset = str.lower(args.dataset.strip()) 26 | args.seed = 1114 27 | 28 | set_seed(args.seed) 29 | #1111, 2222, 1114,1116 30 | #不行的: 1115, 1112，1113，1000，1222 31 | print("Start loading the data....") 32 | train_config = get_config(dataset, mode='train', batch_size=args.batch_size) 33 | valid_config = get_config(dataset, mode='valid', batch_size=args.batch_size) 34 | test_config = get_config(dataset, mode='test', batch_size=args.batch_size) 35 | 36 | # pretrained_emb saved in train_config here 37 | train_loader = get_loader(args, train_config, shuffle=True) 38 | print('Training data loaded!') 39 | valid_loader = get_loader(args, valid_config, shuffle=False) 40 | print('Validation data loaded!') 41 | test_loader = get_loader(args, test_config, shuffle=False) 42 | print('Test data loaded!') 43 | print('Finish loading the data....') 44 | 45 | torch.autograd.set_detect_anomaly(True) 46 | 47 | # addintional appending 48 | args.word2id = train_config.word2id 49 | 50 | # architecture parameters 51 | args.d_tin, args.d_vin, args.d_ain = train_config.tva_dim 52 | args.dataset = args.data = dataset 53 | args.when = args.when 54 | args.n_class = output_dim_dict.get(dataset, 1) 55 | args.criterion = criterion_dict.get(dataset, 'MSELoss') 56 | 57 | solver = Solver(args, train_loader=train_loader, dev_loader=valid_loader, 58 | test_loader=test_loader, is_train=True) 59 | solver.train_and_eval() -------------------------------------------------------------------------------- /Multimodal-Infomax/src/test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import argparse 3 | import numpy as np 4 | 5 | from utils import * 6 | from torch.utils.data import DataLoader 7 | from solver import Solver 8 | from config import get_args, get_config, output_dim_dict, criterion_dict 9 | from data_loader import get_loader 10 | from model import MMIM 11 | 12 | from sklearn.metrics import classification_report, accuracy_score, f1_score 13 | 14 | if __name__ == '__main__': 15 | args = get_args() 16 | dataset = str.lower(args.dataset.strip()) 17 | 18 | print("Start loading the data....") 19 | train_config = get_config(dataset, mode='train', batch_size=args.batch_size) 20 | valid_config = get_config(dataset, mode='valid', batch_size=args.batch_size) 21 | test_config = get_config(dataset, mode='test', batch_size=args.batch_size) 22 | 23 | # pretrained_emb saved in train_config here 24 | train_loader = get_loader(args, train_config, shuffle=True) 25 | print('Training data loaded!') 26 | valid_loader = get_loader(args, valid_config, shuffle=False) 27 | print('Validation data loaded!') 28 | test_loader = get_loader(args, test_config, shuffle=False) 29 | print('Test data loaded!') 30 | print('Finish loading the data....') 31 | 32 | torch.autograd.set_detect_anomaly(True) 33 | 34 | # addintional appending 35 | args.word2id = train_config.word2id 36 | 37 | # architecture parameters 38 | args.d_tin, args.d_vin, args.d_ain = train_config.tva_dim 39 | args.dataset = args.data = dataset 40 | args.when = args.when 41 | args.n_class = output_dim_dict.get(dataset, 1) 42 | args.criterion = criterion_dict.get(dataset, 'MSELoss') 43 | 44 | #model 45 | if args.train_method=="missing": 46 | save_mode = f'0' 47 | elif args.train_method=="g_noise": 48 | save_mode = f'N' 49 | elif args.train_method=="hybird": 50 | save_mode = f'H' 51 | else: 52 | raise 53 | save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={save_mode}/' 54 | print("-----------------save_dir") 55 | print(save_dir) 56 | # if not os.path.exists(save_dir): 57 | # os.makedirs(save_dir) 58 | 59 | # save_dir = f'pre_trained_models/{args.data}/best_{int(args.train_changed_pct*100)}%{args.train_changed_modal[0].upper()}={0 if args.train_method=="missing" else "N"}/' 60 | 61 | if args.dataset == 'mosi': 62 | # model = torch.load(f'pre_trained_models/best_mosi.pt') 63 | model = torch.load(f'{save_dir}/best_mosi.pt') 64 | elif args.dataset == 'mosei': 65 | # model = torch.load(f'pre_trained_models/best_mosei.pt') 66 | model = torch.load(f'{save_dir}/best_mosei.pt') 67 | 68 | solver = Solver(args, train_loader=train_loader, dev_loader=valid_loader, 69 | test_loader=test_loader, is_train=False, model=model) 70 | 71 | test_loss, results, truths = solver.evaluate(solver.model, solver.criterion, test=True) 72 | 73 | if args.dataset == 'mosi': 74 | # eval_mosi(results, truths, True) 75 | ########################################################## 76 | y_pred = results 77 | y_true = truths 78 | accuracy = calc_metrics(y_true, y_pred, 'test', True) 79 | print("------------------------------------------------------") 80 | print("y_pred:", y_pred.shape) 81 | print("y_true:", y_true.shape) 82 | 83 | non_zeros = np.array([i for i, e in enumerate(y_true) if e != 0]) 84 | binary_truth = (y_true[non_zeros] > 0) 85 | binary_preds = (y_pred[non_zeros] > 0) 86 | acc2 = accuracy_score(binary_truth.cpu(), binary_preds.cpu()) 87 | print("acc2:", acc2) 88 | # print(torch.eq(binary_truth, binary_preds)) 89 | idx_diff = [] 90 | for i in range(len(binary_preds)): 91 | if (binary_preds[i][0] == binary_truth[i][0]): 92 | pass 93 | else: 94 | idx_diff.append(i) 95 | print("idx_diff:", idx_diff) 96 | ########################################################## 97 | elif args.dataset == 'mosei': 98 | # eval_mosei_senti(results, truths, True) 99 | ########################################################## 100 | y_pred = results 101 | y_true = truths 102 | accuracy = calc_metrics(y_true, y_pred, 'test', True) 103 | ########################################################## 104 | print("------------------------------------------------------") 105 | print("y_pred:", y_pred.shape) 106 | print("y_true:", y_true.shape) 107 | 108 | non_zeros = np.array([i for i, e in enumerate(y_true) if e != 0]) 109 | binary_truth = (y_true[non_zeros] > 0) 110 | binary_preds = (y_pred[non_zeros] > 0) 111 | acc2 = accuracy_score(binary_truth.cpu(), binary_preds.cpu()) 112 | print("acc2:", acc2) 113 | # print(torch.eq(binary_truth, binary_preds)) 114 | idx_diff = [] 115 | for i in range(len(binary_preds)): 116 | if (binary_preds[i][0] == binary_truth[i][0]): 117 | pass 118 | else: 119 | idx_diff.append(i) 120 | print("idx_diff:", idx_diff) 121 | # solver.train_and_eval() -------------------------------------------------------------------------------- /Multimodal-Infomax/src/test.sh: -------------------------------------------------------------------------------- 1 | rm -r /home/yingting/.cache/huggingface/transformers/transformers 2 | CUDA_VISIBLE_DEVICES=1 python test.py --dataset mosi --contrast --lr_main 1e-3 --lr_mmilb 4e-3 --alpha 0.3 --beta 0.1 --batch_size 32 --d_vh 32 --d_ah 32 --train_method missing --train_changed_modal language --train_changed_pct 0.3 --test_method missing --test_changed_modal language --test_changed_pct 0 --is_test 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MSA-Robustness 2 | NAACL 2022 paper on [Analyzing Modality Robustness in Multimodal Sentiment Analysis](https://arxiv.org/pdf/2205.15465.pdf) 3 | 4 | # Setup the environment 5 | Configure the environment of different models respectively, configure the corresponding environment according to the requirements.txt in the model directory. 6 | 7 | # Data Download 8 | - Install [CMU Multimodal SDK](https://github.com/A2Zadeh/CMU-MultimodalSDK). Ensure, you can perform ```from mmsdk import mmdatasdk```. 9 | 10 | # Running the code 11 | Take MISA as an example 12 | 13 | 1. ```cd MISA``` 14 | 2. ```cd src``` 15 | 3. Set ```word_emb_path``` in ```config.py``` to [glove file](http://nlp.stanford.edu/data/glove.840B.300d.zip). 16 | 4. Set ```sdk_dir``` to the path of CMU-MultimodalSDK. 17 | 3. ```bash run.sh``` When doing robustness training, run the "TRAIN" section of run.sh, and when doing diagnostic tests, run the "TEST" section of run.sh. 18 | 19 |     ```--train_method``` means the robustness training method, one of ```{missing, g_noise, hybird}```, ```missing``` means set to zero noise, ```g_noise``` means set to Gaussian Noise, ```hybird``` means the data of train_changed_pct is set to zero_noise, and the data of train_changed_pct is set to Gaussian_Noise. 20 | 21 |     ```--train_changed_modal``` means the modality of change during training, one of ```{language, video, audio}```. 22 | 23 |     ```--train_changed_pct``` means the percentage of change during training, can set between ```0~1```. 24 | 25 |     ```--test_method``` means the diagnostic tests method, one of ```{missing, g_noise, hybird}```, ```missing``` means set to zero noise, ```g_noise``` means set to Gaussian Noise, ```hybird``` means the data of test_changed_pct is set to zero_noise, and the data of test_changed_pct is set to Gaussian_Noise. 26 | 27 |     ```--test_changed_modal``` means the modality of change during testing, one of ```{language, video, audio}```. 28 | 29 |     ```--train_changed_pct``` means the percentage of change during testing, can set between ```0~1```. 30 | 31 | # Citation 32 | 33 | ``` 34 | @article{hazarika2022analyzing, 35 | title={Analyzing Modality Robustness in Multimodal Sentiment Analysis}, 36 | author={Hazarika, Devamanyu and Li, Yingting and Cheng, Bo and Zhao, Shuai and Zimmermann, Roger and Poria, Soujanya}, 37 | publisher={NAACL}, 38 | year={2022} 39 | } 40 | 41 | ``` 42 | 43 | 44 | -------------------------------------------------------------------------------- /Self-MM/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 iyuge2 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Self-MM/README.md: -------------------------------------------------------------------------------- 1 | ![Python 3.6](https://img.shields.io/badge/python-3.6-green.svg) 2 | 3 | ## SELF-MM 4 | > Pytorch implementation for codes in [Learning Modality-Specific Representations with Self-Supervised Multi-Task Learning for Multimodal Sentiment Analysis (AAAI2021)](https://arxiv.org/abs/2102.04830). Please see our another repo [MMSA](https://github.com/thuiar/MMSA) for more details, which is a scalable framework for MSA. 5 | 6 | ### Model 7 | 8 | ![model](assets/MainModel.png) 9 | 10 | ### Usage 11 | 12 | 1. Datasets and pre-trained berts 13 | 14 | Download dataset features and pre-trained berts from the following links. 15 | 16 | - [Baidu Cloud Drive](https://pan.baidu.com/s/1oksuDEkkd3vGg2oBMBxiVw) with code: `ctgs` 17 | - [Google Cloud Drive](https://drive.google.com/drive/folders/1E5kojBirtd5VbfHsFp6FYWkQunk73Nsv?usp=sharing) 18 | 19 | For all features, you can use `SHA-1 Hash Value` to check the consistency. 20 | > `MOSI/unaligned_50.pkl`: `5da0b8440fc5a7c3a457859af27458beb993e088` 21 | > `MOSI/aligned_50.pkl`: `5c62b896619a334a7104c8bef05d82b05272c71c` 22 | > `MOSEI/unaligned_50.pkl`: `db3e2cff4d706a88ee156981c2100975513d4610` 23 | > `MOSEI/aligned_50.pkl`: `ef49589349bc1c2bc252ccc0d4657a755c92a056` 24 | > `SIMS/unaligned_39.pkl`: `a00c73e92f66896403c09dbad63e242d5af756f8` 25 | 26 | Due to the size limitations, the MOSEI features and SIMS raw videos are available in `Baidu Cloud Drive` only. All dataset features are organized as: 27 | 28 | ```python 29 | { 30 | "train": { 31 | "raw_text": [], 32 | "audio": [], 33 | "vision": [], 34 | "id": [], # [video_id$_$clip_id, ..., ...] 35 | "text": [], 36 | "text_bert": [], 37 | "audio_lengths": [], 38 | "vision_lengths": [], 39 | "annotations": [], 40 | "classification_labels": [], # Negative(< 0), Neutral(0), Positive(> 0) 41 | "regression_labels": [] 42 | }, 43 | "valid": {***}, # same as the "train" 44 | "test": {***}, # same as the "train" 45 | } 46 | ``` 47 | 48 | For MOSI and MOSEI, the pre-extracted text features are from BERT, different from the original glove features in the [CMU-Multimodal-SDK](http://immortal.multicomp.cs.cmu.edu/raw_datasets/processed_data/). 49 | 50 | For SIMS, if you want to extract features from raw videos, you need to install [Openface Toolkits](https://github.com/TadasBaltrusaitis/OpenFace/wiki) first, and then refer our codes in the `data/DataPre.py`. 51 | 52 | ``` 53 | python data/DataPre.py --data_dir [path_to_Dataset] --language ** --openface2Path [path_to_FeatureExtraction] 54 | ``` 55 | 56 | For bert models, you also can download [Bert-Base, Chinese](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip) from [Google-Bert](https://github.com/google-research/bert). And then, convert tensorflow into pytorch using [transformers-cli](https://huggingface.co/transformers/converting_tensorflow_models.html) 57 | 58 | 2. Clone this repo and install requirements. 59 | ``` 60 | git clone https://github.com/thuiar/Self-MM 61 | cd Self-MM 62 | conda create --name self_mm python=3.7 63 | source activate self_mm 64 | pip install -r requirements.txt 65 | ``` 66 | 67 | 3. Make some changes 68 | Modify the `config/config_tune.py` and `config/config_regression.py` to update dataset pathes. 69 | 70 | 4. Run codes 71 | ``` 72 | python run.py --modelName self_mm --datasetName mosi 73 | ``` 74 | 75 | ### Results 76 | 77 | > Detailed results are shown in [MMSA](https://github.com/thuiar/MMSA) > [results/result-stat.md](https://github.com/thuiar/MMSA/blob/master/results/result-stat.md). 78 | 79 | ### Paper 80 | --- 81 | Please cite our paper if you find our work useful for your research: 82 | ``` 83 | @inproceedings{yu2021le, 84 | title={Learning Modality-Specific Representations with Self-Supervised Multi-Task Learning for Multimodal Sentiment Analysis}, 85 | author={Yu, Wenmeng and Xu, Hua and Ziqi, Yuan and Jiele, Wu}, 86 | booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, 87 | year={2021} 88 | } 89 | ``` 90 | -------------------------------------------------------------------------------- /Self-MM/Recent_work.md: -------------------------------------------------------------------------------- 1 | Analyzing Multimodal Sentiment Analysis 2 | 3 | Predominant focus of recent works is towards imporve fusion, or alignment of modalities(text, audio, video) to boost overall performance - CMU-MOSI, MOSEI, MELD, etc. 4 | 5 | 1. MISA 6 | 2. Self-MM 7 | 3. Bi Bi-modal 8 | 4. Mutual Information Maximization 9 | 5. Cross-modal Transformers 10 | 11 | https://github.com/declare-lab/multimodal-deep-learning 12 | 13 | Overlooked aspects are: 1) Missing modality 2) Noise Sensitivity 3) Leveraging Strong Modalities 14 | 15 | # Robustness Direction 16 | 1. RQ1) How robust are the present methods towards missing modalities? If not, is there an easy way to make them robust? 17 | 2. RQ2) How sensitive are modalities towards real-world noise? Are certain modalities more sensitive(e.g. Language modality)? 18 | 19 | Does Multimodal scores hamper the language unimodal score? If yes, can we reduce that. 20 | 21 | # Improving Performance Direction 22 | 3. RQ3) Language operates as a strong modality in Multimodal Sentiment Analysis(MSA) datasets. But most works do not leverage on this fact and attempt to equally model all modalities. Can we focus on language as a dominant modality and the remaining audio and visual cues as complementary modalities? 23 | 24 | Takeaways: We are trying to leverage language modality - in a way that is robust. 25 | 26 | # Revamping the benchmark 27 | 4. RQ4) Benchmarking MSA with modern neural feature extractors - Wav2vec for audio.(Engineering) 28 | 29 | -------------------------------------------------------------------------------- /Self-MM/assets/MainModel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/assets/MainModel.png -------------------------------------------------------------------------------- /Self-MM/config/__pycache__/config_regression.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/config/__pycache__/config_regression.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/config/__pycache__/config_tune.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/config/__pycache__/config_tune.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/data/__init__.py -------------------------------------------------------------------------------- /Self-MM/data/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/data/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/data/__pycache__/load_data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/data/__pycache__/load_data.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/data/getLengths.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle as plk 3 | import numpy as np 4 | 5 | from tqdm import tqdm 6 | 7 | def get_lengths(mode, feature_name): 8 | fd = data[mode][feature_name] 9 | max_len = fd.shape[1] 10 | 11 | c_sum = np.sum(fd, axis=-1) 12 | lengths = [] 13 | for i in tqdm(range(fd.shape[0])): 14 | null = True 15 | zeros = np.zeros([fd.shape[1], fd.shape[2]]) 16 | cur_length = max_len 17 | for j in range(max_len): 18 | if c_sum[i][j] == 0: 19 | cur_length = j 20 | null = False 21 | break 22 | if cur_length == 0: 23 | cur_length = 1 24 | lengths.append(cur_length) 25 | return lengths 26 | 27 | with open('/home/sharing/disk3/dataset/multimodal-sentiment-dataset/ALL/mosei/unaligned_50.pkl', 'rb') as lf: 28 | data = plk.load(lf) 29 | 30 | def handleData(mode): 31 | # data[mode]['audio_lengths'], _ = get_lengths(mode, 'feature_A') 32 | # data[mode]['vision_lengths'], _ = get_lengths(mode, 'feature_V') 33 | data[mode]['audio_lengths'] = get_lengths(mode, 'audio') 34 | data[mode]['vision_lengths'] = get_lengths(mode, 'vision') 35 | 36 | handleData('train') 37 | handleData('valid') 38 | handleData('test') 39 | 40 | with open('/home/sharing/disk3/dataset/multimodal-sentiment-dataset/ALL/mosei/unaligned_50.pkl', 'wb') as df: 41 | plk.dump(data, df, protocol = 4) -------------------------------------------------------------------------------- /Self-MM/models/AMIO.py: -------------------------------------------------------------------------------- 1 | """ 2 | AIO -- All Model in One 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from torch.nn.parameter import Parameter 9 | from torch.nn.init import xavier_uniform, xavier_normal, orthogonal 10 | 11 | from models.subNets.AlignNets import AlignSubNet 12 | from models.multiTask import * 13 | 14 | __all__ = ['AMIO'] 15 | 16 | MODEL_MAP = { 17 | 'self_mm': SELF_MM 18 | } 19 | 20 | class AMIO(nn.Module): 21 | def __init__(self, args): 22 | super(AMIO, self).__init__() 23 | self.need_model_aligned = args.need_model_aligned 24 | # simulating word-align network (for seq_len_T == seq_len_A == seq_len_V) 25 | if(self.need_model_aligned): 26 | self.alignNet = AlignSubNet(args, 'avg_pool') 27 | if 'seq_lens' in args.keys(): 28 | args.seq_lens = self.alignNet.get_seq_len() 29 | lastModel = MODEL_MAP[args.modelName] 30 | self.Model = lastModel(args) 31 | 32 | def forward(self, text_x, audio_x, video_x): 33 | if(self.need_model_aligned): 34 | text_x, audio_x, video_x = self.alignNet(text_x, audio_x, video_x) 35 | return self.Model(text_x, audio_x, video_x) -------------------------------------------------------------------------------- /Self-MM/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/__init__.py -------------------------------------------------------------------------------- /Self-MM/models/__pycache__/AMIO.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/__pycache__/AMIO.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/multiTask/__init__.py: -------------------------------------------------------------------------------- 1 | from models.multiTask.SELF_MM import SELF_MM 2 | 3 | __all__ = ['SELF_MM'] -------------------------------------------------------------------------------- /Self-MM/models/multiTask/__pycache__/SELF_MM.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/multiTask/__pycache__/SELF_MM.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/multiTask/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/multiTask/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/subNets/AlignNets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.autograd import Variable 5 | from torch.nn.parameter import Parameter 6 | from torch.nn.init import xavier_uniform, xavier_normal, orthogonal 7 | 8 | __all__ = ['AlignSubNet'] 9 | 10 | class CTCModule(nn.Module): 11 | def __init__(self, in_dim, out_seq_len): 12 | ''' 13 | This module is performing alignment from A (e.g., audio) to B (e.g., text). 14 | :param in_dim: Dimension for input modality A 15 | :param out_seq_len: Sequence length for output modality B 16 | From: https://github.com/yaohungt/Multimodal-Transformer 17 | ''' 18 | super(CTCModule, self).__init__() 19 | # Use LSTM for predicting the position from A to B 20 | self.pred_output_position_inclu_blank = nn.LSTM(in_dim, out_seq_len+1, num_layers=2, batch_first=True) # 1 denoting blank 21 | 22 | self.out_seq_len = out_seq_len 23 | 24 | self.softmax = nn.Softmax(dim=2) 25 | 26 | def forward(self, x): 27 | ''' 28 | :input x: Input with shape [batch_size x in_seq_len x in_dim] 29 | ''' 30 | # NOTE that the index 0 refers to blank. 31 | pred_output_position_inclu_blank, _ = self.pred_output_position_inclu_blank(x) 32 | 33 | prob_pred_output_position_inclu_blank = self.softmax(pred_output_position_inclu_blank) # batch_size x in_seq_len x out_seq_len+1 34 | prob_pred_output_position = prob_pred_output_position_inclu_blank[:, :, 1:] # batch_size x in_seq_len x out_seq_len 35 | prob_pred_output_position = prob_pred_output_position.transpose(1,2) # batch_size x out_seq_len x in_seq_len 36 | pseudo_aligned_out = torch.bmm(prob_pred_output_position, x) # batch_size x out_seq_len x in_dim 37 | 38 | # pseudo_aligned_out is regarded as the aligned A (w.r.t B) 39 | # return pseudo_aligned_out, (pred_output_position_inclu_blank) 40 | return pseudo_aligned_out 41 | 42 | class AlignSubNet(nn.Module): 43 | def __init__(self, args, mode): 44 | """ 45 | mode: the way of aligning 46 | avg_pool, ctc, conv1d 47 | """ 48 | super(AlignSubNet, self).__init__() 49 | assert mode in ['avg_pool', 'ctc', 'conv1d'] 50 | 51 | in_dim_t, in_dim_a, in_dim_v = args.feature_dims 52 | seq_len_t, seq_len_a, seq_len_v = args.seq_lens 53 | self.dst_len = seq_len_t 54 | self.mode = mode 55 | 56 | self.ALIGN_WAY = { 57 | 'avg_pool': self.__avg_pool, 58 | 'ctc': self.__ctc, 59 | 'conv1d': self.__conv1d 60 | } 61 | 62 | if mode == 'conv1d': 63 | self.conv1d_T = nn.Conv1d(seq_len_t, self.dst_len, kernel_size=1, bias=False) 64 | self.conv1d_A = nn.Conv1d(seq_len_a, self.dst_len, kernel_size=1, bias=False) 65 | self.conv1d_V = nn.Conv1d(seq_len_v, self.dst_len, kernel_size=1, bias=False) 66 | elif mode == 'ctc': 67 | self.ctc_t = CTCModule(in_dim_t, self.dst_len) 68 | self.ctc_a = CTCModule(in_dim_a, self.dst_len) 69 | self.ctc_v = CTCModule(in_dim_v, self.dst_len) 70 | 71 | def get_seq_len(self): 72 | return self.dst_len 73 | 74 | def __ctc(self, text_x, audio_x, video_x): 75 | text_x = self.ctc_t(text_x) if text_x.size(1) != self.dst_len else text_x 76 | audio_x = self.ctc_a(audio_x) if audio_x.size(1) != self.dst_len else audio_x 77 | video_x = self.ctc_v(video_x) if video_x.size(1) != self.dst_len else video_x 78 | return text_x, audio_x, video_x 79 | 80 | def __avg_pool(self, text_x, audio_x, video_x): 81 | def align(x): 82 | raw_seq_len = x.size(1) 83 | if raw_seq_len == self.dst_len: 84 | return x 85 | if raw_seq_len // self.dst_len == raw_seq_len / self.dst_len: 86 | pad_len = 0 87 | pool_size = raw_seq_len // self.dst_len 88 | else: 89 | pad_len = self.dst_len - raw_seq_len % self.dst_len 90 | pool_size = raw_seq_len // self.dst_len + 1 91 | pad_x = x[:, -1, :].unsqueeze(1).expand([x.size(0), pad_len, x.size(-1)]) 92 | x = torch.cat([x, pad_x], dim=1).view(x.size(0), pool_size, self.dst_len, -1) 93 | x = x.mean(dim=1) 94 | return x 95 | text_x = align(text_x) 96 | audio_x = align(audio_x) 97 | video_x = align(video_x) 98 | return text_x, audio_x, video_x 99 | 100 | def __conv1d(self, text_x, audio_x, video_x): 101 | text_x = self.conv1d_T(text_x) if text_x.size(1) != self.dst_len else text_x 102 | audio_x = self.conv1d_A(text_x) if audio_x.size(1) != self.dst_len else audio_x 103 | video_x = self.conv1d_V(text_x) if video_x.size(1) != self.dst_len else video_x 104 | return text_x, audio_x, video_x 105 | 106 | def forward(self, text_x, audio_x, video_x): 107 | # already aligned 108 | if text_x.size(1) == audio_x.size(1) == video_x.size(1): 109 | return text_x, audio_x, video_x 110 | return self.ALIGN_WAY[self.mode](text_x, audio_x, video_x) -------------------------------------------------------------------------------- /Self-MM/models/subNets/BertTextEncoder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import collections 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | from transformers import BertTokenizer, BertModel 10 | 11 | __all__ = ['BertTextEncoder'] 12 | 13 | class BertTextEncoder(nn.Module): 14 | def __init__(self, language='en', use_finetune=False): 15 | """ 16 | language: en / cn 17 | """ 18 | super(BertTextEncoder, self).__init__() 19 | 20 | assert language in ['en', 'cn'] 21 | 22 | tokenizer_class = BertTokenizer 23 | model_class = BertModel 24 | # directory is fine 25 | # pretrained_weights = '/home/sharing/disk3/pretrained_embedding/Chinese/bert/pytorch' 26 | if language == 'en': 27 | self.tokenizer = tokenizer_class.from_pretrained('pretrained_model/bert_en', do_lower_case=True) 28 | self.model = model_class.from_pretrained('pretrained_model/bert_en') 29 | elif language == 'cn': 30 | self.tokenizer = tokenizer_class.from_pretrained('pretrained_model/bert_cn') 31 | self.model = model_class.from_pretrained('pretrained_model/bert_cn') 32 | 33 | self.use_finetune = use_finetune 34 | 35 | def get_tokenizer(self): 36 | return self.tokenizer 37 | 38 | def from_text(self, text): 39 | """ 40 | text: raw data 41 | """ 42 | input_ids = self.get_id(text) 43 | with torch.no_grad(): 44 | last_hidden_states = self.model(input_ids)[0] # Models outputs are now tuples 45 | return last_hidden_states.squeeze() 46 | 47 | def forward(self, text): 48 | """ 49 | text: (batch_size, 3, seq_len) 50 | 3: input_ids, input_mask, segment_ids 51 | input_ids: input_ids, 52 | input_mask: attention_mask, 53 | segment_ids: token_type_ids 54 | """ 55 | input_ids, input_mask, segment_ids = text[:,0,:].long(), text[:,1,:].float(), text[:,2,:].long() 56 | if self.use_finetune: 57 | last_hidden_states = self.model(input_ids=input_ids, 58 | attention_mask=input_mask, 59 | token_type_ids=segment_ids)[0] # Models outputs are now tuples 60 | else: 61 | with torch.no_grad(): 62 | last_hidden_states = self.model(input_ids=input_ids, 63 | attention_mask=input_mask, 64 | token_type_ids=segment_ids)[0] # Models outputs are now tuples 65 | return last_hidden_states 66 | 67 | if __name__ == "__main__": 68 | bert_normal = BertTextEncoder() 69 | -------------------------------------------------------------------------------- /Self-MM/models/subNets/FeatureNets.py: -------------------------------------------------------------------------------- 1 | """ 2 | Ref paper: Tensor Fusion Network for Multimodal Sentiment Analysis 3 | Ref url: https://github.com/Justin1904/TensorFusionNetworks 4 | """ 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | __all__ = ['SubNet', 'TextSubNet'] 10 | 11 | class SubNet(nn.Module): 12 | ''' 13 | The subnetwork that is used in TFN for video and audio in the pre-fusion stage 14 | ''' 15 | 16 | def __init__(self, in_size, hidden_size, dropout): 17 | ''' 18 | Args: 19 | in_size: input dimension 20 | hidden_size: hidden layer dimension 21 | dropout: dropout probability 22 | Output: 23 | (return value in forward) a tensor of shape (batch_size, hidden_size) 24 | ''' 25 | super(SubNet, self).__init__() 26 | self.norm = nn.BatchNorm1d(in_size) 27 | self.drop = nn.Dropout(p=dropout) 28 | self.linear_1 = nn.Linear(in_size, hidden_size) 29 | self.linear_2 = nn.Linear(hidden_size, hidden_size) 30 | self.linear_3 = nn.Linear(hidden_size, hidden_size) 31 | 32 | def forward(self, x): 33 | ''' 34 | Args: 35 | x: tensor of shape (batch_size, in_size) 36 | ''' 37 | normed = self.norm(x) 38 | dropped = self.drop(normed) 39 | y_1 = F.relu(self.linear_1(dropped)) 40 | y_2 = F.relu(self.linear_2(y_1)) 41 | y_3 = F.relu(self.linear_3(y_2)) 42 | 43 | return y_3 44 | 45 | 46 | class TextSubNet(nn.Module): 47 | ''' 48 | The LSTM-based subnetwork that is used in TFN for text 49 | ''' 50 | 51 | def __init__(self, in_size, hidden_size, out_size, num_layers=1, dropout=0.2, bidirectional=False): 52 | ''' 53 | Args: 54 | in_size: input dimension 55 | hidden_size: hidden layer dimension 56 | num_layers: specify the number of layers of LSTMs. 57 | dropout: dropout probability 58 | bidirectional: specify usage of bidirectional LSTM 59 | Output: 60 | (return value in forward) a tensor of shape (batch_size, out_size) 61 | ''' 62 | super(TextSubNet, self).__init__() 63 | if num_layers == 1: 64 | dropout = 0.0 65 | self.rnn = nn.LSTM(in_size, hidden_size, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional, batch_first=True) 66 | self.dropout = nn.Dropout(dropout) 67 | self.linear_1 = nn.Linear(hidden_size, out_size) 68 | 69 | def forward(self, x): 70 | ''' 71 | Args: 72 | x: tensor of shape (batch_size, sequence_len, in_size) 73 | ''' 74 | _, final_states = self.rnn(x) 75 | h = self.dropout(final_states[0].squeeze()) 76 | y_1 = self.linear_1(h) 77 | return y_1 78 | -------------------------------------------------------------------------------- /Self-MM/models/subNets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/subNets/__init__.py -------------------------------------------------------------------------------- /Self-MM/models/subNets/__pycache__/AlignNets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/subNets/__pycache__/AlignNets.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/subNets/__pycache__/BertTextEncoder.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/subNets/__pycache__/BertTextEncoder.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/subNets/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/subNets/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/models/subNets/transformers_encoder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/models/subNets/transformers_encoder/__init__.py -------------------------------------------------------------------------------- /Self-MM/models/subNets/transformers_encoder/position_embedding.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | # Code adapted from the fairseq repo. 7 | 8 | def make_positions(tensor, padding_idx, left_pad): 9 | """Replace non-padding symbols with their position numbers. 10 | Position numbers begin at padding_idx+1. 11 | Padding symbols are ignored, but it is necessary to specify whether padding 12 | is added on the left side (left_pad=True) or right side (left_pad=False). 13 | """ 14 | max_pos = padding_idx + 1 + tensor.size(1) 15 | device = tensor.get_device() 16 | buf_name = f'range_buf_{device}' 17 | if not hasattr(make_positions, buf_name): 18 | setattr(make_positions, buf_name, tensor.new()) 19 | setattr(make_positions, buf_name, getattr(make_positions, buf_name).type_as(tensor)) 20 | if getattr(make_positions, buf_name).numel() < max_pos: 21 | torch.arange(padding_idx + 1, max_pos, out=getattr(make_positions, buf_name)) 22 | mask = tensor.ne(padding_idx) 23 | positions = getattr(make_positions, buf_name)[:tensor.size(1)].expand_as(tensor) 24 | if left_pad: 25 | positions = positions - mask.size(1) + mask.long().sum(dim=1).unsqueeze(1) 26 | new_tensor = tensor.clone() 27 | return new_tensor.masked_scatter_(mask, positions[mask]).long() 28 | 29 | 30 | class SinusoidalPositionalEmbedding(nn.Module): 31 | """This module produces sinusoidal positional embeddings of any length. 32 | Padding symbols are ignored, but it is necessary to specify whether padding 33 | is added on the left side (left_pad=True) or right side (left_pad=False). 34 | """ 35 | 36 | def __init__(self, embedding_dim, padding_idx=0, left_pad=0, init_size=128): 37 | super().__init__() 38 | self.embedding_dim = embedding_dim 39 | self.padding_idx = padding_idx 40 | self.left_pad = left_pad 41 | self.weights = dict() # device --> actual weight; due to nn.DataParallel :-( 42 | self.register_buffer('_float_tensor', torch.FloatTensor(1)) 43 | 44 | @staticmethod 45 | def get_embedding(num_embeddings, embedding_dim, padding_idx=None): 46 | """Build sinusoidal embeddings. 47 | This matches the implementation in tensor2tensor, but differs slightly 48 | from the description in Section 3.5 of "Attention Is All You Need". 49 | """ 50 | half_dim = embedding_dim // 2 51 | emb = math.log(10000) / (half_dim - 1) 52 | emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb) 53 | emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) 54 | emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1) 55 | if embedding_dim % 2 == 1: 56 | # zero pad 57 | emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) 58 | if padding_idx is not None: 59 | emb[padding_idx, :] = 0 60 | return emb 61 | 62 | def forward(self, input): 63 | """Input is expected to be of size [bsz x seqlen].""" 64 | bsz, seq_len = input.size() 65 | max_pos = self.padding_idx + 1 + seq_len 66 | device = input.get_device() 67 | if device not in self.weights or max_pos > self.weights[device].size(0): 68 | # recompute/expand embeddings if needed 69 | self.weights[device] = SinusoidalPositionalEmbedding.get_embedding( 70 | max_pos, 71 | self.embedding_dim, 72 | self.padding_idx, 73 | ) 74 | self.weights[device] = self.weights[device].type_as(self._float_tensor).to(input.device) 75 | positions = make_positions(input, self.padding_idx, self.left_pad) 76 | return self.weights[device].index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach() 77 | 78 | def max_positions(self): 79 | """Maximum number of supported positions.""" 80 | return int(1e5) # an arbitrary large number -------------------------------------------------------------------------------- /Self-MM/plt.py: -------------------------------------------------------------------------------- 1 | 2 | # import matplotlib.pyplot as plt 3 | 4 | # num_list = [1.5,0.6,7.8,6] 5 | # name_list = ['L', 'V', 'A'] 6 | 7 | # bars = plt.bar([1,2,3], num_list, color='grey', width=0.2, tick_label=name_list) 8 | 9 | # for bar in bars: 10 | # height = bar.get_height() 11 | # plt.text(bar.get_x() + bar.get_width() / 2, height, str(int(height))) 12 | 13 | 14 | # # plt.bar(range(len(num_list)), num_list) 15 | # plt.show() 16 | 17 | fig1,ax = plt.subplots(figsize=(10,6)) 18 | 19 | ax.bar(df.index+0.0, df.iloc[:,0],width=0.1,label='BKNG') 20 | ax.bar(df.index+0.1, df.iloc[:,1],width=0.1,label='MCD') 21 | ax.bar(df.index+0.2, df.iloc[:,2],width=0.1,label='YUM') 22 | plt.grid(True) 23 | ax.set_yscale('symlog') 24 | plt.legend() 25 | plt.xlabel('date') 26 | plt.ylabel('value') 27 | plt.title('ROE') -------------------------------------------------------------------------------- /Self-MM/requirements.txt: -------------------------------------------------------------------------------- 1 | face-recognition==1.2.3 2 | facenet_pytorch==0.4.0 3 | numpy==1.22.0 4 | pandas==0.25.1 5 | Pillow==9.3.0 6 | scikit-learn==0.21.3 7 | scipy==1.3.1 8 | six==1.12.0 9 | torch==1.13.1 10 | torchvision==0.2.1 11 | tqdm==4.36.1 12 | transformers==2.10.0 13 | pytorch_transformers==1.0.0 14 | 15 | # conda create --name self_mm python=3.7 16 | # source activate self_mm 17 | # pip install -r requirements.txt 18 | -------------------------------------------------------------------------------- /Self-MM/run.sh: -------------------------------------------------------------------------------- 1 | ####TRAIN 2 | python run.py --modelName self_mm \ 3 | --datasetName mosi \ 4 | --train_method missing \ 5 | --train_changed_modal language \ 6 | --train_changed_pct 0.3 7 | ###TEST 8 | # python test.py --modelName self_mm \ 9 | # --datasetName mosi \ 10 | # --train_method missing \ 11 | # --train_changed_modal language \ 12 | # --train_changed_pct 0.3 \ 13 | # --test_method missing \ 14 | # --test_changed_modal language \ 15 | # --test_changed_pct 0 \ 16 | # --is_test -------------------------------------------------------------------------------- /Self-MM/trains/ATIO.py: -------------------------------------------------------------------------------- 1 | """ 2 | AIO -- All Trains in One 3 | """ 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from torch.nn.parameter import Parameter 9 | from torch.nn.init import xavier_uniform, xavier_normal, orthogonal 10 | 11 | from trains.multiTask import * 12 | 13 | __all__ = ['ATIO'] 14 | 15 | class ATIO(): 16 | def __init__(self): 17 | self.TRAIN_MAP = { 18 | 'self_mm': SELF_MM, 19 | } 20 | 21 | def getTrain(self, args): 22 | return self.TRAIN_MAP[args.modelName.lower()](args) 23 | -------------------------------------------------------------------------------- /Self-MM/trains/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/trains/__init__.py -------------------------------------------------------------------------------- /Self-MM/trains/__pycache__/ATIO.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/trains/__pycache__/ATIO.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/trains/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/trains/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/trains/multiTask/__init__.py: -------------------------------------------------------------------------------- 1 | from trains.multiTask.SELF_MM import SELF_MM 2 | 3 | __all__ = ['SELF_MM'] -------------------------------------------------------------------------------- /Self-MM/trains/multiTask/__pycache__/SELF_MM.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/trains/multiTask/__pycache__/SELF_MM.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/trains/multiTask/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/trains/multiTask/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/utils/__init__.py -------------------------------------------------------------------------------- /Self-MM/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/utils/__pycache__/functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/utils/__pycache__/functions.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/utils/__pycache__/metricsTop.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/declare-lab/MSA-Robustness/2de83fb56d57657577246e0577988a9447fce618/Self-MM/utils/__pycache__/metricsTop.cpython-37.pyc -------------------------------------------------------------------------------- /Self-MM/utils/functions.py: -------------------------------------------------------------------------------- 1 | def dict_to_str(src_dict): 2 | dst_str = "" 3 | for key in src_dict.keys(): 4 | dst_str += " %s: %.4f " %(key, src_dict[key]) 5 | return dst_str 6 | 7 | class Storage(dict): 8 | """ 9 | A Storage object is like a dictionary except `obj.foo` can be used inadition to `obj['foo']` 10 | ref: https://blog.csdn.net/a200822146085/article/details/88430450 11 | """ 12 | def __getattr__(self, key): 13 | try: 14 | return self[key] if key in self else False 15 | except KeyError as k: 16 | raise AttributeError(k) 17 | 18 | def __setattr__(self, key, value): 19 | self[key] = value 20 | 21 | def __delattr__(self, key): 22 | try: 23 | del self[key] 24 | except KeyError as k: 25 | raise AttributeError(k) 26 | 27 | def __str__(self): 28 | return "<" + self.__class__.__name__ + dict.__repr__(self) + ">" 29 | 30 | --------------------------------------------------------------------------------