├── .gitignore ├── README.md ├── __init__.py ├── configs ├── CSL-Daily.yaml ├── baseline.yaml ├── phoenix2014-T.yaml └── phoenix2014.yaml ├── dataset ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── dataloader_videoall.cpython-38.pyc │ └── video_augmentation_torch.cpython-38.pyc └── dataloader_video.py ├── main.py ├── modules ├── BiLSTM.py ├── __init__.py ├── criterions.py ├── gcn_lib │ ├── __init__.py │ ├── pos_embed.py │ ├── temgraph.py │ ├── torch_edge.py │ ├── torch_nn.py │ └── torch_vertex.py ├── loss.py ├── resnet.py └── tconv.py ├── preprocess ├── CSL-Daily │ ├── CSL-Daily-groundtruth-dev.stm │ ├── CSL-Daily-groundtruth-test.stm │ ├── CSL-Daily-groundtruth-train.stm │ ├── dev_info.npy │ ├── gloss_dict.npy │ ├── split_1.txt │ ├── test_info.npy │ ├── train_info.npy │ └── video_map.txt ├── phoenix2014-T │ ├── dev_info.npy │ ├── gloss_dict.npy │ ├── phoenix2014-T-groundtruth-dev.stm │ ├── phoenix2014-T-groundtruth-test.stm │ ├── phoenix2014-T-groundtruth-train.stm │ ├── test_info.npy │ └── train_info.npy └── phoenix2014 │ ├── dev_info.npy │ ├── gloss_dict.npy │ ├── phoenix2014-groundtruth-dev.stm │ ├── phoenix2014-groundtruth-test.stm │ ├── phoenix2014-groundtruth-train.stm │ ├── test_info.npy │ └── train_info.npy ├── requirements.txt ├── seq_scripts.py ├── slr_network.py ├── test.py └── utils ├── Rouge.py ├── __init__.py ├── decode.py ├── device.py ├── metrics.py ├── misc.py ├── optimizer.py ├── pack_code.py ├── parameters.py ├── random_state.py ├── record.py ├── sacrebleu.py └── video_augmentation.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pkl 2 | /weight -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SignGraph: A Sign Sequence is Worth Graphs of Nodes 2 | An implementation of the paper: SignGraph: A Sign Sequence is Worth Graphs of Nodes. (CVPR 2024) [[paper]](https://openaccess.thecvf.com/content/CVPR2024/papers/Gan_SignGraph_A_Sign_Sequence_is_Worth_Graphs_of_Nodes_CVPR_2024_paper.pdf) 3 | 4 | ## Prerequisites 5 | 6 | - This project is implemented in Pytorch (>1.8). Thus please install Pytorch first. 7 | 8 | - ctcdecode==0.4 [[parlance/ctcdecode]](https://github.com/parlance/ctcdecode)，for beam search decode. 9 | 10 | - For these who failed install ctcdecode (and it always does), you can download [ctcdecode here](https://drive.google.com/file/d/1LjbJz60GzT4qK6WW59SIB1Zi6Sy84wOS/view?usp=sharing), unzip it, and try 11 | `cd ctcdecode` and `pip install .` 12 | 13 | - Pealse follow [this link](https://pytorch-geometric.readthedocs.io/en/latest/install/installation.html) to install pytorch geometric 14 | 15 | - You can install other required modules by conducting 16 | `pip install -r requirements.txt` 17 | 18 | 19 | 20 | 21 | 22 | ## Data Preparation 23 | 24 | 1. PHOENIX2014 dataset: Download the RWTH-PHOENIX-Weather 2014 Dataset [[download link]](https://www-i6.informatik.rwth-aachen.de/~koller/RWTH-PHOENIX/). 25 | 26 | 2. PHOENIX2014-T datasetDownload the RWTH-PHOENIX-Weather 2014 Dataset [[download link]](https://www-i6.informatik.rwth-aachen.de/~koller/RWTH-PHOENIX-2014-T/) 27 | 28 | 3. CSL dataset： Request the CSL Dataset from this website [[download link]](https://ustc-slr.github.io/openresources/cslr-dataset-2015/index.html) 29 | 30 | 31 | Download datasets and extract them, no further data preprocessing needed. 32 | 33 | ## Weights 34 | 35 | We make some imporvments of our code, and provide newest checkpoionts and better performance. 36 | 37 | |Dataset | Backbone | Dev WER | Del / Ins | Test WER | Del / Ins | Pretrained model | 38 | | --------| -------- | ---------- | ----------- | ----------- | -----------| --- | 39 | |Phoenix14T | SignGraph | 17.00|4.99/2.32| 19.44| 5.14/3.38|[[Google Drive]](https://drive.google.com/drive/folders/1FVvbXV7f2-5lJhVlCm-bqzyZ55C1LQ-g?usp=sharing) | 40 | |Phoenix14 |SignGraph|17.13|6.00/2.17| 18.17|5.65/2.23|[[Google Drive]](https://drive.google.com/drive/folders/1O5JBkmnu2TO8Domzd60tqql8l1zNCzHc?usp=sharing) | 41 | |CSL-Daily |SignGraph|26.38|9.92/2.62| 25.84|9.39/2.58|[[Google Drive]](https://drive.google.com/drive/folders/1t09Ixpiujw6WJrkSF8gwexvKJie4RGsh?usp=sharing) | 42 | 43 | 44 | 45 | ​To evaluate the pretrained model, choose the dataset from phoenix2014/phoenix2014-T/CSL/CSL-Daily in line 3 in ./config/baseline.yaml first, and run the command below： 46 | `python main.py --device your_device --load-weights path_to_weight.pt --phase test` 47 | 48 | ### Training 49 | 50 | The priorities of configuration files are: command line > config file > default values of argparse. To train the SLR model, run the command below: 51 | 52 | `python main.py --device your_device` 53 | 54 | Note that you can choose the target dataset from phoenix2014/phoenix2014-T/CSL/CSL-Daily in line 3 in ./config/baseline.yaml. 55 | 56 | ### Thanks 57 | 58 | This repo is based on [VAC (ICCV 2021)](https://openaccess.thecvf.com/content/ICCV2021/html/Min_Visual_Alignment_Constraint_for_Continuous_Sign_Language_Recognition_ICCV_2021_paper.html), [VIT (NIPS 2022)](https://arxiv.org/abs/2206.00272) and [RTG-Net (ACM MM2023)](https://dl.acm.org/doi/10.1145/3581783.3611820)！ 59 | 60 | ### Citation 61 | 62 | If you find this repo useful in your research works, please consider citing: 63 | 64 | ```latex 65 | @inproceedings{gan2024signgraph, 66 | title={SignGraph: A Sign Sequence is Worth Graphs of Nodes}, 67 | author={Gan, Shiwei and Yin, Yafeng and Jiang, Zhiwei and Wen, Hongkai and Xie, Lei and Lu, Sanglu}, 68 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 69 | pages={13470--13479}, 70 | year={2024} 71 | } 72 | 73 | @inproceedings{gan2023towards, 74 | title={Towards Real-Time Sign Language Recognition and Translation on Edge Devices}, 75 | author={Gan, Shiwei and Yin, Yafeng and Jiang, Zhiwei and Xie, Lei and Lu, Sanglu}, 76 | booktitle={Proceedings of the 31st ACM International Conference on Multimedia}, 77 | pages={4502--4512}, 78 | year={2023} 79 | } 80 | 81 | @inproceedings{gan2023contrastive, 82 | title={Contrastive learning for sign language recognition and translation}, 83 | author={Gan, Shiwei and Yin, Yafeng and Jiang, Zhiwei and Xia, Kang and Xie, Lei and Lu, Sanglu}, 84 | booktitle={Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence, IJCAI-23}, 85 | pages={763--772}, 86 | year={2023} 87 | } 88 | 89 | @article{han2022vision, 90 | title={Vision gnn: An image is worth graph of nodes}, 91 | author={Han, Kai and Wang, Yunhe and Guo, Jianyuan and Tang, Yehui and Wu, Enhua}, 92 | journal={Advances in neural information processing systems}, 93 | volume={35}, 94 | pages={8291--8303}, 95 | year={2022} 96 | } 97 | ``` 98 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/__init__.py -------------------------------------------------------------------------------- /configs/CSL-Daily.yaml: -------------------------------------------------------------------------------- 1 | dataset_root: /CSL-Daily 2 | dict_path: ./preprocess/CSL-Daily/gloss_dict.npy 3 | evaluation_dir: ./evaluation/slr_eval 4 | evaluation_prefix: CSL-Daily-groundtruth 5 | -------------------------------------------------------------------------------- /configs/baseline.yaml: -------------------------------------------------------------------------------- 1 | feeder: dataset.dataloader_video.BaseFeeder 2 | phase: train 3 | dataset: phoenix2014-T 4 | #CSL-Daily 5 | # dataset: phoenix14-si5 6 | 7 | work_dir: ./work_dirt/ 8 | batch_size: 4 9 | random_seed: 0 10 | test_batch_size: 4 11 | num_worker: 20 12 | device: 0 13 | log_interval: 10000 14 | eval_interval: 1 15 | save_interval: 10 16 | 17 | evaluate_tool: python 18 | loss_weights: 19 | SeqCTC: 1.0 20 | ConvCTC: 1.0 21 | Dist: 25.0 22 | load_weights: False 23 | load_checkpoints: False 24 | optimizer_args: 25 | optimizer: Adam 26 | learning_rate: 27 | base_lr: 0.0001 28 | step: [20, 30, 35] 29 | learning_ratio: 1 30 | scheduler: ScheaL #consine 31 | weight_decay: 0.0001 32 | start_epoch: 0 33 | num_epoch: 101 #50 34 | nesterov: False 35 | 36 | feeder_args: 37 | mode: 'train' 38 | datatype: 'video' 39 | num_gloss: -1 40 | drop_ratio: 1.0 41 | frame_interval: 1 42 | image_scale: 1.0 # 0-1 represents ratio, >1 represents absolute value 43 | input_size: 224 44 | 45 | model: slr_network.SLRModel 46 | decode_mode: beam 47 | model_args: 48 | num_classes: 1296 49 | c2d_type: resnet18 50 | conv_type: 2 51 | use_bn: 1 52 | share_classifier: True 53 | weight_norm: True 54 | -------------------------------------------------------------------------------- /configs/phoenix2014-T.yaml: -------------------------------------------------------------------------------- 1 | dataset_root: "/data/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/" 2 | 3 | dict_path: ./preprocess/phoenix2014-T/gloss_dict.npy 4 | evaluation_dir: ./evaluation/slr_eval 5 | evaluation_prefix: phoenix2014-T-groundtruth # 6 | -------------------------------------------------------------------------------- /configs/phoenix2014.yaml: -------------------------------------------------------------------------------- 1 | dataset_root: /phoenix2014-release/phoenix-2014-multisigner 2 | dict_path: ./preprocess/phoenix2014/gloss_dict.npy 3 | evaluation_dir: ./evaluation/slr_eval 4 | evaluation_prefix: phoenix2014-groundtruth 5 | -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dataset/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/dataset/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/dataloader_videoall.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/dataset/__pycache__/dataloader_videoall.cpython-38.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/video_augmentation_torch.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/dataset/__pycache__/video_augmentation_torch.cpython-38.pyc -------------------------------------------------------------------------------- /dataset/dataloader_video.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import sys 4 | import pdb 5 | import six 6 | import glob 7 | import time 8 | import torch 9 | import random 10 | import pandas 11 | import warnings 12 | 13 | warnings.simplefilter(action='ignore', category=FutureWarning) 14 | 15 | import numpy as np 16 | # import pyarrow as pa 17 | from PIL import Image 18 | import torch.utils.data as data 19 | import matplotlib.pyplot as plt 20 | from utils import video_augmentation,clean_phoenix_2014_trans,clean_phoenix_2014 21 | from torch.utils.data.sampler import Sampler 22 | 23 | sys.path.append("..") 24 | global kernel_sizes 25 | 26 | class BaseFeeder(data.Dataset): 27 | def __init__(self, prefix, gloss_dict, dataset='phoenix2014', drop_ratio=1, num_gloss=-1, mode="train", transform_mode=True, 28 | datatype="lmdb", frame_interval=1, image_scale=1.0, kernel_size=1, input_size=224): 29 | self.mode = mode 30 | self.ng = num_gloss 31 | self.prefix = prefix 32 | self.dict = gloss_dict 33 | self.data_type = datatype 34 | self.dataset = dataset 35 | self.input_size = input_size 36 | global kernel_sizes 37 | kernel_sizes = kernel_size 38 | self.frame_interval = frame_interval # not implemented for read_features() 39 | self.image_scale = image_scale # not implemented for read_features() 40 | self.feat_prefix = f"{prefix}/features/fullFrame-256x256px/{mode}" 41 | #/data1/gsw/CSL-Daily/sentence/frames_512x512 42 | self.transform_mode = "train" if transform_mode else "test" 43 | self.inputs_list = np.load(f"./preprocess/{dataset}/{mode}_info.npy", allow_pickle=True).item() 44 | print(mode, len(self)) 45 | self.data_aug = self.transform() 46 | print("") 47 | 48 | def __getitem__(self, idx): 49 | if self.data_type == "video": 50 | input_data, label, fi = self.read_video(idx) 51 | input_data, label = self.normalize(input_data, label) 52 | # input_data, label = self.normalize(input_data, label, fi['fileid']) 53 | return input_data, torch.LongTensor(label), fi #self.inputs_list[idx]['original_info'] 54 | elif self.data_type == "lmdb": 55 | input_data, label, fi = self.read_lmdb(idx) 56 | input_data, label = self.normalize(input_data, label) 57 | return input_data, torch.LongTensor(label), self.inputs_list[idx]['original_info'] 58 | else: 59 | input_data, label = self.read_features(idx) 60 | return input_data, label, self.inputs_list[idx]['original_info'] 61 | 62 | def read_video(self, index): 63 | # load file info 64 | fi = self.inputs_list[index] 65 | if 'phoenix' in self.dataset: 66 | img_folder = os.path.join(self.prefix, "features/fullFrame-210x260px/" + fi['folder']) 67 | 68 | elif self.dataset == 'CSL-Daily': 69 | img_folder = os.path.join(self.prefix, "sentence/frames_512x512/" + fi['folder']) 70 | img_list = sorted(glob.glob(img_folder)) 71 | img_list = img_list[int(torch.randint(0, self.frame_interval, [1]))::self.frame_interval] 72 | label_list = [] 73 | if self.dataset=='phoenix2014': 74 | fi['label'] = clean_phoenix_2014(fi['label']) 75 | if self.dataset=='phoenix2014-T': 76 | fi['label']=clean_phoenix_2014_trans(fi['label']) 77 | for phase in fi['label'].split(" "): 78 | if phase == '': 79 | continue 80 | if phase in self.dict.keys(): 81 | label_list.append(self.dict[phase][0]) 82 | return [cv2.cvtColor(cv2.resize(cv2.imread(img_path), (256, 256), interpolation=cv2.INTER_LANCZOS4), 83 | cv2.COLOR_BGR2RGB) for img_path in img_list], label_list, fi 84 | 85 | def read_features(self, index): 86 | # load file info 87 | fi = self.inputs_list[index] 88 | data = np.load(f"./features/{self.mode}/{fi['fileid']}_features.npy", allow_pickle=True).item() 89 | return data['features'], data['label'] 90 | 91 | def normalize(self, video, label, file_id=None): 92 | video, label = self.data_aug(video, label, file_id) 93 | video = video.float() / 127.5 - 1 94 | return video, label 95 | 96 | def transform(self): 97 | if self.transform_mode == "train": 98 | print("Apply training transform.") 99 | return video_augmentation.Compose([ 100 | # video_augmentation.CenterCrop(224), 101 | # video_augmentation.WERAugment('/lustre/wangtao/current_exp/exp/baseline/boundary.npy'), 102 | video_augmentation.RandomCrop(self.input_size), 103 | video_augmentation.RandomHorizontalFlip(0.5), 104 | video_augmentation.Resize(self.image_scale), 105 | video_augmentation.ToTensor(), 106 | video_augmentation.TemporalRescale(0.2, self.frame_interval), 107 | ]) 108 | else: 109 | print("Apply testing transform.") 110 | return video_augmentation.Compose([ 111 | video_augmentation.CenterCrop(self.input_size), 112 | video_augmentation.Resize(self.image_scale), 113 | video_augmentation.ToTensor(), 114 | ]) 115 | 116 | def byte_to_img(self, byteflow): 117 | unpacked = pa.deserialize(byteflow) 118 | imgbuf = unpacked[0] 119 | buf = six.BytesIO() 120 | buf.write(imgbuf) 121 | buf.seek(0) 122 | img = Image.open(buf).convert('RGB') 123 | return img 124 | 125 | @staticmethod 126 | def collate_fn(batch): 127 | batch = [item for item in sorted(batch, key=lambda x: len(x[0]), reverse=True)] 128 | video, label, info = list(zip(*batch)) 129 | 130 | left_pad = 0 131 | last_stride = 1 132 | total_stride = 1 133 | global kernel_sizes 134 | for layer_2idx, ks in enumerate(kernel_sizes): 135 | if ks[0] == 'K': 136 | left_pad = left_pad * last_stride 137 | left_pad += int((int(ks[1])-1)/2) 138 | elif ks[0] == 'P': 139 | last_stride = int(ks[1]) 140 | total_stride = total_stride * last_stride 141 | if len(video[0].shape) > 3: 142 | max_len = len(video[0]) 143 | video_length = torch.LongTensor([np.ceil(len(vid) / total_stride) * total_stride + 2*left_pad for vid in video]) 144 | right_pad = int(np.ceil(max_len / total_stride)) * total_stride - max_len + left_pad 145 | max_len = max_len + left_pad + right_pad 146 | padded_video = [torch.cat( 147 | ( 148 | vid[0][None].expand(left_pad, -1, -1, -1), 149 | vid, 150 | vid[-1][None].expand(max_len - len(vid) - left_pad, -1, -1, -1), 151 | ) 152 | , dim=0) 153 | for vid in video] 154 | padded_video = torch.stack(padded_video) 155 | else: 156 | max_len = len(video[0]) 157 | video_length = torch.LongTensor([len(vid) for vid in video]) 158 | padded_video = [torch.cat( 159 | ( 160 | vid, 161 | vid[-1][None].expand(max_len - len(vid), -1), 162 | ) 163 | , dim=0) 164 | for vid in video] 165 | padded_video = torch.stack(padded_video).permute(0, 2, 1) 166 | label_length = torch.LongTensor([len(lab) for lab in label]) 167 | if max(label_length) == 0: 168 | return padded_video, video_length, [], [], info 169 | else: 170 | padded_label = [] 171 | for lab in label: 172 | padded_label.extend(lab) 173 | padded_label = torch.LongTensor(padded_label) 174 | return padded_video, video_length, padded_label, label_length, info 175 | 176 | def __len__(self): 177 | return len(self.inputs_list) - 1 178 | 179 | def record_time(self): 180 | self.cur_time = time.time() 181 | return self.cur_time 182 | 183 | def split_time(self): 184 | split_time = time.time() - self.cur_time 185 | self.record_time() 186 | return split_time 187 | 188 | 189 | if __name__ == "__main__": 190 | feeder = BaseFeeder() 191 | dataloader = torch.utils.data.DataLoader( 192 | dataset=feeder, 193 | batch_size=1, 194 | shuffle=True, 195 | drop_last=True, 196 | num_workers=0, 197 | ) 198 | for data in dataloader: 199 | pdb.set_trace() 200 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 4 | import pdb 5 | import sys 6 | import cv2 7 | import yaml 8 | import torch 9 | import random 10 | import importlib 11 | import faulthandler 12 | import numpy as np 13 | import torch.nn as nn 14 | import shutil 15 | import inspect 16 | import time 17 | from collections import OrderedDict 18 | 19 | faulthandler.enable() 20 | import utils 21 | from seq_scripts import seq_train, seq_eval 22 | from torch.cuda.amp import autocast as autocast 23 | from utils.misc import * 24 | class Processor(): 25 | def __init__(self, arg): 26 | self.arg = arg 27 | if not os.path.exists(self.arg.work_dir): 28 | os.makedirs(self.arg.work_dir) 29 | shutil.copy2(__file__, self.arg.work_dir) 30 | shutil.copy2('./configs/baseline.yaml', self.arg.work_dir) 31 | shutil.copy2('./modules/tconv.py', self.arg.work_dir) 32 | shutil.copy2('./modules/resnet.py', self.arg.work_dir) 33 | shutil.copy2('./modules/gcn_lib/temgraph.py', self.arg.work_dir) 34 | torch.backends.cudnn.benchmark = True 35 | if type(self.arg.device) is not int: 36 | init_distributed_mode(self.arg) 37 | self.recoder = utils.Recorder(self.arg.work_dir, self.arg.print_log, self.arg.log_interval) 38 | self.save_arg() 39 | if self.arg.random_fix: 40 | self.rng = utils.RandomState(seed=self.arg.random_seed) 41 | self.device = utils.GpuDataParallel() 42 | 43 | 44 | self.recoder = utils.Recorder(self.arg.work_dir, self.arg.print_log, self.arg.log_interval) 45 | self.dataset = {} 46 | self.data_loader = {} 47 | self.gloss_dict = np.load(self.arg.dataset_info['dict_path'], allow_pickle=True).item() 48 | self.arg.model_args['num_classes'] = len(self.gloss_dict) + 1 49 | self.model, self.optimizer = self.loading() 50 | 51 | 52 | 53 | 54 | def start(self): 55 | if self.arg.phase == 'train': 56 | best_dev = {"wer":200.0, "del":100.0,"ins":100.0} 57 | best_tes = {"wer": 200.0, "del": 100.0, "ins": 100.0} 58 | best_epoch = 0 59 | total_time = 0 60 | epoch_time = 0 61 | if is_main_process(): 62 | self.recoder.print_log('Parameters:\n{}\n'.format(str(vars(self.arg)))) 63 | seq_model_list = [] 64 | for epoch in range(self.arg.optimizer_args['start_epoch'], self.arg.optimizer_args['num_epoch']): 65 | save_model = epoch % self.arg.save_interval == 0 66 | eval_model = epoch % self.arg.eval_interval == 0 67 | epoch_time = time.time() 68 | seq_train(self.data_loader['train'], self.model, self.optimizer, 69 | self.device, epoch, self.recoder) 70 | dev_wer={} 71 | dev_wer['wer']=0 72 | if is_main_process(): 73 | if eval_model: 74 | dev_wer = seq_eval(self.arg, self.data_loader['dev'], self.model, self.device, 75 | 'dev', epoch, self.arg.work_dir, self.recoder, self.arg.evaluate_tool) 76 | test_wer = seq_eval(self.arg, self.data_loader['test'], self.model, self.device, 77 | 'test', epoch, self.arg.work_dir, self.recoder, self.arg.evaluate_tool) 78 | self.recoder.print_log("Dev WER: {:05.2f}% DEV del {:05.2f}% DEV ins {:05.2f}%".format(dev_wer['wer'], dev_wer['del'], dev_wer['ins'])) 79 | self.recoder.print_log("Test WER: {:05.2f}% Test del {:05.2f}% Test ins {:05.2f}%".format(test_wer['wer'], test_wer['del'], 80 | test_wer['ins'])) 81 | 82 | if dev_wer["wer"] < best_dev["wer"]: 83 | best_dev = dev_wer 84 | best_tes = test_wer 85 | best_epoch = epoch 86 | model_path = "{}_best_model.pt".format(self.arg.work_dir) 87 | self.save_model(epoch, model_path) 88 | self.recoder.print_log('Save best model') 89 | self.recoder.print_log('Best_dev: {:05.2f}, {:05.2f}, {:05.2f}, ' 90 | 'Best_test: {:05.2f}, {:05.2f}, {:05.2f},' 91 | 'Epoch : {}'.format(best_dev["wer"], best_dev["del"], best_dev["ins"], 92 | best_tes["wer"],best_tes["del"],best_tes["ins"], best_epoch)) 93 | if save_model: 94 | model_path = "{}dev_{:05.2f}_epoch{}_model.pt".format(self.arg.work_dir, dev_wer['wer'], epoch) 95 | seq_model_list.append(model_path) 96 | print("seq_model_list", seq_model_list) 97 | self.save_model(epoch, model_path) 98 | epoch_time = time.time() - epoch_time 99 | total_time += epoch_time 100 | torch.cuda.empty_cache() 101 | self.recoder.print_log('Epoch {} costs {} mins {} seconds'.format(epoch, int(epoch_time)//60, int(epoch_time)%60)) 102 | self.recoder.print_log('Training costs {} hours {} mins {} seconds'.format(int(total_time)//60//60, int(total_time)//60%60, int(total_time)%60)) 103 | elif self.arg.phase == 'test' and is_main_process(): 104 | if self.arg.load_weights is None and self.arg.load_checkpoints is None: 105 | print('Please appoint --weights.') 106 | self.recoder.print_log('Model: {}.'.format(self.arg.model)) 107 | self.recoder.print_log('Weights: {}.'.format(self.arg.load_weights)) 108 | train_wer = seq_eval(self.arg, self.data_loader["train_eval"], self.model, self.device, 109 | "train", 6667, self.arg.work_dir, self.recoder, self.arg.evaluate_tool) 110 | dev_wer = seq_eval(self.arg, self.data_loader["dev"], self.model, self.device, 111 | "dev", 6667, self.arg.work_dir, self.recoder, self.arg.evaluate_tool) 112 | test_wer = seq_eval(self.arg, self.data_loader["test"], self.model, self.device, 113 | "test", 6667, self.arg.work_dir, self.recoder, self.arg.evaluate_tool) 114 | self.recoder.print_log('Evaluation Done.\n') 115 | elif self.arg.phase == "features": 116 | for mode in ["train", "dev", "test"]: 117 | seq_feature_generation( 118 | self.data_loader[mode + "_eval" if mode == "train" else mode], 119 | self.model, self.device, mode, self.arg.work_dir, self.recoder 120 | ) 121 | 122 | def save_arg(self): 123 | arg_dict = vars(self.arg) 124 | if not os.path.exists(self.arg.work_dir): 125 | os.makedirs(self.arg.work_dir) 126 | with open('{}/config.yaml'.format(self.arg.work_dir), 'w') as f: 127 | yaml.dump(arg_dict, f) 128 | 129 | def save_model(self, epoch, save_path): 130 | if len(self.device.gpu_list)>1: 131 | model = self.model.module 132 | else: 133 | model= self.model 134 | torch.save({ 135 | 'epoch': epoch, 136 | 'model_state_dict': model.state_dict(), 137 | 'optimizer_state_dict': self.optimizer.state_dict(), 138 | 'scheduler_state_dict': self.optimizer.scheduler.state_dict(), 139 | 'rng_state': self.rng.save_rng_state(), 140 | }, save_path) 141 | 142 | def loading(self): 143 | self.device.set_device(self.arg.device) 144 | 145 | 146 | 147 | print("Loading model") 148 | model_class = import_class(self.arg.model) 149 | model = model_class( 150 | **self.arg.model_args, 151 | gloss_dict=self.gloss_dict, 152 | loss_weights=self.arg.loss_weights, 153 | ) 154 | shutil.copy2(inspect.getfile(model_class), self.arg.work_dir) 155 | optimizer = utils.Optimizer(model, self.arg.optimizer_args) 156 | 157 | if self.arg.load_weights: 158 | self.load_model_weights(model, self.arg.load_weights) 159 | elif self.arg.load_checkpoints: 160 | self.load_checkpoint_weights(model, optimizer) 161 | 162 | self.kernel_sizes = model.conv1d.kernel_size 163 | model = self.model_to_device(model) 164 | print("Loading model finished.") 165 | self.load_data() 166 | return model, optimizer 167 | 168 | def model_to_device(self, model): 169 | model = model.to(self.device.output_device) 170 | if len(self.device.gpu_list) > 1: 171 | print("using dataparalleling...") 172 | model = nn.SyncBatchNorm.convert_sync_batchnorm(model.to(self.arg.local_rank)) 173 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[self.arg.local_rank]) 174 | else: 175 | model.cuda() 176 | # model.cuda() 177 | return model 178 | 179 | def load_model_weights(self, model, weight_path): 180 | state_dict = torch.load(weight_path) 181 | if len(self.arg.ignore_weights): 182 | for w in self.arg.ignore_weights: 183 | if state_dict.pop(w, None) is not None: 184 | print('Successfully Remove Weights: {}.'.format(w)) 185 | else: 186 | print('Can Not Remove Weights: {}.'.format(w)) 187 | weights = self.modified_weights(state_dict['model_state_dict'], False) 188 | # weights = self.modified_weights(state_dict['model_state_dict']) 189 | s_dict = model.state_dict() 190 | for name in weights: 191 | if name not in s_dict: 192 | print(name) 193 | continue 194 | if s_dict[name].shape == weights[name].shape: 195 | s_dict[name] = weights[name] 196 | model.load_state_dict(s_dict, strict=True) 197 | 198 | @staticmethod 199 | def modified_weights(state_dict, modified=False): 200 | state_dict = OrderedDict([(k.replace('.module', ''), v) for k, v in state_dict.items()]) 201 | if not modified: 202 | return state_dict 203 | modified_dict = dict() 204 | return modified_dict 205 | 206 | def load_checkpoint_weights(self, model, optimizer): 207 | self.load_model_weights(model, self.arg.load_checkpoints) 208 | state_dict = torch.load(self.arg.load_checkpoints) 209 | 210 | if len(torch.cuda.get_rng_state_all()) == len(state_dict['rng_state']['cuda']): 211 | print("Loading random seeds...") 212 | self.rng.set_rng_state(state_dict['rng_state']) 213 | if "optimizer_state_dict" in state_dict.keys(): 214 | print("Loading optimizer parameters...") 215 | optimizer.load_state_dict(state_dict["optimizer_state_dict"]) 216 | optimizer.to(self.arg.local_rank) 217 | if "scheduler_state_dict" in state_dict.keys(): 218 | print("Loading scheduler parameters...") 219 | optimizer.scheduler.load_state_dict(state_dict["scheduler_state_dict"]) 220 | 221 | self.arg.optimizer_args['start_epoch'] = state_dict["epoch"] + 1 222 | self.recoder.print_log("Resuming from checkpoint: epoch {self.arg.optimizer_args['start_epoch']}") 223 | 224 | def load_data(self): 225 | print("Loading Dataprocessing") 226 | self.feeder = import_class(self.arg.feeder) 227 | shutil.copy2(inspect.getfile(self.feeder), self.arg.work_dir) 228 | if self.arg.dataset == 'CSL': 229 | dataset_list = zip(["train", "dev"], [True, False]) 230 | elif 'phoenix' in self.arg.dataset: 231 | dataset_list = zip(["train", "train_eval", "dev", "test"], [True, False, False, False]) 232 | elif self.arg.dataset == 'CSL-Daily': 233 | dataset_list = zip(["train", "train_eval", "dev", "test"], [True, False, False, False]) 234 | for idx, (mode, train_flag) in enumerate(dataset_list): 235 | arg = self.arg.feeder_args 236 | arg["prefix"] = self.arg.dataset_info['dataset_root'] 237 | arg["mode"] = mode.split("_")[0] 238 | arg["transform_mode"] = train_flag 239 | self.dataset[mode] = self.feeder(gloss_dict=self.gloss_dict, kernel_size= self.kernel_sizes, dataset=self.arg.dataset, **arg) 240 | self.data_loader[mode] = self.build_dataloader(self.dataset[mode], mode, train_flag) 241 | print("Loading Dataprocessing finished.") 242 | def init_fn(self, worker_id): 243 | np.random.seed(int(self.arg.random_seed)+worker_id) 244 | 245 | 246 | def build_dataloader(self, dataset, mode, train_flag): 247 | if len(self.device.gpu_list) > 1: 248 | if train_flag: 249 | sampler = torch.utils.data.distributed.DistributedSampler(dataset, shuffle=train_flag) 250 | else: 251 | sampler = torch.utils.data.SequentialSampler(dataset) 252 | batch_size = self.arg.batch_size if mode == "train" else self.arg.test_batch_size 253 | loader = torch.utils.data.DataLoader( 254 | dataset, 255 | sampler=sampler, 256 | batch_size=batch_size, 257 | collate_fn=self.feeder.collate_fn, 258 | num_workers=self.arg.num_worker, 259 | pin_memory=True, 260 | worker_init_fn=self.init_fn, 261 | ) 262 | return loader 263 | else: 264 | return torch.utils.data.DataLoader( 265 | dataset, 266 | batch_size= self.arg.batch_size if mode == "train" else self.arg.test_batch_size, 267 | shuffle=train_flag, 268 | drop_last=train_flag, 269 | num_workers=self.arg.num_worker, # if train_flag else 0 270 | collate_fn=self.feeder.collate_fn, 271 | pin_memory=True, 272 | worker_init_fn=self.init_fn, 273 | ) 274 | 275 | 276 | def import_class(name): 277 | components = name.rsplit('.', 1) 278 | mod = importlib.import_module(components[0]) 279 | mod = getattr(mod, components[1]) 280 | return mod 281 | 282 | 283 | if __name__ == '__main__': 284 | sparser = utils.get_parser() 285 | p = sparser.parse_args() 286 | if p.config is not None: 287 | with open(p.config, 'r') as f: 288 | try: 289 | default_arg = yaml.load(f, Loader=yaml.FullLoader) 290 | except AttributeError: 291 | default_arg = yaml.load(f) 292 | key = vars(p).keys() 293 | for k in default_arg.keys(): 294 | if k not in key: 295 | print('WRONG ARG: {}'.format(k)) 296 | assert (k in key) 297 | sparser.set_defaults(**default_arg) 298 | args = sparser.parse_args() 299 | with open(f"./configs/{args.dataset}.yaml", 'r') as f: 300 | args.dataset_info = yaml.load(f, Loader=yaml.FullLoader) 301 | processor = Processor(args) 302 | utils.pack_code("./", args.work_dir) 303 | processor.start() 304 | 305 | -------------------------------------------------------------------------------- /modules/BiLSTM.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | 7 | class BiLSTMLayer(nn.Module): 8 | def __init__(self, input_size, debug=False, hidden_size=512, num_layers=1, dropout=0.3, 9 | bidirectional=True, rnn_type='LSTM', num_classes=-1): 10 | super(BiLSTMLayer, self).__init__() 11 | 12 | self.dropout = dropout 13 | self.num_layers = num_layers 14 | self.input_size = input_size 15 | self.bidirectional = bidirectional 16 | self.num_directions = 2 if bidirectional else 1 17 | self.hidden_size = int(hidden_size / self.num_directions) 18 | self.rnn_type = rnn_type 19 | self.debug = debug 20 | self.rnn = getattr(nn, self.rnn_type)( 21 | input_size=self.input_size, 22 | hidden_size=self.hidden_size, 23 | num_layers=self.num_layers, 24 | dropout=self.dropout, 25 | bidirectional=self.bidirectional) 26 | # for name, param in self.rnn.named_parameters(): 27 | # if name[:6] == 'weight': 28 | # nn.init.orthogonal_(param) 29 | 30 | def forward(self, src_feats, src_lens, hidden=None): 31 | """ 32 | Args: 33 | - src_feats: (max_src_len, batch_size, D) 34 | - src_lens: (batch_size) 35 | Returns: 36 | - outputs: (max_src_len, batch_size, hidden_size * num_directions) 37 | - hidden : (num_layers, batch_size, hidden_size * num_directions) 38 | """ 39 | # (max_src_len, batch_size, D) 40 | packed_emb = nn.utils.rnn.pack_padded_sequence(src_feats, src_lens) 41 | 42 | # rnn(gru) returns: 43 | # - packed_outputs: shape same as packed_emb 44 | # - hidden: (num_layers * num_directions, batch_size, hidden_size) 45 | if hidden is not None and self.rnn_type == 'LSTM': 46 | half = int(hidden.size(0) / 2) 47 | hidden = (hidden[:half], hidden[half:]) 48 | packed_outputs, hidden = self.rnn(packed_emb, hidden) 49 | 50 | # outputs: (max_src_len, batch_size, hidden_size * num_directions) 51 | rnn_outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs) 52 | 53 | if self.bidirectional: 54 | # (num_layers * num_directions, batch_size, hidden_size) 55 | # => (num_layers, batch_size, hidden_size * num_directions) 56 | hidden = self._cat_directions(hidden) 57 | 58 | if isinstance(hidden, tuple): 59 | # cat hidden and cell states 60 | hidden = torch.cat(hidden, 0) 61 | 62 | return { 63 | "predictions": rnn_outputs, 64 | "hidden": hidden 65 | } 66 | 67 | def _cat_directions(self, hidden): 68 | """ If the encoder is bidirectional, do the following transformation. 69 | Ref: https://github.com/IBM/pytorch-seq2seq/blob/master/seq2seq/models/DecoderRNN.py#L176 70 | ----------------------------------------------------------- 71 | In: (num_layers * num_directions, batch_size, hidden_size) 72 | (ex: num_layers=2, num_directions=2) 73 | 74 | layer 1: forward__hidden(1) 75 | layer 1: backward_hidden(1) 76 | layer 2: forward__hidden(2) 77 | layer 2: backward_hidden(2) 78 | 79 | ----------------------------------------------------------- 80 | Out: (num_layers, batch_size, hidden_size * num_directions) 81 | 82 | layer 1: forward__hidden(1) backward_hidden(1) 83 | layer 2: forward__hidden(2) backward_hidden(2) 84 | """ 85 | 86 | def _cat(h): 87 | return torch.cat([h[0:h.size(0):2], h[1:h.size(0):2]], 2) 88 | 89 | if isinstance(hidden, tuple): 90 | # LSTM hidden contains a tuple (hidden state, cell state) 91 | hidden = tuple([_cat(h) for h in hidden]) 92 | else: 93 | # GRU hidden 94 | hidden = _cat(hidden) 95 | 96 | return hidden 97 | -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .BiLSTM import BiLSTMLayer 2 | from .tconv import TemporalConv 3 | from .gcn_lib.temgraph import * 4 | from .gcn_lib.torch_edge import * 5 | from .gcn_lib.torch_nn import * -------------------------------------------------------------------------------- /modules/criterions.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | class SeqKD(nn.Module): 7 | """ 8 | NLL loss with label smoothing. 9 | """ 10 | 11 | def __init__(self, T=1): 12 | super(SeqKD, self).__init__() 13 | self.kdloss = nn.KLDivLoss(reduction='batchmean') 14 | self.T = T 15 | 16 | def forward(self, prediction_logits, ref_logits, use_blank=True): 17 | start_idx = 0 if use_blank else 1 18 | prediction_logits = F.log_softmax(prediction_logits[:, :, start_idx:]/self.T, dim=-1) \ 19 | .view(-1, ref_logits.shape[2] - start_idx) 20 | ref_probs = F.softmax(ref_logits[:, :, start_idx:]/self.T, dim=-1) \ 21 | .view(-1, ref_logits.shape[2] - start_idx) 22 | loss = self.kdloss(prediction_logits, ref_probs)*self.T*self.T 23 | return loss 24 | 25 | -------------------------------------------------------------------------------- /modules/gcn_lib/__init__.py: -------------------------------------------------------------------------------- 1 | # 2022.06.17-Changed for building ViG model 2 | # Huawei Technologies Co., Ltd. 3 | from .torch_nn import * 4 | from .torch_edge import * 5 | from .torch_vertex import * 6 | from .temgraph import * 7 | -------------------------------------------------------------------------------- /modules/gcn_lib/pos_embed.py: -------------------------------------------------------------------------------- 1 | # 2022.06.17-Changed for building ViG model 2 | # Huawei Technologies Co., Ltd. 3 | # modified from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py 4 | # Copyright (c) Meta Platforms, Inc. and affiliates. 5 | # All rights reserved. 6 | 7 | # This source code is licensed under the license found in the 8 | # LICENSE file in the root directory of this source tree. 9 | # -------------------------------------------------------- 10 | # Position embedding utils 11 | # -------------------------------------------------------- 12 | 13 | import numpy as np 14 | 15 | import torch 16 | 17 | # -------------------------------------------------------- 18 | # relative position embedding 19 | # References: https://arxiv.org/abs/2009.13658 20 | # -------------------------------------------------------- 21 | def get_2d_relative_pos_embed(embed_dim, grid_size): 22 | """ 23 | grid_size: int of the grid height and width 24 | return: 25 | pos_embed: [grid_size*grid_size, grid_size*grid_size] 26 | """ 27 | pos_embed = get_2d_sincos_pos_embed(embed_dim, grid_size) 28 | relative_pos = 2 * np.matmul(pos_embed, pos_embed.transpose()) / pos_embed.shape[1] 29 | return relative_pos 30 | 31 | 32 | # -------------------------------------------------------- 33 | # 2D sine-cosine position embedding 34 | # References: 35 | # Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py 36 | # MoCo v3: https://github.com/facebookresearch/moco-v3 37 | # -------------------------------------------------------- 38 | def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): 39 | """ 40 | grid_size: int of the grid height and width 41 | return: 42 | pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) 43 | """ 44 | grid_h = np.arange(grid_size, dtype=np.float32) 45 | grid_w = np.arange(grid_size, dtype=np.float32) 46 | grid = np.meshgrid(grid_w, grid_h) # here w goes first 47 | grid = np.stack(grid, axis=0) 48 | 49 | grid = grid.reshape([2, 1, grid_size, grid_size]) 50 | pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) 51 | if cls_token: 52 | pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0) 53 | return pos_embed 54 | 55 | 56 | def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): 57 | assert embed_dim % 2 == 0 58 | 59 | # use half of dimensions to encode grid_h 60 | emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) 61 | emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) 62 | 63 | emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) 64 | return emb 65 | 66 | 67 | def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): 68 | """ 69 | embed_dim: output dimension for each position 70 | pos: a list of positions to be encoded: size (M,) 71 | out: (M, D) 72 | """ 73 | assert embed_dim % 2 == 0 74 | omega = np.arange(embed_dim // 2, dtype=np.float) 75 | omega /= embed_dim / 2. 76 | omega = 1. / 10000**omega # (D/2,) 77 | 78 | pos = pos.reshape(-1) # (M,) 79 | out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product 80 | 81 | emb_sin = np.sin(out) # (M, D/2) 82 | emb_cos = np.cos(out) # (M, D/2) 83 | 84 | emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) 85 | return emb 86 | -------------------------------------------------------------------------------- /modules/gcn_lib/temgraph.py: -------------------------------------------------------------------------------- 1 | from torch_geometric.nn import GCNConv, SAGEConv, EdgeConv 2 | import torch.nn as nn 3 | import torch 4 | from timm.models.layers import DropPath 5 | from einops import rearrange 6 | import numpy as np 7 | import random 8 | import torch.nn.functional as F 9 | 10 | 11 | 12 | def ForEucDis(x, y): 13 | with torch.no_grad(): 14 | b, c, t, n = x.shape 15 | x = x.permute(0, 2,3, 1) #b t n c 16 | y = y.permute(0, 2,3, 1) 17 | x = x.reshape(b, t, n, c) 18 | y = y.reshape(b, t, n, c) 19 | return torch.cdist(x, y) 20 | 21 | class TemporalGraph(nn.Module): 22 | def __init__(self, in_channels, k=4, drop_path=0.0): 23 | super(TemporalGraph, self).__init__() 24 | self.k = k 25 | self.reduction_channel = in_channels 26 | self.down_conv = nn.Sequential( 27 | nn.Conv3d(in_channels, self.reduction_channel, kernel_size=(3,1,1), bias=False,padding=(1,0,0)), 28 | nn.BatchNorm3d(self.reduction_channel) 29 | ) 30 | self.up_conv = nn.Sequential( 31 | nn.Conv3d(in_channels, self.reduction_channel, kernel_size=(3,1,1), bias=False,padding=(1,0,0)), 32 | nn.BatchNorm3d(self.reduction_channel) 33 | ) 34 | self.gconv = GCNConv(self.reduction_channel, self.reduction_channel) 35 | # self.gconv = SAGEConv(self.reduction_channel, self.reduction_channel) 36 | 37 | def forward(self, x, batch): 38 | tlen, c, h, w = x.shape 39 | x = rearrange(x.view(batch, tlen // batch, c, h, w), "b v c h w-> b c v h w") 40 | x = self.down_conv(x) 41 | x = rearrange(x, "b c v h w-> b c v (h w)") 42 | x1, x2 = x[:, :, :-1, :], x[:,:,1:,:] # b c t-1 hw 43 | sim = -ForEucDis(x1, x2) 44 | b, t_1, hw, hw = sim.shape 45 | sim = F.normalize(sim.view(b,t_1, -1), dim=-1) 46 | sim = torch.where(sim < 0.05, 100, sim) 47 | _, topk_indices = torch.topk(sim, k=self.k) 48 | row_indices,col_indices = topk_indices // hw, topk_indices % hw 49 | finaledge = torch.zeros((b, t_1, self.k, 2), dtype=torch.int) 50 | for i in range(t_1): 51 | finaledge[:,i, :, 0] = row_indices[:, i,:]+ i* hw 52 | finaledge[:, i, :, 1] = col_indices[:, i, :] + (i+1) * hw 53 | finaledge = finaledge.view(b, t_1*self.k, 2) 54 | finaledge_re = torch.stack((finaledge[:,:,1], finaledge[:,:,0]), dim=-1) 55 | # torch.save(finaledge, "./work_dir/test/temproal.pt") 56 | 57 | finaledge = torch.cat((finaledge, finaledge_re), dim=1).permute(0,2,1).detach() 58 | x = rearrange(x, "b c v n-> b (v n) c") 59 | out = torch.zeros_like(x).to(x.device) 60 | for i in range(batch): 61 | out[i] = self.gconv(x[i], finaledge[i].to(x.device)) 62 | x = out.permute(0,2,1).view(b, self.reduction_channel, tlen//b, h, w) 63 | x = self.up_conv(x).permute(0, 2, 1, 3, 4).contiguous().view(tlen, c, h, w) 64 | return x 65 | 66 | 67 | # y = model(x, 2) 68 | # 69 | # print(y.shape) 70 | -------------------------------------------------------------------------------- /modules/gcn_lib/torch_edge.py: -------------------------------------------------------------------------------- 1 | # 2022.06.17-Changed for building ViG model 2 | # Huawei Technologies Co., Ltd. 3 | import math 4 | import torch 5 | from torch import nn 6 | import torch.nn.functional as F 7 | 8 | 9 | def pairwise_distance(x): 10 | """ 11 | Compute pairwise distance of a point cloud. 12 | Args: 13 | x: tensor (batch_size, num_points, num_dims) 14 | Returns: 15 | pairwise distance: (batch_size, num_points, num_points) 16 | """ 17 | with torch.no_grad(): 18 | x_inner = -2*torch.matmul(x, x.transpose(2, 1)) 19 | x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) 20 | return x_square + x_inner + x_square.transpose(2, 1) 21 | 22 | 23 | def part_pairwise_distance(x, start_idx=0, end_idx=1): 24 | """ 25 | Compute pairwise distance of a point cloud. 26 | Args: 27 | x: tensor (batch_size, num_points, num_dims) 28 | Returns: 29 | pairwise distance: (batch_size, num_points, num_points) 30 | """ 31 | with torch.no_grad(): 32 | x_part = x[:, start_idx:end_idx] 33 | x_square_part = torch.sum(torch.mul(x_part, x_part), dim=-1, keepdim=True) 34 | x_inner = -2*torch.matmul(x_part, x.transpose(2, 1)) 35 | x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) 36 | return x_square_part + x_inner + x_square.transpose(2, 1) 37 | 38 | 39 | def xy_pairwise_distance(x, y): 40 | """ 41 | Compute pairwise distance of a point cloud. 42 | Args: 43 | x: tensor (batch_size, num_points, num_dims) 44 | Returns: 45 | pairwise distance: (batch_size, num_points, num_points) 46 | """ 47 | with torch.no_grad(): 48 | xy_inner = -2*torch.matmul(x, y.transpose(2, 1)) 49 | x_square = torch.sum(torch.mul(x, x), dim=-1, keepdim=True) 50 | y_square = torch.sum(torch.mul(y, y), dim=-1, keepdim=True) 51 | return x_square + xy_inner + y_square.transpose(2, 1) 52 | 53 | 54 | def dense_knn_matrix(x, k=16, relative_pos=None): 55 | """Get KNN based on the pairwise distance. 56 | Args: 57 | x: (batch_size, num_dims, num_points, 1) 58 | k: int 59 | Returns: 60 | nearest neighbors: (batch_size, num_points, k) (batch_size, num_points, k) 61 | """ 62 | with torch.no_grad(): 63 | x = x.transpose(2, 1).squeeze(-1) 64 | batch_size, n_points, n_dims = x.shape 65 | ### memory efficient implementation ### 66 | n_part = 10000 67 | if n_points > n_part: 68 | nn_idx_list = [] 69 | groups = math.ceil(n_points / n_part) 70 | for i in range(groups): 71 | start_idx = n_part * i 72 | end_idx = min(n_points, n_part * (i + 1)) 73 | dist = part_pairwise_distance(x.detach(), start_idx, end_idx) 74 | if relative_pos is not None: 75 | dist += relative_pos[:, start_idx:end_idx] 76 | _, nn_idx_part = torch.topk(-dist, k=k) 77 | nn_idx_list += [nn_idx_part] 78 | nn_idx = torch.cat(nn_idx_list, dim=1) 79 | else: 80 | dist = pairwise_distance(x.detach()) 81 | if relative_pos is not None: 82 | dist += relative_pos 83 | _, nn_idx = torch.topk(-dist, k=k) # b, n, k 84 | ###### 85 | center_idx = torch.arange(0, n_points, device=x.device).repeat(batch_size, k, 1).transpose(2, 1) 86 | return torch.stack((nn_idx, center_idx), dim=0) 87 | 88 | 89 | def xy_dense_knn_matrix(x, y, k=16, relative_pos=None): 90 | """Get KNN based on the pairwise distance. 91 | Args: 92 | x: (batch_size, num_dims, num_points, 1) 93 | k: int 94 | Returns: 95 | nearest neighbors: (batch_size, num_points, k) (batch_size, num_points, k) 96 | """ 97 | with torch.no_grad(): 98 | x = x.transpose(2, 1).squeeze(-1) 99 | y = y.transpose(2, 1).squeeze(-1) 100 | batch_size, n_points, n_dims = x.shape 101 | dist = xy_pairwise_distance(x.detach(), y.detach()) 102 | if relative_pos is not None: 103 | dist += relative_pos 104 | _, nn_idx = torch.topk(-dist, k=k) 105 | center_idx = torch.arange(0, n_points, device=x.device).repeat(batch_size, k, 1).transpose(2, 1) 106 | return torch.stack((nn_idx, center_idx), dim=0) 107 | 108 | 109 | class DenseDilated(nn.Module): 110 | """ 111 | Find dilated neighbor from neighbor list 112 | 113 | edge_index: (2, batch_size, num_points, k) 114 | """ 115 | def __init__(self, k=9, dilation=1, stochastic=False, epsilon=0.0): 116 | super(DenseDilated, self).__init__() 117 | self.dilation = dilation 118 | self.stochastic = stochastic 119 | self.epsilon = epsilon 120 | self.k = k 121 | 122 | def forward(self, edge_index): 123 | # print("debug torch_edge line123=", edge_index, self.dilation) 124 | if self.stochastic: 125 | if torch.rand(1) < self.epsilon and self.training: 126 | num = self.k * self.dilation 127 | randnum = torch.randperm(num)[:self.k] 128 | edge_index = edge_index[:, :, :, randnum] 129 | else: 130 | edge_index = edge_index[:, :, :, ::self.dilation] 131 | else: 132 | edge_index = edge_index[:, :, :, ::self.dilation] 133 | # print("debug torch_edge line133=", edge_index, self.dilation) 134 | return edge_index 135 | 136 | 137 | class DenseDilatedKnnGraph(nn.Module): 138 | """ 139 | Find the neighbors' indices based on dilated knn 140 | """ 141 | def __init__(self, k=9, dilation=1, stochastic=False, epsilon=0.0): 142 | super(DenseDilatedKnnGraph, self).__init__() 143 | self.dilation = dilation 144 | self.stochastic = stochastic 145 | self.epsilon = epsilon 146 | self.k = k 147 | self._dilated = DenseDilated(k, dilation, stochastic, epsilon) 148 | 149 | def forward(self, x, y=None, relative_pos=None): 150 | if y is not None: 151 | #### normalize 152 | x = F.normalize(x, p=2.0, dim=1) 153 | y = F.normalize(y, p=2.0, dim=1) 154 | #### 155 | edge_index = xy_dense_knn_matrix(x, y, self.k * self.dilation, relative_pos) 156 | else: 157 | #### normalize 158 | x = F.normalize(x, p=2.0, dim=1) 159 | #### 160 | edge_index = dense_knn_matrix(x, self.k * self.dilation, relative_pos) 161 | return self._dilated(edge_index) 162 | -------------------------------------------------------------------------------- /modules/gcn_lib/torch_nn.py: -------------------------------------------------------------------------------- 1 | # 2022.06.17-Changed for building ViG model 2 | # Huawei Technologies Co., Ltd. 3 | import torch 4 | from torch import nn 5 | from torch.nn import Sequential as Seq, Linear as Lin, Conv2d 6 | 7 | 8 | ############################## 9 | # Basic layers 10 | ############################## 11 | def act_layer(act, inplace=False, neg_slope=0.2, n_prelu=1): 12 | # activation layer 13 | 14 | act = act.lower() 15 | if act == 'relu': 16 | layer = nn.ReLU(inplace) 17 | elif act == 'leakyrelu': 18 | layer = nn.LeakyReLU(neg_slope, inplace) 19 | elif act == 'prelu': 20 | layer = nn.PReLU(num_parameters=n_prelu, init=neg_slope) 21 | elif act == 'gelu': 22 | layer = nn.GELU() 23 | elif act == 'hswish': 24 | layer = nn.Hardswish(inplace) 25 | else: 26 | raise NotImplementedError('activation layer [%s] is not found' % act) 27 | return layer 28 | 29 | 30 | def norm_layer(norm, nc): 31 | # normalization layer 2d 32 | norm = norm.lower() 33 | if norm == 'batch': 34 | layer = nn.BatchNorm2d(nc, affine=True) 35 | elif norm == 'instance': 36 | layer = nn.InstanceNorm2d(nc, affine=False) 37 | else: 38 | raise NotImplementedError('normalization layer [%s] is not found' % norm) 39 | return layer 40 | 41 | 42 | class MLP(Seq): 43 | def __init__(self, channels, act='relu', norm=None, bias=True): 44 | m = [] 45 | for i in range(1, len(channels)): 46 | m.append(Lin(channels[i - 1], channels[i], bias)) 47 | if act is not None and act.lower() != 'none': 48 | m.append(act_layer(act)) 49 | if norm is not None and norm.lower() != 'none': 50 | m.append(norm_layer(norm, channels[-1])) 51 | super(MLP, self).__init__(*m) 52 | 53 | 54 | class BasicConv(Seq): 55 | def __init__(self, channels, act='relu', norm=None, bias=True, drop=0.): 56 | m = [] 57 | for i in range(1, len(channels)): 58 | m.append(Conv2d(channels[i - 1], channels[i], 1, bias=bias, groups=4)) 59 | if norm is not None and norm.lower() != 'none': 60 | m.append(norm_layer(norm, channels[-1])) 61 | if act is not None and act.lower() != 'none': 62 | m.append(act_layer(act)) 63 | if drop > 0: 64 | m.append(nn.Dropout2d(drop)) 65 | 66 | super(BasicConv, self).__init__(*m) 67 | 68 | self.reset_parameters() 69 | 70 | def reset_parameters(self): 71 | for m in self.modules(): 72 | if isinstance(m, nn.Conv2d): 73 | nn.init.kaiming_normal_(m.weight) 74 | if m.bias is not None: 75 | nn.init.zeros_(m.bias) 76 | elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.InstanceNorm2d): 77 | m.weight.data.fill_(1) 78 | m.bias.data.zero_() 79 | 80 | 81 | def batched_index_select(x, idx): 82 | r"""fetches neighbors features from a given neighbor idx 83 | 84 | Args: 85 | x (Tensor): input feature Tensor 86 | :math:`\mathbf{X} \in \mathbb{R}^{B \times C \times N \times 1}`. 87 | idx (Tensor): edge_idx 88 | :math:`\mathbf{X} \in \mathbb{R}^{B \times N \times l}`. 89 | Returns: 90 | Tensor: output neighbors features 91 | :math:`\mathbf{X} \in \mathbb{R}^{B \times C \times N \times k}`. 92 | """ 93 | batch_size, num_dims, num_vertices_reduced = x.shape[:3] 94 | _, num_vertices, k = idx.shape 95 | idx_base = torch.arange(0, batch_size, device=idx.device).view(-1, 1, 1) * num_vertices_reduced 96 | idx = idx + idx_base 97 | idx = idx.contiguous().view(-1) 98 | 99 | x = x.transpose(2, 1) 100 | feature = x.contiguous().view(batch_size * num_vertices_reduced, -1)[idx, :] 101 | feature = feature.view(batch_size, num_vertices, k, num_dims).permute(0, 3, 1, 2).contiguous() 102 | return feature 103 | -------------------------------------------------------------------------------- /modules/gcn_lib/torch_vertex.py: -------------------------------------------------------------------------------- 1 | # 2022.06.17-Changed for building ViG model 2 | # Huawei Technologies Co., Ltd. 3 | import numpy as np 4 | import torch 5 | from torch import nn 6 | from .torch_nn import BasicConv, batched_index_select, act_layer 7 | from .torch_edge import DenseDilatedKnnGraph 8 | from .pos_embed import get_2d_relative_pos_embed 9 | import torch.nn.functional as F 10 | from timm.models.layers import DropPath 11 | 12 | 13 | class MRConv2d(nn.Module): 14 | """ 15 | Max-Relative Graph Convolution (Paper: https://arxiv.org/abs/1904.03751) for dense Dataprocessing type 16 | """ 17 | def __init__(self, in_channels, out_channels, act='relu', norm=None, bias=True): 18 | super(MRConv2d, self).__init__() 19 | self.nn = BasicConv([in_channels*2, out_channels], act, norm, bias) 20 | 21 | def forward(self, x, edge_index, y=None): 22 | x_i = batched_index_select(x, edge_index[1]) 23 | if y is not None: 24 | x_j = batched_index_select(y, edge_index[0]) 25 | else: 26 | x_j = batched_index_select(x, edge_index[0]) 27 | x_j, _ = torch.max(x_j - x_i, -1, keepdim=True) 28 | b, c, n, _ = x.shape 29 | x = torch.cat([x.unsqueeze(2), x_j.unsqueeze(2)], dim=2).reshape(b, 2 * c, n, _) 30 | return self.nn(x) 31 | 32 | 33 | class EdgeConv2d(nn.Module): 34 | """ 35 | Edge convolution layer (with activation, batch normalization) for dense Dataprocessing type 36 | """ 37 | def __init__(self, in_channels, out_channels, act='relu', norm=None, bias=True): 38 | super(EdgeConv2d, self).__init__() 39 | self.nn = BasicConv([in_channels * 2, out_channels], act, norm, bias) 40 | 41 | def forward(self, x, edge_index, y=None): 42 | x_i = batched_index_select(x, edge_index[1]) 43 | if y is not None: 44 | x_j = batched_index_select(y, edge_index[0]) 45 | else: 46 | x_j = batched_index_select(x, edge_index[0]) 47 | max_value, _ = torch.max(self.nn(torch.cat([x_i, x_j - x_i], dim=1)), -1, keepdim=True) 48 | return max_value 49 | 50 | 51 | class GraphSAGE(nn.Module): 52 | """ 53 | GraphSAGE Graph Convolution (Paper: https://arxiv.org/abs/1706.02216) for dense Dataprocessing type 54 | """ 55 | def __init__(self, in_channels, out_channels, act='relu', norm=None, bias=True): 56 | super(GraphSAGE, self).__init__() 57 | self.nn1 = BasicConv([in_channels, in_channels], act, norm, bias) 58 | self.nn2 = BasicConv([in_channels*2, out_channels], act, norm, bias) 59 | 60 | def forward(self, x, edge_index, y=None): 61 | if y is not None: 62 | x_j = batched_index_select(y, edge_index[0]) 63 | else: 64 | x_j = batched_index_select(x, edge_index[0]) 65 | x_j, _ = torch.max(self.nn1(x_j), -1, keepdim=True) 66 | return self.nn2(torch.cat([x, x_j], dim=1)) 67 | 68 | 69 | class GINConv2d(nn.Module): 70 | """ 71 | GIN Graph Convolution (Paper: https://arxiv.org/abs/1810.00826) for dense Dataprocessing type 72 | """ 73 | def __init__(self, in_channels, out_channels, act='relu', norm=None, bias=True): 74 | super(GINConv2d, self).__init__() 75 | self.nn = BasicConv([in_channels, out_channels], act, norm, bias) 76 | eps_init = 0.0 77 | self.eps = nn.Parameter(torch.Tensor([eps_init])) 78 | 79 | def forward(self, x, edge_index, y=None): 80 | if y is not None: 81 | x_j = batched_index_select(y, edge_index[0]) 82 | else: 83 | x_j = batched_index_select(x, edge_index[0]) 84 | x_j = torch.sum(x_j, -1, keepdim=True) 85 | return self.nn((1 + self.eps) * x + x_j) 86 | 87 | 88 | class GraphConv2d(nn.Module): 89 | """ 90 | Static graph convolution layer 91 | """ 92 | def __init__(self, in_channels, out_channels, conv='edge', act='relu', norm=None, bias=True): 93 | super(GraphConv2d, self).__init__() 94 | if conv == 'edge': 95 | self.gconv = EdgeConv2d(in_channels, out_channels, act, norm, bias) 96 | elif conv == 'mr': 97 | self.gconv = MRConv2d(in_channels, out_channels, act, norm, bias) 98 | elif conv == 'sage': 99 | self.gconv = GraphSAGE(in_channels, out_channels, act, norm, bias) 100 | elif conv == 'gin': 101 | self.gconv = GINConv2d(in_channels, out_channels, act, norm, bias) 102 | else: 103 | raise NotImplementedError('conv:{} is not supported'.format(conv)) 104 | 105 | def forward(self, x, edge_index, y=None): 106 | return self.gconv(x, edge_index, y) 107 | 108 | 109 | class DyGraphConv2d(GraphConv2d): 110 | """ 111 | Dynamic graph convolution layer 112 | """ 113 | def __init__(self, in_channels, out_channels, kernel_size=9, dilation=1, conv='edge', act='relu', 114 | norm=None, bias=True, stochastic=False, epsilon=0.0, r=1): 115 | super(DyGraphConv2d, self).__init__(in_channels, out_channels, conv, act, norm, bias) 116 | self.k = kernel_size 117 | self.d = dilation 118 | self.r = r 119 | self.dilated_knn_graph = DenseDilatedKnnGraph(kernel_size, dilation, stochastic, epsilon) 120 | 121 | def forward(self, x, relative_pos=None): 122 | B, C, H, W = x.shape 123 | y = None 124 | if self.r > 1: 125 | y = F.avg_pool2d(x, self.r, self.r) 126 | y = y.reshape(B, C, -1, 1).contiguous() 127 | x = x.reshape(B, C, -1, 1).contiguous() 128 | edge_index = self.dilated_knn_graph(x, y, relative_pos) 129 | 130 | # print(edge_index,edge_index.shape, x.shape) 131 | # torch.save(edge_index, "./work_dir/test/local.pt") 132 | 133 | x = super(DyGraphConv2d, self).forward(x, edge_index, y) 134 | return x.reshape(B, -1, H, W).contiguous() 135 | 136 | 137 | class Grapher(nn.Module): 138 | """ 139 | Grapher module with graph convolution and fc layers 140 | """ 141 | def __init__(self, in_channels, kernel_size=9, dilation=1, conv='edge', act='relu', norm=None, 142 | bias=True, stochastic=False, epsilon=0.0, r=1, n=196, drop_path=0.0, relative_pos=False): 143 | super(Grapher, self).__init__() 144 | self.channels = in_channels 145 | self.n = n 146 | self.r = r 147 | self.fc1 = nn.Sequential( 148 | nn.Conv2d(in_channels, in_channels, 1, stride=1, padding=0), 149 | nn.BatchNorm2d(in_channels), 150 | ) 151 | self.graph_conv = DyGraphConv2d(in_channels, in_channels * 2, kernel_size, dilation, conv, 152 | act, norm, bias, stochastic, epsilon, r) 153 | self.fc2 = nn.Sequential( 154 | nn.Conv2d(in_channels * 2, in_channels, 1, stride=1, padding=0), 155 | nn.BatchNorm2d(in_channels), 156 | ) 157 | self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() 158 | self.relative_pos = None 159 | if relative_pos: 160 | # print('using relative_pos') 161 | relative_pos_tensor = torch.from_numpy(np.float32(get_2d_relative_pos_embed(in_channels, 162 | int(n**0.5)))).unsqueeze(0).unsqueeze(1) 163 | relative_pos_tensor = F.interpolate( 164 | relative_pos_tensor, size=(n, n//(r*r)), mode='bicubic', align_corners=False) 165 | self.relative_pos = nn.Parameter(-relative_pos_tensor.squeeze(1), requires_grad=False) 166 | 167 | def _get_relative_pos(self, relative_pos, H, W): 168 | if relative_pos is None or H * W == self.n: 169 | return relative_pos 170 | else: 171 | N = H * W 172 | N_reduced = N // (self.r * self.r) 173 | return F.interpolate(relative_pos.unsqueeze(0), size=(N, N_reduced), mode="bicubic").squeeze(0) 174 | 175 | def forward(self, x): 176 | x = self.fc1(x) 177 | B, C, H, W = x.shape 178 | relative_pos = self._get_relative_pos(self.relative_pos, H, W) 179 | x = self.graph_conv(x, relative_pos) 180 | x = self.fc2(x) 181 | x = self.drop_path(x) 182 | return x 183 | 184 | 185 | class GrapherT(nn.Module): 186 | """ 187 | Grapher module with graph convolution and fc layers 188 | """ 189 | def __init__(self, in_channels, kernel_size=9, dilation=1, conv='edge', act='relu', norm=None, 190 | bias=True, stochastic=False, epsilon=0.0, r=1, drop_path=0.0, relative_pos=False): 191 | super(GrapherT, self).__init__() 192 | self.channels = in_channels 193 | 194 | self.r = r 195 | self.fc1 = nn.Sequential( 196 | nn.Conv2d(in_channels, in_channels, (3,1), stride=1, padding=(1,0)), 197 | nn.BatchNorm2d(in_channels), 198 | ) 199 | self.graph_conv = DyGraphConv2d(in_channels, in_channels * 2, kernel_size, dilation, conv, 200 | act, norm, bias, stochastic, epsilon, r) 201 | self.fc2 = nn.Sequential( 202 | nn.Conv2d(in_channels * 2, in_channels, 1, stride=1, padding=0), 203 | nn.BatchNorm2d(in_channels), 204 | ) 205 | self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() 206 | self.relative_pos = None 207 | 208 | 209 | def forward(self, x): 210 | _tmp = x 211 | x = self.fc1(x) 212 | x = self.graph_conv(x, None) 213 | x = self.fc2(x) 214 | x = self.drop_path(x) + _tmp 215 | return x 216 | -------------------------------------------------------------------------------- /modules/loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn, Tensor 3 | from torch.autograd import Variable 4 | import torch.nn.functional as F 5 | 6 | import numpy as np 7 | 8 | 9 | 10 | class XentLoss(nn.Module): 11 | """ 12 | Cross-Entropy Loss with optional label smoothing 13 | """ 14 | 15 | def __init__(self, pad_index: int, smoothing: float = 0.0): 16 | super(XentLoss, self).__init__() 17 | self.smoothing = smoothing 18 | self.pad_index = pad_index 19 | if self.smoothing <= 0.0: 20 | # standard xent loss 21 | self.criterion = nn.NLLLoss(ignore_index=self.pad_index, reduction="sum") 22 | else: 23 | # custom label-smoothed loss, computed with KL divergence loss 24 | self.criterion = nn.KLDivLoss(reduction="sum") 25 | 26 | def _smooth_targets(self, targets: Tensor, vocab_size: int): 27 | """ 28 | Smooth target distribution. All non-reference words get uniform 29 | probability mass according to "smoothing". 30 | :param targets: target indices, batch*seq_len 31 | :param vocab_size: size of the output vocabulary 32 | :return: smoothed target distributions, batch*seq_len x vocab_size 33 | """ 34 | # batch*seq_len x vocab_size 35 | smooth_dist = targets.new_zeros((targets.size(0), vocab_size)).float() 36 | # fill distribution uniformly with smoothing 37 | smooth_dist.fill_(self.smoothing / (vocab_size - 2)) 38 | # assign true label the probability of 1-smoothing ("confidence") 39 | smooth_dist.scatter_(1, targets.unsqueeze(1).data, 1.0 - self.smoothing) 40 | # give padding probability of 0 everywhere 41 | smooth_dist[:, self.pad_index] = 0 42 | # masking out padding area (sum of probabilities for padding area = 0) 43 | padding_positions = torch.nonzero(targets.data == self.pad_index) 44 | # pylint: disable=len-as-condition 45 | if len(padding_positions) > 0: 46 | smooth_dist.index_fill_(0, padding_positions.squeeze(), 0.0) 47 | return Variable(smooth_dist, requires_grad=False) 48 | 49 | # pylint: disable=arguments-differ 50 | def forward(self, log_probs, targets): 51 | """ 52 | Compute the cross-entropy between logits and targets. 53 | If label smoothing is used, target distributions are not one-hot, but 54 | "1-smoothing" for the correct target token and the rest of the 55 | probability mass is uniformly spread across the other tokens. 56 | :param log_probs: log probabilities as predicted by model 57 | :param targets: target indices 58 | :return: 59 | """ 60 | if self.smoothing > 0: 61 | targets = self._smooth_targets( 62 | targets=targets.contiguous().view(-1), vocab_size=log_probs.size(-1) 63 | ) 64 | # targets: distributions with batch*seq_len x vocab_size 65 | assert ( 66 | log_probs.contiguous().view(-1, log_probs.size(-1)).shape 67 | == targets.shape 68 | ) 69 | else: 70 | # targets: indices with batch*seq_len 71 | targets = targets.contiguous().view(-1) 72 | loss = self.criterion( 73 | log_probs.contiguous().view(-1, log_probs.size(-1)), targets 74 | ) 75 | return loss 76 | 77 | 78 | class SeqKD(nn.Module): 79 | """ 80 | NLL loss with label smoothing. 81 | """ 82 | 83 | def __init__(self, T=1): 84 | super(SeqKD, self).__init__() 85 | self.kdloss = nn.KLDivLoss(reduction='batchmean') 86 | self.T = T 87 | 88 | def forward(self, prediction_logits, ref_logits, use_blank=True): 89 | start_idx = 0 if use_blank else 1 90 | prediction_logits = F.log_softmax(prediction_logits[:, :, start_idx:]/self.T, dim=-1) \ 91 | .view(-1, ref_logits.shape[2] - start_idx) 92 | ref_probs = F.softmax(ref_logits[:, :, start_idx:]/self.T, dim=-1) \ 93 | .view(-1, ref_logits.shape[2] - start_idx) 94 | loss = self.kdloss(prediction_logits, ref_probs)*self.T*self.T 95 | return loss 96 | -------------------------------------------------------------------------------- /modules/resnet.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.utils.model_zoo as model_zoo 6 | import torch.nn.functional as F 7 | from einops import rearrange 8 | 9 | __all__ = [ 10 | 'ResNet', 'resnet10', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 11 | 'resnet152', 'resnet200' 12 | ] 13 | model_urls = { 14 | 'resnet18': 'https://download.pytorch.org/models/resnet18-f37072fd.pth', 15 | 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 16 | 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 17 | 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 18 | 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', 19 | } 20 | 21 | from modules.gcn_lib.torch_vertex import Grapher, act_layer 22 | from modules.gcn_lib.temgraph import TemporalGraph 23 | 24 | 25 | def conv3x3(in_planes, out_planes, stride=1): 26 | # 3x3x3 convolution with padding 27 | return nn.Conv3d( 28 | in_planes, 29 | out_planes, 30 | kernel_size=(1, 3, 3), 31 | stride=(1, stride, stride), 32 | padding=(0, 1, 1), 33 | bias=False) 34 | 35 | 36 | class BasicBlock(nn.Module): 37 | expansion = 1 38 | 39 | def __init__(self, inplanes, planes, stride=1, downsample=None): 40 | super(BasicBlock, self).__init__() 41 | self.conv1 = conv3x3(inplanes, planes, stride) 42 | self.bn1 = nn.BatchNorm3d(planes) 43 | self.relu = nn.ReLU(inplace=True) 44 | self.conv2 = conv3x3(planes, planes) 45 | self.bn2 = nn.BatchNorm3d(planes) 46 | self.downsample = downsample 47 | self.stride = stride 48 | 49 | def forward(self, x): 50 | residual = x 51 | 52 | out = self.conv1(x) 53 | out = self.bn1(out) 54 | out = self.relu(out) 55 | 56 | out = self.conv2(out) 57 | out = self.bn2(out) 58 | 59 | if self.downsample is not None: 60 | residual = self.downsample(x) 61 | 62 | out += residual 63 | out = self.relu(out) 64 | 65 | return out 66 | 67 | 68 | class ResNet(nn.Module): 69 | 70 | def __init__(self, block, layers, num_classes=1000): 71 | self.inplanes = 64 72 | super(ResNet, self).__init__() 73 | self.conv1 = nn.Conv3d(3, 64, kernel_size=(1, 7, 7), stride=(1, 2, 2), padding=(0, 3, 3), 74 | bias=False) 75 | self.bn1 = nn.BatchNorm3d(64) 76 | self.relu = nn.ReLU(inplace=True) 77 | self.maxpool = nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) 78 | self.layer1 = self._make_layer(block, 64, layers[0]) 79 | self.layer2 = self._make_layer(block, 128, layers[1], stride=2) 80 | self.layer3 = self._make_layer(block, 256, layers[2], stride=2) 81 | self.layer4 = self._make_layer(block, 512, layers[3], stride=2) 82 | self.avgpool = nn.AvgPool2d(7, stride=1) 83 | self.localG = Grapher(in_channels=256, kernel_size=3, dilation=1, conv='edge', #mr 84 | act='relu', norm="batch", bias=True, stochastic=False, 85 | epsilon=0.0, r=1, n=14 * 14, drop_path=0.0, relative_pos=True) # kernel_size=2 86 | self.localG2 = Grapher(in_channels=512, kernel_size=4, dilation=1, conv='edge', 87 | act='relu', norm="batch", bias=True, stochastic=False, 88 | epsilon=0.0, r=1, n=7 * 7, drop_path=0.0, relative_pos=True) # kernel_size=2 89 | self.temporalG = TemporalGraph(k=14 * 14 // 4, in_channels=256, drop_path=0) 90 | self.temporalG2 = TemporalGraph(k=7 * 7, in_channels=512, drop_path=0) 91 | self.alpha = nn.Parameter(torch.ones(4), requires_grad=True) 92 | self.fc = nn.Linear(512 * block.expansion, num_classes) 93 | 94 | 95 | for m in self.modules(): 96 | if isinstance(m, nn.Conv3d) or isinstance(m, nn.Conv2d): 97 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 98 | elif isinstance(m, nn.BatchNorm3d) or isinstance(m, nn.BatchNorm2d): 99 | nn.init.constant_(m.weight, 1) 100 | nn.init.constant_(m.bias, 0) 101 | 102 | def _make_layer(self, block, planes, blocks, stride=1): 103 | downsample = None 104 | if stride != 1 or self.inplanes != planes * block.expansion: 105 | downsample = nn.Sequential( 106 | nn.Conv3d(self.inplanes, planes * block.expansion, 107 | kernel_size=1, stride=(1, stride, stride), bias=False), 108 | nn.BatchNorm3d(planes * block.expansion), 109 | ) 110 | 111 | layers = [] 112 | layers.append(block(self.inplanes, planes, stride, downsample)) 113 | self.inplanes = planes * block.expansion 114 | for i in range(1, blocks): 115 | layers.append(block(self.inplanes, planes)) 116 | return nn.Sequential(*layers) 117 | 118 | def forward(self, x): 119 | N, C, T, H, W = x.size() 120 | x = self.conv1(x) 121 | x = self.bn1(x) 122 | x = self.relu(x) 123 | x = self.maxpool(x) # torch.Size([1, 64, 100, 56, 56]) 124 | x = self.layer1(x) # ([1, 64, 100, 56, 56]) 125 | x = self.layer2(x) # ize([1, 128, 100, 28, 28]) 126 | x = self.layer3(x) # e([1, 256, 100, 14, 14]) 127 | # 128 | N, C, T, H, W = x.size() 129 | x = rearrange(x, 'N C T H W -> (N T) C H W') # [78, 256, 14, 14]) 130 | x = x + self.localG(x) * self.alpha[0] 131 | x = x + self.temporalG(x, N) * self.alpha[1] 132 | x = x.view(N, T, C, H, W).permute(0, 2, 1, 3, 4) 133 | # # 134 | x = self.layer4(x) # [1, 512, 100, 7, 7]) 135 | # # 136 | N, C, T, H, W = x.size() 137 | x = rearrange(x, 'N C T H W -> (N T) C H W') # [78, 256, 14, 14]) 138 | x = x + self.localG2(x) * self.alpha[2] 139 | x = x + self.temporalG2(x, N) * self.alpha[3] 140 | x = x.view(N, T, C, H, W).permute(0, 2, 1, 3, 4) 141 | # 142 | 143 | x = x.transpose(1, 2).contiguous() # debug5= torch.Size([1, 100, 512, 7, 7]) 144 | x = x.view((-1,) + x.size()[2:]) # bt,c,h,w #ze([100, 512, 7, 7]) 145 | x = self.avgpool(x) 146 | x = x.view(x.size(0), -1) # bt,c 147 | x = self.fc(x) # bt,c 148 | 149 | return x 150 | 151 | 152 | def resnet18(**kwargs): 153 | """Constructs a ResNet-18 based model. 154 | """ 155 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) 156 | checkpoint = model_zoo.load_url(model_urls['resnet18']) 157 | layer_name = list(checkpoint.keys()) 158 | for ln in layer_name: 159 | if 'conv' in ln or 'downsample.0.weight' in ln: 160 | checkpoint[ln] = checkpoint[ln].unsqueeze(2) 161 | model.load_state_dict(checkpoint, strict=False) 162 | return model 163 | 164 | 165 | def resnet34(**kwargs): 166 | """Constructs a ResNet-34 model. 167 | """ 168 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) 169 | return model 170 | 171 | 172 | -------------------------------------------------------------------------------- /modules/tconv.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import copy 3 | import torch 4 | import collections 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | class MultiScale_TemporalConv(nn.Module): 9 | def __init__(self, in_channels, out_channels, kernel_size=3, dilations=[1,2,3,4],): 10 | 11 | super().__init__() 12 | 13 | # Multiple branches of temporal convolution 14 | self.num_branches = 4 15 | 16 | # Temporal Convolution branches 17 | self.branches = nn.ModuleList([ 18 | nn.Sequential( 19 | nn.Conv1d( 20 | in_channels, 21 | out_channels//self.num_branches, 22 | kernel_size=kernel_size, 23 | dilation=dilation, 24 | padding = dilation), 25 | nn.BatchNorm1d(out_channels//self.num_branches) 26 | ) 27 | for dilation in dilations 28 | ]) 29 | 30 | #self.fuse = nn.Conv1d(in_channels * self.num_branches, out_channels, kernel_size=1) 31 | #self.fuse = nn.Conv2d(in_channels, out_channels, kernel_size=(4,1)) 32 | #self.bn = nn.BatchNorm1d(out_channels) 33 | 34 | def forward(self, x): 35 | # Input dim: (N,C,T,V) 36 | branch_outs = [] 37 | for tempconv in self.branches: 38 | out = tempconv(x) 39 | branch_outs.append(out) 40 | 41 | out = torch.cat(branch_outs, dim=1) 42 | #out = torch.stack(branch_outs, dim=2) 43 | #out = self.fuse(out).squeeze(2) 44 | #out = self.bn(out) 45 | return out 46 | 47 | 48 | class TemporalConv(nn.Module): 49 | def __init__(self, input_size, hidden_size, conv_type=2, use_bn=False, num_classes=-1): 50 | super(TemporalConv, self).__init__() 51 | self.use_bn = use_bn 52 | self.input_size = input_size 53 | self.hidden_size = hidden_size 54 | self.num_classes = num_classes 55 | self.conv_type = conv_type 56 | 57 | if self.conv_type == 0: 58 | self.kernel_size = ['K3'] 59 | elif self.conv_type == 1: 60 | self.kernel_size = ['K5', "P2"] 61 | elif self.conv_type == 2: 62 | self.kernel_size = ['K5', "P2", 'K5', "P2"] 63 | elif self.conv_type == 3: 64 | self.kernel_size = ['K5', 'K5', "P2"] 65 | elif self.conv_type == 4: 66 | self.kernel_size = ['K5', 'K5'] 67 | elif self.conv_type == 5: 68 | self.kernel_size = ['K5', "P2", 'K5'] 69 | elif self.conv_type == 6: 70 | self.kernel_size = ["P2", 'K5', 'K5'] 71 | elif self.conv_type == 7: 72 | self.kernel_size = ["P2", 'K5', "P2", 'K5'] 73 | elif self.conv_type == 8: 74 | self.kernel_size = ["P2", "P2", 'K5', 'K5'] 75 | 76 | modules = [] 77 | for layer_idx, ks in enumerate(self.kernel_size): 78 | input_sz = self.input_size if layer_idx == 0 or self.conv_type == 6 and layer_idx == 1 or self.conv_type == 7 and layer_idx == 1 or self.conv_type == 8 and layer_idx == 2 else self.hidden_size 79 | if ks[0] == 'P': 80 | modules.append(nn.MaxPool1d(kernel_size=int(ks[1]), ceil_mode=False)) 81 | elif ks[0] == 'K': 82 | modules.append( 83 | nn.Conv1d(input_sz, self.hidden_size, kernel_size=int(ks[1]), stride=1, padding=0) 84 | #MultiScale_TemporalConv(input_sz, self.hidden_size) 85 | ) 86 | modules.append(nn.BatchNorm1d(self.hidden_size)) 87 | modules.append(nn.ReLU(inplace=True)) 88 | self.temporal_conv = nn.Sequential(*modules) 89 | 90 | if self.num_classes != -1: 91 | self.fc = nn.Linear(self.hidden_size, self.num_classes) 92 | 93 | def update_lgt(self, lgt): 94 | feat_len = copy.deepcopy(lgt) 95 | for ks in self.kernel_size: 96 | if ks[0] == 'P': 97 | feat_len = torch.div(feat_len, 2) 98 | else: 99 | feat_len -= int(ks[1]) - 1 100 | #pass 101 | return feat_len 102 | 103 | def forward(self, frame_feat, lgt): 104 | visual_feat = self.temporal_conv(frame_feat) 105 | lgt = self.update_lgt(lgt) 106 | logits = None if self.num_classes == -1 \ 107 | else self.fc(visual_feat.transpose(1, 2)).transpose(1, 2) 108 | return { 109 | "visual_feat": visual_feat.permute(2, 0, 1), 110 | "conv_logits": logits.permute(2, 0, 1), 111 | "feat_len": lgt, 112 | } 113 | -------------------------------------------------------------------------------- /preprocess/CSL-Daily/dev_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/CSL-Daily/dev_info.npy -------------------------------------------------------------------------------- /preprocess/CSL-Daily/gloss_dict.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/CSL-Daily/gloss_dict.npy -------------------------------------------------------------------------------- /preprocess/CSL-Daily/test_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/CSL-Daily/test_info.npy -------------------------------------------------------------------------------- /preprocess/CSL-Daily/train_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/CSL-Daily/train_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014-T/dev_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014-T/dev_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014-T/gloss_dict.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014-T/gloss_dict.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014-T/phoenix2014-T-groundtruth-dev.stm: -------------------------------------------------------------------------------- 1 | 11August_2010_Wednesday_tagesschau-2 1 Signer08 0.0 1.79769e+308 DRUCK TIEF KOMMEN 2 | 11August_2010_Wednesday_tagesschau-3 1 Signer08 0.0 1.79769e+308 ES-BEDEUTET VIEL WOLKE UND KOENNEN REGEN GEWITTER KOENNEN 3 | 11August_2010_Wednesday_tagesschau-8 1 Signer08 0.0 1.79769e+308 WIND MAESSIG SCHWACH REGION WENN GEWITTER WIND KOENNEN 4 | 25October_2010_Monday_tagesschau-22 1 Signer01 0.0 1.79769e+308 MITTWOCH REGEN KOENNEN NORDWEST WAHRSCHEINLICH NORD STARK WIND 5 | 05May_2011_Thursday_tagesschau-25 1 Signer08 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN FREITAG SECHSTE MAI ZEIGEN-BILDSCHIRM 6 | 15December_2010_Wednesday_tagesschau-40 1 Signer05 0.0 1.79769e+308 DANN STARK SCHNEE SCHNEIEN KOMMEN 7 | 10March_2011_Thursday_heute-51 1 Signer01 0.0 1.79769e+308 SUEDWEST KOMMEN WARM MORGEN SCHON FRANKREICH REGION FUENFZEHN BIS ZWANZIG GRAD WOCHENENDE KOMMEN WARM 8 | 14August_2009_Friday_tagesschau-72 1 Signer05 0.0 1.79769e+308 MORGEN DAENEMARK IX ZWANZIG MAXIMAL DREISSIG GRAD 9 | 26January_2010_Tuesday_heute-107 1 Signer03 0.0 1.79769e+308 SCHOEN ABEND WUENSCHEN 10 | 12February_2010_Friday_tagesschau-125 1 Signer09 0.0 1.79769e+308 MORGEN MAL SONNE SPEZIELL ALPEN 11 | 12February_2010_Friday_tagesschau-126 1 Signer09 0.0 1.79769e+308 MAL BISSCHEN SCHNEE NORD MEHR STARK SCHNEE 12 | 01November_2010_Monday_tagesschau-140 1 Signer05 0.0 1.79769e+308 NORDWEST AUCH TEIL IX REGEN 13 | 28November_2011_Monday_tagesschau-165 1 Signer07 0.0 1.79769e+308 TAG SUED MITTE WOLKE KRAEFTIG NEBEL 14 | 28November_2011_Monday_tagesschau-168 1 Signer07 0.0 1.79769e+308 NACHT FUENF GRAD SEE MINUS VIER BERG IX 15 | 28November_2011_Monday_tagesschau-172 1 Signer07 0.0 1.79769e+308 ABER REGEN ERST FREITAG ABEND SUED KOMMEN REGEN 16 | 24November_2011_Thursday_heute-222 1 Signer05 0.0 1.79769e+308 NORD NUR WOLKE FROST SUED 17 | 24November_2011_Thursday_heute-223 1 Signer05 0.0 1.79769e+308 DRESDEN IX REGION BERG NULL MINUS SECHS NORD neg-FROST neg-HABEN MILD 18 | 19October_2009_Monday_tagesschau-229 1 Signer09 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN 19 | 12July_2010_Monday_heute-244 1 Signer01 0.0 1.79769e+308 DEUTSCHLAND ZWEI TEIL REGEN SCHWUEL FRISCH KUEHL 20 | 21August_2011_Sunday_tagesschau-264 1 Signer08 0.0 1.79769e+308 SUEDWEST SOWIESO STERN KOENNEN SEHEN 21 | 21August_2011_Sunday_tagesschau-265 1 Signer08 0.0 1.79769e+308 MITTE REGION WOLKE MORGEN KOENNEN REGEN GEWITTER 22 | 22November_2010_Monday_heute-279 1 Signer01 0.0 1.79769e+308 IX KOENNEN GEFAHR GLATT BESONDERS MITTE REGION SUED DANN KALT 23 | 14October_2010_Thursday_tagesschau-298 1 Signer07 0.0 1.79769e+308 SAMSTAG REGEN BERG REGEN TEILWEISE UMWANDELN SCHNEE SCHNEIEN 24 | 14October_2010_Thursday_tagesschau-299 1 Signer07 0.0 1.79769e+308 SCHOEN WO MEER REGION 25 | 06April_2010_Tuesday_tagesschau-308 1 Signer03 0.0 1.79769e+308 WIND SCHWACH MAESSIG WEHEN 26 | 12May_2010_Wednesday_tagesschau-318 1 Signer01 0.0 1.79769e+308 TAG DANN SUED SUEDOST REGEN NACH MITTAG IX AUCH GEWITTER 27 | 17April_2010_Saturday_tagesschau-396 1 Signer03 0.0 1.79769e+308 LUFT DRUCK HOCH DESHALB MORGEN SONNE UEBERALL 28 | 01October_2009_Thursday_tagesschau-417 1 Signer03 0.0 1.79769e+308 MORGEN ZEHN GRAD IX SIEBZEHN GRAD OBER FLUSS 29 | 04May_2010_Tuesday_heute-453 1 Signer04 0.0 1.79769e+308 HOCH NORD SONNE DESHALB GRENZE HOCH TIEF 30 | 30January_2013_Wednesday_tagesschau-475 1 Signer05 0.0 1.79769e+308 FREITAG DANN KOMMEN REGEN IX UEBERWIEGEND WIND 31 | 30January_2013_Wednesday_tagesschau-477 1 Signer05 0.0 1.79769e+308 SONNTAG WAHRSCHEINLICH FREUNDLICH IM-VERLAUF WEST SCHNEE SCHNEIEN REGEN 32 | 14October_2009_Wednesday_tagesschau-529 1 Signer01 0.0 1.79769e+308 BESONDERS NORDWEST WIND STARK 33 | 20November_2011_Sunday_tagesschau-536 1 Signer04 0.0 1.79769e+308 MISCHUNG NEBEL HOCH NEBEL SONNE BLEIBEN 34 | 25February_2011_Friday_tagesschau-575 1 Signer08 0.0 1.79769e+308 DIENSTAG RUHIG TROCKEN WETTER 35 | 18October_2010_Monday_heute-629 1 Signer04 0.0 1.79769e+308 NUR SUED MEHR FREUNDLICH 36 | 18October_2010_Monday_heute-630 1 Signer04 0.0 1.79769e+308 JETZT WUENSCHEN SCHOEN ABEND 37 | 06October_2010_Wednesday_tagesschau-632 1 Signer01 0.0 1.79769e+308 IN-KOMMEND BLEIBEN LUFT DRUCK HOCH KOMMEN BEDEUTEN NEBEL HIMMEL UND SONNE MISCHUNG 38 | 06October_2010_Wednesday_tagesschau-640 1 Signer01 0.0 1.79769e+308 MORGEN DREIZEHN REGION SUEDWEST IX VIER ZWANZIG GRAD PASSEN FAST WIE SOMMER 39 | 04August_2011_Thursday_tagesschau-718 1 Signer05 0.0 1.79769e+308 MONTAG AUCH WECHSELHAFT UND MEHR KUEHL 40 | 21October_2009_Wednesday_tagesschau-735 1 Signer04 0.0 1.79769e+308 FRANKREICH KOMMEN WOLKE OST NIEDERSACHSEN BAYERN SUEDWEST REGEN 41 | 29May_2011_Sunday_tagesschau-764 1 Signer01 0.0 1.79769e+308 WEHEN SCHWACH MAESSIG 42 | 19April_2010_Monday_heute-781 1 Signer08 0.0 1.79769e+308 SUED MEHR RUHIG FREUNDLICH BISSCHEN MILD 43 | 30August_2011_Tuesday_heute-783 1 Signer07 0.0 1.79769e+308 LIEB ZUSCHAUER GUT ABEND 44 | 30August_2011_Tuesday_heute-784 1 Signer07 0.0 1.79769e+308 SOMMER ZWEITAUSEND ELF FAST VORBEI DEUTSCH LAND BISSCHEN WARM MEHR WENIG SONNE ABER ENORM VIEL REGEN 45 | 04December_2009_Friday_tagesschau-841 1 Signer05 0.0 1.79769e+308 MANCHMAL SCHWACH MAESSIG IX WIND IX WEST STARK WIND 46 | 22January_2010_Friday_tagesschau-907 1 Signer04 0.0 1.79769e+308 TAG VIEL WOLKE NEBEL BLEIBEN TROCKEN IX SONNE MOEGLICH 47 | 28September_2012_Friday_tagesschau-916 1 Signer06 0.0 1.79769e+308 SUEDWEST EUROPA SUED BEKOMMEN REGEN 48 | 28September_2012_Friday_tagesschau-917 1 Signer06 0.0 1.79769e+308 REGION LANG FREUNDLICH 49 | 28September_2012_Friday_tagesschau-922 1 Signer06 0.0 1.79769e+308 SUED WIND SCHWACH NORD MAESSIG FRISCH SCHAUER KUESTE BERG TEIL STURM ORKAN 50 | 03February_2011_Thursday_tagesschau-938 1 Signer08 0.0 1.79769e+308 BAYERN WALD HEUTE ABEND MINUS FUENF FLUSS DREI GRAD 51 | 03February_2011_Thursday_tagesschau-939 1 Signer08 0.0 1.79769e+308 MORGEN ZWEI GRAD REGION FLUSS NEUN GRAD 52 | 04April_2011_Monday_heute-965 1 Signer01 0.0 1.79769e+308 MORGEN MEHR WARM IX ALS HEUTE 53 | 04April_2011_Monday_heute-966 1 Signer01 0.0 1.79769e+308 VIERZEHN BIS NEUNZEHN GRAD KOENNEN 54 | 23November_2011_Wednesday_tagesschau-973 1 Signer05 0.0 1.79769e+308 JETZT WIE-AUSSEHEN WETTER MORGEN DONNERSTAG VIER ZWANZIG NOVEMBER 55 | 23November_2011_Wednesday_tagesschau-980 1 Signer05 0.0 1.79769e+308 NORDWEST WOLKE MANCHMAL SONNE BISSCHEN IX S+H IX REGEN 56 | 23November_2011_Wednesday_tagesschau-984 1 Signer05 0.0 1.79769e+308 DONNERSTAG DANN WIEDER MEHR WIND IX WIND KOMMEN ANFANG IX REGEN KOMMEN GLEICH 57 | 19August_2010_Thursday_tagesschau-995 1 Signer03 0.0 1.79769e+308 SAMSTAG HAUPTSAECHLICH SONNE NORD BISSCHEN REGEN 58 | 29October_2009_Thursday_tagesschau-1011 1 Signer05 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN DREISSIG OKTOBER 59 | 06May_2010_Thursday_heute-1054 1 Signer04 0.0 1.79769e+308 NACHMITTAG KOENNEN SONNE KOENNEN 60 | 05October_2011_Wednesday_heute-1069 1 Signer01 0.0 1.79769e+308 LIEB ZUSCHAUER poss-EUCH GUT ABEND 61 | 23May_2011_Monday_tagesschau-1097 1 Signer08 0.0 1.79769e+308 IX NORD TIEF KOMMEN WOLKE KOMMEN 62 | 19May_2010_Wednesday_heute-1119 1 Signer03 0.0 1.79769e+308 WEST FLUSS WEST REGION NACH MITTAG BESSER SONNE MOEGLICH 63 | 09August_2009_Sunday_tagesschau-1179 1 Signer01 0.0 1.79769e+308 MORGEN WEST TIEF KOMMEN RICHTUNG OST AUFZIEHEN 64 | 09August_2009_Sunday_tagesschau-1180 1 Signer01 0.0 1.79769e+308 REGEN GEWITTER STARK 65 | 09August_2009_Sunday_tagesschau-1181 1 Signer01 0.0 1.79769e+308 NORDWEST SPAETER AUFLOESEN 66 | 09August_2009_Sunday_tagesschau-1182 1 Signer01 0.0 1.79769e+308 JETZT IN-KOMMEND IX REGEN GEWITTER KOENNEN STARK 67 | 05July_2010_Monday_tagesschau-1200 1 Signer01 0.0 1.79769e+308 TAG SONNE WOLKE WECHSELHAFT TEILWEISE AUCH LANG SONNE 68 | 20June_2011_Monday_tagesschau-1253 1 Signer07 0.0 1.79769e+308 TAG IX REGEN AUCH KOMMEN OST 69 | 21June_2011_Tuesday_tagesschau-1264 1 Signer07 0.0 1.79769e+308 SUEDOST SCHOEN REST neg-REGION SCHON REGEN 70 | 12July_2011_Tuesday_heute-1274 1 Signer01 0.0 1.79769e+308 GEWITTER ENORM SUEDWEST BIS-MORGEN NACHT KOMMEN 71 | 12July_2011_Tuesday_heute-1279 1 Signer01 0.0 1.79769e+308 MORGEN GEWITTER MEISTENS WO SUEDOST REGION BRAND BURG IX NORD TEILWEISE REGEN 72 | 12July_2011_Tuesday_heute-1280 1 Signer01 0.0 1.79769e+308 WEST LANGSAM VERSCHWINDEN 73 | 12July_2011_Tuesday_heute-1281 1 Signer01 0.0 1.79769e+308 WIND KUESTE AUCH 74 | 25May_2010_Tuesday_tagesschau-1294 1 Signer05 0.0 1.79769e+308 HEUTE NACHT FLUSS VIERZEHN GRAD NORD MAXIMAL NULL 75 | 18April_2011_Monday_tagesschau-1316 1 Signer01 0.0 1.79769e+308 HOCH REGION BIS FREITAG MEISTENS WETTER FREUNDLICH SONNE TEMPERATUR STEIGEN 76 | 31January_2010_Sunday_tagesschau-1333 1 Signer05 0.0 1.79769e+308 MORGEN NORDWEST ZWEI SONST REGION MINUS FUENF ZWISCHEN EINS ZWISCHEN 77 | 22August_2010_Sunday_tagesschau-1345 1 Signer05 0.0 1.79769e+308 WIND MAESSIG FRISCH KUESTE IX STARK WENN GEWITTER IX DANN STURM MOEGLICH 78 | 02October_2009_Friday_tagesschau-1354 1 Signer03 0.0 1.79769e+308 DAZU MORGEN NORD WETTER WECHSELHAFT SONNTAG AUCH MITTE 79 | 02October_2009_Friday_tagesschau-1355 1 Signer03 0.0 1.79769e+308 REGION SO 80 | 02October_2009_Friday_tagesschau-1362 1 Signer03 0.0 1.79769e+308 MONTAG NORD REGION REGEN REGION BESSER IM-VERLAUF REGION REGEN 81 | 10January_2011_Monday_heute-1375 1 Signer01 0.0 1.79769e+308 REGEN DANN MITTWOCH REGEN DONNERSTAG STARK REGEN 82 | 13August_2011_Saturday_tagesschau-1409 1 Signer08 0.0 1.79769e+308 SPEZIELL SUEDOST KOENNEN ORT UNWETTER REGEN HAGEL STURM KOENNEN 83 | 13August_2011_Saturday_tagesschau-1410 1 Signer08 0.0 1.79769e+308 NORMAL WIND LEICHT MAESSIG KOMMEN WEHEN 84 | 23December_2010_Thursday_tagesschau-1430 1 Signer07 0.0 1.79769e+308 JETZT WETTER VORAUSSAGE MORGEN FREITAG VIER ZWANZIG DEZEMBER 85 | 10August_2010_Tuesday_heute-1455 1 Signer05 0.0 1.79769e+308 OST REGION SONNE MEHR WARM MAXIMAL SECHS ZWANZIG SUED MAXIMAL SIEBEN ZWANZIG GRAD 86 | 25January_2010_Monday_heute-1467 1 Signer03 0.0 1.79769e+308 SUED REGION MEHR WOLKE FLUSS ALPEN MOEGLICH SCHNEE ABER IM-VERLAUF WENIGER 87 | 30July_2010_Friday_tagesschau-1520 1 Signer04 0.0 1.79769e+308 KUEHL SIEBEN GRAD ALLGAEU IX SECHSZEHN NORD 88 | 06December_2011_Tuesday_heute-1553 1 Signer07 0.0 1.79769e+308 STURM TAGSUEBER DABEI 89 | 06December_2011_Tuesday_heute-1554 1 Signer07 0.0 1.79769e+308 IHR WUENSCHEN ABEND MACHEN GUT 90 | 19July_2009_Sunday_tagesschau-1568 1 Signer09 0.0 1.79769e+308 MONTAG MEISTENS TIEF DRUCK REGION LUFT MEER LUFT KOMMEN 91 | 28November_2011_Monday_heute-1587 1 Signer07 0.0 1.79769e+308 AUCH MORGEN WETTER SUPER neg-HABEN FUENF GRAD NORDOST MILD WO SUEDWEST 92 | 07February_2011_Monday_tagesschau-1612 1 Signer07 0.0 1.79769e+308 NACHT NORD MEHR STURM AUCH KUESTE ORKAN 93 | 07February_2011_Monday_tagesschau-1613 1 Signer07 0.0 1.79769e+308 TAG SCHWACH MAESSIG WEHEN NORD NORDOST FRISCH MAESSIG WEHEN OST STARK WEHEN 94 | 07April_2010_Wednesday_heute-1631 1 Signer05 0.0 1.79769e+308 FREUNDLICH BLEIBEN REGION SUED BISSCHEN SCHAUER 95 | 07April_2010_Wednesday_heute-1632 1 Signer05 0.0 1.79769e+308 SCHOEN ABEND AUSRICHTEN 96 | 03July_2011_Sunday_tagesschau-1673 1 Signer04 0.0 1.79769e+308 MITTWOCH WECHSELHAFT 97 | 03July_2011_Sunday_tagesschau-1674 1 Signer04 0.0 1.79769e+308 AUCH DONNERSTAG WEITER WECHSELHAFT 98 | 13April_2011_Wednesday_tagesschau-1683 1 Signer07 0.0 1.79769e+308 REST REGION SONNE WOLKE NUR ORT BISSCHEN REGEN 99 | 12January_2011_Wednesday_tagesschau-1732 1 Signer05 0.0 1.79769e+308 HEUTE NACHT SIEBEN SONST REGION SECHS ZWISCHEN EINS 100 | 09March_2011_Wednesday_heute-1742 1 Signer01 0.0 1.79769e+308 AUCH WIND STARK STROEMEN REGION STURM ORKAN KUESTE KOENNEN 101 | 20November_2010_Saturday_tagesschau-1823 1 Signer04 0.0 1.79769e+308 AB MONTAG DAS-IST-ES DEUTSCH EINFLUSS REGEN 102 | 20November_2010_Saturday_tagesschau-1824 1 Signer04 0.0 1.79769e+308 IX WIND UND SCHNEE STUFENWEISE GRENZE 103 | 15February_2010_Monday_tagesschau-1843 1 Signer01 0.0 1.79769e+308 NORDWEST SCHNEE BISSCHEN KOENNEN 104 | 14July_2010_Wednesday_tagesschau-1865 1 Signer01 0.0 1.79769e+308 SONNTAG RUHIG VERSCHWINDEN 105 | 14April_2010_Wednesday_heute-1879 1 Signer05 0.0 1.79769e+308 GUT ABEND LIEB ZUSCHAUER 106 | 14April_2010_Wednesday_heute-1880 1 Signer05 0.0 1.79769e+308 IN-KOMMEND FRUEHLING WETTER MORGEN WARM 107 | 14April_2010_Wednesday_heute-1881 1 Signer05 0.0 1.79769e+308 WOHER WARM 108 | 14April_2010_Wednesday_heute-1892 1 Signer05 0.0 1.79769e+308 SCHOEN ABEND WUENSCHEN MACHEN GUT WUENSCHEN 109 | 31May_2011_Tuesday_heute-1899 1 Signer01 0.0 1.79769e+308 HOFFEN REGION JETZT NORDOST VERLAUFEN VERSCHWINDEN 110 | 28December_2011_Wednesday_tagesschau-1911 1 Signer04 0.0 1.79769e+308 BERG MORGEN NOCH REGEN IX SCHAUER NORD GEWITTER 111 | 09January_2010_Saturday_tagesschau-1919 1 Signer09 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SONNTAG ZEHNTE JANUAR 112 | 11May_2010_Tuesday_tagesschau-1960 1 Signer01 0.0 1.79769e+308 MORGEN WETTER WIE-AUSSEHEN MITTWOCH ZWOELF MAI 113 | 11May_2010_Tuesday_tagesschau-1964 1 Signer01 0.0 1.79769e+308 TAG DANN NORD REGION REGEN SUEDOST DOCH BISSCHEN FREUNDLICH 114 | 11May_2010_Tuesday_tagesschau-1971 1 Signer01 0.0 1.79769e+308 SAMSTAG NORD SUEDOST REGEN ABER GLEICH ZEIT WIND 115 | 03July_2009_Friday_tagesschau-2003 1 Signer04 0.0 1.79769e+308 GERADE RHEIN-PFALZ SAARLAND NORDRHEIN-WESTFALEN GEWITTER 116 | 03July_2009_Friday_tagesschau-2008 1 Signer04 0.0 1.79769e+308 IM-MOMENT DOCH GEWITTER DEUTSCH WETTER DIENST WARNUNG SCHON FERTIG BEKANNTGEBEN 117 | 02November_2010_Tuesday_tagesschau-2031 1 Signer05 0.0 1.79769e+308 SUED WAHRSCHEINLICH HOCH DRUCK RUHIG FREUNDLICH HERBST 118 | 02November_2010_Tuesday_tagesschau-2039 1 Signer05 0.0 1.79769e+308 AUCH SAMSTAG NAECHSTE GLEICH IX KOMMEN MEHR KUEHL 119 | 22January_2011_Saturday_tagesschau-2080 1 Signer04 0.0 1.79769e+308 WIND SCHWACH WEHEN 120 | 22January_2011_Saturday_tagesschau-2082 1 Signer04 0.0 1.79769e+308 REGION MORGEN SECHS BERG ALLGAEU KALT MINUS VIER 121 | 06July_2009_Monday_tagesschau-2109 1 Signer09 0.0 1.79769e+308 MORGEN WERT EIFEL ACHTZEHN BIS SECHS ZWANZIG LAND 122 | 22April_2010_Thursday_heute-2120 1 Signer03 0.0 1.79769e+308 MORGEN SONNE UEBERALL KUESTE REGION WOLKE MOEGLICH REGEN neg-VIEL 123 | 22April_2010_Thursday_heute-2124 1 Signer03 0.0 1.79769e+308 TROCKEN ENORM 124 | 22April_2010_Thursday_heute-2125 1 Signer03 0.0 1.79769e+308 ABEND WUENSCHEN 125 | 17October_2009_Saturday_tagesschau-2127 1 Signer04 0.0 1.79769e+308 TIEF REGION SCHWACH MORGEN SOLL EINFLUSS OST REGION SUED 126 | 17October_2009_Saturday_tagesschau-2133 1 Signer04 0.0 1.79769e+308 WIND SCHWACH WEHEN 127 | 17October_2009_Saturday_tagesschau-2138 1 Signer04 0.0 1.79769e+308 MITTWOCH WEST WOLKE ACHT BIS SECHSZEHN 128 | 17October_2009_Saturday_tagesschau-2139 1 Signer04 0.0 1.79769e+308 NEBEL KUEHL 129 | 18November_2011_Friday_tagesschau-2143 1 Signer04 0.0 1.79769e+308 WOLKE REGION SCHAFFEN NUR NORDWEST KOMMEN 130 | 27September_2009_Sunday_tagesschau-2169 1 Signer04 0.0 1.79769e+308 SUED SCHWACH WEHEN IX MAESSIG KUESTE FRISCH OST STURM 131 | 27September_2009_Sunday_tagesschau-2170 1 Signer04 0.0 1.79769e+308 NACHT TEIL BODEN FROST 132 | 06August_2010_Friday_tagesschau-2172 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SAMSTAG SIEBTE AUGUST 133 | 01March_2011_Tuesday_tagesschau-2209 1 Signer01 0.0 1.79769e+308 MORGEN MEISTENS SONNE 134 | 01March_2011_Tuesday_tagesschau-2210 1 Signer01 0.0 1.79769e+308 NORD KOENNEN BEWOELKT 135 | 01March_2011_Tuesday_tagesschau-2211 1 Signer01 0.0 1.79769e+308 REGION WEHEN SUED VIEL FRISCH WIND BISSCHEN STURM BERG STURM ORKAN KOENNEN 136 | 30December_2010_Thursday_tagesschau-2236 1 Signer01 0.0 1.79769e+308 HEUTE NACHT IX REGION NULL GRAD WENN STERN KOENNEN SEHEN BIS MINUS ZWEI ZWANZIG GRAD 137 | 30October_2009_Friday_tagesschau-2275 1 Signer03 0.0 1.79769e+308 WIND MAESSIG WEHEN 138 | 30October_2009_Friday_tagesschau-2276 1 Signer03 0.0 1.79769e+308 WEST REGION HEUTE NACHT FUENF GRAD REGION HAUPTSAECHLICH FROST MOEGLICH GLATT VORSICHT 139 | 07January_2010_Thursday_tagesschau-2301 1 Signer09 0.0 1.79769e+308 HEUTE NACHT TEMPERATUR MINUS DREI MINUS SIEBZEHN WENN KLAR TIEF 140 | 07January_2010_Thursday_tagesschau-2302 1 Signer09 0.0 1.79769e+308 TAG MINUS SIEBEN NORD NULL 141 | 07June_2010_Monday_heute-2322 1 Signer04 0.0 1.79769e+308 DAZU WARM LUFT TATSAECHLICH WARM HEISS NAECHSTE-WOCHE KUEHL 142 | 07June_2010_Monday_heute-2327 1 Signer04 0.0 1.79769e+308 OST SUED BERLIN SUED DEUTSCHLAND VIER ZWANZIG GRAD BIS ACHT ZWANZIG GRAD 143 | 26September_2009_Saturday_tagesschau-2333 1 Signer05 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SONNTAG SIEBEN ZWANZIG SEPTEMBER 144 | 26September_2009_Saturday_tagesschau-2335 1 Signer05 0.0 1.79769e+308 NUR NORD BISSCHEN WOLKE 145 | 26September_2009_Saturday_tagesschau-2342 1 Signer05 0.0 1.79769e+308 HEUTE NACHT NORD ZWOELF LAND IX VIER GRAD 146 | 26September_2009_Saturday_tagesschau-2343 1 Signer05 0.0 1.79769e+308 MORGEN TAG ACHTZEHN MAXIMAL DREI ZWANZIG GRAD 147 | 12March_2011_Saturday_tagesschau-2455 1 Signer07 0.0 1.79769e+308 TAG TEIL NUR EINS GRAD NORD SEE 148 | 15September_2010_Wednesday_heute-2476 1 Signer05 0.0 1.79769e+308 IM-VERLAUF FRUEH HERBST 149 | 10February_2010_Wednesday_tagesschau-2524 1 Signer03 0.0 1.79769e+308 SONNTAG SCHNEE WENIG NORD REGION MOEGLICH SONNE 150 | 25November_2010_Thursday_tagesschau-2529 1 Signer05 0.0 1.79769e+308 TAG SUEDOST SCHNEE SCHNEIEN FLACH AUCH SCHNEE SCHNEIEN 151 | 25November_2010_Thursday_tagesschau-2530 1 Signer05 0.0 1.79769e+308 NORDWEST SCHNEE SCHAUER VERSCHWINDEN 152 | 25November_2010_Thursday_tagesschau-2536 1 Signer05 0.0 1.79769e+308 SONNTAG IX WOLKE TEIL FREUNDLICH TROCKEN MINUS FUENF ZWISCHEN MAXIMAL ZWEI GRAD 153 | 12January_2010_Tuesday_heute-2545 1 Signer03 0.0 1.79769e+308 IM-VERLAUF SCHNEE UMKEHREN REGEN ABER EIS REGEN VORSICHT STRASSE GLATT MOEGLICH 154 | 12January_2010_Tuesday_heute-2546 1 Signer03 0.0 1.79769e+308 MORGEN TAG IM-VERLAUF SUEDWEST REGION BLEIBEN SO MOEGLICH SCHNEE 155 | 09December_2009_Wednesday_tagesschau-2561 1 Signer02 0.0 1.79769e+308 NORD REGION SCHWACH MAESSIG WIND 156 | 09December_2009_Wednesday_tagesschau-2562 1 Signer02 0.0 1.79769e+308 HEUTE NACHT RHEIN ACHT GRAD ALPEN MINUS SECHS 157 | 12July_2009_Sunday_tagesschau-2573 1 Signer01 0.0 1.79769e+308 MORGEN SONNE WOLKE WECHSELHAFT NORD REGION SONNE OFT 158 | 12July_2009_Sunday_tagesschau-2574 1 Signer01 0.0 1.79769e+308 IX SCHAUER SUEDWEST ABEND GEWITTER KOENNEN 159 | 06July_2010_Tuesday_tagesschau-2653 1 Signer05 0.0 1.79769e+308 NORDOST BISSCHEN SCHAUER ABEND AUFLOESEN 160 | 06July_2010_Tuesday_tagesschau-2654 1 Signer05 0.0 1.79769e+308 EIN-PAAR HARMLOS NEBEL DANN KLAR 161 | 05January_2010_Tuesday_tagesschau-2672 1 Signer01 0.0 1.79769e+308 HEUTE NACHT MINUS ZWEI NORD MITTE MINUS EINS ZWANZIG GRAD 162 | 19January_2011_Wednesday_heute-2680 1 Signer07 0.0 1.79769e+308 IN-KOMMEND TAG BLEIBEN KALT MITTE EUROPA UNGEFAEHR NULL ZWISCHEN MITTE MEER ZEHN BIS FUENFZEHN 163 | 01October_2010_Friday_tagesschau-2691 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SAMSTAG ZWEITE OKTOBER 164 | 31January_2013_Thursday_tagesschau-2710 1 Signer01 0.0 1.79769e+308 IX DANN SCHNEE KOENNEN ABEND DANN NORD SCHNEE FLACH SCHNEE 165 | 02November_2010_Tuesday_heute-2742 1 Signer05 0.0 1.79769e+308 GUT ABEND LIEB DU ZUSCHAUER poss-EUCH 166 | 02November_2010_Tuesday_heute-2754 1 Signer05 0.0 1.79769e+308 MILD NORD AUCH ZWOELF ZWISCHEN VIERZEHN SUED UNGEFAEHR VIERZEHN MAXIMAL SECHSZEHN GRAD 167 | 01July_2011_Friday_tagesschau-2763 1 Signer04 0.0 1.79769e+308 NORD SONNE WOLKE WECHSELHAFT BISSCHEN SCHAUER SUEDWEST BESONDERS SONNE MOEGLICH 168 | 01July_2011_Friday_tagesschau-2765 1 Signer04 0.0 1.79769e+308 HEUTE NACHT DREIZEHN NORD VIER BERG 169 | 01July_2011_Friday_tagesschau-2766 1 Signer04 0.0 1.79769e+308 AM-TAG DREIZEHN OST EINS ZWANZIG FLUSS 170 | 03August_2010_Tuesday_tagesschau-2776 1 Signer01 0.0 1.79769e+308 DANN KOMMEN SCHAUER STARK MIT GEWITTER 171 | 26January_2010_Tuesday_tagesschau-2787 1 Signer03 0.0 1.79769e+308 UND IM-VERLAUF UEBERALL WECHSELHAFT MOEGLICH SCHNEE 172 | 08February_2010_Monday_tagesschau-2796 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN DIENSTAG NEUNTE FEBRUAR 173 | 28January_2010_Thursday_tagesschau-2817 1 Signer05 0.0 1.79769e+308 WIE-AUSSEHEN IN-KOMMEND SAMSTAG SCHNEE SCHNEIEN BESONDERS NORD WAHRSCHEINLICH SONNE 174 | 05February_2010_Friday_tagesschau-2919 1 Signer01 0.0 1.79769e+308 HEUTE NACHT TEMPERATUR MINUS FUENF NORD BIS PLUS DREI WEST FLUSS 175 | 05February_2010_Friday_tagesschau-2920 1 Signer01 0.0 1.79769e+308 IX PLUS SECHS GRAD VOR POMMERN REGION BISSCHEN FROST 176 | 20September_2010_Monday_heute-2934 1 Signer01 0.0 1.79769e+308 DABEI NORD MEHR MILD 177 | 20September_2010_Monday_heute-2935 1 Signer01 0.0 1.79769e+308 ABER TAGSUEBER IX MEHR KUEHL 178 | 29September_2012_Saturday_tagesschau-2966 1 Signer06 0.0 1.79769e+308 MONTAG VIEL REGION FREUNDLICH KUESTE SUEDOST KURZ WOLKE TEIL REGEN 179 | 29September_2012_Saturday_tagesschau-2967 1 Signer06 0.0 1.79769e+308 DIENSTAG WEST MEHR WOLKE SCHAUER 180 | 09August_2010_Monday_tagesschau-3000 1 Signer05 0.0 1.79769e+308 NORDWEST SPAETER AUCH WOLKE DANN DABEI GEWITTER SCHAUER 181 | 09August_2010_Monday_tagesschau-3004 1 Signer05 0.0 1.79769e+308 MITTWOCH SINKEN KUEHLER WECHSELHAFT SONNE WOLKE SCHAUER GEWITTER WEST 182 | 07October_2012_Sunday_tagesschau-3026 1 Signer08 0.0 1.79769e+308 TAG KLAR STERN SEHEN KOENNEN FROST 183 | 01December_2011_Thursday_heute-3058 1 Signer05 0.0 1.79769e+308 STUNDE AUCH MEISTENS REGEN ERST WEST DANN REGION 184 | 01December_2011_Thursday_heute-3060 1 Signer05 0.0 1.79769e+308 UND TAG BLEIBEN KUEHL SECHS GRAD BAYERN IX REGION AUCH S+H IX AUCH 185 | 01December_2011_Thursday_heute-3061 1 Signer05 0.0 1.79769e+308 ABER DAZWISCHEN REGION MILD NEUN BIS VIERZEHN GRAD 186 | 20April_2011_Wednesday_tagesschau-3084 1 Signer05 0.0 1.79769e+308 SUEDWEST DEUTSCH LAND BISSCHEN FEUCHT KOMMEN IX MOEGLICH SCHAUER GEWITTER WAHRSCHEINLICH 187 | 21April_2010_Wednesday_heute-3122 1 Signer08 0.0 1.79769e+308 WOCHE WARM AUCH SONNE JEDEN-TAG GRAD STEIGEN 188 | 21April_2010_Wednesday_heute-3124 1 Signer08 0.0 1.79769e+308 SCHOEN ABEND WUENSCHEN 189 | 05September_2010_Sunday_tagesschau-3132 1 Signer04 0.0 1.79769e+308 AUCH OBER FLUSS BIS ABEND MEHR WOLKE 190 | 24May_2010_Monday_tagesschau-3142 1 Signer05 0.0 1.79769e+308 NACHT TEIL WOLKE TEIL KLAR SCHAUER GEWITTER 191 | 20May_2010_Thursday_tagesschau-3151 1 Signer03 0.0 1.79769e+308 JETZT WETTER MORGEN FREITAG EINS ZWANZIG MAI 192 | 20February_2011_Sunday_tagesschau-3164 1 Signer04 0.0 1.79769e+308 WETTER WIE-AUSSEHEN MORGEN MONTAG EINS ZWANZIG FEBRUAR 193 | 20February_2011_Sunday_tagesschau-3166 1 Signer04 0.0 1.79769e+308 AM-TAG REGION SCHNEE FLUSS REGEN 194 | 20December_2010_Monday_tagesschau-3199 1 Signer01 0.0 1.79769e+308 NORD KALT KOMMEN ZONE DAZWISCHEN 195 | 20December_2010_Monday_tagesschau-3204 1 Signer01 0.0 1.79769e+308 NORD FLUSS SUED TROCKEN IX 196 | 20December_2010_Monday_tagesschau-3205 1 Signer01 0.0 1.79769e+308 ZONE KOENNEN REGEN TEILWEISE SCHNEE REGEN UMWANDELN GEFRIEREN 197 | 20December_2010_Monday_tagesschau-3208 1 Signer01 0.0 1.79769e+308 NEUN GRAD REGION MINUS ACHT NORDOST 198 | 26August_2009_Wednesday_tagesschau-3225 1 Signer01 0.0 1.79769e+308 FREITAG SUEDOST SONNE FREITAG DANN MEHR WOLKE REGEN KOENNEN 199 | 27August_2009_Thursday_tagesschau-3276 1 Signer01 0.0 1.79769e+308 NORD SUEDOST AUCH KOENNEN REGEN GEWITTER DABEI 200 | 27August_2009_Thursday_tagesschau-3284 1 Signer01 0.0 1.79769e+308 SONNTAG UEBERWIEGEND SONNE 201 | 27August_2009_Thursday_tagesschau-3285 1 Signer01 0.0 1.79769e+308 NORD SCHAUER 202 | 27August_2009_Thursday_tagesschau-3286 1 Signer01 0.0 1.79769e+308 MONTAG AEHNLICH WETTER ABER BISSCHEN MEHR WARM 203 | 04July_2009_Saturday_tagesschau-3345 1 Signer04 0.0 1.79769e+308 MONTAG UEBERWIEGEND SCHAUER OST GEWITTER SUED WEST LANG FREUNDLICH 204 | 26April_2010_Monday_heute-3398 1 Signer01 0.0 1.79769e+308 SUEDWEST KLAR MORGEN VIER BIS NEUN GRAD 205 | 23September_2010_Thursday_tagesschau-3444 1 Signer01 0.0 1.79769e+308 REGION BIS ABEND TROCKEN 206 | 02September_2010_Thursday_heute-3456 1 Signer05 0.0 1.79769e+308 NORD DEUTSCHLAND IX KLAR LUFT IX NEBEL UND TEILWEISE BISSCHEN DAZWISCHEN REGEN HERZ BERG BAYERN REGION 207 | 02September_2010_Thursday_heute-3457 1 Signer05 0.0 1.79769e+308 MORGEN NORD FREUNDLICH 208 | 25September_2010_Saturday_tagesschau-3480 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SONNTAG SECHS ZWANZIG SEPTEMBER 209 | 02October_2012_Tuesday_heute-3494 1 Signer05 0.0 1.79769e+308 HEUTE NACHT neg-KAUM NUR NORDWEST BISSCHEN REGEN SONST REGION RUHIG 210 | 11December_2009_Friday_tagesschau-3509 1 Signer04 0.0 1.79769e+308 ALPEN LANG SCHNEE 211 | 06January_2010_Wednesday_heute-3523 1 Signer01 0.0 1.79769e+308 HEUTE NACHT MEHR SCHNEE NORD SUEDOST 212 | 06January_2010_Wednesday_heute-3525 1 Signer01 0.0 1.79769e+308 MORGEN BESONDERS NORD REGION UND WEST REGEN KOENNEN 213 | 06January_2010_Wednesday_heute-3530 1 Signer01 0.0 1.79769e+308 WIND KOMMEN DANN SCHNEE WITTERUNG 214 | 05April_2010_Monday_tagesschau-3546 1 Signer03 0.0 1.79769e+308 JETZT WETTER MORGEN DIENSTAG SECHSTE APRIL 215 | 20October_2009_Tuesday_tagesschau-3586 1 Signer09 0.0 1.79769e+308 JETZT WIE-AUSSEHEN WETTER MORGEN MITTWOCH EINS ZWANZIG OKTOBER 216 | 20October_2009_Tuesday_tagesschau-3594 1 Signer09 0.0 1.79769e+308 ABEND FLUSS WEST REGION MOEGLICH REGEN 217 | 20October_2009_Tuesday_tagesschau-3595 1 Signer09 0.0 1.79769e+308 WIND WEHEN MAESSIG WEHEN 218 | 02December_2010_Thursday_tagesschau-3636 1 Signer07 0.0 1.79769e+308 NORD NORDWEST MEHR VIEL WOLKE 219 | 02December_2010_Thursday_tagesschau-3637 1 Signer07 0.0 1.79769e+308 SCHNEE MEHR KUESTE REGION 220 | 22November_2011_Tuesday_tagesschau-3647 1 Signer07 0.0 1.79769e+308 MORGEN WIEDER WETTER RUHIG FREUNDLICH WETTER HOCH DRUCK 221 | 23April_2010_Friday_tagesschau-3667 1 Signer04 0.0 1.79769e+308 NACHT MINUS DREI BIS PLUS VIER NORD PLUS FUENF IX 222 | 23April_2010_Friday_tagesschau-3668 1 Signer04 0.0 1.79769e+308 MORGEN ZEHN BIS FUENFZEHN REGION SECHSZEHN BIS DREI ZWANZIG 223 | 30May_2011_Monday_heute-3677 1 Signer01 0.0 1.79769e+308 OST WARM REGION BIS DREISSIG GRAD REGION UEBERSPRINGEN REGION KUEHL NUR FUENFZEHN SECHSZEHN GRAD 224 | 24January_2011_Monday_tagesschau-3712 1 Signer01 0.0 1.79769e+308 NORD EUROPA TIEF KOMMEN 225 | 24January_2011_Monday_tagesschau-3720 1 Signer01 0.0 1.79769e+308 WIND WEHEN FRISCH KOMMEN HOCH STURM 226 | 23February_2011_Wednesday_heute-3780 1 Signer05 0.0 1.79769e+308 SCHOEN ABEND MACHEN poss-EUCH GUT 227 | 10August_2010_Tuesday_tagesschau-3865 1 Signer05 0.0 1.79769e+308 OST SUED HEUTE NACHT TROCKEN AUCH KLAR DANN TEIL REGION SCHAUER 228 | 25June_2010_Friday_tagesschau-3884 1 Signer01 0.0 1.79769e+308 FUENFZEHN GRAD REGION SAUER LAND NUR SECHS GRAD 229 | 25June_2010_Friday_tagesschau-3887 1 Signer01 0.0 1.79769e+308 MONTAG SONNE WARM WEITER 230 | 12September_2010_Sunday_tagesschau-3919 1 Signer01 0.0 1.79769e+308 IN-KOMMEND WETTER WECHSELHAFT 231 | 08April_2010_Thursday_tagesschau-3949 1 Signer05 0.0 1.79769e+308 LUFT KUEHL REGION TIEF HOCH REGION KOMMEN KUEHL 232 | 24April_2010_Saturday_tagesschau-3963 1 Signer04 0.0 1.79769e+308 MORGEN AUCH HOCH LUFT VIEL SONNE 233 | 08January_2010_Friday_tagesschau-3984 1 Signer09 0.0 1.79769e+308 WIE-AUSSEHEN IN-KOMMEND SONNTAG OST SPEZIELL SCHNEE NORD BLEIBEN WIND 234 | 04May_2011_Wednesday_heute-3996 1 Signer01 0.0 1.79769e+308 LANGSAM STEIGEN WARM FLUSS ACHTZEHN BIS ZWANZIG GRAD REGION ZWOELF BIS ACHTZEHN GRAD KUEHL poss-SEIN 235 | 08April_2010_Thursday_heute-4004 1 Signer05 0.0 1.79769e+308 REGEN negalp-AUCH ABER KUEHL 236 | 30September_2012_Sunday_tagesschau-4033 1 Signer06 0.0 1.79769e+308 NACHT SYLT DREIZEHN GRAD MITTE BERG TAL NULL GRAD 237 | 30September_2012_Sunday_tagesschau-4037 1 Signer06 0.0 1.79769e+308 MITTWOCH NAECHSTE DASSELBE 238 | 30September_2012_Sunday_tagesschau-4038 1 Signer06 0.0 1.79769e+308 DONNERSTAG DEUTSCH LAND WECHSELHAFT UEBERWIEGEND REGEN MOEGLICH STARK WIND 239 | 24June_2011_Friday_tagesschau-4071 1 Signer05 0.0 1.79769e+308 IX MORGEN IM-VERLAUF IX ANFANG FREUNDLICH ABER WOLKE KOMMEN OST 240 | 24June_2011_Friday_tagesschau-4073 1 Signer05 0.0 1.79769e+308 MORGEN IX WEHEN IX NORD FRISCH WIND 241 | 05September_2009_Saturday_tagesschau-4078 1 Signer04 0.0 1.79769e+308 WETTER WIE-AUSSEHEN MORGEN SONNTAG SECHS SEPTEMBER 242 | 19February_2010_Friday_tagesschau-4101 1 Signer09 0.0 1.79769e+308 NAECHSTE WOCHE MILD ABER WECHSELHAFT 243 | 10April_2010_Saturday_tagesschau-4140 1 Signer08 0.0 1.79769e+308 MORGEN VIER GRAD REGION SECHS BIS ELF GRAD FLUSS BIS ZWOELF GRAD 244 | 07December_2010_Tuesday_tagesschau-4152 1 Signer07 0.0 1.79769e+308 TAG MINUS DREI NORDOST PLUS VIERZEHN GRAD SUED 245 | 18September_2010_Saturday_tagesschau-4197 1 Signer07 0.0 1.79769e+308 MITTWOCH ALLE GUT SONNE SCHOEN SECHSZEHN BIS FUENF ZWANZIG GRAD 246 | 08December_2009_Tuesday_heute-4211 1 Signer02 0.0 1.79769e+308 ANDERE TEMPERATUR 247 | 08December_2009_Tuesday_heute-4212 1 Signer02 0.0 1.79769e+308 GLEICH OST VIER GRAD 248 | 08December_2009_Tuesday_heute-4213 1 Signer02 0.0 1.79769e+308 MILD ACHT BIS NEUN GRAD 249 | 29September_2011_Thursday_heute-4235 1 Signer05 0.0 1.79769e+308 SCHOEN ABEND poss-EUCH MACHEN GUT poss-EUCH 250 | 26August_2010_Thursday_tagesschau-4265 1 Signer03 0.0 1.79769e+308 JETZT WETTER MORGEN FREITAG SIEBEN ZWANZIG AUGUST 251 | 24October_2009_Saturday_tagesschau-4283 1 Signer01 0.0 1.79769e+308 SUEDOST REGEN VIEL 252 | 24October_2009_Saturday_tagesschau-4284 1 Signer01 0.0 1.79769e+308 IX REGEN KOENNEN BESONDERS NORD 253 | 24October_2009_Saturday_tagesschau-4285 1 Signer01 0.0 1.79769e+308 TAG DANN WEST FREUNDLICH SONNE DABEI 254 | 31May_2011_Tuesday_tagesschau-4294 1 Signer01 0.0 1.79769e+308 DEUTSCH WETTER DIENST WARNUNG 255 | 31May_2011_Tuesday_tagesschau-4295 1 Signer01 0.0 1.79769e+308 NORDWEST IX AUCH KOMMEN SONNE MEISTENS 256 | 13July_2010_Tuesday_tagesschau-4317 1 Signer01 0.0 1.79769e+308 OST DRUCKFLAECHE STARK BESONDERS IN-KOMMEND KOENNEN GEWITTER REGEN 257 | 22June_2011_Wednesday_heute-4381 1 Signer07 0.0 1.79769e+308 SO GEWITTER VON HOCH IX REGION NORD EUROPA REGION WARM LUFT KOMMEN KUEHL LUFT WEST KOMMEN 258 | 23August_2010_Monday_tagesschau-4429 1 Signer08 0.0 1.79769e+308 WOLKE DABEI HEUTE NACHT KOENNEN REGEN KOMMEN SUEDOST REGEN 259 | 23August_2010_Monday_tagesschau-4430 1 Signer08 0.0 1.79769e+308 DAZU KOENNEN GEWITTER 260 | 23August_2010_Monday_tagesschau-4438 1 Signer08 0.0 1.79769e+308 SUED MEHR FREUNDLICH 261 | 29June_2011_Wednesday_tagesschau-4482 1 Signer07 0.0 1.79769e+308 DARUM WARNUNG DEUTSCH WETTER DIENST WARNUNG INFORMIEREN 262 | 09February_2010_Tuesday_tagesschau-4495 1 Signer03 0.0 1.79769e+308 AUCH OBER FLUSS HOCH FLUSS FROST 263 | 09February_2010_Tuesday_tagesschau-4496 1 Signer03 0.0 1.79769e+308 AUCH SCHNEE KOMMEN 264 | 09February_2010_Tuesday_tagesschau-4497 1 Signer03 0.0 1.79769e+308 BLEIBEN IM-VERLAUF 265 | 01July_2009_Wednesday_tagesschau-4555 1 Signer09 0.0 1.79769e+308 MORGEN NORD KAUM GEWITTER REGION SONNE 266 | 11October_2010_Monday_tagesschau-4607 1 Signer07 0.0 1.79769e+308 AUCH NAH FLUSS UND ALPEN AUCH MOEGLICH NEBEL 267 | 11October_2010_Monday_tagesschau-4612 1 Signer07 0.0 1.79769e+308 TAG ZEHN GRAD BISSCHEN BEWOELKT ACHTZEHN GRAD FLUSS IX 268 | 11October_2010_Monday_tagesschau-4615 1 Signer07 0.0 1.79769e+308 FREITAG BISSCHEN VERSCHWINDEN BISSCHEN VERSCHWINDEN NORD BISSCHEN REGEN BISSCHEN STELLENWEISE VIERZEHN GRAD UNGEFAEHR 269 | 30November_2010_Tuesday_tagesschau-4617 1 Signer08 0.0 1.79769e+308 SKANDINAVIEN HOCH KOMMEN KALT KOMMEN 270 | 30November_2010_Tuesday_tagesschau-4618 1 Signer08 0.0 1.79769e+308 FRANKREICH KOMMEN TIEF KOMMEN WOLKE KOMMEN KOENNEN SCHNEE REGEN 271 | 30November_2010_Tuesday_tagesschau-4620 1 Signer08 0.0 1.79769e+308 KUESTE HEUTE NACHT SCHNEE REGION WOLKE BISSCHEN TROCKEN 272 | 30November_2010_Tuesday_tagesschau-4621 1 Signer08 0.0 1.79769e+308 NORD MORGEN KOENNEN SONNE 273 | 01February_2011_Tuesday_heute-4637 1 Signer08 0.0 1.79769e+308 SUED VERAENDERN KAUM WIE HEUTE SONNE ODER NEBEL 274 | 28September_2009_Monday_tagesschau-4679 1 Signer04 0.0 1.79769e+308 NACHT VIERZEHN NORD SECHS ALLGAEU 275 | 28September_2009_Monday_tagesschau-4680 1 Signer04 0.0 1.79769e+308 TAG VIERZEHN NORD REGEN EINS ZWANZIG FLUSS 276 | 19March_2011_Saturday_tagesschau-4690 1 Signer04 0.0 1.79769e+308 MORGEN FRUEH VERSCHWINDEN LANGSAM SCHNELL NORDOST BISSCHEN WOLKE BLEIBEN TROCKEN 277 | 19March_2011_Saturday_tagesschau-4697 1 Signer04 0.0 1.79769e+308 KUESTE BISSCHEN WOLKE ABER neg-REGEN neg-HABEN 278 | 05December_2009_Saturday_tagesschau-4701 1 Signer01 0.0 1.79769e+308 IX AUCH WIND DABEI TEMPERATUR FUENF BIS UNGEFAEHR DREIZEHN GRAD REGION 279 | 21January_2011_Friday_tagesschau-4890 1 Signer04 0.0 1.79769e+308 NORDWEST REGEN ODER SCHNEE VORSICHT GLATT KOENNEN 280 | 21November_2009_Saturday_tagesschau-4905 1 Signer04 0.0 1.79769e+308 KOMMEN LOCH ABER TROTZDEM SCHAUER 281 | 21November_2009_Saturday_tagesschau-4906 1 Signer04 0.0 1.79769e+308 SUEDOST SCHWACH WEHEN MAESSIG 282 | 21November_2009_Saturday_tagesschau-4911 1 Signer04 0.0 1.79769e+308 NAECHSTE WOCHE WECHSELHAFT WOLKE TEIL REGEN 283 | 25April_2010_Sunday_tagesschau-4942 1 Signer01 0.0 1.79769e+308 KOENNEN IX MEHR KUEHL KOMMEN AB DIENSTAG LUFT HOCH HOEHER TEMPERATUR HOEHER MEHR FREUNDLICH 284 | 25April_2010_Sunday_tagesschau-4951 1 Signer01 0.0 1.79769e+308 DONNERSTAG BIS SECHS ZWANZIG GRAD VIEL SONNE 285 | 25April_2010_Sunday_tagesschau-4952 1 Signer01 0.0 1.79769e+308 WEST MEHR SCHAUER GEWITTER 286 | 22March_2011_Tuesday_tagesschau-4956 1 Signer08 0.0 1.79769e+308 NACHT WOLKE KOMMEN BLEIBEN KLAR HIMMEL STERN SEHEN 287 | 01February_2011_Tuesday_tagesschau-4975 1 Signer08 0.0 1.79769e+308 IN-KOMMEND SCHNEE 288 | 01February_2011_Tuesday_tagesschau-4976 1 Signer08 0.0 1.79769e+308 FREITAG SAMSTAG NORD WIND REGEN SUED FREUNDLICH TROCKEN 289 | 28February_2011_Monday_tagesschau-4984 1 Signer01 0.0 1.79769e+308 WEHEN WIND SCHWACH MAESSIG BERG FRISCH WIND 290 | 01June_2010_Tuesday_tagesschau-5002 1 Signer01 0.0 1.79769e+308 MORGEN WETTER WIE-AUSSEHEN MITTWOCH ZWEITE JUNI 291 | 10March_2011_Thursday_tagesschau-5086 1 Signer01 0.0 1.79769e+308 SONNTAG WEST NORDWEST WIEDER VIEL WOLKE BISSCHEN REGEN OST KOMMEN FREUNDLICH 292 | 24December_2010_Friday_tagesschau-5128 1 Signer07 0.0 1.79769e+308 MEHR SCHNEE SCHNEIEN MOEGLICH SCHNEIEN AUCH MOEGLICH 293 | 24December_2010_Friday_tagesschau-5129 1 Signer07 0.0 1.79769e+308 WETTER WARNUNG VON DEUTSCH WETTER DIENST 294 | 31August_2010_Tuesday_tagesschau-5174 1 Signer01 0.0 1.79769e+308 WIND SCHWACH WEHEN ZEIGEN-BILDSCHIRM 295 | 15September_2010_Wednesday_tagesschau-5209 1 Signer05 0.0 1.79769e+308 SAMSTAG NORD SCHAUER REGION FREUNDLICH SUED IM-VERLAUF REGEN 296 | 15September_2010_Wednesday_tagesschau-5210 1 Signer05 0.0 1.79769e+308 SONNTAG ALPEN IX AUCH REGEN NORD SCHAUER MITTE MEHR FREUNDLICH SONNE 297 | 16May_2010_Sunday_tagesschau-5227 1 Signer08 0.0 1.79769e+308 NORD HEUTE NACHT KLAR HIMMEL STERN KOENNEN SEHEN SUEDWEST REGEN KOENNEN NEBEL BEREICH 298 | 20July_2010_Tuesday_heute-5251 1 Signer03 0.0 1.79769e+308 JETZT IM-MOMENT NACHT NOCH WARM IX MORGEN FRUEH MEHR KUEHL 299 | 27July_2010_Tuesday_heute-5261 1 Signer08 0.0 1.79769e+308 LIEB ZUSCHAUER GUT ABEND 300 | 23August_2010_Monday_heute-5319 1 Signer08 0.0 1.79769e+308 FRISCH MORGEN ANGENEHM DEUTSCH LAND 301 | 23August_2010_Monday_heute-5320 1 Signer08 0.0 1.79769e+308 ABER IM-MOMENT DANN VIEL REGEN REGION OST REGION 302 | 23August_2010_Monday_heute-5321 1 Signer08 0.0 1.79769e+308 OST LANGSAM ABER SUED WEITER REGEN GEWITTER 303 | 08October_2009_Thursday_tagesschau-5348 1 Signer01 0.0 1.79769e+308 HEUTE NACHT NORD MITTE KLAR STERN KOENNEN SEHEN 304 | 08October_2009_Thursday_tagesschau-5349 1 Signer01 0.0 1.79769e+308 REGION NEBEL 305 | 08October_2009_Thursday_tagesschau-5350 1 Signer01 0.0 1.79769e+308 SUED REGEN KOENNEN DABEI BLITZ DONNER 306 | 08October_2009_Thursday_tagesschau-5351 1 Signer01 0.0 1.79769e+308 MORGEN REGION REGEN DABEI NORD DANN AUFLOESEN DANN SONNE DABEI 307 | 07December_2011_Wednesday_tagesschau-5399 1 Signer08 0.0 1.79769e+308 NACH MITTAG NORDWEST WOLKE KOENNEN REGEN 308 | 10December_2011_Saturday_tagesschau-5420 1 Signer01 0.0 1.79769e+308 AB MONTAG IN-KOMMEND WETTER WECHSELHAFT WIND 309 | 10December_2011_Saturday_tagesschau-5421 1 Signer01 0.0 1.79769e+308 AB DIENSTAG STURM KOENNEN 310 | 10December_2011_Saturday_tagesschau-5422 1 Signer01 0.0 1.79769e+308 TEIL REGEN BERG MOEGLICH SCHNEE 311 | 10December_2011_Saturday_tagesschau-5423 1 Signer01 0.0 1.79769e+308 ABER IN-KOMMEND MILD DANN WIEDER SCHNEE GRENZE AUF 312 | 21November_2011_Monday_heute-5433 1 Signer07 0.0 1.79769e+308 SUED NEBEL REGION KOENNEN LANG NEBEL 313 | 21November_2011_Monday_heute-5434 1 Signer07 0.0 1.79769e+308 WENN SONNE HABEN DANN SCHAFFEN ELF GRAD NEBEL NUR DREI GRAD 314 | 22November_2009_Sunday_tagesschau-5490 1 Signer04 0.0 1.79769e+308 NACHT TEMPERATUR VIER BIS NEUN 315 | 23July_2011_Saturday_tagesschau-5498 1 Signer04 0.0 1.79769e+308 SPAETER REGEN UEBERALL REGION 316 | 08June_2010_Tuesday_tagesschau-5527 1 Signer03 0.0 1.79769e+308 HEUTE NACHT ELF GRAD BERG ALLGAEU BIS SIEBZEHN FLUSS 317 | 06October_2011_Thursday_heute-5532 1 Signer01 0.0 1.79769e+308 LIEB ZUSCHAUER GUT ABEND 318 | 17January_2010_Sunday_tagesschau-5554 1 Signer05 0.0 1.79769e+308 TAG HAUPTSAECHLICH WOLKE NEBEL DANN TEIL SCHNEE SCHNEIEN REGEN 319 | 17January_2010_Sunday_tagesschau-5555 1 Signer05 0.0 1.79769e+308 SUEDWEST BISSCHEN SONNE 320 | 18November_2009_Wednesday_tagesschau-5583 1 Signer05 0.0 1.79769e+308 WENN KLAR MOEGLICH NEBEL NORD 321 | 18November_2009_Wednesday_tagesschau-5584 1 Signer05 0.0 1.79769e+308 MORGEN UEBERWIEGEND WOLKE TROPFEN 322 | 18November_2009_Wednesday_tagesschau-5585 1 Signer05 0.0 1.79769e+308 SPAETER VERSCHWINDEN 323 | 18November_2009_Wednesday_tagesschau-5586 1 Signer05 0.0 1.79769e+308 WENN NEBEL neg-HABEN FREUNDLICH 324 | 18November_2009_Wednesday_tagesschau-5587 1 Signer05 0.0 1.79769e+308 OST NACHT GEFAHR WIND 325 | 12December_2010_Sunday_tagesschau-5597 1 Signer05 0.0 1.79769e+308 IX NACHT WAHRSCHEINLICH LOCKER ORT NEBEL HABEN LANG BLEIBEN 326 | 12December_2010_Sunday_tagesschau-5605 1 Signer05 0.0 1.79769e+308 TAG VOGEL LAND MINUS ACHT NORD MINUS DREI GRAD 327 | 29November_2011_Tuesday_tagesschau-5612 1 Signer07 0.0 1.79769e+308 WOLKE HEUTE NACHT NORDWEST WOLKE BISSCHEN REGEN 328 | 29November_2011_Tuesday_tagesschau-5619 1 Signer07 0.0 1.79769e+308 NACHT SECHS GRAD SEE MINUS DREI FLUSS 329 | 07December_2010_Tuesday_heute-5626 1 Signer07 0.0 1.79769e+308 LEIDER DURCHGEHEND DAUER REGEN NACHT SUED DEUTSCHLAND REGEN 330 | 30March_2010_Tuesday_tagesschau-5694 1 Signer01 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN MITTWOCH EINS DREISSIG MAERZ 331 | 26March_2011_Saturday_tagesschau-5730 1 Signer08 0.0 1.79769e+308 BERG IX DURCHGEHEND REGEN 332 | 26March_2011_Saturday_tagesschau-5731 1 Signer08 0.0 1.79769e+308 DIENSTAG UEBERALL VIEL SONNE WOLKE LOCKER TROCKEN 333 | 30November_2009_Monday_tagesschau-5749 1 Signer03 0.0 1.79769e+308 SUED VIERHUNDERT SIEBENHUNDERT METER HOEHE MOEGLICH SCHNEE ENORM 334 | 30November_2009_Monday_tagesschau-5755 1 Signer03 0.0 1.79769e+308 MORGEN TEMPERATUR NULL BIS SIEBEN GRAD FLUSS MOEGLICH ACHT GRAD 335 | 26October_2010_Tuesday_tagesschau-5789 1 Signer01 0.0 1.79769e+308 MORGEN NORD WEST REGEN REGION NEBEL WOLKE HIMMEL 336 | 26October_2010_Tuesday_tagesschau-5796 1 Signer01 0.0 1.79769e+308 KUESTE WIND HABEN 337 | 12December_2011_Monday_heute-5806 1 Signer08 0.0 1.79769e+308 UND DAZU STURM NORD KUESTE STURM 338 | 26November_2011_Saturday_tagesschau-5848 1 Signer07 0.0 1.79769e+308 NACHT SEE SCHWER STURM MORGEN ORKAN HABEN DESHALB FRISCH MAESSIG WEHEN 339 | 09August_2010_Monday_heute-5905 1 Signer05 0.0 1.79769e+308 AUCH BIS FREITAG WIEDER SINKEN KUEHL 340 | 09August_2010_Monday_heute-5906 1 Signer05 0.0 1.79769e+308 SCHOEN ABEND MACHEN GUT 341 | 16December_2009_Wednesday_tagesschau-5918 1 Signer05 0.0 1.79769e+308 FREITAG MEHR WOLKE neg-SONNE NORDOST SCHNEE SCHNEIEN 342 | 04October_2010_Monday_heute-5945 1 Signer01 0.0 1.79769e+308 LIEB ZUSCHAUER poss-EUCH GUT ABEND 343 | 04October_2010_Monday_heute-5954 1 Signer01 0.0 1.79769e+308 ZWANZIG BIS UEBER ZWANZIG GRAD 344 | 25August_2010_Wednesday_heute-6001 1 Signer03 0.0 1.79769e+308 HEUTE NACHT SCHON WEST REGEN LANGSAM KOMMEN FLUSS REGION NOCH 345 | 25August_2010_Wednesday_heute-6008 1 Signer03 0.0 1.79769e+308 WIE HERBST 346 | 25August_2010_Wednesday_heute-6009 1 Signer03 0.0 1.79769e+308 SCHOEN ABEND WUENSCHEN 347 | 07April_2010_Wednesday_tagesschau-6029 1 Signer05 0.0 1.79769e+308 REGION SUEDOST SONNE NORDWEST UEBERWIEGEND WOLKE IX REGEN SUEDOST AUCH SCHAUER 348 | 13February_2010_Saturday_tagesschau-6036 1 Signer09 0.0 1.79769e+308 WETTER WIE-AUSSEHEN MORGEN SONNTAG VIERZEHN FEBRUAR 349 | 13February_2010_Saturday_tagesschau-6044 1 Signer09 0.0 1.79769e+308 WIE-AUSSEHEN IN-KOMMEND MONTAG MAL SONNE MAL WOLKE WEST BISSCHEN SCHNEE 350 | 24September_2009_Thursday_heute-6077 1 Signer05 0.0 1.79769e+308 ZWEI TIEF 351 | 16February_2010_Tuesday_tagesschau-6098 1 Signer01 0.0 1.79769e+308 WEST TAG REGEN IX KOENNEN SCHNEE REGEN KALT FROST OST WOLKE BEWOELKT FREUNDLICH 352 | 27October_2009_Tuesday_tagesschau-6148 1 Signer05 0.0 1.79769e+308 NORDOST TIEF KOMMEN WOLKE REGEN 353 | 26July_2010_Monday_tagesschau-6256 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN DIENSTAG SIEBEN ZWANZIG JULI 354 | 16September_2010_Thursday_tagesschau-6271 1 Signer05 0.0 1.79769e+308 UNGEMUETLICH SUED BESSER ALS NORD 355 | 16September_2010_Thursday_tagesschau-6272 1 Signer05 0.0 1.79769e+308 HEUTE NACHT NORD MITTE 356 | 16September_2010_Thursday_tagesschau-6276 1 Signer05 0.0 1.79769e+308 HEUTE NACHT ZWISCHEN FUENF ZWISCHEN TAG MITTE BERG DREIZEHN IX NEUNZEHN GRAD 357 | 30November_2011_Wednesday_tagesschau-6289 1 Signer01 0.0 1.79769e+308 TAG FUENF GRAD NEBEL UND FUENFZEHN GRAD FOEHN REGION 358 | 30November_2011_Wednesday_tagesschau-6291 1 Signer01 0.0 1.79769e+308 SUEDOST poss-SEIN FREUNDLICH SONNE 359 | 30November_2011_Wednesday_tagesschau-6292 1 Signer01 0.0 1.79769e+308 SAMSTAG SONNTAG DANN WECHSELHAFT IX REGEN BESONDERS SONNTAG KOENNEN STURM 360 | 10August_2011_Wednesday_tagesschau-6303 1 Signer05 0.0 1.79769e+308 IM-VERLAUF IM-VERLAUF TEMPERATUR REGION 361 | 10August_2011_Wednesday_tagesschau-6304 1 Signer05 0.0 1.79769e+308 SUED MEHR FREUNDLICH IX NORD NICHT ORT AUCH SCHAUER GEWITTER KURZ 362 | 10August_2011_Wednesday_tagesschau-6305 1 Signer05 0.0 1.79769e+308 SONNTAG BIS DREISSIG GRAD MOEGLICH WIEDER UNWETTER 363 | 17February_2011_Thursday_tagesschau-6307 1 Signer07 0.0 1.79769e+308 WETTER SO KNAPP VERAENDERN MEISTENS BLEIBEN WOLKE TROCKEN WETTER 364 | 24January_2013_Thursday_tagesschau-6323 1 Signer07 0.0 1.79769e+308 OST SUEDOST UEBERWIEGEND WOLKE BISSCHEN SCHNEE 365 | 24January_2013_Thursday_tagesschau-6327 1 Signer07 0.0 1.79769e+308 HEUTE NACHT KLAR HIMMEL KOENNEN MINUS FUENFZEHN GRAD NORDOST WEHEN MINUS DREI GRAD 366 | 30January_2011_Sunday_tagesschau-6345 1 Signer07 0.0 1.79769e+308 JETZT WETTER VORAUS INFORMIEREN MORGEN MONTAG EINS DREISSIG JANUAR 367 | 10January_2011_Monday_tagesschau-6404 1 Signer01 0.0 1.79769e+308 HEUTE-NACHT TROCKEN IX REGEN MEISTENS REGION 368 | 10January_2011_Monday_tagesschau-6409 1 Signer01 0.0 1.79769e+308 HEUTE-NACHT IX EINS GRAD MINUS SIEBEN REGION 369 | 05April_2011_Tuesday_tagesschau-6420 1 Signer01 0.0 1.79769e+308 REGION KOMMEN SONNE SEHEN KOENNEN 370 | 06May_2011_Friday_tagesschau-6438 1 Signer01 0.0 1.79769e+308 MONTAG WEITER SONNE WEST SCHAUER GEWITTER 371 | 04July_2011_Monday_heute-6440 1 Signer07 0.0 1.79769e+308 IHR WUENSCHEN WISSEN SOMMER WEITER WIE LAUFEN WIE-AUSSEHEN DAMEN UND HERREN poss-EUCH 372 | 04July_2011_Monday_heute-6441 1 Signer07 0.0 1.79769e+308 BLEIBEN HOCH TIEF HOCH TIEF 373 | 23May_2011_Monday_heute-6459 1 Signer08 0.0 1.79769e+308 DANN REGEN VERSCHWINDEN WIND STURM WEST KOMMEN 374 | 23May_2011_Monday_heute-6461 1 Signer08 0.0 1.79769e+308 DANN ABEND KOENNEN BISSCHEN REGEN GEWITTER BERG IX 375 | 23May_2011_Monday_heute-6466 1 Signer08 0.0 1.79769e+308 VIEL REGEN neg-HABEN 376 | 23September_2010_Thursday_heute-6476 1 Signer01 0.0 1.79769e+308 WEST AUS FUENF ZWANZIG VIERZEHN BIS ACHTZEHN GRAD 377 | 29January_2010_Friday_tagesschau-6489 1 Signer05 0.0 1.79769e+308 MORGEN WECHSELHAFT ABER IX MORGEN BISSCHEN BESSER RUHIG 378 | 29January_2010_Friday_tagesschau-6490 1 Signer05 0.0 1.79769e+308 NACHT SCHNEE MEHR REGION KOMMEN DANN MORGEN SCHNEE SCHNEIEN NORDOST LANG SCHNEE SCHNEIEN 379 | 03November_2010_Wednesday_tagesschau-6505 1 Signer07 0.0 1.79769e+308 FLUSS SUED MOEGLICH SONNE BLEIBEN TROCKEN 380 | 03November_2010_Wednesday_tagesschau-6506 1 Signer07 0.0 1.79769e+308 SUED WEHEN BISSCHEN WEHEN NORD FRISCH NATUR STARK STURM SEE MEER SCHWER STURM BERG E+R+Z BERG AUCH ORKAN 381 | 03November_2010_Wednesday_tagesschau-6507 1 Signer07 0.0 1.79769e+308 KOELN REGION IX DREIZEHN GRAD ALPEN DREI GRAD 382 | 20June_2011_Monday_heute-6514 1 Signer07 0.0 1.79769e+308 DANN WETTER WECHSELHAFT 383 | 20June_2011_Monday_heute-6521 1 Signer07 0.0 1.79769e+308 MITTE ZONE neg-MOEGEN STARK WOLKE ODER REGEN 384 | 16December_2011_Friday_tagesschau-6529 1 Signer08 0.0 1.79769e+308 KOMMEN IX KALT KOMMEN IX SCHNEE GLATT KOENNEN 385 | 16December_2011_Friday_tagesschau-6532 1 Signer08 0.0 1.79769e+308 MORGEN NORDOST NOCH REGEN SCHNEE REGEN BERG IX NOCH SCHNEE DURCHGEHEND 386 | 16April_2010_Friday_tagesschau-6577 1 Signer03 0.0 1.79769e+308 HOCH KOMMEN OST KOMMEN MORGEN DEUTSCH LAND VIEL SONNE TROCKEN 387 | 12September_2009_Saturday_tagesschau-6589 1 Signer04 0.0 1.79769e+308 DESHALB NAECHSTE WOCHE KOMMEN 388 | 12September_2009_Saturday_tagesschau-6590 1 Signer04 0.0 1.79769e+308 IM-VERLAUF NACHT NORD WOLKE BISSCHEN REGEN 389 | 12September_2009_Saturday_tagesschau-6591 1 Signer04 0.0 1.79769e+308 TAG SUED FREUNDLICH UNTEN KOMMEN MEHR REGEN TROPFEN 390 | 24July_2011_Sunday_tagesschau-6621 1 Signer04 0.0 1.79769e+308 DARUNTER TAG IM-VERLAUF MEHR WARM 391 | 24July_2011_Sunday_tagesschau-6622 1 Signer04 0.0 1.79769e+308 TROTZDEM BLEIBEN SCHAUER GEWITTER NOCH 392 | 01April_2010_Thursday_heute-6697 1 Signer04 0.0 1.79769e+308 ICH OSTERN WETTER ZUFRIEDEN 393 | 01April_2010_Thursday_heute-6698 1 Signer04 0.0 1.79769e+308 MITTAG TEMPERATUR SUED WARM poss-MEIN NICHT 394 | 12November_2009_Thursday_tagesschau-6770 1 Signer05 0.0 1.79769e+308 NORD MORGEN BEWOELKT REGION REGEN 395 | 12November_2009_Thursday_tagesschau-6771 1 Signer05 0.0 1.79769e+308 SUED MEISTENS TROCKEN EINIGE WOLKE NEBEL MAL SONNE 396 | 12November_2009_Thursday_tagesschau-6772 1 Signer05 0.0 1.79769e+308 IM-VERLAUF MITTE AUCH WOLKE AUFLOESEN 397 | 02June_2010_Wednesday_heute-6796 1 Signer01 0.0 1.79769e+308 SAMSTAG AUCH DANN SONNTAG WEST BISSCHEN SCHAUER GEWITTER 398 | 13December_2009_Sunday_tagesschau-6822 1 Signer04 0.0 1.79769e+308 SUEDOST SCHNEE REGION TROCKEN MINUS ACHT BIS PLUS EINS ZWISCHEN 399 | 06December_2011_Tuesday_tagesschau-6842 1 Signer07 0.0 1.79769e+308 BERG MEISTENS SCHNEE REGEN BERG ABER SUED REGION REGEN UNGEFAEHR TAUSEND ZWEIHUNDERT IX SCHNEE 400 | 06July_2011_Wednesday_tagesschau-6870 1 Signer07 0.0 1.79769e+308 TAG EINS ZWANZIG GRAD SYLT IX UND NEUN ZWANZIG GRAD REGION UND REGION 401 | 10August_2009_Monday_heute-6891 1 Signer01 0.0 1.79769e+308 ABER JETZT OST KOMMEN SCHWUEL LUFT AUFZIEHEN 402 | 10August_2009_Monday_heute-6892 1 Signer01 0.0 1.79769e+308 NEU KOMMEN REGEN ABER MEHR ANGENEHM NASE LUFT 403 | 06February_2011_Sunday_tagesschau-6914 1 Signer04 0.0 1.79769e+308 HEUTE NACHT AB MORGEN NACHMITTAG NORD HARZ REGION STURM 404 | 06February_2011_Sunday_tagesschau-6917 1 Signer04 0.0 1.79769e+308 NORD MORGEN SIEBEN REGION FLUSS BIS FUENFZEHN 405 | 24January_2011_Monday_heute-6980 1 Signer01 0.0 1.79769e+308 MORGEN MEISTENS ZWEI BIS SECHS GRAD ABER BAYERN REGION BISSCHEN KALT MITTE REGION AUCH 406 | 24January_2011_Monday_heute-6983 1 Signer01 0.0 1.79769e+308 REGEN REGION UND IX TEILWEISE AUCH SONNE DABEI 407 | 24January_2011_Monday_heute-6984 1 Signer01 0.0 1.79769e+308 MITTWOCH SONNE NICHT NUR WEST SCHNEE REGEN MISCHUNG DANN BEISEITE 408 | 17January_2011_Monday_tagesschau-7001 1 Signer07 0.0 1.79769e+308 SUED NOCH BISSCHEN NOCH WETTER GUT WETTER 409 | 17January_2011_Monday_tagesschau-7003 1 Signer07 0.0 1.79769e+308 REST REGION MEHR MEISTENS WOLKE BESONDERS KUESTE IX MEHR REGEN 410 | 20September_2010_Monday_tagesschau-7029 1 Signer01 0.0 1.79769e+308 BESONDERS NAH FLUSS NEBEL FELD 411 | 29November_2009_Sunday_tagesschau-7043 1 Signer03 0.0 1.79769e+308 MORGEN REGEN KOMMEN SCHWARZ WALD BERG REGION SCHNEE SIEBENHUNDERT FUENFHUNDERT METER BERG SCHON SCHNEE 412 | 03March_2011_Thursday_tagesschau-7064 1 Signer05 0.0 1.79769e+308 MORGEN IX MILD ZWOELF NORD ZWEI GRAD 413 | 19January_2011_Wednesday_tagesschau-7068 1 Signer07 0.0 1.79769e+308 JETZT WETTER VORAUS INFORMIEREN MORGEN DONNERSTAG ZWANZIG JANUAR 414 | 25August_2009_Tuesday_tagesschau-7090 1 Signer01 0.0 1.79769e+308 HEUTE-NACHT STURM KOENNEN 415 | 25August_2009_Tuesday_tagesschau-7091 1 Signer01 0.0 1.79769e+308 MORGEN SCHWACH MAESSIG ZEIGEN-BILDSCHIRM WIND 416 | 25August_2009_Tuesday_tagesschau-7093 1 Signer01 0.0 1.79769e+308 MORGEN EINS ZWANZIG REGION BIS ACHT ZWANZIG 417 | 25August_2009_Tuesday_tagesschau-7094 1 Signer01 0.0 1.79769e+308 UNTEN REGION IN-KOMMEND WIE-AUSSEHEN TEIL WECHSELHAFT SONNE WOLKE UNTERSCHIED GEWITTER KOENNEN 418 | 09July_2009_Thursday_tagesschau-7110 1 Signer01 0.0 1.79769e+308 MORGEN WETTER WIE-AUSSEHEN FREITAG ZEHNTE JULI 419 | 09July_2009_Thursday_tagesschau-7122 1 Signer01 0.0 1.79769e+308 SAMSTAG SUEDWEST UEBERWIEGEND TROCKEN IX SONNE SCHAUER NORDWEST DANN GEWITTER 420 | 30March_2011_Wednesday_tagesschau-7127 1 Signer05 0.0 1.79769e+308 SAMSTAG SONNTAG HOCH SPANIEN IX VERBREITEN SONNE AUCH TEIL TATSAECHLICH SCHON SOMMER TEMPERATUR 421 | 30March_2011_Wednesday_tagesschau-7130 1 Signer05 0.0 1.79769e+308 OST REGION IX SONNE WAHRSCHEINLICH DANN ABEND IX REGEN IX 422 | 31October_2009_Saturday_tagesschau-7147 1 Signer03 0.0 1.79769e+308 MOEGLICH HEUTE NACHT FROST GLATT VORSICHT FLUSS MOEGLICH PLUS ACHT 423 | 02June_2010_Wednesday_tagesschau-7160 1 Signer01 0.0 1.79769e+308 HEUTE NACHT SECHS BIS DREIZEHN GRAD TAG TEMPERATUR STEIGEN ZEHN GRAD ALPEN BIS FUENF ZWANZIG GRAD WEST 424 | 01May_2010_Saturday_tagesschau-7197 1 Signer05 0.0 1.79769e+308 MITTWOCH SUED NORD BISSCHEN REGEN SONST REGION FREUNDLICH 425 | 04July_2010_Sunday_tagesschau-7198 1 Signer04 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN MONTAG FUENFTE JULI 426 | 04July_2010_Sunday_tagesschau-7199 1 Signer04 0.0 1.79769e+308 TIEF NORD EUROPA DIESE NORDWEST MORGEN SCHAUER SUEDOST GEWITTER 427 | 10November_2009_Tuesday_tagesschau-7250 1 Signer04 0.0 1.79769e+308 DONNERSTAG NEBEL AUFLOESEN WOLKE BEWOELKT HABEN2 428 | 24January_2013_Thursday_heute-7254 1 Signer07 0.0 1.79769e+308 ENDLICH SONNE ABER DEUTSCH KAUM 429 | 24January_2013_Thursday_heute-7265 1 Signer07 0.0 1.79769e+308 ABER MILD MEHR MEER IX ODER SUEDWEST MINUS EINS MINUS ZWEI GRAD UNGEFAEHR MEHR 430 | 26April_2010_Monday_tagesschau-7271 1 Signer01 0.0 1.79769e+308 TEMPERATUR IN-KOMMEND MEHR 431 | 26April_2010_Monday_tagesschau-7273 1 Signer01 0.0 1.79769e+308 TAG BESONDERS NORDOST ALPEN SCHAUER KOENNEN DANN SONNE WOLKE WECHSELHAFT SUEDWEST HAUPTSAECHLICH SONNE 432 | 20March_2010_Saturday_heute-7288 1 Signer01 0.0 1.79769e+308 MORGEN WEST VERBREITEN 433 | 20March_2010_Saturday_heute-7296 1 Signer01 0.0 1.79769e+308 IN-KOMMEND MEHR KUEHL WENIGER WIND STARK 434 | 20March_2010_Saturday_heute-7297 1 Signer01 0.0 1.79769e+308 BERG REGION REGEN 435 | 27June_2010_Sunday_tagesschau-7386 1 Signer03 0.0 1.79769e+308 DIENSTAG MITTWOCH TIEF KOMMEN MOEGLICH REGEN GEWITTER 436 | 04January_2010_Monday_heute-7399 1 Signer01 0.0 1.79769e+308 SCHWARZ WALD FUENF ZWANZIG C+M HARZ BIS SECHZIG C+M 437 | 04January_2010_Monday_heute-7401 1 Signer01 0.0 1.79769e+308 REGION ANDERS MILD HINREICHEND HEUTE NACHT HINREICHEND SCHNEE 438 | 29July_2010_Thursday_tagesschau-7411 1 Signer05 0.0 1.79769e+308 A HOCH KOMMEN SAMSTAG SONNTAG MEHR SONNE TEMPERATUR STEIGEN 439 | 15February_2011_Tuesday_tagesschau-7468 1 Signer08 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN MITTWOCH SECHSZEHN FEBRUAR ZEIGEN-BILDSCHIRM 440 | 10December_2009_Thursday_tagesschau-7482 1 Signer02 0.0 1.79769e+308 JETZT WIE-AUSSEHEN WETTER MORGEN FREITAG ELF DEZEMBER 441 | 28May_2010_Friday_tagesschau-7496 1 Signer08 0.0 1.79769e+308 HOCH KOMMEN OST KOMMEN ES-BEDEUTET MORGEN FREUNDLICH WETTER 442 | 13July_2009_Monday_tagesschau-7513 1 Signer05 0.0 1.79769e+308 REGION SONNE SCHAUER GEWITTER SUED IX ENORM HAGEL STARK REGEN STURM 443 | 16July_2011_Saturday_tagesschau-7524 1 Signer05 0.0 1.79769e+308 NACHT KOMMEN REGEN STARK OST SUEDOST KLAR SEHEN STERN KOENNEN 444 | 09February_2011_Wednesday_heute-7538 1 Signer07 0.0 1.79769e+308 HEUTE NACHT AUFKLAREN MORGEN NACH MITTAG SCHON NORD SEE SCHON BISSCHEN REGEN IX MILD LUFT 445 | 23September_2009_Wednesday_tagesschau-7582 1 Signer01 0.0 1.79769e+308 WOCHE DANN HOCH KOMMEN WARM 446 | 23September_2009_Wednesday_tagesschau-7583 1 Signer01 0.0 1.79769e+308 MITTE NORD HEUTE NACHT BISSCHEN REGEN IX 447 | 23September_2009_Wednesday_tagesschau-7593 1 Signer01 0.0 1.79769e+308 SONNTAG SCHEINEN ALPEN AUCH SCHAUER MOEGLICH 448 | 09April_2010_Friday_tagesschau-7623 1 Signer08 0.0 1.79769e+308 JETZT WETTER MORGEN SAMSTAG ZEHNTE APRIL WIE-AUSSEHEN ZEIGEN-BILDSCHIRM 449 | 09April_2010_Friday_tagesschau-7632 1 Signer08 0.0 1.79769e+308 NACHT NULL BIS ZEHN FLUSS BIS SIEBEN GRAD 450 | 03October_2012_Wednesday_tagesschau-7651 1 Signer05 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN DONNERSTAG VIERTE OKTOBER 451 | 03October_2012_Wednesday_tagesschau-7655 1 Signer05 0.0 1.79769e+308 SUEDOST MEISTENS TROCKEN 452 | 03October_2012_Wednesday_tagesschau-7656 1 Signer05 0.0 1.79769e+308 MORGEN HAUPTSAECHLICH WOLKE VERSCHWINDEN IX 453 | 03October_2012_Wednesday_tagesschau-7657 1 Signer05 0.0 1.79769e+308 SONST REGION HAUPTSAECHLICH REGEN 454 | 29June_2010_Tuesday_tagesschau-7704 1 Signer03 0.0 1.79769e+308 WIND NORD SCHWACH MAESSIG WEHEN SCHWACH WEHEN 455 | 29June_2010_Tuesday_tagesschau-7705 1 Signer03 0.0 1.79769e+308 HEUTE NACHT TEMPERATUR ELF BIS ACHTZEHN GRAD 456 | 15October_2009_Thursday_tagesschau-7719 1 Signer03 0.0 1.79769e+308 MORGEN OST REGION ZWEI GRAD REGION DREIZEHN GRAD 457 | 06January_2010_Wednesday_tagesschau-7726 1 Signer01 0.0 1.79769e+308 GROSSBRITANNIEN REGION FRANKREICH MITTE EUROPA REGION TIEF KOMMEN SCHNEIEN SCHNEE WENIG NUR IX 458 | 11February_2011_Friday_tagesschau-7755 1 Signer01 0.0 1.79769e+308 WOCHENENDE IM-VERLAUF MONTAG DIENSTAG NORDOST DANN KALT 459 | 25January_2013_Friday_tagesschau-7813 1 Signer06 0.0 1.79769e+308 REGION LUFT TIEF KOMMEN DANN SONNTAG SCHNEE REGEN GEFRIEREN 460 | 25January_2013_Friday_tagesschau-7815 1 Signer06 0.0 1.79769e+308 MORGEN TEIL UEBERWIEGEND WOLKE TEIL SONNE OST UEBERWIEGEND MEHR SONNE DABEI 461 | 23January_2011_Sunday_tagesschau-7834 1 Signer04 0.0 1.79769e+308 AUCH DIENSTAG WOLKE MEHR REGEN ODER SCHNEE 462 | 05February_2011_Saturday_tagesschau-7855 1 Signer04 0.0 1.79769e+308 HEUTE NACHT NORD TEIL STURM BERG ORKAN 463 | 26May_2010_Wednesday_heute-7867 1 Signer05 0.0 1.79769e+308 MORGEN REGEN NORDOST SCHNELL VERSCHWINDEN SCHON SCHAUER 464 | 20July_2010_Tuesday_tagesschau-7877 1 Signer03 0.0 1.79769e+308 HOCH OST KOMMEN MORGEN SOLL EINFLUSS DEUTSCH LAND 465 | 20July_2010_Tuesday_tagesschau-7883 1 Signer03 0.0 1.79769e+308 HEUTE NACHT RUHRGEBIET ZWANZIG GRAD MITTE BERG MOEGLICH ELF GRAD 466 | 13May_2010_Thursday_tagesschau-7916 1 Signer03 0.0 1.79769e+308 JETZT WETTER MORGEN FREITAG VIERZEHN MAI 467 | 13May_2010_Thursday_tagesschau-7926 1 Signer03 0.0 1.79769e+308 REGION MEHR SONNE 468 | 04February_2010_Thursday_heute-7966 1 Signer01 0.0 1.79769e+308 SCHNEE WARUM HEUTE NACHT KALT ZEIGEN-BILDSCHIRM LEICHT FROST MINUS EINS BIS MINUS VIER NUR WEST FROST KEIN 469 | 04February_2010_Thursday_heute-7970 1 Signer01 0.0 1.79769e+308 NORD TAG REGEN KOMMEN NOCH-NICHT SONNE FREUNDLICH SONNE DABEI UND STURM WIND 470 | 02December_2011_Friday_tagesschau-8009 1 Signer04 0.0 1.79769e+308 NACHT IX BISSCHEN REGEN HOCH SCHNEE 471 | 02December_2011_Friday_tagesschau-8010 1 Signer04 0.0 1.79769e+308 REGION TEIL KLAR NORD GEBEN SCHAUER 472 | 21July_2009_Tuesday_tagesschau-8061 1 Signer04 0.0 1.79769e+308 DIESE WARNUNG UNWETTER DEUTSCH WETTER DIENST 473 | 21July_2009_Tuesday_tagesschau-8062 1 Signer04 0.0 1.79769e+308 HAUPTSAECHLICH NORD MITTE REGEN GEWITTER 474 | 16January_2010_Saturday_tagesschau-8073 1 Signer05 0.0 1.79769e+308 WEST TEIL SCHNEE SCHNEIEN TEIL REGEN HEUTE NACHT MOEGLICH REGEN MIT EIS GLATT GEFAHR 475 | 29November_2011_Tuesday_heute-8095 1 Signer07 0.0 1.79769e+308 SCHOEN ABEND poss-EUCH 476 | 07April_2011_Thursday_tagesschau-8102 1 Signer01 0.0 1.79769e+308 SUED WEHEN IX WEHEN KOENNEN STURM KOENNEN NORD SCHWER STURM 477 | 29March_2011_Tuesday_tagesschau-8117 1 Signer01 0.0 1.79769e+308 ROSTOCK ACHT REGION LEIPZIG ACHTZEHN GRAD ZONE 478 | 27May_2011_Friday_tagesschau-8122 1 Signer08 0.0 1.79769e+308 MORGEN A HOCH KOMMEN REGION FREUNDLICH 479 | 27May_2011_Friday_tagesschau-8133 1 Signer08 0.0 1.79769e+308 DIENSTAG KOMMEN REGEN KOENNEN GEWITTER UNWETTER REGEN HAGEL KRAEFTIG WIND KOENNEN 480 | 11January_2011_Tuesday_heute-8176 1 Signer05 0.0 1.79769e+308 AB MITTAG ZWISCHEN KOMMEN WOLKE MEISTENS REGEN 481 | 22July_2010_Thursday_tagesschau-8194 1 Signer05 0.0 1.79769e+308 JETZT WETTER MORGEN FREITAG DREI ZWANZIG JULI 482 | 22July_2010_Thursday_tagesschau-8195 1 Signer05 0.0 1.79769e+308 SKANDINAVIEN NORD WOLKE TIEF AUCH DEUTSCHLAND REGION KOMMEN 483 | 09June_2010_Wednesday_heute-8266 1 Signer08 0.0 1.79769e+308 FREITAG BLEIBEN WARM NASS SAMSTAG DANN SUEDWEST REGEN UND KUEHL 484 | 12December_2011_Monday_tagesschau-8281 1 Signer08 0.0 1.79769e+308 MORGEN DREI GRAD REGION BAYERN WALD IX ZWOELF KOELN 485 | 12December_2011_Monday_tagesschau-8282 1 Signer08 0.0 1.79769e+308 MITTWOCH WIND KOENNEN STURM REGEN 486 | 25January_2010_Monday_tagesschau-8298 1 Signer03 0.0 1.79769e+308 UND JETZT WETTER MORGEN DIENSTAG SECHS ZWANZIG JANUAR 487 | 25January_2010_Monday_tagesschau-8305 1 Signer03 0.0 1.79769e+308 WIND NORD WEHEN WENIG UNTEN STARK WEHEN 488 | 29March_2010_Monday_tagesschau-8384 1 Signer01 0.0 1.79769e+308 JETZT MORGEN WETTER WIE-AUSSEHEN DIENSTAG DREISSIG MAERZ 489 | 03September_2010_Friday_tagesschau-8423 1 Signer05 0.0 1.79769e+308 KUEHL KOMMEN 490 | 03September_2010_Friday_tagesschau-8424 1 Signer05 0.0 1.79769e+308 TEMPERATUR SAMSTAG SONNTAG IX GLEICH 491 | 03September_2010_Friday_tagesschau-8432 1 Signer05 0.0 1.79769e+308 TAG NORDOST SECHSZEHN REGION IX DREI ZWANZIG GRAD 492 | 02December_2010_Thursday_heute-8444 1 Signer07 0.0 1.79769e+308 GUT ABEND LIEB ZUSCHAUER 493 | 02December_2010_Thursday_heute-8445 1 Signer07 0.0 1.79769e+308 SO VIEL SCHNEE ANFANG DEZEMBER TJA SELTEN BISHER WENN UEBER SICHT NULL FAST REGION DEUTSCHLAND REGION SCHNEE HABEN2 494 | 17May_2010_Monday_tagesschau-8473 1 Signer08 0.0 1.79769e+308 NORD AUCH REGEN DABEI SUEDWEST BLEIBEN TROCKEN 495 | 17May_2010_Monday_tagesschau-8477 1 Signer08 0.0 1.79769e+308 WIE-AUSSEHEN IN-KOMMEND MITTWOCH BLEIBEN WOLKE IX KOENNEN REGEN WEST KOENNEN SONNE 496 | 13August_2009_Thursday_tagesschau-8487 1 Signer05 0.0 1.79769e+308 WIND SCHWACH MAESSIG IX WEHEN 497 | 20October_2010_Wednesday_heute-8493 1 Signer05 0.0 1.79769e+308 HALLO GUT ABEND 498 | 20October_2010_Wednesday_heute-8494 1 Signer05 0.0 1.79769e+308 WINTER SCHON HABEN MITTE BERG SCHON WEISS BODEN 499 | 20October_2010_Wednesday_heute-8501 1 Signer05 0.0 1.79769e+308 IX SUED DANN RUHIG MEHR TROCKEN AUCH BISSCHEN SONNE 500 | 28July_2011_Thursday_tagesschau-8521 1 Signer01 0.0 1.79769e+308 WEST SUEDWEST TIEF KOMMEN SCHAUER GEWITTER KOENNEN DEUTSCH WETTER DIENST WARNUNG 501 | 29September_2009_Tuesday_tagesschau-8538 1 Signer04 0.0 1.79769e+308 SUED SCHWACH NORD MAESSIG KUESTE STARK STURM 502 | 09May_2011_Monday_tagesschau-8583 1 Signer01 0.0 1.79769e+308 MORGEN ZWANZIG GRAD NORD BIS ACHT ZWANZIG REGION 503 | 09May_2011_Monday_tagesschau-8585 1 Signer01 0.0 1.79769e+308 DONNERSTAG SONNE TEILWEISE REGEN STARK GEWITTER 504 | 09May_2011_Monday_tagesschau-8586 1 Signer01 0.0 1.79769e+308 FREITAG RUHIG MEHR KUEHL 505 | 30July_2011_Saturday_tagesschau-8591 1 Signer01 0.0 1.79769e+308 REGEN REGION VIEL HIMMEL REGION SUEDWEST AUCH LOCH neg-WOLKE negalp-KEIN 506 | 13November_2009_Friday_tagesschau-8610 1 Signer09 0.0 1.79769e+308 HEUTE NACHT DREIZEHN BIS DREI GRAD ALPEN WAHRSCHEINLICH ZWISCHEN NULL 507 | 29April_2010_Thursday_heute-8622 1 Signer03 0.0 1.79769e+308 MORGEN NACH MITTAG GEWITTER REGEN AUCH 508 | 29April_2010_Thursday_heute-8623 1 Signer03 0.0 1.79769e+308 MORGEN TEMPERATUR ACHT BIS SECHSZEHN GRAD 509 | 29April_2010_Thursday_heute-8625 1 Signer03 0.0 1.79769e+308 OST REGION ZUERST WARM VIER AUCH SUED BAYERN WARM 510 | 11April_2010_Sunday_tagesschau-8639 1 Signer08 0.0 1.79769e+308 DANN STURM WEHEN OST FRISCH WIND BERG KOENNEN STURM 511 | 11December_2010_Saturday_tagesschau-8657 1 Signer08 0.0 1.79769e+308 JETZT WETTER MORGEN SONNTAG ZWOELF DEZEMBER WIE-AUSSEHEN ZEIGEN-BILDSCHIRM 512 | 11December_2010_Saturday_tagesschau-8659 1 Signer08 0.0 1.79769e+308 MORGEN SONNE IX KOMMEN 513 | 24January_2010_Sunday_tagesschau-8710 1 Signer04 0.0 1.79769e+308 SUED SCHWACH WEHEN IX MAESSIG WEHEN 514 | 15July_2011_Friday_tagesschau-8726 1 Signer08 0.0 1.79769e+308 IX MORGEN NEUNZEHN GRAD REGION SIEBEN ZWANZIG GRAD 515 | 06October_2012_Saturday_tagesschau-8729 1 Signer08 0.0 1.79769e+308 JETZT WETTER WIE-AUSSEHEN MORGEN SONNTAG SIEBTE OKTOBER 516 | 06October_2012_Saturday_tagesschau-8730 1 Signer08 0.0 1.79769e+308 MORGEN DEUTSCH LAND IX TIEF KOMMEN KUEHL KOMMEN 517 | 06October_2012_Saturday_tagesschau-8744 1 Signer08 0.0 1.79769e+308 MITTWOCH TEIL SONNE TEIL WOLKE BLEIBEN TROCKEN 518 | 23March_2011_Wednesday_tagesschau-8751 1 Signer05 0.0 1.79769e+308 NORDOST WIEDER WOLKE ABER SONST NUR BISSCHEN WOLKE 519 | 27January_2013_Sunday_tagesschau-8836 1 Signer01 0.0 1.79769e+308 HEUTE NACHT PLUS DREI REGION NULL ZWISCHEN LEICHT FROST MINUS EINS MINUS ZWEI 520 | -------------------------------------------------------------------------------- /preprocess/phoenix2014-T/test_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014-T/test_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014-T/train_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014-T/train_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014/dev_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014/dev_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014/gloss_dict.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014/gloss_dict.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014/test_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014/test_info.npy -------------------------------------------------------------------------------- /preprocess/phoenix2014/train_info.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gswycf/SignGraph/417b9124d27228588c2e5252cfcc591b03db9f19/preprocess/phoenix2014/train_info.npy -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.4.3 2 | numpy==1.20.3 3 | opencv_python==4.5.5.64 4 | pandas==1.3.4 5 | Pillow==9.4.0 6 | PyYAML==6.0 7 | scipy==1.7.1 8 | six==1.16.0 9 | tqdm==4.62.3 10 | ctcdecode 11 | -------------------------------------------------------------------------------- /seq_scripts.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pdb 3 | import sys 4 | import copy 5 | import torch 6 | import numpy as np 7 | import torch.nn as nn 8 | from tqdm import tqdm 9 | import torch.nn.functional as F 10 | import matplotlib.pyplot as plt 11 | from torch.cuda.amp import autocast as autocast 12 | from torch.cuda.amp import GradScaler 13 | from einops import rearrange 14 | from collections import defaultdict 15 | from utils.metrics import wer_list 16 | from utils.misc import * 17 | 18 | 19 | 20 | def seq_train(loader, model, optimizer, device, epoch_idx, recoder): 21 | model.train() 22 | optimizer.scheduler.step(epoch_idx) 23 | loss_value = [] 24 | clr = [group['lr'] for group in optimizer.optimizer.param_groups] 25 | scaler = GradScaler() 26 | for batch_idx, data in enumerate(tqdm(loader)): 27 | vid = device.data_to_device(data[0]) 28 | vid_lgt = device.data_to_device(data[1]) 29 | label = device.data_to_device(data[2]) 30 | label_lgt = device.data_to_device(data[3]) 31 | optimizer.zero_grad() 32 | with autocast(): 33 | 34 | ret_dict = model(vid, vid_lgt, label=label, label_lgt=label_lgt) 35 | if len(device.gpu_list)>1: 36 | loss = model.module.criterion_calculation(ret_dict, label, label_lgt) 37 | else: 38 | loss = model.criterion_calculation(ret_dict, label, label_lgt) 39 | 40 | if np.isinf(loss.item()) or np.isnan(loss.item()): 41 | print('loss is nan') 42 | print(str(data[1])+' frames', str(data[3])+' glosses') 43 | continue 44 | scaler.scale(loss).backward() 45 | scaler.step(optimizer.optimizer) 46 | scaler.update() 47 | if len(device.gpu_list)>1: 48 | torch.cuda.synchronize() 49 | torch.distributed.reduce(loss, dst=0) 50 | 51 | loss_value.append(loss.item()) 52 | if batch_idx % recoder.log_interval == 0 and is_main_process(): 53 | recoder.print_log( 54 | '\tEpoch: {}, Batch({}/{}) done. Loss: {:.8f} lr:{:.6f}' 55 | .format(epoch_idx, batch_idx, len(loader), loss.item(), clr[0])) 56 | del ret_dict 57 | del loss 58 | optimizer.scheduler.step() 59 | if is_main_process(): 60 | recoder.print_log('\tMean training loss: {:.10f}.'.format(np.mean(loss_value))) 61 | return 62 | 63 | 64 | def seq_eval(cfg, loader, model, device, mode, epoch, work_dir, recoder, evaluate_tool="python"): 65 | model.eval() 66 | results=defaultdict(dict) 67 | 68 | for batch_idx, data in enumerate(tqdm(loader)): 69 | recoder.record_timer("device") 70 | vid = device.data_to_device(data[0]) 71 | vid_lgt = device.data_to_device(data[1]) 72 | label = device.data_to_device(data[2]) 73 | label_lgt = device.data_to_device(data[3]) 74 | info = [d['fileid'] for d in data[-1]] 75 | gloss = [d['label'] for d in data[-1]] 76 | with torch.no_grad(): 77 | ret_dict = model(vid, vid_lgt, label=label, label_lgt=label_lgt) 78 | for inf, conv_sents, recognized_sents, gl in zip(info, ret_dict['conv_sents'], ret_dict['recognized_sents'], gloss): 79 | results[inf]['conv_sents'] = conv_sents 80 | results[inf]['recognized_sents'] = recognized_sents 81 | results[inf]['gloss'] = gl 82 | gls_hyp = [' '.join(results[n]['conv_sents']) for n in results] 83 | gls_ref = [results[n]['gloss'] for n in results] 84 | wer_results_con = wer_list(hypotheses=gls_hyp, references=gls_ref) 85 | gls_hyp = [' '.join(results[n]['recognized_sents']) for n in results] 86 | wer_results = wer_list(hypotheses=gls_hyp, references=gls_ref) 87 | if wer_results['wer'] < wer_results_con['wer']: 88 | reg_per = wer_results 89 | else: 90 | reg_per = wer_results_con 91 | recoder.print_log('\tEpoch: {} {} done. Conv wer: {:.4f} ins:{:.4f}, del:{:.4f}'.format( 92 | epoch, mode, wer_results_con['wer'], wer_results_con['ins'], wer_results_con['del']), 93 | f"{work_dir}/{mode}.txt") 94 | recoder.print_log('\tEpoch: {} {} done. LSTM wer: {:.4f} ins:{:.4f}, del:{:.4f}'.format( 95 | epoch, mode, wer_results['wer'], wer_results['ins'], wer_results['del']), f"{work_dir}/{mode}.txt") 96 | 97 | return {"wer":reg_per['wer'], "ins":reg_per['ins'], 'del':reg_per['del']} 98 | 99 | -------------------------------------------------------------------------------- /slr_network.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import copy 3 | import utils 4 | import torch 5 | import types 6 | import numpy as np 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import torchvision.models as models 10 | from modules.criterions import SeqKD 11 | from modules import BiLSTMLayer, TemporalConv 12 | import modules.resnet as resnet 13 | 14 | class Identity(nn.Module): 15 | def __init__(self): 16 | super(Identity, self).__init__() 17 | 18 | def forward(self, x): 19 | return x 20 | 21 | 22 | class NormLinear(nn.Module): 23 | def __init__(self, in_dim, out_dim): 24 | super(NormLinear, self).__init__() 25 | self.weight = nn.Parameter(torch.Tensor(in_dim, out_dim)) 26 | nn.init.xavier_uniform_(self.weight, gain=nn.init.calculate_gain('relu')) 27 | 28 | def forward(self, x): 29 | outputs = torch.matmul(x, F.normalize(self.weight, dim=0)) 30 | return outputs 31 | 32 | 33 | class SLRModel(nn.Module): 34 | def __init__( 35 | self, num_classes, c2d_type, conv_type, use_bn=False, 36 | hidden_size=1024, gloss_dict=None, loss_weights=None, 37 | weight_norm=True, share_classifier=True 38 | ): 39 | super(SLRModel, self).__init__() 40 | self.decoder = None 41 | self.loss = dict() 42 | self.criterion_init() 43 | self.num_classes = num_classes 44 | self.loss_weights = loss_weights 45 | self.conv2d = getattr(resnet, c2d_type)() 46 | self.conv2d.fc = Identity() 47 | 48 | self.conv1d = TemporalConv(input_size=512, 49 | hidden_size=hidden_size, 50 | conv_type=conv_type, 51 | use_bn=use_bn, 52 | num_classes=num_classes) 53 | self.decoder = utils.Decode(gloss_dict, num_classes, 'beam') 54 | self.temporal_model = BiLSTMLayer(rnn_type='LSTM', input_size=hidden_size, hidden_size=hidden_size, 55 | num_layers=2, bidirectional=True) 56 | if weight_norm: 57 | self.classifier = NormLinear(hidden_size, self.num_classes) 58 | self.conv1d.fc = NormLinear(hidden_size, self.num_classes) 59 | else: 60 | self.classifier = nn.Linear(hidden_size, self.num_classes) 61 | self.conv1d.fc = nn.Linear(hidden_size, self.num_classes) 62 | if share_classifier: 63 | self.conv1d.fc = self.classifier 64 | 65 | def backward_hook(self, module, grad_input, grad_output): 66 | for g in grad_input: 67 | g[g != g] = 0 68 | 69 | def masked_bn(self, inputs, len_x): 70 | def pad(tensor, length): 71 | return torch.cat([tensor, tensor.new(length - tensor.size(0), *tensor.size()[1:]).zero_()]) 72 | 73 | x = torch.cat([inputs[len_x[0] * idx:len_x[0] * idx + lgt] for idx, lgt in enumerate(len_x)]) 74 | x = self.conv2d(x) 75 | x = torch.cat([pad(x[sum(len_x[:idx]):sum(len_x[:idx + 1])], len_x[0]) 76 | for idx, lgt in enumerate(len_x)]) 77 | return x 78 | 79 | def forward(self, x, len_x, label=None, label_lgt=None): 80 | 81 | if len(x.shape) == 5: 82 | # videos 83 | batch, temp, channel, height, width = x.shape 84 | framewise = self.conv2d(x.permute(0,2,1,3,4)).view(batch, temp, -1).permute(0,2,1) # btc -> bct 85 | else: 86 | framewise = x 87 | conv1d_outputs = self.conv1d(framewise, len_x) 88 | # x: T, B, C 89 | x = conv1d_outputs['visual_feat'] 90 | lgt = conv1d_outputs['feat_len'].cpu() 91 | tm_outputs = self.temporal_model(x, lgt) 92 | outputs = self.classifier(tm_outputs['predictions']) 93 | pred = None if self.training \ 94 | else self.decoder.decode(outputs, lgt, batch_first=False, probs=False) 95 | conv_pred = None if self.training \ 96 | else self.decoder.decode(conv1d_outputs['conv_logits'], lgt, batch_first=False, probs=False) 97 | return { 98 | "framewise_features": framewise, 99 | "visual_features": x, 100 | "temproal_features": tm_outputs['predictions'], 101 | "feat_len": lgt, 102 | "conv_logits": conv1d_outputs['conv_logits'], 103 | "sequence_logits": outputs, 104 | "conv_sents": conv_pred, 105 | "recognized_sents": pred, 106 | } 107 | 108 | def criterion_calculation(self, ret_dict, label, label_lgt): 109 | loss = 0 110 | for k, weight in self.loss_weights.items(): 111 | if k == 'ConvCTC': 112 | loss += weight * self.loss['CTCLoss'](ret_dict["conv_logits"].log_softmax(-1), 113 | label.cpu().int(), ret_dict["feat_len"].cpu().int(), 114 | label_lgt.cpu().int()).mean() 115 | elif k == 'SeqCTC': 116 | loss += weight * self.loss['CTCLoss'](ret_dict["sequence_logits"].log_softmax(-1), 117 | label.cpu().int(), ret_dict["feat_len"].cpu().int(), 118 | label_lgt.cpu().int()).mean() 119 | elif k == 'Dist': 120 | loss += weight * self.loss['distillation'](ret_dict["conv_logits"], 121 | ret_dict["sequence_logits"].detach(), 122 | use_blank=False) 123 | return loss 124 | 125 | def criterion_init(self): 126 | self.loss['CTCLoss'] = torch.nn.CTCLoss(reduction='none', zero_infinity=False) 127 | self.loss['distillation'] = SeqKD(T=8) 128 | return self.loss 129 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | from dataset.dataloader_videoall import BaseFeeder 2 | import torch 3 | def test(): 4 | feeder = BaseFeeder(None, gloss_dict=None, kernel_size=['K5', "P2", 'K5', "P2"]) 5 | print(feeder) 6 | dataloader = torch.utils.data.DataLoader( 7 | 8 | dataset=feeder, 9 | batch_size=4, 10 | shuffle=True, 11 | drop_last=True, 12 | num_workers=0, 13 | collate_fn=feeder.collate_fn, 14 | ) 15 | for data in dataloader: 16 | pdb.set_trace() 17 | 18 | if __name__ == '__main__': 19 | test() -------------------------------------------------------------------------------- /utils/Rouge.py: -------------------------------------------------------------------------------- 1 | """ROUGE metric implementation. 2 | Copy from tf_seq2seq/seq2seq/metrics/rouge.py. 3 | This is a modified and slightly extended verison of 4 | https://github.com/miso-belica/sumy/blob/dev/sumy/evaluation/rouge.py. 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import division 9 | from __future__ import print_function 10 | from __future__ import unicode_literals 11 | 12 | import itertools 13 | import numpy as np 14 | 15 | 16 | # pylint: disable=C0103 17 | 18 | 19 | def _get_ngrams(n, text): 20 | """Calculates n-grams. 21 | Args: 22 | n: which n-grams to calculate 23 | text: An array of tokens 24 | Returns: 25 | A set of n-grams 26 | """ 27 | ngram_set = set() 28 | text_length = len(text) 29 | max_index_ngram_start = text_length - n 30 | for i in range(max_index_ngram_start + 1): 31 | ngram_set.add(tuple(text[i:i + n])) 32 | return ngram_set 33 | 34 | 35 | def _split_into_words(sentences): 36 | """Splits multiple sentences into words and flattens the result""" 37 | return list(itertools.chain(*[_.split(" ") for _ in sentences])) 38 | 39 | 40 | def _get_word_ngrams(n, sentences): 41 | """Calculates word n-grams for multiple sentences. 42 | """ 43 | assert len(sentences) > 0 44 | assert n > 0 45 | 46 | words = _split_into_words(sentences) 47 | return _get_ngrams(n, words) 48 | 49 | 50 | def _len_lcs(x, y): 51 | """ 52 | Returns the length of the Longest Common Subsequence between sequences x 53 | and y. 54 | Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence 55 | Args: 56 | x: sequence of words 57 | y: sequence of words 58 | Returns 59 | integer: Length of LCS between x and y 60 | """ 61 | table = _lcs(x, y) 62 | n, m = len(x), len(y) 63 | return table[n, m] 64 | 65 | 66 | def _lcs(x, y): 67 | """ 68 | Computes the length of the longest common subsequence (lcs) between two 69 | strings. The implementation below uses a DP programming algorithm and runs 70 | in O(nm) time where n = len(x) and m = len(y). 71 | Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence 72 | Args: 73 | x: collection of words 74 | y: collection of words 75 | Returns: 76 | Table of dictionary of coord and len lcs 77 | """ 78 | n, m = len(x), len(y) 79 | table = dict() 80 | for i in range(n + 1): 81 | for j in range(m + 1): 82 | if i == 0 or j == 0: 83 | table[i, j] = 0 84 | elif x[i - 1] == y[j - 1]: 85 | table[i, j] = table[i - 1, j - 1] + 1 86 | else: 87 | table[i, j] = max(table[i - 1, j], table[i, j - 1]) 88 | return table 89 | 90 | 91 | def _recon_lcs(x, y): 92 | """ 93 | Returns the Longest Subsequence between x and y. 94 | Source: http://www.algorithmist.com/index.php/Longest_Common_Subsequence 95 | Args: 96 | x: sequence of words 97 | y: sequence of words 98 | Returns: 99 | sequence: LCS of x and y 100 | """ 101 | i, j = len(x), len(y) 102 | table = _lcs(x, y) 103 | 104 | def _recon(i, j): 105 | """private recon calculation""" 106 | if i == 0 or j == 0: 107 | return [] 108 | elif x[i - 1] == y[j - 1]: 109 | return _recon(i - 1, j - 1) + [(x[i - 1], i)] 110 | elif table[i - 1, j] > table[i, j - 1]: 111 | return _recon(i - 1, j) 112 | else: 113 | return _recon(i, j - 1) 114 | 115 | recon_tuple = tuple(map(lambda x: x[0], _recon(i, j))) 116 | return recon_tuple 117 | 118 | 119 | def rouge_n(evaluated_sentences, reference_sentences, n=2): 120 | """ 121 | Computes ROUGE-N of two text collections of sentences. 122 | Source: http://research.microsoft.com/en-us/um/people/cyl/download/ 123 | papers/rouge-working-note-v1.3.1.pdf 124 | Args: 125 | evaluated_sentences: The sentences that have been picked by the summarizer 126 | reference_sentences: The sentences from the referene set 127 | n: Size of ngram. Defaults to 2. 128 | Returns: 129 | A tuple (f1, precision, recall) for ROUGE-N 130 | Raises: 131 | ValueError: raises exception if a param has len <= 0 132 | """ 133 | if len(evaluated_sentences) <= 0 or len(reference_sentences) <= 0: 134 | raise ValueError("Collections must contain at least 1 sentence.") 135 | 136 | evaluated_ngrams = _get_word_ngrams(n, evaluated_sentences) 137 | reference_ngrams = _get_word_ngrams(n, reference_sentences) 138 | reference_count = len(reference_ngrams) 139 | evaluated_count = len(evaluated_ngrams) 140 | 141 | # Gets the overlapping ngrams between evaluated and reference 142 | overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams) 143 | overlapping_count = len(overlapping_ngrams) 144 | 145 | # Handle edge case. This isn't mathematically correct, but it's good enough 146 | if evaluated_count == 0: 147 | precision = 0.0 148 | else: 149 | precision = overlapping_count / evaluated_count 150 | 151 | if reference_count == 0: 152 | recall = 0.0 153 | else: 154 | recall = overlapping_count / reference_count 155 | 156 | f1_score = 2.0 * ((precision * recall) / (precision + recall + 1e-8)) 157 | 158 | # return overlapping_count / reference_count 159 | return f1_score, precision, recall 160 | 161 | 162 | def _f_p_r_lcs(llcs, m, n): 163 | """ 164 | Computes the LCS-based F-measure score 165 | Source: http://research.microsoft.com/en-us/um/people/cyl/download/papers/ 166 | rouge-working-note-v1.3.1.pdf 167 | Args: 168 | llcs: Length of LCS 169 | m: number of words in reference summary 170 | n: number of words in candidate summary 171 | Returns: 172 | Float. LCS-based F-measure score 173 | """ 174 | r_lcs = llcs / m 175 | p_lcs = llcs / n 176 | beta = p_lcs / (r_lcs + 1e-12) 177 | num = (1 + (beta ** 2)) * r_lcs * p_lcs 178 | denom = r_lcs + ((beta ** 2) * p_lcs) 179 | f_lcs = num / (denom + 1e-12) 180 | return f_lcs, p_lcs, r_lcs 181 | 182 | 183 | def rouge_l_sentence_level(evaluated_sentences, reference_sentences): 184 | """ 185 | Computes ROUGE-L (sentence level) of two text collections of sentences. 186 | http://research.microsoft.com/en-us/um/people/cyl/download/papers/ 187 | rouge-working-note-v1.3.1.pdf 188 | Calculated according to: 189 | R_lcs = LCS(X,Y)/m 190 | P_lcs = LCS(X,Y)/n 191 | F_lcs = ((1 + beta^2)*R_lcs*P_lcs) / (R_lcs + (beta^2) * P_lcs) 192 | where: 193 | X = reference summary 194 | Y = Candidate summary 195 | m = length of reference summary 196 | n = length of candidate summary 197 | Args: 198 | evaluated_sentences: The sentences that have been picked by the summarizer 199 | reference_sentences: The sentences from the referene set 200 | Returns: 201 | A float: F_lcs 202 | Raises: 203 | ValueError: raises exception if a param has len <= 0 204 | """ 205 | if len(evaluated_sentences) <= 0 or len(reference_sentences) <= 0: 206 | raise ValueError("Collections must contain at least 1 sentence.") 207 | reference_words = _split_into_words(reference_sentences) 208 | evaluated_words = _split_into_words(evaluated_sentences) 209 | m = len(reference_words) 210 | n = len(evaluated_words) 211 | lcs = _len_lcs(evaluated_words, reference_words) 212 | return _f_p_r_lcs(lcs, m, n) 213 | 214 | 215 | def _union_lcs(evaluated_sentences, reference_sentence): 216 | """ 217 | Returns LCS_u(r_i, C) which is the LCS score of the union longest common 218 | subsequence between reference sentence ri and candidate summary C. For example 219 | if r_i= w1 w2 w3 w4 w5, and C contains two sentences: c1 = w1 w2 w6 w7 w8 and 220 | c2 = w1 w3 w8 w9 w5, then the longest common subsequence of r_i and c1 is 221 | "w1 w2" and the longest common subsequence of r_i and c2 is "w1 w3 w5". The 222 | union longest common subsequence of r_i, c1, and c2 is "w1 w2 w3 w5" and 223 | LCS_u(r_i, C) = 4/5. 224 | Args: 225 | evaluated_sentences: The sentences that have been picked by the summarizer 226 | reference_sentence: One of the sentences in the reference summaries 227 | Returns: 228 | float: LCS_u(r_i, C) 229 | ValueError: 230 | Raises exception if a param has len <= 0 231 | """ 232 | if len(evaluated_sentences) <= 0: 233 | raise ValueError("Collections must contain at least 1 sentence.") 234 | 235 | lcs_union = set() 236 | reference_words = _split_into_words([reference_sentence]) 237 | combined_lcs_length = 0 238 | for eval_s in evaluated_sentences: 239 | evaluated_words = _split_into_words([eval_s]) 240 | lcs = set(_recon_lcs(reference_words, evaluated_words)) 241 | combined_lcs_length += len(lcs) 242 | lcs_union = lcs_union.union(lcs) 243 | 244 | union_lcs_count = len(lcs_union) 245 | union_lcs_value = union_lcs_count / combined_lcs_length 246 | return union_lcs_value 247 | 248 | 249 | def rouge_l_summary_level(evaluated_sentences, reference_sentences): 250 | """ 251 | Computes ROUGE-L (summary level) of two text collections of sentences. 252 | http://research.microsoft.com/en-us/um/people/cyl/download/papers/ 253 | rouge-working-note-v1.3.1.pdf 254 | Calculated according to: 255 | R_lcs = SUM(1, u)[LCS(r_i,C)]/m 256 | P_lcs = SUM(1, u)[LCS(r_i,C)]/n 257 | F_lcs = ((1 + beta^2)*R_lcs*P_lcs) / (R_lcs + (beta^2) * P_lcs) 258 | where: 259 | SUM(i,u) = SUM from i through u 260 | u = number of sentences in reference summary 261 | C = Candidate summary made up of v sentences 262 | m = number of words in reference summary 263 | n = number of words in candidate summary 264 | Args: 265 | evaluated_sentences: The sentences that have been picked by the summarizer 266 | reference_sentence: One of the sentences in the reference summaries 267 | Returns: 268 | A float: F_lcs 269 | Raises: 270 | ValueError: raises exception if a param has len <= 0 271 | """ 272 | if len(evaluated_sentences) <= 0 or len(reference_sentences) <= 0: 273 | raise ValueError("Collections must contain at least 1 sentence.") 274 | 275 | # total number of words in reference sentences 276 | m = len(_split_into_words(reference_sentences)) 277 | 278 | # total number of words in evaluated sentences 279 | n = len(_split_into_words(evaluated_sentences)) 280 | 281 | union_lcs_sum_across_all_references = 0 282 | for ref_s in reference_sentences: 283 | union_lcs_sum_across_all_references += _union_lcs(evaluated_sentences, 284 | ref_s) 285 | return _f_p_r_lcs(union_lcs_sum_across_all_references, m, n) 286 | 287 | 288 | def rouge(hypotheses, references): 289 | """Calculates average rouge scores for a list of hypotheses and 290 | references""" 291 | 292 | # Filter out hyps that are of 0 length 293 | # hyps_and_refs = zip(hypotheses, references) 294 | # hyps_and_refs = [_ for _ in hyps_and_refs if len(_[0]) > 0] 295 | # hypotheses, references = zip(*hyps_and_refs) 296 | 297 | # Calculate ROUGE-1 F1, precision, recall scores 298 | rouge_1 = [ 299 | rouge_n([hyp], [ref], 1) for hyp, ref in zip(hypotheses, references) 300 | ] 301 | rouge_1_f, rouge_1_p, rouge_1_r = map(np.mean, zip(*rouge_1)) 302 | 303 | # Calculate ROUGE-2 F1, precision, recall scores 304 | rouge_2 = [ 305 | rouge_n([hyp], [ref], 2) for hyp, ref in zip(hypotheses, references) 306 | ] 307 | rouge_2_f, rouge_2_p, rouge_2_r = map(np.mean, zip(*rouge_2)) 308 | 309 | # Calculate ROUGE-L F1, precision, recall scores 310 | rouge_l = [ 311 | rouge_l_sentence_level([hyp], [ref]) 312 | for hyp, ref in zip(hypotheses, references) 313 | ] 314 | rouge_l_f, rouge_l_p, rouge_l_r = map(np.mean, zip(*rouge_l)) 315 | 316 | return { 317 | "rouge_1/f_score": rouge_1_f, 318 | "rouge_1/r_score": rouge_1_r, 319 | "rouge_1/p_score": rouge_1_p, 320 | "rouge_2/f_score": rouge_2_f, 321 | "rouge_2/r_score": rouge_2_r, 322 | "rouge_2/p_score": rouge_2_p, 323 | "rouge_l/f_score": rouge_l_f, 324 | "rouge_l/r_score": rouge_l_r, 325 | "rouge_l/p_score": rouge_l_p, 326 | } -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .device import GpuDataParallel 2 | from .decode import Decode 3 | from .optimizer import Optimizer 4 | from .pack_code import pack_code 5 | from .parameters import get_parser 6 | from .random_state import RandomState 7 | from .record import Recorder 8 | from .misc import * 9 | from .Rouge import * 10 | from .sacrebleu import * 11 | from .video_augmentation import * 12 | from .metrics import * -------------------------------------------------------------------------------- /utils/decode.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pdb 3 | import time 4 | import torch 5 | import ctcdecode 6 | import numpy as np 7 | from itertools import groupby 8 | import torch.nn.functional as F 9 | 10 | 11 | class Decode(object): 12 | def __init__(self, gloss_dict, num_classes, search_mode, blank_id=0, beam_width=10): 13 | self.i2g_dict = dict((v[0], k) for k, v in gloss_dict.items()) 14 | self.g2i_dict = {v: k for k, v in self.i2g_dict.items()} 15 | self.num_classes = num_classes 16 | self.search_mode = search_mode 17 | self.blank_id = blank_id 18 | vocab = [chr(x) for x in range(20000, 20000 + num_classes)] 19 | self.ctc_decoder = ctcdecode.CTCBeamDecoder(vocab, beam_width=beam_width, blank_id=blank_id, 20 | num_processes=10) 21 | 22 | def decode(self, nn_output, vid_lgt, batch_first=True, probs=False): 23 | if not batch_first: 24 | nn_output = nn_output.permute(1, 0, 2) 25 | if self.search_mode == "max": 26 | return self.MaxDecode(nn_output, vid_lgt) 27 | else: 28 | return self.BeamSearch(nn_output, vid_lgt, probs) 29 | 30 | def BeamSearch(self, nn_output, vid_lgt, probs=False): 31 | ''' 32 | CTCBeamDecoder Shape: 33 | - Input: nn_output (B, T, N), which should be passed through a softmax layer 34 | - Output: beam_resuls (B, N_beams, T), int, need to be decoded by i2g_dict 35 | beam_scores (B, N_beams), p=1/np.exp(beam_score) 36 | timesteps (B, N_beams) 37 | out_lens (B, N_beams) 38 | ''' 39 | 40 | index_list = torch.argmax(nn_output.cpu(), axis=2) 41 | batchsize, lgt = index_list.shape 42 | blank_rate =[] 43 | for batch_idx in range(batchsize): 44 | group_result = [x.item() for x in index_list[batch_idx][:int(vid_lgt[batch_idx])]] 45 | blank_rate.append(group_result) 46 | 47 | 48 | if not probs: 49 | nn_output = nn_output.softmax(-1).cpu() 50 | vid_lgt = vid_lgt.cpu() 51 | beam_result, beam_scores, timesteps, out_seq_len = self.ctc_decoder.decode(nn_output, vid_lgt) 52 | ret_list = [] 53 | for batch_idx in range(len(nn_output)): 54 | first_result = beam_result[batch_idx][0][:out_seq_len[batch_idx][0]] 55 | if len(first_result) != 0: 56 | first_result = torch.stack([x[0] for x in groupby(first_result)]) 57 | # ret_list.append([str(int(gloss_id)) for idx, gloss_id in enumerate(first_result)]) 58 | ret_list.append([self.i2g_dict[int(gloss_id)] for idx, gloss_id in 59 | enumerate(first_result)]) 60 | return ret_list 61 | 62 | def MaxDecode(self, nn_output, vid_lgt): 63 | index_list = torch.argmax(nn_output, axis=2) 64 | batchsize, lgt = index_list.shape 65 | ret_list = [] 66 | for batch_idx in range(batchsize): 67 | group_result = [x[0] for x in groupby(index_list[batch_idx][:vid_lgt[batch_idx]])] 68 | filtered = [*filter(lambda x: x != self.blank_id, group_result)] 69 | if len(filtered) > 0: 70 | max_result = torch.stack(filtered) 71 | max_result = [x[0] for x in groupby(max_result)] 72 | else: 73 | max_result = filtered 74 | ret_list.append([(self.i2g_dict[int(gloss_id)], idx) for idx, gloss_id in 75 | enumerate(max_result)]) 76 | return ret_list 77 | -------------------------------------------------------------------------------- /utils/device.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pdb 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class GpuDataParallel(object): 8 | def __init__(self): 9 | self.gpu_list = [] 10 | self.output_device = None 11 | 12 | def set_device(self, device): 13 | device = str(device) 14 | if device != 'None': 15 | self.gpu_list = [i for i in range(len(device.split(',')))] 16 | print(self.gpu_list) 17 | os.environ["CUDA_VISIBLE_DEVICES"] = device 18 | output_device = self.gpu_list[0] 19 | 20 | self.occupy_gpu(self.gpu_list) 21 | self.output_device = output_device if len(self.gpu_list) > 0 else "cpu" 22 | 23 | def model_to_device(self, model): 24 | # model = convert_model(model) 25 | model = model.to(self.output_device) 26 | if len(self.gpu_list) > 1: 27 | model = nn.DataParallel( 28 | model, 29 | device_ids=self.gpu_list, 30 | output_device=self.output_device) 31 | return model 32 | 33 | def data_to_device(self, data): 34 | if isinstance(data, torch.FloatTensor): 35 | return data.to(self.output_device) 36 | elif isinstance(data, torch.DoubleTensor): 37 | return data.float().to(self.output_device) 38 | elif isinstance(data, torch.ByteTensor): 39 | return data.long().to(self.output_device) 40 | elif isinstance(data, torch.LongTensor): 41 | return data.to(self.output_device) 42 | elif isinstance(data, list) or isinstance(data, tuple): 43 | return [self.data_to_device(d) for d in data] 44 | else: 45 | raise ValueError(data.shape, "Unknown Dtype: {}".format(data.dtype)) 46 | 47 | def criterion_to_device(self, loss): 48 | return loss.to(self.output_device) 49 | 50 | def occupy_gpu(self, gpus=None): 51 | """ 52 | make program appear on nvidia-smi. 53 | """ 54 | if len(gpus) == 0: 55 | torch.zeros(1).cuda() 56 | else: 57 | gpus = [gpus] if isinstance(gpus, int) else list(gpus) 58 | for g in gpus: 59 | torch.zeros(1).cuda(g) 60 | -------------------------------------------------------------------------------- /utils/metrics.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | """ 3 | This module holds various MT evaluation metrics. 4 | """ 5 | 6 | from utils import Rouge,sacrebleu 7 | import numpy as np 8 | 9 | WER_COST_DEL = 3 10 | WER_COST_INS = 3 11 | WER_COST_SUB = 4 12 | 13 | 14 | def chrf(references, hypotheses): 15 | """ 16 | Character F-score from sacrebleu 17 | :param hypotheses: list of hypotheses (strings) 18 | :param references: list of references (strings) 19 | :return: 20 | """ 21 | return ( 22 | sacrebleu.corpus_chrf(hypotheses=hypotheses, references=references).score * 100 23 | ) 24 | 25 | 26 | def bleu(references, hypotheses, level='word'): 27 | """ 28 | Raw corpus BLEU from sacrebleu (without tokenization) 29 | :param hypotheses: list of hypotheses (strings) 30 | :param references: list of references (strings) 31 | :return: 32 | """ 33 | if level=='char': 34 | #split word 35 | references = [' '.join(list(r)) for r in references] 36 | hypotheses = [' '.join(list(r)) for r in hypotheses] 37 | bleu_scores = sacrebleu.raw_corpus_bleu( 38 | sys_stream=hypotheses, ref_streams=[references] 39 | ).scores 40 | scores = {} 41 | for n in range(len(bleu_scores)): 42 | scores["bleu" + str(n + 1)] = bleu_scores[n] 43 | return scores 44 | 45 | 46 | def token_accuracy(references, hypotheses, level="word"): 47 | """ 48 | Compute the accuracy of hypothesis tokens: correct tokens / all tokens 49 | Tokens are correct if they appear in the same position in the reference. 50 | :param hypotheses: list of hypotheses (strings) 51 | :param references: list of references (strings) 52 | :param level: segmentation level, either "word", "bpe", or "char" 53 | :return: 54 | """ 55 | correct_tokens = 0 56 | all_tokens = 0 57 | split_char = " " if level in ["word", "bpe"] else "" 58 | assert len(hypotheses) == len(references) 59 | for hyp, ref in zip(hypotheses, references): 60 | all_tokens += len(hyp) 61 | for h_i, r_i in zip(hyp.split(split_char), ref.split(split_char)): 62 | # min(len(h), len(r)) tokens considered 63 | if h_i == r_i: 64 | correct_tokens += 1 65 | return (correct_tokens / all_tokens) * 100 if all_tokens > 0 else 0.0 66 | 67 | 68 | def sequence_accuracy(references, hypotheses): 69 | """ 70 | Compute the accuracy of hypothesis tokens: correct tokens / all tokens 71 | Tokens are correct if they appear in the same position in the reference. 72 | :param hypotheses: list of hypotheses (strings) 73 | :param references: list of references (strings) 74 | :return: 75 | """ 76 | assert len(hypotheses) == len(references) 77 | correct_sequences = sum( 78 | [1 for (hyp, ref) in zip(hypotheses, references) if hyp == ref] 79 | ) 80 | return (correct_sequences / len(hypotheses)) * 100 if hypotheses else 0.0 81 | 82 | 83 | def rouge_deprecated(references, hypotheses, level='word'): 84 | #beta 1.2 85 | rouge_score = 0 86 | n_seq = len(hypotheses) 87 | if level=='char': 88 | #split word 89 | references = [' '.join(list(r)) for r in references] 90 | hypotheses = [' '.join(list(r)) for r in hypotheses] 91 | 92 | for h, r in zip(hypotheses, references): 93 | rouge_score += mscoco_rouge.calc_score(hypotheses=[h], references=[r]) / n_seq 94 | 95 | return rouge_score * 100 96 | 97 | def rouge(references, hypotheses, level='word'): 98 | if level=='char': 99 | hyp = [list(x) for x in hypotheses] 100 | ref = [list(x) for x in references] 101 | else: 102 | hyp = [x.split() for x in hypotheses] 103 | ref = [x.split() for x in references] 104 | a = Rouge.rouge([' '.join(x) for x in hyp], [' '.join(x) for x in ref]) 105 | return a['rouge_l/f_score']*100 106 | 107 | def wer_list(references, hypotheses): 108 | total_error = total_del = total_ins = total_sub = total_ref_len = 0 109 | 110 | for r, h in zip(references, hypotheses): 111 | res = wer_single(r=r, h=h) 112 | # print("debug==", r, "||", h) 113 | total_error += res["num_err"] 114 | total_del += res["num_del"] 115 | total_ins += res["num_ins"] 116 | total_sub += res["num_sub"] 117 | total_ref_len += res["num_ref"] 118 | 119 | wer = (total_error / total_ref_len) * 100 120 | del_rate = (total_del / total_ref_len) * 100 121 | ins_rate = (total_ins / total_ref_len) * 100 122 | sub_rate = (total_sub / total_ref_len) * 100 123 | 124 | return { 125 | "wer": wer, 126 | "del": del_rate, 127 | "ins": ins_rate, 128 | "sub": sub_rate, 129 | # "del":total_del, 130 | # "ins":total_ins, 131 | # "sub":total_sub, 132 | # "ref_len":total_ref_len, 133 | # "error":total_error, 134 | } 135 | 136 | 137 | def wer_single(r, h): 138 | # print("debug==", r, h) 139 | r = r.strip().split() 140 | h = h.strip().split() 141 | edit_distance_matrix = edit_distance(r=r, h=h) 142 | alignment, alignment_out = get_alignment(r=r, h=h, d=edit_distance_matrix) 143 | num_cor = np.sum([s == "C" for s in alignment]) 144 | num_del = np.sum([s == "D" for s in alignment]) 145 | num_ins = np.sum([s == "I" for s in alignment]) 146 | num_sub = np.sum([s == "S" for s in alignment]) 147 | num_err = num_del + num_ins + num_sub 148 | num_ref = len(r) 149 | 150 | return { 151 | "alignment": alignment, 152 | "alignment_out": alignment_out, 153 | "num_cor": num_cor, 154 | "num_del": num_del, 155 | "num_ins": num_ins, 156 | "num_sub": num_sub, 157 | "num_err": num_err, 158 | "num_ref": num_ref, 159 | } 160 | 161 | 162 | def edit_distance(r, h): 163 | """ 164 | Original Code from https://github.com/zszyellow/WER-in-python/blob/master/wer.py 165 | This function is to calculate the edit distance of reference sentence and the hypothesis sentence. 166 | Main algorithm used is dynamic programming. 167 | Attributes: 168 | r -> the list of words produced by splitting reference sentence. 169 | h -> the list of words produced by splitting hypothesis sentence. 170 | """ 171 | d = np.zeros((len(r) + 1) * (len(h) + 1), dtype=np.uint8).reshape( 172 | (len(r) + 1, len(h) + 1) 173 | ) 174 | for i in range(len(r) + 1): 175 | for j in range(len(h) + 1): 176 | if i == 0: 177 | # d[0][j] = j 178 | d[0][j] = j * WER_COST_INS 179 | elif j == 0: 180 | d[i][0] = i * WER_COST_DEL 181 | for i in range(1, len(r) + 1): 182 | for j in range(1, len(h) + 1): 183 | if r[i - 1] == h[j - 1]: 184 | d[i][j] = d[i - 1][j - 1] 185 | else: 186 | substitute = d[i - 1][j - 1] + WER_COST_SUB 187 | insert = d[i][j - 1] + WER_COST_INS 188 | delete = d[i - 1][j] + WER_COST_DEL 189 | d[i][j] = min(substitute, insert, delete) 190 | return d 191 | 192 | 193 | 194 | def get_alignment(r, h, d): 195 | """ 196 | Original Code from https://github.com/zszyellow/WER-in-python/blob/master/wer.py 197 | This function is to get the list of steps in the process of dynamic programming. 198 | Attributes: 199 | r -> the list of words produced by splitting reference sentence. 200 | h -> the list of words produced by splitting hypothesis sentence. 201 | d -> the matrix built when calculating the editing distance of h and r. 202 | """ 203 | x = len(r) 204 | y = len(h) 205 | max_len = 3 * (x + y) 206 | 207 | alignlist = [] 208 | align_ref = "" 209 | align_hyp = "" 210 | alignment = "" 211 | 212 | while True: 213 | if (x <= 0 and y <= 0) or (len(alignlist) > max_len): 214 | break 215 | elif x >= 1 and y >= 1 and d[x][y] == d[x - 1][y - 1] and r[x - 1] == h[y - 1]: 216 | align_hyp = " " + h[y - 1] + align_hyp 217 | align_ref = " " + r[x - 1] + align_ref 218 | alignment = " " * (len(r[x - 1]) + 1) + alignment 219 | alignlist.append("C") 220 | x = max(x - 1, 0) 221 | y = max(y - 1, 0) 222 | elif x >= 1 and y >= 1 and d[x][y] == d[x - 1][y - 1] + WER_COST_SUB: 223 | ml = max(len(h[y - 1]), len(r[x - 1])) 224 | align_hyp = " " + h[y - 1].ljust(ml) + align_hyp 225 | align_ref = " " + r[x - 1].ljust(ml) + align_ref 226 | alignment = " " + "S" + " " * (ml - 1) + alignment 227 | alignlist.append("S") 228 | x = max(x - 1, 0) 229 | y = max(y - 1, 0) 230 | elif y >= 1 and d[x][y] == d[x][y - 1] + WER_COST_INS: 231 | align_hyp = " " + h[y - 1] + align_hyp 232 | align_ref = " " + "*" * len(h[y - 1]) + align_ref 233 | alignment = " " + "I" + " " * (len(h[y - 1]) - 1) + alignment 234 | alignlist.append("I") 235 | x = max(x, 0) 236 | y = max(y - 1, 0) 237 | else: 238 | align_hyp = " " + "*" * len(r[x - 1]) + align_hyp 239 | align_ref = " " + r[x - 1] + align_ref 240 | alignment = " " + "D" + " " * (len(r[x - 1]) - 1) + alignment 241 | alignlist.append("D") 242 | x = max(x - 1, 0) 243 | y = max(y, 0) 244 | 245 | align_ref = align_ref[1:] 246 | align_hyp = align_hyp[1:] 247 | alignment = alignment[1:] 248 | 249 | return ( 250 | alignlist[::-1], 251 | {"align_ref": align_ref, "align_hyp": align_hyp, "alignment": alignment}, 252 | ) -------------------------------------------------------------------------------- /utils/misc.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import glob 3 | import os 4 | import os.path 5 | import errno 6 | import shutil 7 | import random 8 | import logging 9 | from sys import platform 10 | from logging import Logger 11 | from typing import Callable, Optional 12 | import numpy as np 13 | 14 | import torch 15 | from torch import nn, Tensor 16 | import yaml 17 | from torch.utils.tensorboard import SummaryWriter 18 | import wandb 19 | 20 | def make_wandb(model_dir, cfg): 21 | if is_main_process(): 22 | wandb.login(key='provide your key here') 23 | run = wandb.init(project='TwoStreamSLT', config=cfg, reinit=True) 24 | wandb.run.name = '/'.join(model_dir.split('/')[-2:]) 25 | wandb.run.save() 26 | return run 27 | else: 28 | return None 29 | 30 | def neq_load_customized(model, pretrained_dict, verbose=False): 31 | ''' load pre-trained model in a not-equal way, 32 | when new model has been partially modified ''' 33 | model_dict = model.state_dict() 34 | tmp = {} 35 | if verbose: 36 | # print("print info misc.py line36 ",list(model_dict.keys())) 37 | print('\n=======Check Weights Loading======') 38 | print('Weights not used from pretrained file:') 39 | for k, v in pretrained_dict.items(): 40 | if k in model_dict and model_dict[k].shape==v.shape: 41 | tmp[k] = v 42 | else: 43 | if verbose: 44 | continue 45 | # print(k) 46 | if verbose: 47 | print('---------------------------') 48 | print('Weights not loaded into new model:') 49 | for k, v in model_dict.items(): 50 | if k not in pretrained_dict: 51 | print(k) 52 | elif model_dict[k].shape != pretrained_dict[k].shape: 53 | print(k, 'shape mis-matched, not loaded') 54 | print('===================================\n') 55 | 56 | del pretrained_dict 57 | model_dict.update(tmp) 58 | del tmp 59 | model.load_state_dict(model_dict) 60 | return model 61 | 62 | 63 | 64 | def move_to_device(batch, device): 65 | for k, v in batch.items(): 66 | if type(v)==dict: 67 | batch[k] = move_to_device(v, device) 68 | elif type(v)==torch.Tensor: 69 | batch[k] = v.to(device) 70 | elif type(v)==list and type(v[0])==torch.Tensor: 71 | batch[k] = [e.to(device) for e in v] 72 | return batch 73 | 74 | def make_model_dir(model_dir: str, overwrite: bool = False) -> str: 75 | """ 76 | Create a new directory for the model. 77 | :param model_dir: path to model directory 78 | :param overwrite: whether to overwrite an existing directory 79 | :return: path to model directory 80 | """ 81 | if is_main_process(): 82 | if not os.path.exists(model_dir): 83 | os.makedirs(model_dir) 84 | elif overwrite: 85 | shutil.rmtree(model_dir) 86 | os.makedirs(model_dir) 87 | synchronize() 88 | return model_dir 89 | 90 | def get_logger(): 91 | return logger 92 | 93 | def make_logger(model_dir: str, log_file: str = "train.log") -> Logger: 94 | """ 95 | Create a logger for logging the training process. 96 | :param model_dir: path to logging directory 97 | :param log_file: path to logging file 98 | :return: logger object 99 | """ 100 | global logger 101 | logger = logging.getLogger(__name__) 102 | if not logger.handlers: 103 | logger.setLevel(level=logging.DEBUG) 104 | fh = logging.FileHandler("{}/{}".format(model_dir, log_file)) 105 | fh.setLevel(level=logging.DEBUG) 106 | logger.addHandler(fh) 107 | formatter = logging.Formatter("%(asctime)s %(message)s") 108 | fh.setFormatter(formatter) 109 | if platform == "linux": 110 | sh = logging.StreamHandler() 111 | if not is_main_process(): 112 | sh.setLevel(logging.ERROR) 113 | sh.setFormatter(formatter) 114 | logging.getLogger("").addHandler(sh) 115 | return logger 116 | 117 | def make_writer(model_dir): 118 | if is_main_process(): 119 | writer = SummaryWriter(log_dir=os.path.join(model_dir + "/tensorboard/")) 120 | else: 121 | writer = None 122 | return writer 123 | 124 | def log_cfg(cfg: dict, logger: Logger, prefix: str = "cfg"): 125 | """ 126 | Write configuration to log. 127 | :param cfg: configuration to log 128 | :param logger: logger that defines where log is written to 129 | :param prefix: prefix for logging 130 | """ 131 | for k, v in cfg.items(): 132 | if isinstance(v, dict): 133 | p = ".".join([prefix, k]) 134 | log_cfg(v, logger, prefix=p) 135 | else: 136 | p = ".".join([prefix, k]) 137 | logger.info("{:34s} : {}".format(p, v)) 138 | 139 | def set_seed(seed: int): 140 | """ 141 | Set the random seed for modules torch, numpy and random. 142 | :param seed: random seed 143 | """ 144 | torch.manual_seed(seed) 145 | torch.cuda.manual_seed(seed) 146 | torch.cuda.manual_seed_all(seed) 147 | np.random.seed(seed) 148 | random.seed(seed) 149 | os.environ['PYTHONHASHSEED'] = str(seed) 150 | torch.backends.cudnn.benchmark = False 151 | torch.backends.cudnn.deterministic = True 152 | 153 | def load_config(path="configs/default.yaml") -> dict: 154 | """ 155 | Loads and parses a YAML configuration file. 156 | :param path: path to YAML configuration file 157 | :return: configuration dictionary 158 | """ 159 | with open(path, "r", encoding="utf-8") as ymlfile: 160 | cfg = yaml.safe_load(ymlfile) 161 | 162 | 163 | if 'RecognitionNetwork' in cfg['model']: 164 | if 'keypoint' in cfg['Dataprocessing'].get('input_streams', ['rgb']): 165 | assert 'keypoint_s3d' in cfg['model']['RecognitionNetwork'] 166 | from dataset.Dataset import get_keypoints_num 167 | keypoints_num = get_keypoints_num( 168 | keypoint_file=cfg['Dataprocessing']['keypoint_file'], use_keypoints=cfg['Dataprocessing']['use_keypoints']) 169 | if 'keypoint_s3d' in cfg['model']['RecognitionNetwork']: 170 | cfg['model']['RecognitionNetwork']['keypoint_s3d']['in_channel'] = keypoints_num 171 | print(f'Overwrite cfg.model.RecognitionNetwork.keypoint_s3d.in_channel -> {keypoints_num}') 172 | return cfg 173 | 174 | def get_latest_checkpoint(ckpt_dir: str) -> Optional[str]: 175 | """ 176 | Returns the latest checkpoint (by time) from the given directory. 177 | If there is no checkpoint in this directory, returns None 178 | :param ckpt_dir: 179 | :return: latest checkpoint file 180 | """ 181 | list_of_files = glob.glob("{}/*.ckpt".format(ckpt_dir)) 182 | latest_checkpoint = None 183 | if list_of_files: 184 | latest_checkpoint = max(list_of_files, key=os.path.getctime) 185 | return latest_checkpoint 186 | 187 | def load_checkpoint(path: str, map_location: str='cpu') -> dict: 188 | """ 189 | Load model from saved checkpoint. 190 | :param path: path to checkpoint 191 | :param use_cuda: using cuda or not 192 | :return: checkpoint (dict) 193 | """ 194 | assert os.path.isfile(path), "Checkpoint %s not found" % path 195 | checkpoint = torch.load(path, map_location=map_location) 196 | return checkpoint 197 | 198 | def freeze_params(module: nn.Module): 199 | """ 200 | Freeze the parameters of this module, 201 | i.e. do not update them during training 202 | :param module: freeze parameters of this module 203 | """ 204 | for _, p in module.named_parameters(): 205 | p.requires_grad = False 206 | 207 | 208 | def symlink_update(target, link_name): 209 | os.system('cp {} {}'.format(target, link_name)) 210 | # try: 211 | # os.symlink(target, link_name) 212 | # except FileExistsError as e: 213 | # if e.errno == errno.EEXIST: 214 | # os.remove(link_name) 215 | # os.symlink(target, link_name) 216 | # else: 217 | # raise e 218 | 219 | def is_main_process(): 220 | return 'WORLD_SIZE' not in os.environ or os.environ['WORLD_SIZE']=='1' or os.environ['LOCAL_RANK']=='0' 221 | 222 | 223 | 224 | def init_distributed_mode(args): 225 | if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: 226 | args.rank = int(os.environ["RANK"]) 227 | args.world_size = int(os.environ['WORLD_SIZE']) 228 | args.gpu = int(os.environ['LOCAL_RANK']) 229 | elif 'SLURM_PROCID' in os.environ: 230 | args.rank = int(os.environ['SLURM_PROCID']) 231 | args.gpu = args.rank % torch.cuda.device_count() 232 | 233 | os.environ['RANK'] = str(args.rank) 234 | os.environ['LOCAL_RANK'] = str(args.gpu) 235 | os.environ['WORLD_SIZE'] = str(args.world_size) 236 | else: 237 | print('Not using distributed mode') 238 | return 239 | 240 | torch.cuda.set_device(args.gpu) 241 | args.dist_backend = 'nccl' 242 | print('| distributed init (rank {}): {}, gpu {}'.format( 243 | args.rank, args.dist_url, args.gpu), flush=True) 244 | torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 245 | world_size=args.world_size, rank=args.rank) 246 | torch.distributed.barrier() 247 | setup_for_distributed(args.rank == 0) 248 | 249 | def init_DDP(): 250 | local_rank = int(os.environ['LOCAL_RANK']) 251 | torch.cuda.set_device(local_rank) 252 | device = torch.device('cuda:{}'.format(local_rank)) 253 | torch.distributed.init_process_group(backend="nccl", init_method="env://") 254 | return local_rank, int(os.environ['WORLD_SIZE']), device 255 | 256 | def synchronize(): 257 | torch.distributed.barrier() 258 | 259 | def freeze_params(module: nn.Module): 260 | """ 261 | Freeze the parameters of this module, 262 | i.e. do not update them during training 263 | :param module: freeze parameters of this module 264 | """ 265 | for _, p in module.named_parameters(): 266 | p.requires_grad = False 267 | 268 | 269 | # Copyright (c) Meta Platforms, Inc. and affiliates. 270 | 271 | # All rights reserved. 272 | 273 | # This source code is licensed under the license found in the 274 | # LICENSE file in the root directory of this source tree. 275 | 276 | 277 | import torch 278 | import os 279 | import torch.distributed as dist 280 | 281 | 282 | def setup_for_distributed(is_master): 283 | """ 284 | This function disables printing when not in master process 285 | """ 286 | import builtins as __builtin__ 287 | builtin_print = __builtin__.print 288 | 289 | def print(*args, **kwargs): 290 | force = kwargs.pop('force', False) 291 | if is_master or force: 292 | builtin_print(*args, **kwargs) 293 | 294 | __builtin__.print = print 295 | 296 | 297 | def is_dist_avail_and_initialized(): 298 | if not dist.is_available(): 299 | return False 300 | if not dist.is_initialized(): 301 | return False 302 | return True 303 | 304 | 305 | def get_world_size(): 306 | if not is_dist_avail_and_initialized(): 307 | return 1 308 | return dist.get_world_size() 309 | 310 | 311 | def get_rank(): 312 | if not is_dist_avail_and_initialized(): 313 | return 0 314 | return dist.get_rank() 315 | 316 | 317 | # def is_main_process(): 318 | # return get_rank() == 0 319 | 320 | def is_main_process(): 321 | return 'WORLD_SIZE' not in os.environ or os.environ['WORLD_SIZE']=='1' or os.environ['LOCAL_RANK']=='0' 322 | 323 | 324 | def save_on_master(*args, **kwargs): 325 | if is_main_process(): 326 | torch.save(*args, **kwargs) 327 | 328 | 329 | def init_distributed_mode(args): 330 | if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: 331 | args.rank = int(os.environ["RANK"]) 332 | args.world_size = int(os.environ['WORLD_SIZE']) 333 | args.gpu = int(os.environ['LOCAL_RANK']) 334 | elif 'SLURM_PROCID' in os.environ: 335 | args.rank = int(os.environ['SLURM_PROCID']) 336 | args.gpu = args.rank % torch.cuda.device_count() 337 | 338 | os.environ['RANK'] = str(args.rank) 339 | os.environ['LOCAL_RANK'] = str(args.gpu) 340 | os.environ['WORLD_SIZE'] = str(args.world_size) 341 | else: 342 | print('Not using distributed mode') 343 | return 344 | 345 | torch.cuda.set_device(args.gpu) 346 | args.dist_backend = 'nccl' 347 | print('| distributed init (rank {}): {}, gpu {}'.format( 348 | args.rank, args.dist_url, args.gpu), flush=True) 349 | torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, 350 | world_size=args.world_size, rank=args.rank) 351 | torch.distributed.barrier() 352 | setup_for_distributed(args.rank == 0) -------------------------------------------------------------------------------- /utils/optimizer.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import torch 3 | import numpy as np 4 | import torch.optim as optim 5 | 6 | 7 | class Optimizer(object): 8 | def __init__(self, model, optim_dict): 9 | self.optim_dict = optim_dict 10 | if self.optim_dict["optimizer"] == 'SGD': 11 | self.optimizer = optim.SGD( 12 | model, 13 | lr=self.optim_dict['base_lr'], 14 | momentum=0.9, 15 | nesterov=self.optim_dict['nesterov'], 16 | weight_decay=self.optim_dict['weight_decay'] 17 | ) 18 | elif self.optim_dict["optimizer"] == 'Adam': 19 | parameters = [] 20 | base_lr = self.optim_dict['learning_rate'].pop('base_lr') 21 | for n, p in model.named_children(): 22 | lr_ = base_lr 23 | for m, lr in self.optim_dict['learning_rate'].items(): 24 | if m in n: 25 | lr_ = lr 26 | print('learning rate {}={}'.format(n, lr_)) 27 | parameters.append({'params': p.parameters(), 'lr': lr_}) 28 | 29 | self.optimizer = optim.Adam( 30 | model.parameters(), 31 | # parameters, 32 | lr=base_lr, 33 | weight_decay=self.optim_dict['weight_decay'] 34 | ) 35 | else: 36 | raise ValueError() 37 | self.scheduler = self.define_lr_scheduler(self.optimizer, self.optim_dict['step']) 38 | 39 | def define_lr_scheduler(self, optimizer, milestones): 40 | if self.optim_dict['scheduler'] == "consine": 41 | print("using CosineAnnealingLR....") 42 | return optim.lr_scheduler.CosineAnnealingLR( 43 | optimizer=optimizer, 44 | eta_min=self.optim_dict['start_epoch'], 45 | T_max=self.optim_dict['num_epoch'], 46 | ) 47 | if self.optim_dict["optimizer"] in ['SGD', 'Adam']: 48 | lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.2) 49 | return lr_scheduler 50 | else: 51 | raise ValueError() 52 | 53 | def zero_grad(self): 54 | self.optimizer.zero_grad() 55 | 56 | def step(self): 57 | self.optimizer.step() 58 | 59 | def state_dict(self): 60 | return self.optimizer.state_dict() 61 | 62 | def load_state_dict(self, state_dict): 63 | self.optimizer.load_state_dict(state_dict) 64 | 65 | def to(self, device): 66 | for state in self.optimizer.state.values(): 67 | for k, v in state.items(): 68 | if isinstance(v, torch.Tensor): 69 | state[k] = v.to(device) 70 | -------------------------------------------------------------------------------- /utils/pack_code.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | import logging 4 | import os 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def pack_code(git_root, run_dir): 10 | if os.path.isdir(f"{git_root}/.git"): 11 | subprocess.run( 12 | ['git', 'archive', '-o', f"{run_dir}/code.tar.gz", 'HEAD'], 13 | check=True, 14 | ) 15 | diff_process = subprocess.run( 16 | ['git', 'diff', 'HEAD'], 17 | check=True, stdout=subprocess.PIPE, text=True, 18 | ) 19 | if diff_process.stdout: 20 | logger.warning('Working tree is dirty. Patch:\n%s', diff_process.stdout) 21 | with open(f"{run_dir}/dirty.patch", 'w') as f: 22 | f.write(diff_process.stdout) 23 | else: 24 | logger.warning('.git does not exist in current dir') 25 | 26 | -------------------------------------------------------------------------------- /utils/parameters.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def get_parser(): 5 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 6 | # parameter priority: command line > config > default 7 | parser = argparse.ArgumentParser( 8 | description='The pytorch implementation for Visual Alignment Constraint ' 9 | 'for Continuous Sign Language Recognition.') 10 | parser.add_argument( 11 | '--work-dir', 12 | default='./work_dir/temp', 13 | help='the work folder for storing results') 14 | parser.add_argument( 15 | '--config', 16 | default='./configs/baseline.yaml', 17 | help='path to the configuration file') 18 | parser.add_argument( 19 | '--random_fix', 20 | type=str2bool, 21 | default=True, 22 | help='fix random seed or not') 23 | parser.add_argument( 24 | '--device', 25 | type=str, 26 | default=0, 27 | help='the indexes of GPUs for training or testing') 28 | 29 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 30 | # processor 31 | parser.add_argument( 32 | '--phase', default='train', help='can be train, test and features') 33 | 34 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 35 | # debug 36 | parser.add_argument( 37 | '--save-interval', 38 | type=int, 39 | default=200, 40 | help='the interval for storing models (#epochs)') 41 | parser.add_argument( 42 | '--random-seed', 43 | type=int, 44 | default=0, 45 | help='the default value for random seed.') 46 | parser.add_argument( 47 | '--eval-interval', 48 | type=int, 49 | default=100, 50 | help='the interval for evaluating models (#epochs)') 51 | parser.add_argument( 52 | '--print-log', 53 | type=str2bool, 54 | default=True, 55 | help='print logging or not') 56 | parser.add_argument( 57 | '--log-interval', 58 | type=int, 59 | default=20, 60 | help='the interval for printing messages (#iteration)') 61 | parser.add_argument( 62 | '--evaluate-tool', default="python", help='sclite or python') 63 | 64 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 65 | # feeder 66 | parser.add_argument( 67 | '--feeder', default='dataloader_video.BaseFeeder', help='Dataprocessing loader will be used') 68 | parser.add_argument( 69 | '--dataset', 70 | default=None, 71 | help='Dataprocessing loader will be used' 72 | ) 73 | parser.add_argument( 74 | '--dataset-info', 75 | default=dict(), 76 | help='Dataprocessing loader will be used' 77 | ) 78 | parser.add_argument( 79 | '--num-worker', 80 | type=int, 81 | default=4, 82 | help='the number of worker for Dataprocessing loader') 83 | parser.add_argument( 84 | '--feeder-args', 85 | default=dict(), 86 | help='the arguments of Dataprocessing loader') 87 | 88 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 89 | # model 90 | parser.add_argument('--model', default=None, help='the model will be used') 91 | parser.add_argument( 92 | '--model-args', 93 | type=dict, 94 | default=dict(), 95 | help='the arguments of model') 96 | parser.add_argument( 97 | '--load-weights', 98 | default=None, 99 | help='load weights for network initialization') 100 | parser.add_argument( 101 | '--load-checkpoints', 102 | default=None, 103 | help='load checkpoints for continue training') 104 | parser.add_argument( 105 | '--decode-mode', 106 | default="max", 107 | help='search mode for decode, max or beam') 108 | parser.add_argument( 109 | '--ignore-weights', 110 | type=str, 111 | default=[], 112 | nargs='+', 113 | help='the name of weights which will be ignored in the initialization') 114 | 115 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 116 | # optim 117 | parser.add_argument( 118 | '--batch-size', type=int, default=16, help='training batch size') 119 | parser.add_argument( 120 | '--test-batch-size', type=int, default=8, help='test batch size') 121 | 122 | default_optimizer_dict = { 123 | "base_lr": 1e-2, 124 | "optimizer": "SGD", 125 | "nesterov": False, 126 | "step": [5, 10], 127 | "weight_decay": 0.00005, 128 | "start_epoch": 1, 129 | } 130 | default_loss_dict = { 131 | "SeqCTC": 1.0, 132 | } 133 | 134 | parser.add_argument( 135 | '--loss-weights', 136 | default=default_loss_dict, 137 | help='loss selection' 138 | ) 139 | 140 | parser.add_argument( 141 | '--optimizer-args', 142 | default=default_optimizer_dict, 143 | help='the arguments of optimizer') 144 | parser.add_argument( 145 | '--num-epoch', 146 | type=int, 147 | default=80, 148 | help='stop training in which epoch') 149 | 150 | 151 | 152 | parser.add_argument('--world-size', default=1, type=int, 153 | help='number of distributed processes') 154 | parser.add_argument('--local_rank', default=0, type=int) 155 | parser.add_argument('--dist-url', default='env://', 156 | help='url used to set up distributed training') 157 | 158 | 159 | 160 | 161 | return parser 162 | 163 | 164 | def str2bool(v): 165 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 166 | return True 167 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 168 | return False 169 | else: 170 | raise argparse.ArgumentTypeError('Boolean value expected.') 171 | -------------------------------------------------------------------------------- /utils/random_state.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import torch 3 | import random 4 | import numpy as np 5 | import os 6 | 7 | 8 | class RandomState(object): 9 | def __init__(self, seed): 10 | torch.set_num_threads(1) 11 | os.environ['PYTHONHASHSEED'] = str(seed) 12 | torch.backends.cudnn.deterministic = True 13 | torch.backends.cudnn.benchmark = False 14 | torch.manual_seed(seed) 15 | torch.cuda.manual_seed(seed) 16 | torch.cuda.manual_seed_all(seed) 17 | np.random.seed(seed) 18 | random.seed(seed) 19 | 20 | def save_rng_state(self): 21 | rng_dict = {} 22 | rng_dict["torch"] = torch.get_rng_state() 23 | rng_dict["cuda"] = torch.cuda.get_rng_state_all() 24 | rng_dict["numpy"] = np.random.get_state() 25 | rng_dict["random"] = random.getstate() 26 | return rng_dict 27 | 28 | def set_rng_state(self, rng_dict): 29 | torch.set_rng_state(rng_dict["torch"]) 30 | torch.cuda.set_rng_state_all(rng_dict["cuda"]) 31 | np.random.set_state(rng_dict["numpy"]) 32 | random.setstate(rng_dict["random"]) 33 | -------------------------------------------------------------------------------- /utils/record.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | import time 3 | 4 | 5 | class Recorder(object): 6 | def __init__(self, work_dir, print_log, log_interval): 7 | self.cur_time = time.time() 8 | self.print_log_flag = print_log 9 | self.log_interval = log_interval 10 | self.log_path = '{}/log.txt'.format(work_dir) 11 | self.timer = dict(dataloader=0.001, device=0.001, forward=0.001, backward=0.001) 12 | 13 | def print_time(self): 14 | localtime = time.asctime(time.localtime(time.time())) 15 | self.print_log("Local current time : " + localtime) 16 | 17 | def print_log(self, str, path=None, print_time=True): 18 | if path is None: 19 | path = self.log_path 20 | if print_time: 21 | localtime = time.asctime(time.localtime(time.time())) 22 | str = "[ " + localtime + ' ] ' + str 23 | print(str) 24 | if self.print_log_flag: 25 | with open(path, 'a') as f: 26 | f.writelines(str) 27 | f.writelines("\n") 28 | 29 | def record_time(self): 30 | self.cur_time = time.time() 31 | return self.cur_time 32 | 33 | def split_time(self): 34 | split_time = time.time() - self.cur_time 35 | self.record_time() 36 | return split_time 37 | 38 | def timer_reset(self): 39 | self.cur_time = time.time() 40 | self.timer = dict(dataloader=0.001, device=0.001, forward=0.001, backward=0.001) 41 | 42 | def record_timer(self, key): 43 | self.timer[key] += self.split_time() 44 | 45 | def print_time_statistics(self): 46 | proportion = { 47 | k: '{:02d}%'.format(int(round(v * 100 / sum(self.timer.values())))) 48 | for k, v in self.timer.items()} 49 | self.print_log( 50 | '\tTime consumption: [Data]{dataloader}, [GPU]{device}, [Forward]{forward}, [Backward]{backward}'.format( 51 | **proportion)) 52 | -------------------------------------------------------------------------------- /utils/video_augmentation.py: -------------------------------------------------------------------------------- 1 | 2 | import PIL 3 | import copy 4 | import scipy.misc 5 | import torch 6 | import random 7 | import numbers 8 | import numpy as np 9 | from torchvision.transforms import transforms 10 | from itertools import groupby 11 | import re 12 | import numpy as np 13 | 14 | def clean_phoenix_2014(prediction): 15 | # TODO (Cihan): Python version of the evaluation script provided 16 | # by the phoenix2014 dataset (not phoenix2014t). This should work 17 | # as intended but further tests are required to make sure it is 18 | # consistent with the bash/sed based clean up script. 19 | 20 | prediction = prediction.strip() 21 | prediction = re.sub(r"loc-", "", prediction) 22 | prediction = re.sub(r"cl-", "", prediction) 23 | prediction = re.sub(r"qu-", "", prediction) 24 | prediction = re.sub(r"poss-", "", prediction) 25 | prediction = re.sub(r"lh-", "", prediction) 26 | prediction = re.sub(r"S0NNE", "SONNE", prediction) 27 | prediction = re.sub(r"HABEN2", "HABEN", prediction) 28 | prediction = re.sub(r"__EMOTION__", "", prediction) 29 | prediction = re.sub(r"__PU__", "", prediction) 30 | prediction = re.sub(r"__LEFTHAND__", "", prediction) 31 | prediction = re.sub(r"WIE AUSSEHEN", "WIE-AUSSEHEN", prediction) 32 | prediction = re.sub(r"ZEIGEN ", "ZEIGEN-BILDSCHIRM ", prediction) 33 | prediction = re.sub(r"ZEIGEN$", "ZEIGEN-BILDSCHIRM", prediction) 34 | prediction = re.sub(r"^([A-Z]) ([A-Z][+ ])", r"\1+\2", prediction) 35 | prediction = re.sub(r"[ +]([A-Z]) ([A-Z]) ", r" \1+\2 ", prediction) 36 | prediction = re.sub(r"([ +][A-Z]) ([A-Z][ +])", r"\1+\2", prediction) 37 | prediction = re.sub(r"([ +][A-Z]) ([A-Z][ +])", r"\1+\2", prediction) 38 | prediction = re.sub(r"([ +][A-Z]) ([A-Z][ +])", r"\1+\2", prediction) 39 | prediction = re.sub(r"([ +]SCH) ([A-Z][ +])", r"\1+\2", prediction) 40 | prediction = re.sub(r"([ +]NN) ([A-Z][ +])", r"\1+\2", prediction) 41 | prediction = re.sub(r"([ +][A-Z]) (NN[ +])", r"\1+\2", prediction) 42 | prediction = re.sub(r"([ +][A-Z]) ([A-Z])$", r"\1+\2", prediction) 43 | prediction = re.sub(r"([A-Z][A-Z])RAUM", r"\1", prediction) 44 | prediction = re.sub(r"-PLUSPLUS", "", prediction) 45 | prediction = re.sub(r" +", " ", prediction) 46 | prediction = re.sub(r"(? im_w: 214 | pad = crop_w - im_w 215 | clip = [np.pad(img, ((0, 0), (pad // 2, pad - pad // 2), (0, 0)), 'constant', constant_values=0) for img in 216 | clip] 217 | w1 = 0 218 | else: 219 | w1 = random.randint(0, im_w - crop_w) 220 | 221 | if crop_h > im_h: 222 | pad = crop_h - im_h 223 | clip = [np.pad(img, ((pad // 2, pad - pad // 2), (0, 0), (0, 0)), 'constant', constant_values=0) for img in 224 | clip] 225 | h1 = 0 226 | else: 227 | h1 = random.randint(0, im_h - crop_h) 228 | 229 | if isinstance(clip[0], np.ndarray): 230 | return [img[h1:h1 + crop_h, w1:w1 + crop_w, :] for img in clip] 231 | elif isinstance(clip[0], PIL.Image.Image): 232 | return [img.crop((w1, h1, w1 + crop_w, h1 + crop_h)) for img in clip] 233 | 234 | 235 | class CenterCrop(object): 236 | def __init__(self, size): 237 | if isinstance(size, numbers.Number): 238 | self.size = (int(size), int(size)) 239 | else: 240 | self.size = size 241 | 242 | def __call__(self, clip): 243 | try: 244 | im_h, im_w, im_c = clip[0].shape 245 | except ValueError: 246 | print(clip[0].shape) 247 | new_h, new_w = self.size 248 | new_h = im_h if new_h >= im_h else new_h 249 | new_w = im_w if new_w >= im_w else new_w 250 | top = int(round((im_h - new_h) / 2.)) 251 | left = int(round((im_w - new_w) / 2.)) 252 | return [img[top:top + new_h, left:left + new_w] for img in clip] 253 | 254 | 255 | class RandomHorizontalFlip(object): 256 | def __init__(self, prob): 257 | self.prob = prob 258 | 259 | def __call__(self, clip): 260 | # B, H, W, 3 261 | flag = random.random() < self.prob 262 | if flag: 263 | clip = np.flip(clip, axis=2) 264 | clip = np.ascontiguousarray(copy.deepcopy(clip)) 265 | return np.array(clip) 266 | 267 | 268 | class RandomRotation(object): 269 | """ 270 | Rotate entire clip randomly by a random angle within 271 | given bounds 272 | Args: 273 | degrees (sequence or int): Range of degrees to select from 274 | If degrees is a number instead of sequence like (min, max), 275 | the range of degrees, will be (-degrees, +degrees). 276 | """ 277 | 278 | def __init__(self, degrees): 279 | if isinstance(degrees, numbers.Number): 280 | if degrees < 0: 281 | raise ValueError('If degrees is a single number,' 282 | 'must be positive') 283 | degrees = (-degrees, degrees) 284 | else: 285 | if len(degrees) != 2: 286 | raise ValueError('If degrees is a sequence,' 287 | 'it must be of len 2.') 288 | self.degrees = degrees 289 | 290 | def __call__(self, clip): 291 | """ 292 | Args: 293 | img (PIL.Image or numpy.ndarray): List of images to be cropped 294 | in format (h, w, c) in numpy.ndarray 295 | Returns: 296 | PIL.Image or numpy.ndarray: Cropped list of images 297 | """ 298 | angle = random.uniform(self.degrees[0], self.degrees[1]) 299 | if isinstance(clip[0], np.ndarray): 300 | rotated = [scipy.misc.imrotate(img, angle) for img in clip] 301 | elif isinstance(clip[0], PIL.Image.Image): 302 | rotated = [img.rotate(angle) for img in clip] 303 | else: 304 | raise TypeError('Expected numpy.ndarray or PIL.Image' + 305 | 'but got list of {0}'.format(type(clip[0]))) 306 | return rotated 307 | 308 | 309 | class TemporalRescale(object): 310 | def __init__(self, temp_scaling=0.2, frame_interval=1): 311 | self.min_len = 32 312 | self.max_len = int(np.ceil(230/frame_interval)) 313 | self.L = 1.0 - temp_scaling 314 | self.U = 1.0 + temp_scaling 315 | 316 | def __call__(self, clip): 317 | vid_len = len(clip) 318 | new_len = int(vid_len * (self.L + (self.U - self.L) * np.random.random())) 319 | if new_len < self.min_len: 320 | new_len = self.min_len 321 | if new_len > self.max_len: 322 | new_len = self.max_len 323 | if (new_len - 4) % 4 != 0: 324 | new_len += 4 - (new_len - 4) % 4 325 | if new_len <= vid_len: 326 | index = sorted(random.sample(range(vid_len), new_len)) 327 | else: 328 | index = sorted(random.choices(range(vid_len), k=new_len)) 329 | return clip[index] 330 | 331 | 332 | class RandomResize(object): 333 | """ 334 | Resize video bysoomingin and out. 335 | Args: 336 | rate (float): Video is scaled uniformly between 337 | [1 - rate, 1 + rate]. 338 | interp (string): Interpolation to use for re-sizing 339 | ('nearest', 'lanczos', 'bilinear', 'bicubic' or 'cubic'). 340 | """ 341 | 342 | def __init__(self, rate=0.0, interp='bilinear'): 343 | self.rate = rate 344 | self.interpolation = interp 345 | 346 | def __call__(self, clip): 347 | scaling_factor = random.uniform(1 - self.rate, 1 + self.rate) 348 | 349 | if isinstance(clip[0], np.ndarray): 350 | im_h, im_w, im_c = clip[0].shape 351 | elif isinstance(clip[0], PIL.Image.Image): 352 | im_w, im_h = clip[0].size 353 | 354 | new_w = int(im_w * scaling_factor) 355 | new_h = int(im_h * scaling_factor) 356 | new_size = (new_h, new_w) 357 | if isinstance(clip[0], np.ndarray): 358 | return [scipy.misc.imresize(img, size=(new_h, new_w), interp=self.interpolation) for img in clip] 359 | elif isinstance(clip[0], PIL.Image.Image): 360 | return [img.resize(size=(new_w, new_h), resample=self._get_PIL_interp(self.interpolation)) for img in clip] 361 | else: 362 | raise TypeError('Expected numpy.ndarray or PIL.Image' + 363 | 'but got list of {0}'.format(type(clip[0]))) 364 | 365 | def _get_PIL_interp(self, interp): 366 | if interp == 'nearest': 367 | return PIL.Image.NEAREST 368 | elif interp == 'lanczos': 369 | return PIL.Image.LANCZOS 370 | elif interp == 'bilinear': 371 | return PIL.Image.BILINEAR 372 | elif interp == 'bicubic': 373 | return PIL.Image.BICUBIC 374 | elif interp == 'cubic': 375 | return PIL.Image.CUBIC 376 | 377 | 378 | class Resize(object): 379 | """ 380 | Resize video bysoomingin and out. 381 | Args: 382 | rate (float): Video is scaled uniformly between 383 | [1 - rate, 1 + rate]. 384 | interp (string): Interpolation to use for re-sizing 385 | ('nearest', 'lanczos', 'bilinear', 'bicubic' or 'cubic'). 386 | """ 387 | 388 | def __init__(self, rate=0.0, interp='bilinear'): 389 | self.rate = rate 390 | self.interpolation = interp 391 | 392 | def __call__(self, clip): 393 | if self.rate == 1.0: 394 | return clip 395 | scaling_factor = self.rate 396 | 397 | if isinstance(clip[0], np.ndarray): 398 | im_h, im_w, im_c = clip[0].shape 399 | elif isinstance(clip[0], PIL.Image.Image): 400 | im_w, im_h = clip[0].size 401 | 402 | new_w = int(im_w * scaling_factor) if scaling_factor>0 and scaling_factor<=1 else int(scaling_factor) 403 | new_h = int(im_h * scaling_factor) if scaling_factor>0 and scaling_factor<=1 else int(scaling_factor) 404 | new_size = (new_w, new_h) 405 | if isinstance(clip[0], np.ndarray): 406 | return [np.array(PIL.Image.fromarray(img).resize(new_size)) for img in clip] 407 | elif isinstance(clip[0], PIL.Image.Image): 408 | return [img.resize(size=(new_w, new_h), resample=self._get_PIL_interp(self.interpolation)) for img in clip] 409 | else: 410 | raise TypeError('Expected numpy.ndarray or PIL.Image' + 411 | 'but got list of {0}'.format(type(clip[0]))) 412 | 413 | def _get_PIL_interp(self, interp): 414 | if interp == 'nearest': 415 | return PIL.Image.NEAREST 416 | elif interp == 'lanczos': 417 | return PIL.Image.LANCZOS 418 | elif interp == 'bilinear': 419 | return PIL.Image.BILINEAR 420 | elif interp == 'bicubic': 421 | return PIL.Image.BICUBIC 422 | elif interp == 'cubic': 423 | return PIL.Image.CUBIC 424 | --------------------------------------------------------------------------------